From bdcec3ed2667e985f1b564cfa6a2558b366acb11 Mon Sep 17 00:00:00 2001 From: Anurag Mittal Date: Wed, 6 Nov 2024 09:30:04 +0100 Subject: [PATCH] COSI-14: add-iam-s3-logging-and-cleanup-in-CI 1. IAM and S3 Deployment: Added configuration files and Docker Compose setup for deploying IAM and S3 services in the CI pipeline (cloudserver-config.json and docker-compose.yml). 2. Docker Login: Configured Docker login with updated credentials and personal token for accessing private repositories. 3. Logging and Debugging: - Enabled detailed logging and data collection for debugging. --- .github/s3_and_iam_deployment/.env | 2 + .../cloudserver-config.json | 49 ++++++++++ .../s3_and_iam_deployment/docker-compose.yml | 25 +++++ .../s3_and_iam_deployment/vault-config.json | 92 +++++++++++++++++++ .github/scripts/wait_for_local_port.bash | 28 ++++++ .github/workflows/ci-e2e-tests.yml | 56 ++++++++++- 6 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 .github/s3_and_iam_deployment/.env create mode 100644 .github/s3_and_iam_deployment/cloudserver-config.json create mode 100644 .github/s3_and_iam_deployment/docker-compose.yml create mode 100644 .github/s3_and_iam_deployment/vault-config.json create mode 100644 .github/scripts/wait_for_local_port.bash diff --git a/.github/s3_and_iam_deployment/.env b/.github/s3_and_iam_deployment/.env new file mode 100644 index 0000000..38eddc5 --- /dev/null +++ b/.github/s3_and_iam_deployment/.env @@ -0,0 +1,2 @@ +VAULT_IMAGE="ghcr.io/scality/vault:7.70.26" +CLOUDSERVER_IMAGE="ghcr.io/scality/cloudserver:7.70.55" diff --git a/.github/s3_and_iam_deployment/cloudserver-config.json b/.github/s3_and_iam_deployment/cloudserver-config.json new file mode 100644 index 0000000..d43a7b9 --- /dev/null +++ b/.github/s3_and_iam_deployment/cloudserver-config.json @@ -0,0 +1,49 @@ +{ + "port": 8000, + "listenOn": [], + "restEndpoints": { + "localhost": "us-east-1", + "127.0.0.1": "us-east-1", + "cloudserver-front": "us-east-1", + "s3.docker.test": "us-east-1", + "127.0.0.2": "us-east-1", + "s3.amazonaws.com": "us-east-1" + }, + "websiteEndpoints": [ + "s3-website-us-east-1.amazonaws.com", + "s3-website.us-east-2.amazonaws.com", + "s3-website-us-west-1.amazonaws.com", + "s3-website-us-west-2.amazonaws.com", + "s3-website.ap-south-1.amazonaws.com", + "s3-website.ap-northeast-2.amazonaws.com", + "s3-website-ap-southeast-1.amazonaws.com", + "s3-website-ap-southeast-2.amazonaws.com", + "s3-website-ap-northeast-1.amazonaws.com", + "s3-website.eu-central-1.amazonaws.com", + "s3-website-eu-west-1.amazonaws.com", + "s3-website-sa-east-1.amazonaws.com", + "s3-website.localhost", + "s3-website.scality.test" + ], + "vaultd": { + "host": "localhost", + "port": 8500 + }, + "clusters": 1, + "log": { + "logLevel": "trace", + "dumpLevel": "error" + }, + "healthChecks": { + "allowFrom": ["127.0.0.1/8", "::1"] + }, + "recordLog": { + "enabled": false, + "recordLogName": "s3-recordlog" + }, + "requests": { + "viaProxy": false, + "trustedProxyCIDRs": [], + "extractClientIPFromHeader": "" + } +} diff --git a/.github/s3_and_iam_deployment/docker-compose.yml b/.github/s3_and_iam_deployment/docker-compose.yml new file mode 100644 index 0000000..9bb9e15 --- /dev/null +++ b/.github/s3_and_iam_deployment/docker-compose.yml @@ -0,0 +1,25 @@ +services: + s3: + profiles: ['iam_s3'] + image: ${CLOUDSERVER_IMAGE} + network_mode: host + environment: + S3VAULT: scality + S3_CONFIG_FILE: /conf/config.json + command: /bin/sh -c "yarn run mem_backend > /logs/s3/s3.log 2>&1" + volumes: + - ./cloudserver-config.json:/conf/config.json:ro + - ./logs/s3:/logs/s3 + + iam: + profiles: ['iam_s3'] + image: ${VAULT_IMAGE} + network_mode: host + command: /bin/sh -c "chmod 400 tests/utils/keyfile && yarn start > /logs/iam/iam.log 2>&1" + environment: + VAULT_CONFIG_FILE: /conf/config.json + VAULT_DB_BACKEND: LEVELDB + volumes: + - ./vault-config.json:/conf/config.json:ro + - ./data/vaultdb:/data + - ./logs/iam:/logs/iam diff --git a/.github/s3_and_iam_deployment/vault-config.json b/.github/s3_and_iam_deployment/vault-config.json new file mode 100644 index 0000000..179f841 --- /dev/null +++ b/.github/s3_and_iam_deployment/vault-config.json @@ -0,0 +1,92 @@ +{ + "clusters": 1, + "healthChecks": { + "allowFrom": ["127.0.0.1/8", "::1"] + }, + "interfaces": { + "S3": { + "address": "0.0.0.0", + "port": 8500, + "allowFrom": ["0.0.0.0/8", "::1"] + }, + "administration": { + "address": "0.0.0.0", + "port": 8600 + }, + "sts": { + "address": "127.0.0.1", + "port": 8800 + }, + "sso": { + "address": "127.0.0.1", + "port": 8700, + "cert": "./tests/utils/sso/defaultCert.crt", + "key": "./tests/utils/sso/defaultCert.key" + } + }, + "sso": { + "clients": [ + { + "id": "grafana", + "secret": "123123", + "redirectUri": [ + "http://localhost:3000/login/generic_oauth", + "https://localhost:3000/login/generic_oauth" + ], + "scopes": ["dev", "prod"] + } + ], + "publicClients": [ + { + "id": "identisee", + "redirectUri": "https://127.0.0.1:8700/user/info", + "scopes": ["admin", "superadmin"] + } + ], + "authCodeTTL": 480, + "accessTokenTTL": 7600 + }, + "map": ["127.0.0.1:4300", "127.0.0.2:4301", "127.0.0.3:4302", "127.0.0.4:4303", "127.0.0.5:4304"], + "keyFilePath": "./tests/utils/keyfile", + "adminCredentialsFilePath": "./tests/utils/admincredentials.json.encrypted", + "log": { + "level": "trace", + "dump": "error" + }, + "accountSeeds": [ + { + "role": { + "roleName": "scality-role1", + "trustPolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { "AWS": "arn:aws:iam::000000000000:user/root" }, + "Action": "sts:AssumeRole", + "Condition": {} + } + ] + } + }, + "permissionPolicy": { + "policyName": "scality-policy1", + "policyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "FullAccess", + "Effect": "Allow", + "Action": ["s3:*"], + "Resource": ["*"] + } + ] + } + } + } + ], + "utapi": { + "host": "127.0.0.1", + "port": 8100 + } +} diff --git a/.github/scripts/wait_for_local_port.bash b/.github/scripts/wait_for_local_port.bash new file mode 100644 index 0000000..0ab03d2 --- /dev/null +++ b/.github/scripts/wait_for_local_port.bash @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +wait_for_local_port() { + local port=$1 + local timeout=$2 + local count=0 + local ret=1 + echo "waiting for storage-service:$port" + while [[ "$ret" -eq "1" && "$count" -lt "$timeout" ]] ; do + nc -z -w 1 localhost $port + ret=$? + if [ ! "$ret" -eq "0" ]; then + echo -n . + sleep 1 + count=$(($count+1)) + fi + done + + echo "" + + if [[ "$count" -eq "$timeout" ]]; then + echo "Server did not start in less than $timeout seconds. Exiting..." + exit 1 + fi + + echo "Server got ready in ~${count} seconds. Starting test now..." +} + +wait_for_local_port $1 $2 diff --git a/.github/workflows/ci-e2e-tests.yml b/.github/workflows/ci-e2e-tests.yml index 92ecc23..8aa6f4f 100644 --- a/.github/workflows/ci-e2e-tests.yml +++ b/.github/workflows/ci-e2e-tests.yml @@ -37,13 +37,67 @@ jobs: with: detached: true - - name: Setup COSI Controller, CRDs and Driver run: | pwd chmod +x .github/scripts/setup_cosi_resources.sh .github/scripts/setup_cosi_resources.sh + - name: Login to Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: "${{ github.repository_owner }}" + password: "${{ github.token }}" + + - name: Restore Cached Docker Images + id: cache_docker_images + uses: actions/cache@v4 + with: + path: /tmp/.docker_cache + key: docker-${{ runner.os }}-${{ hashFiles('.github/s3_and_iam_deployment/.env') }} + restore-keys: | + docker-${{ runner.os }}- + + - name: Load Cached Images + run: | + for image in /tmp/.docker_cache/*.tar; do + docker load -i "$image" || true + done + continue-on-error: true + + - name: Setup IAM and S3 Services + run: |- + set -e -o pipefail; + mkdir -p logs/s3 logs/iam logs/cosi_driver data/vaultdb && chown -R runner:docker logs data && chmod -R ugo+rwx logs data + docker compose --profile iam_s3 up -d --quiet-pull + bash ../scripts/wait_for_local_port.bash 8600 30 + bash ../scripts/wait_for_local_port.bash 8000 30 + working-directory: .github/s3_and_iam_deployment + + - name: Save Images to Cache if not present + if: steps.cache_docker_images.outputs.cache-hit != 'true' + run: | + source .github/s3_and_iam_deployment/.env + echo "Vault Image: $VAULT_IMAGE" + echo "CloudServer Image: $CLOUDSERVER_IMAGE" + mkdir -p /tmp/.docker_cache + docker save "$VAULT_IMAGE" -o /tmp/.docker_cache/vault_image.tar + docker save "$CLOUDSERVER_IMAGE" -o /tmp/.docker_cache/cloudserver_image.tar + shell: bash + + - name: Cleaup IAM and S3 Services + run: docker compose --profile iam_s3 down + working-directory: .github/s3_and_iam_deployment + + - name: Move S3 and IAM logs and data to artifacts directory + run: |- + set -e -o pipefail; + mkdir -p .github/e2e_tests/artifacts/logs .github/e2e_tests/artifacts/data + cp -r .github/s3_and_iam_deployment/logs/* .github/e2e_tests/artifacts/logs/ + cp -r .github/s3_and_iam_deployment/data/* .github/e2e_tests/artifacts/data/ + if: always() + - name: Capture Kubernetes Logs in artifacts directory run: | chmod +x .github/scripts/capture_k8s_logs.sh