diff --git a/.github/workflows/singleton.yml b/.github/workflows/singleton.yml new file mode 100644 index 000000000..bb1fdab52 --- /dev/null +++ b/.github/workflows/singleton.yml @@ -0,0 +1,65 @@ +# Find full documentation here https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions +name: Singleton image + +on: + pull_request: + paths: + - 'containers/singleton/**' + push: + branches: + - main + paths: + - 'containers/singleton/**' + + # Manual invocation. + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/singleton + +# Ensure we only ever have one build running at a time. +# If we push twice in quick succession, the first build will be stopped once the second starts. +# This avoids any race conditions. +concurrency: + group: ${{ github.ref }}/singleton + cancel-in-progress: true + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + + - name: Log in to the Container registry + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,format=long + + - name: Build and push Docker image + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 + with: + context: ./ + file: containers/singleton/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/containers/singleton/Dockerfile b/containers/singleton/Dockerfile new file mode 100644 index 000000000..8d7a95285 --- /dev/null +++ b/containers/singleton/Dockerfile @@ -0,0 +1,3 @@ +FROM amazonlinux:2.0.20240412.0 + +RUN yum install -y -q aws-cli jq diff --git a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap index dfa0d30b3..ac225eaa5 100644 --- a/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap +++ b/packages/cdk/lib/__snapshots__/service-catalogue.test.ts.snap @@ -10910,13 +10910,13 @@ spec: "Command": [ "/bin/bash", "-c", - "yum install -y -q jq awscli;ECS_CLUSTER=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Cluster');ECS_FAMILY=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Family');ECS_TASK_ARN=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN');RUNNING=$(aws ecs list-tasks --cluster $ECS_CLUSTER --family $ECS_FAMILY | jq '.taskArns | length');[[ \${RUNNING} > 1 ]] && exit 114 || exit 0", + "ECS_CLUSTER=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Cluster');ECS_FAMILY=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Family');ECS_TASK_ARN=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.TaskARN');RUNNING=$(aws ecs list-tasks --cluster $ECS_CLUSTER --family $ECS_FAMILY | jq '.taskArns | length');[[ \${RUNNING} > 1 ]] && exit 114 || exit 0", ], "EntryPoint": [ "", ], "Essential": false, - "Image": "public.ecr.aws/amazonlinux/amazonlinux:latest", + "Image": "ghcr.io/guardian/service-catalogue/singleton:sha-855e948a9669e1edb9b72a37118f7372bb3282fb", "LogConfiguration": { "LogDriver": "awsfirelens", "Options": { @@ -10931,6 +10931,7 @@ spec: }, }, "Name": "CloudquerySource-OrgWideEc2AwsCli", + "ReadonlyRootFilesystem": true, }, { "Command": [ diff --git a/packages/cdk/lib/cloudquery/images.ts b/packages/cdk/lib/cloudquery/images.ts index 043adb86a..34bd8814b 100644 --- a/packages/cdk/lib/cloudquery/images.ts +++ b/packages/cdk/lib/cloudquery/images.ts @@ -6,8 +6,8 @@ export const Images = { `ghcr.io/guardian/service-catalogue/cloudquery:sha-0f2713edae5157260cfbf4eaa1f2d682e980fe7e`, ), devxLogs: ContainerImage.fromRegistry('ghcr.io/guardian/devx-logs:2'), - amazonLinux: ContainerImage.fromRegistry( - 'public.ecr.aws/amazonlinux/amazonlinux:latest', + singletonImage: ContainerImage.fromRegistry( + 'ghcr.io/guardian/service-catalogue/singleton:sha-855e948a9669e1edb9b72a37118f7372bb3282fb', ), // https://github.com/guardian/cq-source-ns1 ns1Source: ContainerImage.fromRegistry( diff --git a/packages/cdk/lib/cloudquery/task.ts b/packages/cdk/lib/cloudquery/task.ts index cf0fe1267..593cced55 100644 --- a/packages/cdk/lib/cloudquery/task.ts +++ b/packages/cdk/lib/cloudquery/task.ts @@ -11,7 +11,7 @@ import { PropagatedTagSource, Secret, } from 'aws-cdk-lib/aws-ecs'; -import type { Cluster, RepositoryImage } from 'aws-cdk-lib/aws-ecs'; +import type { Cluster, RepositoryImage, Volume } from 'aws-cdk-lib/aws-ecs'; import type { ScheduledFargateTaskProps } from 'aws-cdk-lib/aws-ecs-patterns'; import { ScheduledFargateTask } from 'aws-cdk-lib/aws-ecs-patterns'; import type { IManagedPolicy, PolicyStatement } from 'aws-cdk-lib/aws-iam'; @@ -227,33 +227,38 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { logging: fireLensLogDriver, }); - task.addVolume({ + const configVolume: Volume = { name: 'config-volume', - }); - task.addVolume({ + }; + task.addVolume(configVolume); + + const cqVolume: Volume = { name: 'cloudquery-volume', - }); - task.addVolume({ + }; + task.addVolume(cqVolume); + + const tmpVolume: Volume = { name: 'tmp-volume', - }); + }; + task.addVolume(tmpVolume); cloudqueryTask.addMountPoints( { // So that we can write task config to this directory containerPath: serviceCatalogueConfigDirectory, - sourceVolume: 'config-volume', + sourceVolume: configVolume.name, readOnly: false, }, { // So that Cloudquery can write to this directory containerPath: '/app/.cq', - sourceVolume: 'cloudquery-volume', + sourceVolume: cqVolume.name, readOnly: false, }, { // So that Cloudquery can write temporary data containerPath: '/tmp', - sourceVolume: 'tmp-volume', + sourceVolume: tmpVolume.name, readOnly: false, }, ); @@ -299,15 +304,12 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { const success = 0; const singletonTask = task.addContainer(`${id}AwsCli`, { - image: Images.amazonLinux, + image: Images.singletonImage, entryPoint: [''], command: [ '/bin/bash', '-c', [ - // Install jq to handle JSON, and awscli to query ECS - 'yum install -y -q jq awscli', - // Who am I? `ECS_CLUSTER=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Cluster')`, `ECS_FAMILY=$(curl -s $ECS_CONTAINER_METADATA_URI/task | jq -r '.Family')`, @@ -320,6 +322,7 @@ export class ScheduledCloudqueryTask extends ScheduledFargateTask { `[[ $\{RUNNING} > 1 ]] && exit ${operationInProgress} || exit ${success}`, ].join(';'), ], + readonlyRootFilesystem: true, logging: fireLensLogDriver, /*