Merge branch 'master' of github.com:lunasec-io/lunasec

lunatrace/bsl/backend-cdk/build-containers.sh

@@ -15,90 +15,73 @@ reset-dir() {
   change-dir "$saved_dir"
 }
 
-echo "NOTE: The backend must be running for migrations to be applied successfully"
-
-# Not working, use docker-compose up to apply metadata
-#change-dir ../hasura
-#if ! hasura migrate --database-name lunatrace apply ; then
-#	echo "unable to apply database migrations"
-#	exit 1
-#fi
-#if ! hasura metadata apply ; then
-#	echo "unable to apply metadata to hasura"
-#	exit 1
-#fi
-#if ! hasura metadata reload ; then
-#	echo "unable to reload metadata in hasura"
-#	exit 1
-#fi
-#reset-dir
-
 # Make sure generated code is up to date before building Docker containers
 echo "Making sure generated code is up to date before building Docker containers"
-echo "frontend generation start"
 
+echo "frontend generation start"
 change-dir ../frontend
 if ! yarn run generate ; then
 	echo "unable to generate code for frontend"
 	exit 1
 fi
 reset-dir
-
-echo "Frontend generation done"
+echo "done"
 
 echo "backend generation start"
-
 change-dir ../backend
 if ! yarn run generate ; then
 	echo "unable to generate code for backend"
 	exit 1
 fi
 reset-dir
-
-echo "Backend generation done"
+echo "done"
 
 echo "starting container build. NOTE=You will be asked for your password "
 
 echo "building root docker file for repo"
-# Build Docker containers
 change-dir ../../..
 if ! sudo docker build . -f lunatrace/bsl/repo-bootstrap.dockerfile -t repo-bootstrap ; then
 	echo "unable to build repo-bootstrap"
 	exit 1
 fi
 reset-dir
+echo "done"
 
+echo "building lunatrace-cli"
+change-dir ../../..
+if ! sudo docker build . -f lunatrace/cli/docker/lunatrace.dockerfile -t lunatrace-cli ; then
+	echo "unable to build lunatrace-cli"
+	exit 1
+fi
+reset-dir
 echo "done"
-echo "building docker file for frontend"
 
+echo "building docker file for frontend"
 change-dir ../frontend
 if ! sudo docker build . -t lunatrace-frontend ; then
 	echo "unable to build lunatrace-frontend"
 	exit 1
 fi
 reset-dir
-
-
 echo "done"
-echo "building docker file for backend express server"
 
+echo "building docker file for backend express server"
 change-dir ../backend
 if ! sudo docker build --target backend-express-server . -t lunatrace-backend ; then
 	echo "unable to build lunatrace-backend"
 	exit 1
 fi
 reset-dir
-
-
 echo "done"
-echo "building docker file for backend queue processor"
 
+echo "building docker file for backend queue processor"
 change-dir ../backend
 if ! sudo docker build --target backend-queue-processor . -t lunatrace-backend-queue-processor ; then
 	echo "unable to build lunatrace-backend-queue-processor"
 	exit 1
 fi
 reset-dir
+echo "done"
 
 echo "Saving docker containers to output dir: $OUT_DIR"
 

lunatrace/bsl/backend-cdk/lib/lunatrace-backend-stack.ts

@@ -32,6 +32,7 @@ import { HostedZone } from '@aws-cdk/aws-route53';
 import { Bucket } from '@aws-cdk/aws-s3';
 import { Secret } from '@aws-cdk/aws-secretsmanager';
 import * as cdk from '@aws-cdk/core';
+import { Duration } from '@aws-cdk/core';
 
 import { StackInputsType } from '../bin/lunatrace-backend';
 
@@ -320,6 +321,7 @@ export class LunatraceBackendStack extends cdk.Stack {
       circuitBreaker: {
         rollback: true,
       },
+      healthCheckGracePeriod: Duration.seconds(5),
       desiredCount: 2,
       deploymentController: {
         type: DeploymentControllerType.ECS,
@@ -350,8 +352,11 @@ export class LunatraceBackendStack extends cdk.Stack {
     });
 
     storageStackStage.sbomBucket.grantReadWrite(loadBalancedFargateService.taskDefinition.taskRole);
-    oryConfigBucket.grantReadWrite(loadBalancedFargateService.taskDefinition.taskRole);
     storageStackStage.manifestBucket.grantReadWrite(loadBalancedFargateService.taskDefinition.taskRole);
+    storageStackStage.processWebhookSqsQueue.grantSendMessages(loadBalancedFargateService.taskDefinition.taskRole);
+    storageStackStage.processRepositorySqsQueue.grantSendMessages(loadBalancedFargateService.taskDefinition.taskRole);
+
+    oryConfigBucket.grantReadWrite(loadBalancedFargateService.taskDefinition.taskRole);
 
     WorkerStack.createWorkerStack(this, {
       env: props.env,

lunatrace/bsl/backend-cdk/lib/worker-stack.ts

@@ -19,7 +19,7 @@ import { ApplicationLoadBalancedFargateService } from '@aws-cdk/aws-ecs-patterns
 import { ISecret } from '@aws-cdk/aws-secretsmanager';
 import { Queue } from '@aws-cdk/aws-sqs';
 import * as cdk from '@aws-cdk/core';
-import { Construct } from '@aws-cdk/core';
+import { Construct, Duration } from '@aws-cdk/core';
 
 import { getContainerTarballPath } from './util';
 import { WorkerStorageStackState } from './worker-storage-stack';
@@ -79,8 +79,6 @@ export class WorkerStack extends cdk.Stack {
       throw new Error(`expected non-null storage stack queues: ${inspect(storageStack)}`);
     }
 
-    repositoryQueue.grantSendMessages(props.fargateService.service.taskDefinition.taskRole);
-
     const workerContainerImage = ContainerImage.fromTarball(
       getContainerTarballPath('lunatrace-backend-queue-processor.tar')
     );
@@ -109,13 +107,28 @@ export class WorkerStack extends cdk.Stack {
       {
         name: 'ProcessRepositoryQueue',
         queue: repositoryQueue,
+        visibility: 600,
+      },
+      {
+        name: 'ProcessWebhookQueue',
+        queue: webhookQueue,
+      },
+      {
+        name: 'ProcessManifestQueue',
+        queue: manifestQueue,
+        visibility: 300,
+      },
+      {
+        name: 'ProcessSbomQueue',
+        queue: sbomQueue,
+        visibility: 300,
       },
     ];
 
     queueServices.forEach((queueService) => {
       const queueFargateService = new ecsPatterns.QueueProcessingFargateService(
         context,
-        'ProcessRepositoryQueueService',
+        queueService.name + 'Service',
         {
           cluster: fargateCluster,
           image: workerContainerImage,
@@ -129,10 +142,15 @@ export class WorkerStack extends cdk.Stack {
             ...(queueService.visibility ? { QUEUE_VISIBILITY: queueService.visibility.toString() } : {}),
           },
           secrets: processQueueCommonSecrets,
-          containerName: 'ProcessRepositoryQueueContainer',
+          containerName: queueService.name + 'Container',
           circuitBreaker: {
             rollback: true,
           },
+          // healthCheck: {
+          //   // stub command to just see if the container is actually running
+          //   command: ['CMD-SHELL', 'ls || exit 1'],
+          //   startPeriod: Duration.seconds(5),
+          // },
           minScalingCapacity: 2,
           deploymentController: {
             type: DeploymentControllerType.ECS,
@@ -143,6 +161,7 @@ export class WorkerStack extends cdk.Stack {
       storageStack.manifestBucket.grantReadWrite(queueFargateService.taskDefinition.taskRole);
       webhookQueue.grantSendMessages(queueFargateService.taskDefinition.taskRole);
       webhookQueue.grantConsumeMessages(queueFargateService.taskDefinition.taskRole);
+      repositoryQueue.grantConsumeMessages(queueFargateService.taskDefinition.taskRole);
       repositoryQueue.grantSendMessages(queueFargateService.taskDefinition.taskRole);
     });
 

lunatrace/bsl/backend/Dockerfile

@@ -4,10 +4,8 @@ RUN apt-get update && apt-get install -y wget curl make
 
 RUN curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
 
-# Build the lunatrace CLI
-WORKDIR /usr/repo/lunatrace/cli/
-RUN make lunatrace
-RUN mv build/lunatrace /usr/local/bin
+# copy the lunatrace cli from the locally built container
+COPY --from=lunatrace-cli /lunatrace /usr/local/bin
 
 WORKDIR /usr/repo/lunatrace/bsl/backend
 RUN yarn run compile

lunatrace/bsl/backend/src/github/webhooks/handlers.ts

@@ -92,8 +92,9 @@ async function pullRequestHandler(event: EmitterWebhookEvent<'pull_request'>) {
 
   if (actionName === 'synchronize' || actionName === 'opened' || actionName === 'reopened') {
     if (!event.payload.installation) {
-      log.error(`no installation found in pull request webhook`);
-      log.info(event);
+      log.error(`no installation found in pull request webhook`, {
+        event,
+      });
       return;
     }