Merge remote-tracking branch 'origin/main'

deepfence_file_server/Dockerfile

@@ -17,4 +17,4 @@ RUN rm ./linux_amd64_full.tar.gz
 RUN mkdir -p /data
 
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["weed", "server", "-ip=0.0.0.0", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/etc/seaweed.json", "-volume.max=15"]
+CMD ["weed", "server", "-ip=0.0.0.0", "-dir=/data", "-s3", "-s3.port=9000", "-s3.config=/etc/seaweed.json", "-volume.max=100"]

deepfence_utils/directory/minio.go

@@ -20,7 +20,11 @@ import (
 	_ "github.com/lib/pq"
 )
 
-var minioClientMap sync.Map
+var (
+	MinioBucket         = utils.GetEnvOrDefault("DEEPFENCE_MINIO_BUCKET", string(NonSaaSDirKey))
+	MinioDatabaseBucket = utils.GetEnvOrDefault("DEEPFENCE_MINIO_DB_BUCKET", string(DatabaseDirKey))
+	minioClientMap      sync.Map
+)
 
 func init() {
 	minioClientMap = sync.Map{}
@@ -46,11 +50,12 @@ type FileManager interface {
 	CreatePublicUploadURL(ctx context.Context, filePath string, addFilePathPrefix bool, expires time.Duration, reqParams url.Values) (string, error)
 	Client() interface{}
 	Bucket() string
-	CreatePublicBucket(ctx context.Context) error
+	CreatePublicBucket(ctx context.Context, bucket string) error
 }
 
 type MinioFileManager struct {
 	client    *minio.Client
+	bucket    string
 	namespace string
 }
 
@@ -113,7 +118,7 @@ func (mfm *MinioFileManager) ListFiles(ctx context.Context, pathPrefix string, r
 	var objectsInfo []ObjectInfo
 	for obj := range objects {
 		isDir := strings.HasSuffix(obj.Key, "/")
-		if skipDir == true && isDir == true {
+		if skipDir && isDir {
 			continue
 		}
 		objectsInfo = append(objectsInfo, ObjectInfo{
@@ -273,35 +278,33 @@ func (mfm *MinioFileManager) Client() interface{} {
 }
 
 func (mfm *MinioFileManager) Bucket() string {
-	return mfm.namespace
+	return mfm.bucket
 }
 
 func (mfm *MinioFileManager) createBucketIfNeeded(ctx context.Context) error {
 
-	exists, err := mfm.client.BucketExists(ctx, mfm.namespace)
+	exists, err := mfm.client.BucketExists(ctx, mfm.bucket)
 
 	if err != nil {
 		return err
 	}
 
 	if !exists {
-		err = mfm.client.MakeBucket(ctx, mfm.namespace,
-			minio.MakeBucketOptions{ObjectLocking: false})
-
+		err = mfm.client.MakeBucket(ctx, mfm.bucket, minio.MakeBucketOptions{ObjectLocking: false})
 	}
 	return err
 }
 
-func (mfm *MinioFileManager) CreatePublicBucket(ctx context.Context) error {
+func (mfm *MinioFileManager) CreatePublicBucket(ctx context.Context, bucket string) error {
 
-	exists, err := mfm.client.BucketExists(ctx, mfm.namespace)
+	exists, err := mfm.client.BucketExists(ctx, bucket)
 	if err != nil {
 		return err
 	} else if exists {
 		return nil
 	}
 
-	err = mfm.client.MakeBucket(ctx, mfm.namespace, minio.MakeBucketOptions{ObjectLocking: false})
+	err = mfm.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ObjectLocking: false})
 	if err != nil {
 		return err
 	}
@@ -343,8 +346,14 @@ func MinioClient(ctx context.Context) (FileManager, error) {
 		return nil, err
 	}
 
+	bucket := MinioBucket
+	if ns == DatabaseDirKey {
+		bucket = MinioDatabaseBucket
+	}
+
 	return &MinioFileManager{
 		client:    client,
+		bucket:    bucket,
 		namespace: string(ns),
 	}, err
 }

deepfence_worker/cronscheduler/init_db.go

@@ -82,7 +82,7 @@ func InitMinioDatabase() error {
 	}
 	retries := 3
 	for {
-		if err := mc.CreatePublicBucket(ctx); err != nil {
+		if err := mc.CreatePublicBucket(ctx, directory.MinioDatabaseBucket); err != nil {
 			log.Error().Err(err).Msgf("failed to create bucket")
 			retries -= 1
 			if retries != 0 {

deepfence_worker/tasks/malwarescan/malwarescan.go

@@ -91,7 +91,7 @@ func (s MalwareScan) StartMalwareScan(ctx context.Context, task *asynq.Task) err
 	}
 	log.Info().Msgf("message tenant id %s", string(tenantID))
 
-	log.Info().Msgf(" payload: %s ", string(task.Payload()))
+	log.Info().Msgf("payload: %s ", string(task.Payload()))
 
 	var params utils.MalwareScanParameters
 

deepfence_worker/tasks/sbom/generate_sbom.go

@@ -84,6 +84,12 @@ func (s SbomGenerator) GenerateSbom(ctx context.Context, task *asynq.Task) error
 		{Key: "namespace", Value: []byte(tenantID)},
 	}
 
+	log.Info().Msgf("payload: %s ", string(task.Payload()))
+
+	if err := json.Unmarshal(task.Payload(), &params); err != nil {
+		return err
+	}
+
 	res, scanCtx := tasks.StartStatusReporter(params.ScanId,
 		func(status tasks.ScanStatus) error {
 			sb, err := json.Marshal(status)
@@ -104,6 +110,7 @@ func (s SbomGenerator) GenerateSbom(ctx context.Context, task *asynq.Task) error
 		},
 		time.Minute*20,
 	)
+
 	log.Info().Msgf("Adding scan id to map:%s", params.ScanId)
 	scanMap.Store(params.ScanId, scanCtx)
 	defer func() {
@@ -113,19 +120,13 @@ func (s SbomGenerator) GenerateSbom(ctx context.Context, task *asynq.Task) error
 		close(res)
 	}()
 
-	worker, err := directory.Worker(ctx)
-	if err != nil {
-		return err
-	}
-
-	log.Info().Msgf("payload: %s ", string(task.Payload()))
-
-	if err := json.Unmarshal(task.Payload(), &params); err != nil {
+	if params.RegistryId == "" {
+		log.Error().Msgf("registry id is empty in params %+v", params)
 		return err
 	}
 
-	if params.RegistryId == "" {
-		log.Error().Msgf("registry id is empty in params %+v", params)
+	worker, err := directory.Worker(ctx)
+	if err != nil {
 		return err
 	}
 

deepfence_worker/tasks/sbom/scan_sbom.go

@@ -99,6 +99,8 @@ func (s SbomParser) ScanSBOM(ctx context.Context, task *asynq.Task) error {
 		{Key: "namespace", Value: []byte(tenantID)},
 	}
 
+	log.Info().Msgf("payload: %s ", string(task.Payload()))
+
 	var params utils.SbomParameters
 
 	if err := json.Unmarshal(task.Payload(), &params); err != nil {
@@ -126,6 +128,7 @@ func (s SbomParser) ScanSBOM(ctx context.Context, task *asynq.Task) error {
 		},
 		time.Minute*20,
 	)
+
 	log.Info().Msgf("Adding scan id to map:%s", params.ScanId)
 	scanMap.Store(params.ScanId, scanCtx)
 	defer func() {
@@ -135,8 +138,6 @@ func (s SbomParser) ScanSBOM(ctx context.Context, task *asynq.Task) error {
 		close(res)
 	}()
 
-	log.Info().Msgf("payload: %s ", string(task.Payload()))
-
 	// send inprogress status
 
 	mc, err := directory.MinioClient(ctx)