From 0935282d62f9dfb991ee13ef3d5be57adea0f71d Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 18 Jun 2024 00:27:09 -0700 Subject: [PATCH] fix: support 'mc put --if-not-exists' Also allow 'mc get --version-id' --- cmd/client-s3.go | 5 +++++ cmd/client.go | 1 + cmd/common-methods.go | 2 ++ cmd/cp-main.go | 2 ++ cmd/get-main.go | 12 +++++++++--- cmd/put-main.go | 28 ++++++++++++++++++---------- go.mod | 2 +- go.sum | 4 ++-- 8 files changed, 40 insertions(+), 16 deletions(-) diff --git a/cmd/client-s3.go b/cmd/client-s3.go index 9493f2e1f3..1b522821d6 100644 --- a/cmd/client-s3.go +++ b/cmd/client-s3.go @@ -1168,6 +1168,11 @@ func (c *S3Client) Put(ctx context.Context, reader io.Reader, size int64, progre opts.SendContentMd5 = true } + if putOpts.ifNotExists { + // Only supported in newer MinIO releases. + opts.SetMatchETagExcept("*") + } + ui, e := c.api.PutObject(ctx, bucket, object, reader, size, opts) if e != nil { errResponse := minio.ToErrorResponse(e) diff --git a/cmd/client.go b/cmd/client.go index 351e7b388c..bf787b133f 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -62,6 +62,7 @@ type PutOptions struct { multipartSize uint64 multipartThreads uint concurrentStream bool + ifNotExists bool } // StatOptions holds options of the HEAD operation diff --git a/cmd/common-methods.go b/cmd/common-methods.go index fe06f3b63e..010da70ea8 100644 --- a/cmd/common-methods.go +++ b/cmd/common-methods.go @@ -494,6 +494,7 @@ func uploadSourceToTargetURL(ctx context.Context, uploadOpts uploadSourceToTarge isPreserve: uploadOpts.preserve, multipartSize: multipartSize, multipartThreads: uint(multipartThreads), + ifNotExists: uploadOpts.ifNotExists, } if isReadAt(reader) || length == 0 { @@ -576,4 +577,5 @@ type uploadSourceToTargetURLOpts struct { multipartSize string multipartThreads string updateProgressTotal bool + ifNotExists bool } diff --git a/cmd/cp-main.go b/cmd/cp-main.go index 36fefcc993..fe0ce60e16 100644 --- a/cmd/cp-main.go +++ b/cmd/cp-main.go @@ -264,6 +264,7 @@ func doCopy(ctx context.Context, copyOpts doCopyOpts) URLs { multipartSize: copyOpts.multipartSize, multipartThreads: copyOpts.multipartThreads, updateProgressTotal: copyOpts.updateProgressTotal, + ifNotExists: copyOpts.ifNotExists, }) if copyOpts.isMvCmd && urls.Error == nil { rmManager.add(ctx, sourceAlias, sourceURL.String()) @@ -557,4 +558,5 @@ type doCopyOpts struct { updateProgressTotal bool multipartSize string multipartThreads string + ifNotExists bool } diff --git a/cmd/get-main.go b/cmd/get-main.go index 26c2c90a8e..43060a1e16 100644 --- a/cmd/get-main.go +++ b/cmd/get-main.go @@ -27,7 +27,12 @@ import ( // get command flags. var ( - getFlags = []cli.Flag{} + getFlags = []cli.Flag{ + cli.StringFlag{ + Name: "version-id, vid", + Usage: "get a specific version of an object", + }, + } ) // Get command. @@ -50,10 +55,10 @@ FLAGS: EXAMPLES: 1. Get an object from MinIO storage to local file system - {{.Prompt}} {{.HelpName}} play/mybucket/object path-to/object + {{.Prompt}} {{.HelpName}} play/mybucket/object path-to/object 2. Get an object from MinIO storage using encryption - {{.Prompt}} {{.HelpName}} --enc-c "play/mybucket/object=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" play/mybucket/object path-to/object + {{.Prompt}} {{.HelpName}} --enc-c "play/mybucket/object=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" play/mybucket/object path-to/object `, } @@ -94,6 +99,7 @@ func mainGet(cliCtx *cli.Context) (e error) { targetURL: targetURL, encKeyDB: encryptionKeys, ignoreBucketExistsCheck: true, + versionID: cliCtx.String("version-id"), } for getURLs := range prepareGetURLs(ctx, opts) { diff --git a/cmd/put-main.go b/cmd/put-main.go index 1a6741c037..d92e7067f3 100644 --- a/cmd/put-main.go +++ b/cmd/put-main.go @@ -41,6 +41,11 @@ var ( Usage: "each part size", Value: "16MiB", }, + cli.BoolFlag{ + Name: "if-not-exists", + Usage: "upload only if object does not exist", + Hidden: true, + }, } ) @@ -68,19 +73,19 @@ ENVIRONMENT VARIABLES: EXAMPLES: 1. Put an object from local file system to S3 storage - {{.Prompt}} {{.HelpName}} path-to/object play/mybucket + {{.Prompt}} {{.HelpName}} path-to/object play/mybucket 2. Put an object from local file system to S3 bucket with name - {{.Prompt}} {{.HelpName}} path-to/object play/mybucket/object + {{.Prompt}} {{.HelpName}} path-to/object play/mybucket/object 3. Put an object from local file system to S3 bucket under a prefix - {{.Prompt}} {{.HelpName}} path-to/object play/mybucket/object-prefix/ + {{.Prompt}} {{.HelpName}} path-to/object play/mybucket/object-prefix/ - 4. Put an object to MinIO storage using sse-c encryption - {{.Prompt}} {{.HelpName}} --enc-c "play/mybucket/object=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" path-to/object play/mybucket/object + 4. Put an object to MinIO storage using sse-c encryption + {{.Prompt}} {{.HelpName}} --enc-c "play/mybucket/object=MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" path-to/object play/mybucket/object - 5. Put an object to MinIO storage using sse-kms encryption - {{.Prompt}} {{.HelpName}} --enc-kms path-to/object play/mybucket/object + 5. Put an object to MinIO storage using sse-kms encryption + {{.Prompt}} {{.HelpName}} --enc-kms path-to/object play/mybucket/object `, } @@ -93,11 +98,13 @@ func mainPut(cliCtx *cli.Context) (e error) { ctx, cancelPut := context.WithCancel(globalContext) defer cancelPut() + // part size size := cliCtx.String("s") if size == "" { - size = "16mb" + size = "32MiB" } + _, perr := humanize.ParseBytes(size) if perr != nil { fatalIf(probe.NewError(perr), "Unable to parse part size") @@ -175,10 +182,11 @@ func mainPut(cliCtx *cli.Context) (e error) { encryptionKeys: encryptionKeys, multipartSize: size, multipartThreads: strconv.Itoa(threads), + ifNotExists: cliCtx.Bool("if-not-exists"), }) if urls.Error != nil { - e = urls.Error.ToGoError() - showLastProgressBar(pg, e) + showLastProgressBar(pg, urls.Error.ToGoError()) + fatalIf(urls.Error.Trace(), "unable to upload") return } } diff --git a/go.mod b/go.mod index 8f48a32cd6..2ed4f60896 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/minio/colorjson v1.0.7 github.com/minio/filepath v1.0.0 github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.70 + github.com/minio/minio-go/v7 v7.0.72-0.20240618070918-0b004e328e1e github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/xattr v0.4.9 github.com/posener/complete v1.2.3 diff --git a/go.sum b/go.sum index 11d185a172..db2d52d1f2 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,8 @@ github.com/minio/madmin-go/v3 v3.0.55-0.20240603092915-420a67132c32 h1:9se7/S4Al github.com/minio/madmin-go/v3 v3.0.55-0.20240603092915-420a67132c32/go.mod h1:IFAwr0XMrdsLovxAdCcuq/eoL4nRuMVQQv0iubJANQw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= -github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/minio/minio-go/v7 v7.0.72-0.20240618070918-0b004e328e1e h1:Ye/gsaKgKD9lg1BgjG5fsGZdKevVpJO9bzqQg7badV8= +github.com/minio/minio-go/v7 v7.0.72-0.20240618070918-0b004e328e1e/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA= github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ= github.com/minio/pkg/v3 v3.0.0 h1:0vOKHgwpya//mb7RH0i1lyPMH2IBBF5hJMNY5Bk2WlY=