From fe750d627eafc1c27506b26e56b39dca2c54b753 Mon Sep 17 00:00:00 2001 From: Jonathan Reem Date: Fri, 25 Sep 2015 12:16:28 -0700 Subject: [PATCH 1/4] Provide a new command, gof3r info, which fetches metadata about an object. The new command will print some small bits of metadata about an object on success and will exit with a non-zero exit code on failure. The info command can be used to query for a file's existence without downloading it, and performs only a HEAD request. Fixes #71 --- getter.go | 29 +++++++++++++++++++++++ gof3r/info.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++ gof3r/main_test.go | 9 +++++++ 3 files changed, 97 insertions(+) create mode 100644 gof3r/info.go diff --git a/getter.go b/getter.go index 600dae0..093d126 100644 --- a/getter.go +++ b/getter.go @@ -307,6 +307,35 @@ func (g *getter) Close() error { return nil } +func GetInfo(b *Bucket, c *Config, key string, version string) (*http.Response, error) { + urlStr, err := b.url(key, c) + + if err != nil { + return nil, err + } + + for i := 0; i < c.NTry; i++ { + var req *http.Request + req, err := http.NewRequest("HEAD", urlStr.String(), nil) + + if err != nil { + return nil, err + } + + b.Sign(req) + + resp, err := c.Client.Do(req) + + if err != nil { + return nil, err + } + + return resp, nil + } + + return nil, fmt.Errorf("Could not get info.") +} + func (g *getter) checkMd5() (err error) { calcMd5 := fmt.Sprintf("%x", g.md5.Sum(nil)) md5Path := fmt.Sprint(".md5", g.url.Path, ".md5") diff --git a/gof3r/info.go b/gof3r/info.go new file mode 100644 index 0000000..c188802 --- /dev/null +++ b/gof3r/info.go @@ -0,0 +1,59 @@ +package main + +import ( + "log" + + "github.com/rlmcpherson/s3gof3r" +) + +var info infoOpts + +type infoOpts struct { + Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"` + Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"` + VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)." no-ini:"true"` +} + +func (info *infoOpts) Execute(args []string) (err error) { + conf := new(s3gof3r.Config) + *conf = *s3gof3r.DefaultConfig + k, err := getAWSKeys() + if err != nil { + return err + } + + s3 := s3gof3r.New(get.EndPoint, k) + b := s3.Bucket(info.Bucket) + conf.Concurrency = get.Concurrency + + if get.NoSSL { + conf.Scheme = "http" + } + + resp, err := s3gof3r.GetInfo(b, s3gof3r.DefaultConfig, info.Key, info.VersionID) + + if err != nil { + return err + } + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + log.Println("Found object:") + log.Println(" Status: ", resp.StatusCode) + log.Println(" Last-Modified: ", resp.Header["Last-Modified"]) + log.Println(" Size: ", resp.Header["Content-Length"]) + } else if resp.StatusCode == 403 { + log.Fatal("Access Denied") + } else { + log.Fatal("Non-2XX status code: ", resp.StatusCode) + } + + return nil +} + +func init() { + _, err := parser.AddCommand("info", "check info from S3", "get information about an object from S3", &info) + + if err != nil { + log.Fatal(err) + } +} diff --git a/gof3r/main_test.go b/gof3r/main_test.go index 9177aa7..9dae0a7 100644 --- a/gof3r/main_test.go +++ b/gof3r/main_test.go @@ -35,6 +35,15 @@ var flagTests = []flagTest{ errors.New("expected argument for flag")}, {[]string{"gof3r", "get"}, errors.New("required flags")}, + {[]string{"gof3r", "info"}, + errors.New("required flags")}, + {[]string{"gof3r", "info", "-b"}, + errors.New("expected argument for flag")}, + {[]string{"gof3r", "info", "-b", "fake-bucket", "-k", "test-key"}, + errors.New("Access Denied")}, + {[]string{"gof3r", "info", "-b", "fake-bucket", "-k", "key", + "-c", "1", "-s", "1024", "--debug", "--no-ssl", "--no-md5"}, + errors.New("Access Denied")}, } func TestFlags(t *testing.T) { From 7639ce6879bf896623a8f9b3c59975c0fc6870b8 Mon Sep 17 00:00:00 2001 From: Jonathan Reem Date: Fri, 8 Jan 2016 19:00:10 -0800 Subject: [PATCH 2/4] Add common options to info command. --- gof3r/info.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/gof3r/info.go b/gof3r/info.go index c188802..50dfc5b 100644 --- a/gof3r/info.go +++ b/gof3r/info.go @@ -9,6 +9,7 @@ import ( var info infoOpts type infoOpts struct { + CommonOpts Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"` Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"` VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)." no-ini:"true"` @@ -22,13 +23,8 @@ func (info *infoOpts) Execute(args []string) (err error) { return err } - s3 := s3gof3r.New(get.EndPoint, k) + s3 := s3gof3r.New(info.EndPoint, k) b := s3.Bucket(info.Bucket) - conf.Concurrency = get.Concurrency - - if get.NoSSL { - conf.Scheme = "http" - } resp, err := s3gof3r.GetInfo(b, s3gof3r.DefaultConfig, info.Key, info.VersionID) From a0e47d963556e31380c6a475568449ceb843b70a Mon Sep 17 00:00:00 2001 From: Bogdan Sorlea Date: Mon, 28 Nov 2016 15:31:55 +0100 Subject: [PATCH 3/4] Allow ampersands and colons in key names --- .gitignore | 5 +++++ sign.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/.gitignore b/.gitignore index 14741e7..e9c39f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ *.mprof *.out .env +.idea +out +*.iml +gof3r/gof3r + diff --git a/sign.go b/sign.go index cc158c8..dfd4e9e 100644 --- a/sign.go +++ b/sign.go @@ -125,6 +125,9 @@ func (s *signer) buildCanonicalString() { uri = "/" } + uri = strings.Replace(uri, "@", "%40", -1) + uri = strings.Replace(uri, ":", "%3A", -1) + s.canonicalString = strings.Join([]string{ s.Request.Method, uri, From 781f00bb941ad2a3416d32b9e13ad8af6116d2d1 Mon Sep 17 00:00:00 2001 From: Bogdan Sorlea Date: Mon, 30 Jan 2017 16:04:29 +0100 Subject: [PATCH 4/4] Improved info command to require only ListBucket permissions --- getter.go | 4 ++- gof3r/info.go | 70 +++++++++++++++++++++++++++++++++++++++++------- gof3r/main.go | 5 +++- gof3r/options.go | 7 ++--- s3gof3r.go | 64 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 15 deletions(-) diff --git a/getter.go b/getter.go index 0248478..1fd228b 100644 --- a/getter.go +++ b/getter.go @@ -312,6 +312,8 @@ func (g *getter) Close() error { } func GetInfo(b *Bucket, c *Config, key string, version string) (*http.Response, error) { + + key = fmt.Sprintf("?%s=%s&%s=%s", listTypeParam, listTypeValue, prefixParam, key) urlStr, err := b.url(key, c) if err != nil { @@ -320,7 +322,7 @@ func GetInfo(b *Bucket, c *Config, key string, version string) (*http.Response, for i := 0; i < c.NTry; i++ { var req *http.Request - req, err := http.NewRequest("HEAD", urlStr.String(), nil) + req, err := http.NewRequest("GET", urlStr.String(), nil) if err != nil { return nil, err diff --git a/gof3r/info.go b/gof3r/info.go index 50dfc5b..74e099e 100644 --- a/gof3r/info.go +++ b/gof3r/info.go @@ -2,22 +2,39 @@ package main import ( "log" - + "encoding/xml" + "io/ioutil" "github.com/rlmcpherson/s3gof3r" + "os" + "strings" + "fmt" ) +type ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Contents []Content `xml:"Contents"` + KeyCount int `xml:KeyCount` +} + +type Content struct { + Name string `xml:"Key"` + ETag string `xml:"ETag"` + LastModified string `xml:"LastModified"` + Size string `xml:"Size"` +} + var info infoOpts type infoOpts struct { CommonOpts - Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"` - Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"` - VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)." no-ini:"true"` + DataOpts + Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"` + Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"` + VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)." no-ini:"true"` + ObjectProperty string `long:"object-property" short:"q" description:"The property requested. Valid values are: etag, last-modified, size" no-ini:"true"` } func (info *infoOpts) Execute(args []string) (err error) { - conf := new(s3gof3r.Config) - *conf = *s3gof3r.DefaultConfig k, err := getAWSKeys() if err != nil { return err @@ -32,15 +49,48 @@ func (info *infoOpts) Execute(args []string) (err error) { return err } + defer resp.Body.Close() + + respBody, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return err + } + + var lbr ListBucketResult + if err := xml.Unmarshal(respBody, &lbr); err != nil { + log.Fatal(err) + } + + // the implementation assumes that the of the first in the response XML + // should match the requested Key exactly - based on the alphabetically (lexicographic) + // ascending ordering of the keys in the response from the AWS API + // otherwise it should fail, as the intent of this method is to obtain info on an exact + // file of the bucket, even though the method works based on a prefix approach + + // if the intent is to improve on the assumption or if the AWS API stops responding with + // the keys in alphabetically ascending order and if the ListBucket-access-based approach + // is to be kept for the `info` command, then the approach should be refactored as to + // obtain all results given the requested prefix (ask for all pages of data), iterate over + // them until the requested key is detected - or failing otherwise + if resp.StatusCode >= 200 && resp.StatusCode < 300 { - log.Println("Found object:") - log.Println(" Status: ", resp.StatusCode) - log.Println(" Last-Modified: ", resp.Header["Last-Modified"]) - log.Println(" Size: ", resp.Header["Content-Length"]) + if lbr.KeyCount < 1 { + os.Exit(1) + } + if info.ObjectProperty == "etag" { + fmt.Println(strings.Replace(lbr.Contents[0].ETag, "\"", "", -1)) + } else if info.ObjectProperty == "last-modified" { + fmt.Println(lbr.Contents[0].LastModified) + } else if info.ObjectProperty == "size" { + fmt.Println(lbr.Contents[0].Size) + } } else if resp.StatusCode == 403 { log.Fatal("Access Denied") + os.Exit(1) } else { log.Fatal("Non-2XX status code: ", resp.StatusCode) + os.Exit(1) } return nil diff --git a/gof3r/main.go b/gof3r/main.go index 7ddb16c..a1bc4d2 100644 --- a/gof3r/main.go +++ b/gof3r/main.go @@ -81,7 +81,10 @@ func main() { } os.Exit(1) } - fmt.Fprintf(os.Stderr, "duration: %v\n", time.Since(start)) + + if appOpts.NoStatusPrint == false { + fmt.Fprintf(os.Stderr, "duration: %v\n", time.Since(start)) + } } // getAWSKeys gets the AWS Keys from environment variables or the instance-based metadata on EC2 diff --git a/gof3r/options.go b/gof3r/options.go index 9fcc3be..7cec462 100644 --- a/gof3r/options.go +++ b/gof3r/options.go @@ -37,9 +37,10 @@ type UpOpts struct { } var appOpts struct { - Version func() `long:"version" short:"v" description:"Print version"` - Man func() `long:"manpage" short:"m" description:"Create gof3r.man man page in current directory"` - WriteIni bool `long:"writeini" short:"i" description:"Write .gof3r.ini in current user's home directory" no-ini:"true"` + Version func() `long:"version" short:"v" description:"Print version"` + Man func() `long:"manpage" short:"m" description:"Create gof3r.man man page in current directory"` + WriteIni bool `long:"writeini" short:"i" description:"Write .gof3r.ini in current user's home directory" no-ini:"true"` + NoStatusPrint bool `long:"no-status-print" short:"z" description:"Do not print status output (e.g. duration), except for the info command" no-ini:"true"` } var parser = flags.NewParser(&appOpts, (flags.HelpFlag | flags.PassDoubleDash)) diff --git a/s3gof3r.go b/s3gof3r.go index b430dd1..a05f70e 100644 --- a/s3gof3r.go +++ b/s3gof3r.go @@ -17,6 +17,10 @@ import ( ) const versionParam = "versionId" +const listTypeParam = "list-type" +const prefixParam = "prefix" + +const listTypeValue = "2" var regionMatcher = regexp.MustCompile("s3[-.]([a-z0-9-]+).amazonaws.com([.a-z0-9]*)") @@ -148,6 +152,64 @@ func (b *Bucket) PutWriter(path string, h http.Header, c *Config) (w io.WriteClo // url returns a parsed url to the given path. c must not be nil func (b *Bucket) url(bPath string, c *Config) (*url.URL, error) { + // parse versionID parameter from path, if included + // See https://github.com/rlmcpherson/s3gof3r/issues/84 for rationale + purl, err := url.Parse(bPath) + if err != nil { + return nil, err + } + + var finalParams = "" + var vals url.Values + + // the prefix should not be URL encoded here, so we are treating it independent of the rest of the keys + // the implementation is based around the previous implementation (see rationale), so refactoring might be in order + + if purl.Query().Get(prefixParam) != "" { + finalParams = fmt.Sprintf("%s=%s", prefixParam, purl.Query().Get(prefixParam)) + purl.Query().Del(prefixParam) + } + + if len(purl.Query()) > 0 { + purl.Query() + vals = make(url.Values) + if v := purl.Query().Get(versionParam); v != "" { + vals.Add(versionParam, purl.Query().Get(versionParam)) + } + if v := purl.Query().Get(listTypeParam); v != "" { + vals.Add(listTypeParam, purl.Query().Get(listTypeParam)) + } + } + + if vals.Encode() != "" { + finalParams = fmt.Sprintf("%s&%s", finalParams, vals.Encode()) + } + + bPath = strings.Split(bPath, "?")[0] // remove all params from initial path + + // handling for bucket names containing periods / explicit PathStyle addressing + // http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html for details + if strings.Contains(b.Name, ".") || c.PathStyle { + return &url.URL{ + Host: b.S3.Domain, + Scheme: c.Scheme, + Path: path.Clean(fmt.Sprintf("/%s/%s", b.Name, bPath)), + RawQuery: finalParams, + }, nil + } else { + return &url.URL{ + Scheme: c.Scheme, + Path: path.Clean(fmt.Sprintf("/%s", bPath)), + Host: path.Clean(fmt.Sprintf("%s.%s", b.Name, b.S3.Domain)), + RawQuery: finalParams, + }, nil + } +} + +// urlForBucketRequest returns a parsed url to the given bucket, used for bucket-specific requests +// c must not be nil +func (b *Bucket) urlForBucketRequest(bPath string, c *Config) (*url.URL, error) { + // parse versionID parameter from path, if included // See https://github.com/rlmcpherson/s3gof3r/issues/84 for rationale purl, err := url.Parse(bPath) @@ -180,6 +242,8 @@ func (b *Bucket) url(bPath string, c *Config) (*url.URL, error) { } } + + func (b *Bucket) conf() *Config { c := b.Config if c == nil {