diff --git a/.gitignore b/.gitignore index 14741e7..e9c39f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ *.mprof *.out .env +.idea +out +*.iml +gof3r/gof3r + diff --git a/getter.go b/getter.go index f496106..1fd228b 100644 --- a/getter.go +++ b/getter.go @@ -311,6 +311,37 @@ func (g *getter) Close() error { return nil } +func GetInfo(b *Bucket, c *Config, key string, version string) (*http.Response, error) { + + key = fmt.Sprintf("?%s=%s&%s=%s", listTypeParam, listTypeValue, prefixParam, key) + urlStr, err := b.url(key, c) + + if err != nil { + return nil, err + } + + for i := 0; i < c.NTry; i++ { + var req *http.Request + req, err := http.NewRequest("GET", urlStr.String(), nil) + + if err != nil { + return nil, err + } + + b.Sign(req) + + resp, err := c.Client.Do(req) + + if err != nil { + return nil, err + } + + return resp, nil + } + + return nil, fmt.Errorf("Could not get info.") +} + func (g *getter) checkMd5() (err error) { calcMd5 := fmt.Sprintf("%x", g.md5.Sum(nil)) md5Path := fmt.Sprint(".md5", g.url.Path, ".md5") diff --git a/gof3r/info.go b/gof3r/info.go new file mode 100644 index 0000000..74e099e --- /dev/null +++ b/gof3r/info.go @@ -0,0 +1,105 @@ +package main + +import ( + "log" + "encoding/xml" + "io/ioutil" + "github.com/rlmcpherson/s3gof3r" + "os" + "strings" + "fmt" +) + +type ListBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Contents []Content `xml:"Contents"` + KeyCount int `xml:KeyCount` +} + +type Content struct { + Name string `xml:"Key"` + ETag string `xml:"ETag"` + LastModified string `xml:"LastModified"` + Size string `xml:"Size"` +} + +var info infoOpts + +type infoOpts struct { + CommonOpts + DataOpts + Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"` + Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"` + VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)." no-ini:"true"` + ObjectProperty string `long:"object-property" short:"q" description:"The property requested. Valid values are: etag, last-modified, size" no-ini:"true"` +} + +func (info *infoOpts) Execute(args []string) (err error) { + k, err := getAWSKeys() + if err != nil { + return err + } + + s3 := s3gof3r.New(info.EndPoint, k) + b := s3.Bucket(info.Bucket) + + resp, err := s3gof3r.GetInfo(b, s3gof3r.DefaultConfig, info.Key, info.VersionID) + + if err != nil { + return err + } + + defer resp.Body.Close() + + respBody, err := ioutil.ReadAll(resp.Body) + + if err != nil { + return err + } + + var lbr ListBucketResult + if err := xml.Unmarshal(respBody, &lbr); err != nil { + log.Fatal(err) + } + + // the implementation assumes that the of the first in the response XML + // should match the requested Key exactly - based on the alphabetically (lexicographic) + // ascending ordering of the keys in the response from the AWS API + // otherwise it should fail, as the intent of this method is to obtain info on an exact + // file of the bucket, even though the method works based on a prefix approach + + // if the intent is to improve on the assumption or if the AWS API stops responding with + // the keys in alphabetically ascending order and if the ListBucket-access-based approach + // is to be kept for the `info` command, then the approach should be refactored as to + // obtain all results given the requested prefix (ask for all pages of data), iterate over + // them until the requested key is detected - or failing otherwise + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + if lbr.KeyCount < 1 { + os.Exit(1) + } + if info.ObjectProperty == "etag" { + fmt.Println(strings.Replace(lbr.Contents[0].ETag, "\"", "", -1)) + } else if info.ObjectProperty == "last-modified" { + fmt.Println(lbr.Contents[0].LastModified) + } else if info.ObjectProperty == "size" { + fmt.Println(lbr.Contents[0].Size) + } + } else if resp.StatusCode == 403 { + log.Fatal("Access Denied") + os.Exit(1) + } else { + log.Fatal("Non-2XX status code: ", resp.StatusCode) + os.Exit(1) + } + + return nil +} + +func init() { + _, err := parser.AddCommand("info", "check info from S3", "get information about an object from S3", &info) + + if err != nil { + log.Fatal(err) + } +} diff --git a/gof3r/main.go b/gof3r/main.go index 7ddb16c..a1bc4d2 100644 --- a/gof3r/main.go +++ b/gof3r/main.go @@ -81,7 +81,10 @@ func main() { } os.Exit(1) } - fmt.Fprintf(os.Stderr, "duration: %v\n", time.Since(start)) + + if appOpts.NoStatusPrint == false { + fmt.Fprintf(os.Stderr, "duration: %v\n", time.Since(start)) + } } // getAWSKeys gets the AWS Keys from environment variables or the instance-based metadata on EC2 diff --git a/gof3r/main_test.go b/gof3r/main_test.go index 9177aa7..9dae0a7 100644 --- a/gof3r/main_test.go +++ b/gof3r/main_test.go @@ -35,6 +35,15 @@ var flagTests = []flagTest{ errors.New("expected argument for flag")}, {[]string{"gof3r", "get"}, errors.New("required flags")}, + {[]string{"gof3r", "info"}, + errors.New("required flags")}, + {[]string{"gof3r", "info", "-b"}, + errors.New("expected argument for flag")}, + {[]string{"gof3r", "info", "-b", "fake-bucket", "-k", "test-key"}, + errors.New("Access Denied")}, + {[]string{"gof3r", "info", "-b", "fake-bucket", "-k", "key", + "-c", "1", "-s", "1024", "--debug", "--no-ssl", "--no-md5"}, + errors.New("Access Denied")}, } func TestFlags(t *testing.T) { diff --git a/gof3r/options.go b/gof3r/options.go index 9fcc3be..7cec462 100644 --- a/gof3r/options.go +++ b/gof3r/options.go @@ -37,9 +37,10 @@ type UpOpts struct { } var appOpts struct { - Version func() `long:"version" short:"v" description:"Print version"` - Man func() `long:"manpage" short:"m" description:"Create gof3r.man man page in current directory"` - WriteIni bool `long:"writeini" short:"i" description:"Write .gof3r.ini in current user's home directory" no-ini:"true"` + Version func() `long:"version" short:"v" description:"Print version"` + Man func() `long:"manpage" short:"m" description:"Create gof3r.man man page in current directory"` + WriteIni bool `long:"writeini" short:"i" description:"Write .gof3r.ini in current user's home directory" no-ini:"true"` + NoStatusPrint bool `long:"no-status-print" short:"z" description:"Do not print status output (e.g. duration), except for the info command" no-ini:"true"` } var parser = flags.NewParser(&appOpts, (flags.HelpFlag | flags.PassDoubleDash)) diff --git a/s3gof3r.go b/s3gof3r.go index b430dd1..a05f70e 100644 --- a/s3gof3r.go +++ b/s3gof3r.go @@ -17,6 +17,10 @@ import ( ) const versionParam = "versionId" +const listTypeParam = "list-type" +const prefixParam = "prefix" + +const listTypeValue = "2" var regionMatcher = regexp.MustCompile("s3[-.]([a-z0-9-]+).amazonaws.com([.a-z0-9]*)") @@ -148,6 +152,64 @@ func (b *Bucket) PutWriter(path string, h http.Header, c *Config) (w io.WriteClo // url returns a parsed url to the given path. c must not be nil func (b *Bucket) url(bPath string, c *Config) (*url.URL, error) { + // parse versionID parameter from path, if included + // See https://github.com/rlmcpherson/s3gof3r/issues/84 for rationale + purl, err := url.Parse(bPath) + if err != nil { + return nil, err + } + + var finalParams = "" + var vals url.Values + + // the prefix should not be URL encoded here, so we are treating it independent of the rest of the keys + // the implementation is based around the previous implementation (see rationale), so refactoring might be in order + + if purl.Query().Get(prefixParam) != "" { + finalParams = fmt.Sprintf("%s=%s", prefixParam, purl.Query().Get(prefixParam)) + purl.Query().Del(prefixParam) + } + + if len(purl.Query()) > 0 { + purl.Query() + vals = make(url.Values) + if v := purl.Query().Get(versionParam); v != "" { + vals.Add(versionParam, purl.Query().Get(versionParam)) + } + if v := purl.Query().Get(listTypeParam); v != "" { + vals.Add(listTypeParam, purl.Query().Get(listTypeParam)) + } + } + + if vals.Encode() != "" { + finalParams = fmt.Sprintf("%s&%s", finalParams, vals.Encode()) + } + + bPath = strings.Split(bPath, "?")[0] // remove all params from initial path + + // handling for bucket names containing periods / explicit PathStyle addressing + // http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html for details + if strings.Contains(b.Name, ".") || c.PathStyle { + return &url.URL{ + Host: b.S3.Domain, + Scheme: c.Scheme, + Path: path.Clean(fmt.Sprintf("/%s/%s", b.Name, bPath)), + RawQuery: finalParams, + }, nil + } else { + return &url.URL{ + Scheme: c.Scheme, + Path: path.Clean(fmt.Sprintf("/%s", bPath)), + Host: path.Clean(fmt.Sprintf("%s.%s", b.Name, b.S3.Domain)), + RawQuery: finalParams, + }, nil + } +} + +// urlForBucketRequest returns a parsed url to the given bucket, used for bucket-specific requests +// c must not be nil +func (b *Bucket) urlForBucketRequest(bPath string, c *Config) (*url.URL, error) { + // parse versionID parameter from path, if included // See https://github.com/rlmcpherson/s3gof3r/issues/84 for rationale purl, err := url.Parse(bPath) @@ -180,6 +242,8 @@ func (b *Bucket) url(bPath string, c *Config) (*url.URL, error) { } } + + func (b *Bucket) conf() *Config { c := b.Config if c == nil { diff --git a/sign.go b/sign.go index cc158c8..dfd4e9e 100644 --- a/sign.go +++ b/sign.go @@ -125,6 +125,9 @@ func (s *signer) buildCanonicalString() { uri = "/" } + uri = strings.Replace(uri, "@", "%40", -1) + uri = strings.Replace(uri, ":", "%3A", -1) + s.canonicalString = strings.Join([]string{ s.Request.Method, uri,