Skip to content

Commit

Permalink
resources: Add responseHeaders option to resources.GetRemote
Browse files Browse the repository at this point in the history
* These response headers will be included in `.Data.Headers` if found.
* The header name matching is case insensitive.
* `Data.Headers` is of type `map[string][]string`
* In most cases there will be only one value per header key, but e.g. `Set-Cookie` commonly has multiple values.

Fixes #12521
  • Loading branch information
bep committed Jan 23, 2025
1 parent 43307b0 commit b0a2173
Show file tree
Hide file tree
Showing 2 changed files with 57 additions and 10 deletions.
37 changes: 36 additions & 1 deletion resources/resource_factories/create/create_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,42 @@ func TestGetRemoteHead(t *testing.T) {

b.AssertFileContent("public/index.html",
"Head Content: .",
"Head Data: map[ContentLength:18210 ContentType:image/png Status:200 OK StatusCode:200 TransferEncoding:[]]",
"Head Data: map[ContentLength:18210 ContentType:image/png Headers:map[] Status:200 OK StatusCode:200 TransferEncoding:[]]",
)
}

func TestGetRemoteResponseHeaders(t *testing.T) {
files := `
-- config.toml --
[security]
[security.http]
methods = ['(?i)GET|POST|HEAD']
urls = ['.*gohugo\.io.*']
-- layouts/index.html --
{{ $url := "https://gohugo.io/img/hugo.png" }}
{{ $opts := dict "method" "head" "responseHeaders" (slice "X-Frame-Options" "Server") }}
{{ with try (resources.GetRemote $url $opts) }}
{{ with .Err }}
{{ errorf "Unable to get remote resource: %s" . }}
{{ else with .Value }}
Response Headers: {{ .Data.Headers }}
{{ else }}
{{ errorf "Unable to get remote resource: %s" $url }}
{{ end }}
{{ end }}
`

b := hugolib.NewIntegrationTestBuilder(
hugolib.IntegrationTestConfig{
T: t,
TxtarString: files,
},
)

b.Build()

b.AssertFileContent("public/index.html",
"Response Headers: map[Server:[Netlify] X-Frame-Options:[DENY]]",
)
}

Expand Down
30 changes: 21 additions & 9 deletions resources/resource_factories/create/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

"github.com/gohugoio/httpcache"
"github.com/gohugoio/hugo/common/hashing"
"github.com/gohugoio/hugo/common/hstrings"
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/common/maps"
Expand All @@ -51,18 +52,28 @@ type HTTPError struct {
Body string
}

func responseToData(res *http.Response, readBody bool) map[string]any {
func responseToData(res *http.Response, readBody bool, includeHeaders []string) map[string]any {
var body []byte
if readBody {
body, _ = io.ReadAll(res.Body)
}

responseHeaders := make(map[string][]string)
if true || len(includeHeaders) > 0 {
for k, v := range res.Header {
if hstrings.InSlicEqualFold(includeHeaders, k) {
responseHeaders[k] = v
}
}
}

m := map[string]any{
"StatusCode": res.StatusCode,
"Status": res.Status,
"TransferEncoding": res.TransferEncoding,
"ContentLength": res.ContentLength,
"ContentType": res.Header.Get("Content-Type"),
"Headers": responseHeaders,
}

if readBody {
Expand All @@ -72,7 +83,7 @@ func responseToData(res *http.Response, readBody bool) map[string]any {
return m
}

func toHTTPError(err error, res *http.Response, readBody bool) *HTTPError {
func toHTTPError(err error, res *http.Response, readBody bool, responseHeaders []string) *HTTPError {
if err == nil {
panic("err is nil")
}
Expand All @@ -85,7 +96,7 @@ func toHTTPError(err error, res *http.Response, readBody bool) *HTTPError {

return &HTTPError{
error: err,
Data: responseToData(res, readBody),
Data: responseToData(res, readBody, responseHeaders),
}
}

Expand Down Expand Up @@ -213,7 +224,7 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
}

if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource from '%s': %s", uri, http.StatusText(res.StatusCode)), res, !isHeadMethod)
return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource from '%s': %s", uri, http.StatusText(res.StatusCode)), res, !isHeadMethod, options.ResponseHeaders)
}

var (
Expand Down Expand Up @@ -280,7 +291,7 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
}

userKey = filename[:len(filename)-len(path.Ext(filename))] + "_" + userKey + mediaType.FirstSuffix.FullSuffix
data := responseToData(res, false)
data := responseToData(res, false, options.ResponseHeaders)

return c.rs.NewResource(
resources.ResourceSourceDescriptor{
Expand Down Expand Up @@ -345,9 +356,10 @@ func hasHeaderKey(m http.Header, key string) bool {
}

type fromRemoteOptions struct {
Method string
Headers map[string]any
Body []byte
Method string
Headers map[string]any
Body []byte
ResponseHeaders []string
}

func (o fromRemoteOptions) BodyReader() io.Reader {
Expand Down Expand Up @@ -432,7 +444,7 @@ func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error
if resp != nil {
msg = resp.Status
}
err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD")
err := toHTTPError(fmt.Errorf("retry timeout (configured to %s) fetching remote resource: %s", t.Cfg.Timeout(), msg), resp, req.Method != "HEAD", nil)
return resp, err
}
time.Sleep(nextSleep)
Expand Down

0 comments on commit b0a2173

Please sign in to comment.