Skip to content

Commit

Permalink
🐛fix: update getOffer to consider quality and specificity (#2486)
Browse files Browse the repository at this point in the history 
* feat: getOffer consider q value and specificity

* fix: ignore q=0

* fix: float

* test: client-prefered order and q=0 not acceptable

* fix: always use my insertion sort.

* fix: sort.SliceStable if > 20

* fix: zero allocations

* perf: optimize the sort

* chore: fix lint issue

* fix: consider order

* chore: fix test func name

* chore: fix helper test func name

* chore: revert fix

* perf: use fasthttp.ParseUfloat

* test: GetOffer and SortAcceptedTypes

* chore: remote nil check

* test: sortAcceptedTypes

* fix: use utils.UnsafeBytes

* docs: update docs for fiber PR #2486

* docs: update docs for fiber PR #2486

* test: add test from docs

* fix: yaml
  • Loading branch information
@sixcolors
sixcolors authored Jun 7, 2023
1 parent d91ea9e commit 0f5ffed
Show file tree
Hide file tree
Showing 4 changed files with 252 additions and 26 deletions.
4 changes: 4 additions & 0 deletions ctx_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ func Test_Ctx_Accepts(t *testing.T) {
utils.AssertEqual(t, "", c.Accepts())
utils.AssertEqual(t, ".xml", c.Accepts(".xml"))
utils.AssertEqual(t, "", c.Accepts(".john"))
utils.AssertEqual(t, "application/xhtml+xml", c.Accepts("application/xml", "application/xml+rss", "application/yaml", "application/xhtml+xml"), "must use client-preferred mime type")

c.Request().Header.Set(HeaderAccept, "application/json, text/plain, */*;q=0")
utils.AssertEqual(t, "", c.Accepts("html"), "must treat */*;q=0 as not acceptable")

c.Request().Header.Set(HeaderAccept, "text/*, application/json")
utils.AssertEqual(t, "html", c.Accepts("html"))
Expand Down
14 changes: 13 additions & 1 deletion docs/api/ctx.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,19 +24,31 @@ func (c *Ctx) AcceptsLanguages(offers ...string) string
```

```go title="Example"
// Accept: text/*, application/json
// Accept: text/html, application/json; q=0.8, text/plain; q=0.5; charset="utf-8"

app.Get("/", func(c *fiber.Ctx) error {
c.Accepts("html") // "html"
c.Accepts("text/html") // "text/html"
c.Accepts("json", "text") // "json"
c.Accepts("application/json") // "application/json"
c.Accepts("text/plain", "application/json") // "application/json", due to quality
c.Accepts("image/png") // ""
c.Accepts("png") // ""
// ...
})
```

```go title="Example 2"
// Accept: text/html, text/*, application/json, */*; q=0

app.Get("/", func(c *fiber.Ctx) error {
c.Accepts("text/plain", "application/json") // "application/json", due to specificity
c.Accepts("application/json", "text/html") // "text/html", due to first match
c.Accepts("image/png") // "", due to */* without q factor 0 is Not Acceptable
// ...
})
```

Fiber provides similar functions for the other accept headers.

```go
Expand Down
131 changes: 113 additions & 18 deletions helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@ import (
"github.com/valyala/fasthttp"
)

// acceptType is a struct that holds the parsed value of an Accept header
// along with quality, specificity, and order.
// used for sorting accept headers.
type acceptedType struct {
spec string
quality float64
specificity int
order int
}

// getTLSConfig returns a net listener's tls config
func getTLSConfig(ln net.Listener) *tls.Config {
// Get listener type
Expand Down Expand Up @@ -263,40 +273,125 @@ func acceptsOfferType(spec, offerType string) bool {
func getOffer(header string, isAccepted func(spec, offer string) bool, offers ...string) string {
if len(offers) == 0 {
return ""
} else if header == "" {
}
if header == "" {
return offers[0]
}

for _, offer := range offers {
if len(offer) == 0 {
continue
// Parse header and get accepted types with their quality and specificity
// See: https://www.rfc-editor.org/rfc/rfc9110#name-content-negotiation-fields
spec, commaPos, order := "", 0, 0
acceptedTypes := make([]acceptedType, 0, 20)
for len(header) > 0 {
order++

// Skip spaces
header = utils.TrimLeft(header, ' ')

// Get spec
commaPos = strings.IndexByte(header, ',')
if commaPos != -1 {
spec = utils.Trim(header[:commaPos], ' ')
} else {
spec = utils.TrimLeft(header, ' ')
}
spec, commaPos := "", 0
for len(header) > 0 && commaPos != -1 {
commaPos = strings.IndexByte(header, ',')
if commaPos != -1 {
spec = utils.Trim(header[:commaPos], ' ')
} else {
spec = utils.TrimLeft(header, ' ')
}
if factorSign := strings.IndexByte(spec, ';'); factorSign != -1 {
spec = spec[:factorSign]
}

// isAccepted if the current offer is accepted
if isAccepted(spec, offer) {
return offer
// Get quality
quality := 1.0
if factorSign := strings.IndexByte(spec, ';'); factorSign != -1 {
factor := utils.Trim(spec[factorSign+1:], ' ')
if strings.HasPrefix(factor, "q=") {
if q, err := fasthttp.ParseUfloat(utils.UnsafeBytes(factor[2:])); err == nil {
quality = q
}
}
spec = spec[:factorSign]
}

// Skip if quality is 0.0
// See: https://www.rfc-editor.org/rfc/rfc9110#quality.values
if quality == 0.0 {
if commaPos != -1 {
header = header[commaPos+1:]
} else {
break
}
continue
}

// Get specificity
specificity := 0
// check for wildcard this could be a mime */* or a wildcard character *
if spec == "*/*" || spec == "*" {
specificity = 1
} else if strings.HasSuffix(spec, "/*") {
specificity = 2
} else if strings.IndexByte(spec, '/') != -1 {
specificity = 3
} else {
specificity = 4
}

// Add to accepted types
acceptedTypes = append(acceptedTypes, acceptedType{spec, quality, specificity, order})

// Next
if commaPos != -1 {
header = header[commaPos+1:]
} else {
break
}
}

if len(acceptedTypes) > 1 {
// Sort accepted types by quality and specificity, preserving order of equal elements
sortAcceptedTypes(&acceptedTypes)
}

// Find the first offer that matches the accepted types
for _, acceptedType := range acceptedTypes {
for _, offer := range offers {
if len(offer) == 0 {
continue
}
if isAccepted(acceptedType.spec, offer) {
return offer
}
}
}

return ""
}

// sortAcceptedTypes sorts accepted types by quality and specificity, preserving order of equal elements
//
// Parameters are not supported, they are ignored when sorting by specificity.
//
// See: https://www.rfc-editor.org/rfc/rfc9110#name-content-negotiation-fields
func sortAcceptedTypes(at *[]acceptedType) {
if at == nil || len(*at) < 2 {
return
}
acceptedTypes := *at

for i := 1; i < len(acceptedTypes); i++ {
lo, hi := 0, i-1
for lo <= hi {
mid := (lo + hi) / 2
if acceptedTypes[i].quality < acceptedTypes[mid].quality ||
(acceptedTypes[i].quality == acceptedTypes[mid].quality && acceptedTypes[i].specificity < acceptedTypes[mid].specificity) ||
(acceptedTypes[i].quality == acceptedTypes[mid].quality && acceptedTypes[i].specificity == acceptedTypes[mid].specificity && acceptedTypes[i].order > acceptedTypes[mid].order) {
lo = mid + 1
} else {
hi = mid - 1
}
}
for j := i; j > lo; j-- {
acceptedTypes[j-1], acceptedTypes[j] = acceptedTypes[j], acceptedTypes[j-1]
}
}
}

func matchEtag(s, etag string) bool {
if s == etag || s == "W/"+etag || "W/"+s == etag {
return true
Expand Down
129 changes: 122 additions & 7 deletions helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,128 @@ func Test_Utils_ETag(t *testing.T) {
})
}

func Test_Utils_GetOffer(t *testing.T) {
t.Parallel()
utils.AssertEqual(t, "", getOffer("hello", acceptsOffer))
utils.AssertEqual(t, "1", getOffer("", acceptsOffer, "1"))
utils.AssertEqual(t, "", getOffer("2", acceptsOffer, "1"))

utils.AssertEqual(t, "", getOffer("", acceptsOfferType))
utils.AssertEqual(t, "", getOffer("text/html", acceptsOfferType))
utils.AssertEqual(t, "", getOffer("text/html", acceptsOfferType, "application/json"))
utils.AssertEqual(t, "", getOffer("text/html;q=0", acceptsOfferType, "text/html"))
utils.AssertEqual(t, "", getOffer("application/json, */*; q=0", acceptsOfferType, "image/png"))
utils.AssertEqual(t, "application/xml", getOffer("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", acceptsOfferType, "application/xml", "application/json"))
utils.AssertEqual(t, "text/html", getOffer("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", acceptsOfferType, "text/html"))
utils.AssertEqual(t, "application/pdf", getOffer("text/plain;q=0,application/pdf;q=0.9,*/*;q=0.000", acceptsOfferType, "application/pdf", "application/json"))
utils.AssertEqual(t, "application/pdf", getOffer("text/plain;q=0,application/pdf;q=0.9,*/*;q=0.000", acceptsOfferType, "application/pdf", "application/json"))

utils.AssertEqual(t, "", getOffer("utf-8, iso-8859-1;q=0.5", acceptsOffer))
utils.AssertEqual(t, "", getOffer("utf-8, iso-8859-1;q=0.5", acceptsOffer, "ascii"))
utils.AssertEqual(t, "utf-8", getOffer("utf-8, iso-8859-1;q=0.5", acceptsOffer, "utf-8"))
utils.AssertEqual(t, "iso-8859-1", getOffer("utf-8;q=0, iso-8859-1;q=0.5", acceptsOffer, "utf-8", "iso-8859-1"))

utils.AssertEqual(t, "deflate", getOffer("gzip, deflate", acceptsOffer, "deflate"))
utils.AssertEqual(t, "", getOffer("gzip, deflate;q=0", acceptsOffer, "deflate"))
}

func Benchmark_Utils_GetOffer(b *testing.B) {
headers := []string{
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"application/json",
"utf-8, iso-8859-1;q=0.5",
"gzip, deflate",
}
offers := [][]string{
{"text/html", "application/xml", "application/xml+xhtml"},
{"application/json"},
{"utf-8"},
{"deflate"},
}
for n := 0; n < b.N; n++ {
for i, header := range headers {
getOffer(header, acceptsOfferType, offers[i]...)
}
}
}

func Test_Utils_SortAcceptedTypes(t *testing.T) {
t.Parallel()
acceptedTypes := []acceptedType{
{spec: "text/html", quality: 1, specificity: 3, order: 0},
{spec: "text/*", quality: 0.5, specificity: 2, order: 1},
{spec: "*/*", quality: 0.1, specificity: 1, order: 2},
{spec: "application/json", quality: 0.999, specificity: 3, order: 3},
{spec: "application/xml", quality: 1, specificity: 3, order: 4},
{spec: "application/pdf", quality: 1, specificity: 3, order: 5},
{spec: "image/png", quality: 1, specificity: 3, order: 6},
{spec: "image/jpeg", quality: 1, specificity: 3, order: 7},
{spec: "image/*", quality: 1, specificity: 2, order: 8},
{spec: "image/gif", quality: 1, specificity: 3, order: 9},
{spec: "text/plain", quality: 1, specificity: 3, order: 10},
}
sortAcceptedTypes(&acceptedTypes)
utils.AssertEqual(t, acceptedTypes, []acceptedType{
{spec: "text/html", quality: 1, specificity: 3, order: 0},
{spec: "application/xml", quality: 1, specificity: 3, order: 4},
{spec: "application/pdf", quality: 1, specificity: 3, order: 5},
{spec: "image/png", quality: 1, specificity: 3, order: 6},
{spec: "image/jpeg", quality: 1, specificity: 3, order: 7},
{spec: "image/gif", quality: 1, specificity: 3, order: 9},
{spec: "text/plain", quality: 1, specificity: 3, order: 10},
{spec: "image/*", quality: 1, specificity: 2, order: 8},
{spec: "application/json", quality: 0.999, specificity: 3, order: 3},
{spec: "text/*", quality: 0.5, specificity: 2, order: 1},
{spec: "*/*", quality: 0.1, specificity: 1, order: 2},
})
}

// go test -v -run=^$ -bench=Benchmark_Utils_SortAcceptedTypes_Sorted -benchmem -count=4
func Benchmark_Utils_SortAcceptedTypes_Sorted(b *testing.B) {
acceptedTypes := make([]acceptedType, 3)
for n := 0; n < b.N; n++ {
acceptedTypes[0] = acceptedType{spec: "text/html", quality: 1, specificity: 1, order: 0}
acceptedTypes[1] = acceptedType{spec: "text/*", quality: 0.5, specificity: 1, order: 1}
acceptedTypes[2] = acceptedType{spec: "*/*", quality: 0.1, specificity: 1, order: 2}
sortAcceptedTypes(&acceptedTypes)
}
utils.AssertEqual(b, "text/html", acceptedTypes[0].spec)
utils.AssertEqual(b, "text/*", acceptedTypes[1].spec)
utils.AssertEqual(b, "*/*", acceptedTypes[2].spec)
}

// go test -v -run=^$ -bench=Benchmark_Utils_SortAcceptedTypes_Unsorted -benchmem -count=4
func Benchmark_Utils_SortAcceptedTypes_Unsorted(b *testing.B) {
acceptedTypes := make([]acceptedType, 11)
for n := 0; n < b.N; n++ {
acceptedTypes[0] = acceptedType{spec: "text/html", quality: 1, specificity: 3, order: 0}
acceptedTypes[1] = acceptedType{spec: "text/*", quality: 0.5, specificity: 2, order: 1}
acceptedTypes[2] = acceptedType{spec: "*/*", quality: 0.1, specificity: 1, order: 2}
acceptedTypes[3] = acceptedType{spec: "application/json", quality: 0.999, specificity: 3, order: 3}
acceptedTypes[4] = acceptedType{spec: "application/xml", quality: 1, specificity: 3, order: 4}
acceptedTypes[5] = acceptedType{spec: "application/pdf", quality: 1, specificity: 3, order: 5}
acceptedTypes[6] = acceptedType{spec: "image/png", quality: 1, specificity: 3, order: 6}
acceptedTypes[7] = acceptedType{spec: "image/jpeg", quality: 1, specificity: 3, order: 7}
acceptedTypes[8] = acceptedType{spec: "image/*", quality: 1, specificity: 2, order: 8}
acceptedTypes[9] = acceptedType{spec: "image/gif", quality: 1, specificity: 3, order: 9}
acceptedTypes[10] = acceptedType{spec: "text/plain", quality: 1, specificity: 3, order: 10}
sortAcceptedTypes(&acceptedTypes)
}
utils.AssertEqual(b, acceptedTypes, []acceptedType{
{spec: "text/html", quality: 1, specificity: 3, order: 0},
{spec: "application/xml", quality: 1, specificity: 3, order: 4},
{spec: "application/pdf", quality: 1, specificity: 3, order: 5},
{spec: "image/png", quality: 1, specificity: 3, order: 6},
{spec: "image/jpeg", quality: 1, specificity: 3, order: 7},
{spec: "image/gif", quality: 1, specificity: 3, order: 9},
{spec: "text/plain", quality: 1, specificity: 3, order: 10},
{spec: "image/*", quality: 1, specificity: 2, order: 8},
{spec: "application/json", quality: 0.999, specificity: 3, order: 3},
{spec: "text/*", quality: 0.5, specificity: 2, order: 1},
{spec: "*/*", quality: 0.1, specificity: 1, order: 2},
})
}

// go test -v -run=^$ -bench=Benchmark_App_ETag -benchmem -count=4
func Benchmark_Utils_ETag(b *testing.B) {
app := New()
Expand Down Expand Up @@ -221,13 +343,6 @@ func Test_Utils_Parse_Address(t *testing.T) {
}
}

func Test_Utils_GetOffset(t *testing.T) {
t.Parallel()
utils.AssertEqual(t, "", getOffer("hello", acceptsOffer))
utils.AssertEqual(t, "1", getOffer("", acceptsOffer, "1"))
utils.AssertEqual(t, "", getOffer("2", acceptsOffer, "1"))
}

func Test_Utils_TestConn_Deadline(t *testing.T) {
t.Parallel()
conn := &testConn{}
Expand Down

1 comment on commit 0f5ffed

@ReneWerner87
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 2.

Benchmark suite Current: 0f5ffed Previous: 06ef450 Ratio
Benchmark_App_ETag 7983 ns/op 1044 B/op 3 allocs/op 3699 ns/op 1044 B/op 3 allocs/op 2.16
Benchmark_Ctx_AcceptsCharsets 166.8 ns/op 0 B/op 0 allocs/op 61.23 ns/op 0 B/op 0 allocs/op 2.72
Benchmark_Ctx_AcceptsEncodings 225.8 ns/op 0 B/op 0 allocs/op 80.48 ns/op 0 B/op 0 allocs/op 2.81
Benchmark_Ctx_AcceptsLanguages 337.1 ns/op 0 B/op 0 allocs/op 62.14 ns/op 0 B/op 0 allocs/op 5.42

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.