Skip to content

Commit

Permalink
Pull request #44: Drop compatibility with old multi tenant system
Browse files Browse the repository at this point in the history 
Merge in PRODUCT/squirreldb from PRODUCT-2228-improve-squirreldb-multi-tenant-system-part-2 to master

* commit '898473e61f65a4465227e2382c89b11f32acf8eb':
  Remove support for X-PromQL HTTP headers
  Remove support of providing metric TTL with label
  • Loading branch information
@ozon2 @PierreF
ozon2 authored and PierreF committed Jan 26, 2023
2 parents 3000671 + 898473e commit 854b3eb
Show file tree
Hide file tree
Showing 8 changed files with 21 additions and 245 deletions.
2 changes: 1 addition & 1 deletion api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
const (
httpServerShutdownTimeout = 10 * time.Second

// A read sample limit is already implemented dynamically with the header X-PromQL-Max-Evaluated-Points.
// A read sample limit is already implemented dynamically with the header X-SquirrelDB-Max-Evaluated-Points.
remoteReadSampleLimit = 0 // No limit
remoteReadMaxBytesInFrame = 1048576 // 1 MiB (Prometheus default)
)
Expand Down
20 changes: 0 additions & 20 deletions api/promql/queryable.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,6 @@ func (s Store) newQuerierFromHeaders(ctx context.Context, mint, maxt int64) (que
}

value := r.Header.Get(types.HeaderForcedMatcher)
if value == "" {
value = r.Header.Get(types.HeaderForcedMatcherOld)
}

if value != "" {
part := strings.SplitN(value, "=", 2)
if len(part) != 2 {
Expand Down Expand Up @@ -128,10 +124,6 @@ func (s Store) newQuerierFromHeaders(ctx context.Context, mint, maxt int64) (que
maxEvaluatedSeries := s.DefaultMaxEvaluatedSeries

maxEvaluatedSeriesText := r.Header.Get(types.HeaderMaxEvaluatedSeries)
if maxEvaluatedSeriesText == "" {
maxEvaluatedSeriesText = r.Header.Get(types.HeaderMaxEvaluatedSeriesOld)
}

if maxEvaluatedSeriesText != "" {
tmp, err := strconv.ParseUint(maxEvaluatedSeriesText, 10, 32)
if err != nil {
Expand All @@ -149,10 +141,6 @@ func (s Store) newQuerierFromHeaders(ctx context.Context, mint, maxt int64) (que
maxEvaluatedPoints := s.DefaultMaxEvaluatedPoints

maxEvaluatedPointsText := r.Header.Get(types.HeaderMaxEvaluatedPoints)
if maxEvaluatedPointsText == "" {
maxEvaluatedPointsText = r.Header.Get(types.HeaderMaxEvaluatedPointsOld)
}

if maxEvaluatedPointsText != "" {
tmp, err := strconv.ParseUint(maxEvaluatedPointsText, 10, 64)
if err != nil {
Expand All @@ -177,10 +165,6 @@ func (s Store) newQuerierFromHeaders(ctx context.Context, mint, maxt int64) (que
}

value = r.Header.Get(types.HeaderForcePreAggregated)
if value == "" {
value = r.Header.Get(types.HeaderForcePreAggregatedOld)
}

if value != "" {
tmp, err := strconv.ParseBool(value)
if err != nil {
Expand All @@ -191,10 +175,6 @@ func (s Store) newQuerierFromHeaders(ctx context.Context, mint, maxt int64) (que
}

value = r.Header.Get(types.HeaderForceRaw)
if value == "" {
value = r.Header.Get(types.HeaderForceRawOld)
}

if value != "" {
tmp, err := strconv.ParseBool(value)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion api/remotestorage/remotestorage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ func (r *RemoteStorage) Appender(ctx context.Context) storage.Appender {

timeToLive, err = strconv.ParseInt(ttlRaw, 10, 64)
if err != nil {
return errAppender{fmt.Errorf("can't parse time to live '%s', using default: %w", ttlRaw, err)}
return errAppender{fmt.Errorf("can't parse time to live header '%s': %w", ttlRaw, err)}
}
}

Expand Down
40 changes: 0 additions & 40 deletions cassandra/index/index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,6 @@ const (
expireBatchSize = 1000
)

const timeToLiveLabelName = "__ttl__"

const (
// The expiration of entries in Cassandra starts everyday at 00:00 UTC + expirationStartOffset.
expirationStartOffset = 6 * time.Hour
Expand Down Expand Up @@ -2105,11 +2103,6 @@ func (c *CassandraIndex) lookupIDsFromCache(

for i, req := range requests {
ttlSeconds := req.TTLSeconds
if ttlSeconds == 0 {
// TODO: Compatibility with TTL label, will be removed later.
ttlSeconds = timeToLiveFromLabels(&req.Labels)
}

if ttlSeconds == 0 {
ttlSeconds = int64(c.options.DefaultTimeToLive.Seconds())
}
Expand Down Expand Up @@ -4423,19 +4416,6 @@ func (c *CassandraIndex) selectIDS2LabelsAndExpiration(
)
}

// popLabelsValue get and delete value via its name from a labels.Label list.
func popLabelsValue(labels *labels.Labels, key string) (string, bool) {
for i, label := range *labels {
if label.Name == key {
*labels = append((*labels)[:i], (*labels)[i+1:]...)

return label.Value, true
}
}

return "", false
}

// sortLabels returns the labels.Label list sorted by name.
func sortLabels(labelList labels.Labels) labels.Labels {
sortedLabels := labelList.Copy()
Expand All @@ -4444,26 +4424,6 @@ func sortLabels(labelList labels.Labels) labels.Labels {
return sortedLabels
}

// Returns and delete time to live from a labels.Label list.
func timeToLiveFromLabels(labels *labels.Labels) int64 {
value, exists := popLabelsValue(labels, timeToLiveLabelName)

var timeToLive int64

if exists {
var err error
timeToLive, err = strconv.ParseInt(value, 10, 64)

if err != nil {
log.Warn().Err(err).Msg("Can't get time to live from labels, using default")

return 0
}
}

return timeToLive
}

// InternalUpdateAllShards updates the expiration of all shards.
// Shards older than the TTL will be deleted, other shard will have their expiration created or updated.
func (c *CassandraIndex) InternalUpdateAllShards(ctx context.Context, ttl time.Duration) error {
Expand Down
129 changes: 0 additions & 129 deletions cassandra/index/index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1047,135 +1047,6 @@ func Benchmark_labelsToID(b *testing.B) {
}
}

func Test_timeToLiveFromLabels(t *testing.T) {
tests := []struct {
name string
labels labels.Labels
wantLabels labels.Labels
want int64
}{
{
name: "no ttl",
labels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "job", Value: "scrape"},
},
want: 0,
wantLabels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "job", Value: "scrape"},
},
},
{
name: "with ttl",
labels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "job", Value: "scrape"},
{Name: "__ttl__", Value: "3600"},
},
want: 3600,
wantLabels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "job", Value: "scrape"},
},
},
{
name: "with ttl2",
labels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "__ttl__", Value: "3600"},
{Name: "job", Value: "scrape"},
},
want: 3600,
wantLabels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "job", Value: "scrape"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := timeToLiveFromLabels(&tt.labels)
if got != tt.want {
t.Errorf("timeToLiveFromLabels() got = %v, want %v", got, tt.want)
}
if !reflect.DeepEqual(tt.labels, tt.wantLabels) {
t.Errorf("timeToLiveFromLabels() labels = %v, want %v", tt.labels, tt.wantLabels)
}
})
}
}

func Benchmark_timeToLiveFromLabels(b *testing.B) {
tests := []struct {
name string
labels labels.Labels
wantLabels labels.Labels
wantTTL int64
wantErr bool
}{
{
name: "no ttl",
labels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: "job", Value: "scrape"},
},
},
{
name: "with ttl",
labels: labels.Labels{
{Name: "__name__", Value: "up"},
{Name: ttlLabel, Value: "3600"},
{Name: "job", Value: "scrape"},
},
},
{
name: "12 labels no ttl",
labels: labels.Labels{
{Name: "job", Value: "scrape"},
{Name: "__name__", Value: "up"},
{Name: "labels1", Value: "value1"},
{Name: "labels2", Value: "value2"},
{Name: "labels3", Value: "value3"},
{Name: "labels4", Value: "value4"},
{Name: "labels5", Value: "value5"},
{Name: "labels6", Value: "value6"},
{Name: "labels7", Value: "value7"},
{Name: "labels8", Value: "value8"},
{Name: "labels9", Value: "value9"},
{Name: "labels10", Value: "value10"},
},
},
{
name: "12 labels ttl",
labels: labels.Labels{
{Name: "job", Value: "scrape"},
{Name: "__name__", Value: "up"},
{Name: "labels1", Value: "value1"},
{Name: "labels2", Value: "value2"},
{Name: "labels3", Value: "value3"},
{Name: "labels4", Value: "value4"},
{Name: "labels5", Value: "value5"},
{Name: "labels6", Value: "value6"},
{Name: ttlLabel, Value: "3600"},
{Name: "labels7", Value: "value7"},
{Name: "labels8", Value: "value8"},
{Name: "labels9", Value: "value9"},
{Name: "labels10", Value: "value10"},
},
},
}
for _, tt := range tests {
b.Run(tt.name, func(b *testing.B) {
for n := 0; n < b.N; n++ {
labelsIn := make(labels.Labels, len(tt.labels))
copy(labelsIn, tt.labels)
_ = timeToLiveFromLabels(&labelsIn)
}
})
}
}

func Test_sortLabels(t *testing.T) {
type args struct {
labels labels.Labels
Expand Down
44 changes: 6 additions & 38 deletions dummy/index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,19 @@ package dummy
import (
"context"
"fmt"
"log"
"sort"
"squirreldb/types"
"strconv"
"strings"
"sync"
"time"

"github.com/prometheus/prometheus/model/labels"
)

const postinglabelName = "__label|names__"
const (
postinglabelName = "__label|names__"
defaultTimeToLive = 24 * time.Hour
)

type MetricsLabel struct {
List []types.MetricLabel
Expand Down Expand Up @@ -134,9 +135,9 @@ func (idx *Index) LookupIDs(ctx context.Context, requests []types.LookupRequest)
}

for i, req := range requests {
ttls[i] = timeToLiveFromLabels(&req.Labels)
ttls[i] = req.TTLSeconds
if ttls[i] == 0 {
ttls[i] = int64(86400)
ttls[i] = int64(defaultTimeToLive.Seconds())
}

ids[i] = types.MetricID(req.Labels.Hash())
Expand Down Expand Up @@ -265,36 +266,3 @@ outer:

return list
}

// copied from cassandra/index.
func timeToLiveFromLabels(labels *labels.Labels) int64 {
value, exists := popLabelsValue(labels, "__ttl__")

var timeToLive int64

if exists {
var err error
timeToLive, err = strconv.ParseInt(value, 10, 64)

if err != nil {
log.Printf("Warning: Can't get time to live from labels (%v), using default", err)

return 0
}
}

return timeToLive
}

// copied from cassandra/index.
func popLabelsValue(labels *labels.Labels, key string) (string, bool) {
for i, label := range *labels {
if label.Name == key {
*labels = append((*labels)[:i], (*labels)[i+1:]...)

return label.Value, true
}
}

return "", false
}
14 changes: 8 additions & 6 deletions tests/squirreldb-cassandra-index-bench/bench.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ func makeInsertRequests(now time.Time, shardID string, rnd *rand.Rand) []types.L

// We remove 1 days (and max ttl update delay) so the expiration of the
// metrics is yesterday
negativeTTL := strconv.FormatInt(-86400-int64(index.InternalMaxTTLUpdateDelay().Seconds()), 10)
negativeTTL := -int64((24*time.Hour + index.InternalMaxTTLUpdateDelay()).Seconds())

for n := 0; n < *shardSize; n++ {
userID := strconv.FormatInt(rnd.Int63n(100000), 10)
Expand All @@ -518,21 +518,23 @@ func makeInsertRequests(now time.Time, shardID string, rnd *rand.Rand) []types.L
labelsMap[fmt.Sprintf("label%02d", i)] = strconv.FormatInt(rnd.Int63n(20), 10)
}

if *expiredFaction > 0 && n%*expiredFaction == 0 {
labelsMap["__ttl__"] = negativeTTL
}

promLabel := labels.FromMap(labelsMap)

if *sortInsert {
sort.Sort(promLabel)
}

metrics[n] = types.LookupRequest{
request := types.LookupRequest{
Start: now,
End: now,
Labels: promLabel,
}

if *expiredFaction > 0 && n%*expiredFaction == 0 {
request.TTLSeconds = negativeTTL
}

metrics[n] = request
}

return metrics
Expand Down
Loading

0 comments on commit 854b3eb

Please sign in to comment.