Skip to content

Commit

Permalink
Use stats package to calculate quantiles (kube-burner#537)
Browse files Browse the repository at this point in the history 
Use stats package to calculate percentiles

Signed-off-by: Raul Sevilla <[email protected]>
  • Loading branch information
@rsevilla87
rsevilla87 authored Jan 10, 2024
1 parent 81c0823 commit 7b7a60b
Show file tree
Hide file tree
Showing 6 changed files with 64 additions and 97 deletions.
6 changes: 3 additions & 3 deletions docs/measurements.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ One document, such as the following, is indexed per each pod created by the work
"uuid": "c40b4346-7af7-4c63-9ab4-aae7ccdd0616",
"namespace": "kubelet-density",
"podName": "kubelet-density-13",
"jobName": "kube-burner-job",
"jobConfig": {},
"nodeName": "worker-001"
}
```
Expand All @@ -53,7 +53,7 @@ Pod latency quantile sample:
"avg": 2876.3,
"timestamp": "2020-11-15T22:26:51.553221077+01:00",
"metricName": "podLatencyQuantilesMeasurement",
"jobName": "kubelet-density"
"jobConfig": {}
},
{
"quantileName": "PodScheduled",
Expand All @@ -65,7 +65,7 @@ Pod latency quantile sample:
"avg": 5.38,
"timestamp": "2020-11-15T22:26:51.553225151+01:00",
"metricName": "podLatencyQuantilesMeasurement",
"jobName": "kubelet-density"
"jobConfig": {}
}
```

Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.19
require (
github.com/Masterminds/sprig/v3 v3.2.2
github.com/cloud-bulldozer/go-commons v1.0.12
github.com/montanaflynn/stats v0.7.1
github.com/openshift/client-go v0.0.0-20230718165156-6014fb98e86a
github.com/prometheus/common v0.44.0
github.com/satori/go.uuid v1.2.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
Expand Down
32 changes: 19 additions & 13 deletions pkg/measurements/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/kube-burner/kube-burner/pkg/config"
"github.com/kube-burner/kube-burner/pkg/measurements/types"
"github.com/montanaflynn/stats"
log "github.com/sirupsen/logrus"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
)
Expand All @@ -36,23 +37,10 @@ type LatencyQuantiles struct {
Avg int `json:"avg"`
Timestamp time.Time `json:"timestamp"`
MetricName string `json:"metricName"`
JobName string `json:"jobName"`
JobConfig config.Job `json:"jobConfig"`
Metadata interface{} `json:"metadata,omitempty"`
}

// SetQuantile adds quantile value
func (plq *LatencyQuantiles) SetQuantile(quantile float64, qValue int) {
switch quantile {
case 0.5:
plq.P50 = qValue
case 0.95:
plq.P95 = qValue
case 0.99:
plq.P99 = qValue
}
}

// CheckThreshold checks latency thresholds
// returns a concatenated list of error strings with a new line between each string
func CheckThreshold(thresholds []types.LatencyThreshold, quantiles []interface{}) error {
Expand All @@ -74,3 +62,21 @@ func CheckThreshold(thresholds []types.LatencyThreshold, quantiles []interface{}
}
return utilerrors.NewAggregate(errs)
}

func NewLatencySummary(input []float64, name string) LatencyQuantiles {
latencyQuantiles := LatencyQuantiles{
QuantileName: name,
Timestamp: time.Now().UTC(),
}
val, _ := stats.Percentile(input, 50)
latencyQuantiles.P50 = int(val)
val, _ = stats.Percentile(input, 95)
latencyQuantiles.P95 = int(val)
val, _ = stats.Percentile(input, 99)
latencyQuantiles.P99 = int(val)
val, _ = stats.Max(input)
latencyQuantiles.Max = int(val)
val, _ = stats.Mean(input)
latencyQuantiles.Avg = int(val)
return latencyQuantiles
}
53 changes: 15 additions & 38 deletions pkg/measurements/pod_latency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ package measurements
import (
"context"
"fmt"
"math"
"sort"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -51,7 +49,6 @@ type podMetric struct {
podReady time.Time
PodReadyLatency int `json:"podReadyLatency"`
MetricName string `json:"metricName"`
JobName string `json:"jobName"`
JobConfig config.Job `json:"jobConfig"`
UUID string `json:"uuid"`
Namespace string `json:"namespace"`
Expand Down Expand Up @@ -85,7 +82,6 @@ func (p *podLatency) handleCreatePod(obj interface{}) {
MetricName: podLatencyMeasurement,
UUID: globalCfg.UUID,
JobConfig: *factory.jobConfig,
JobName: factory.jobConfig.Name,
Metadata: factory.metadata,
}
}
Expand Down Expand Up @@ -200,7 +196,6 @@ func (p *podLatency) collect(measurementWg *sync.WaitGroup) {
NodeName: pod.Spec.NodeName,
UUID: globalCfg.UUID,
JobConfig: *factory.jobConfig,
JobName: factory.jobConfig.Name,
Metadata: factory.metadata,
scheduled: scheduled,
initialized: initialized,
Expand Down Expand Up @@ -326,41 +321,23 @@ func (p *podLatency) normalizeMetrics() float64 {
}

func (p *podLatency) calcQuantiles() {
quantiles := []float64{0.5, 0.95, 0.99}
quantileMap := map[corev1.PodConditionType][]int{}
jc := factory.jobConfig
jc.Objects = nil
quantileMap := map[corev1.PodConditionType][]float64{}
for _, normLatency := range p.normLatencies {
quantileMap[corev1.PodScheduled] = append(quantileMap[corev1.PodScheduled], normLatency.(podMetric).SchedulingLatency)
quantileMap[corev1.ContainersReady] = append(quantileMap[corev1.ContainersReady], normLatency.(podMetric).ContainersReadyLatency)
quantileMap[corev1.PodInitialized] = append(quantileMap[corev1.PodInitialized], normLatency.(podMetric).InitializedLatency)
quantileMap[corev1.PodReady] = append(quantileMap[corev1.PodReady], normLatency.(podMetric).PodReadyLatency)
quantileMap[corev1.PodScheduled] = append(quantileMap[corev1.PodScheduled], float64(normLatency.(podMetric).SchedulingLatency))
quantileMap[corev1.ContainersReady] = append(quantileMap[corev1.ContainersReady], float64(normLatency.(podMetric).ContainersReadyLatency))
quantileMap[corev1.PodInitialized] = append(quantileMap[corev1.PodInitialized], float64(normLatency.(podMetric).InitializedLatency))
quantileMap[corev1.PodReady] = append(quantileMap[corev1.PodReady], float64(normLatency.(podMetric).PodReadyLatency))
}
for quantileName, v := range quantileMap {
podQ := metrics.LatencyQuantiles{
QuantileName: string(quantileName),
UUID: globalCfg.UUID,
Timestamp: time.Now().UTC(),
JobName: factory.jobConfig.Name,
JobConfig: *jc,
MetricName: podLatencyQuantilesMeasurement,
Metadata: factory.metadata,
}
sort.Ints(v)
length := len(v)
if length > 1 {
for _, quantile := range quantiles {
qValue := v[int(math.Ceil(float64(length)*quantile))-1]
podQ.SetQuantile(quantile, qValue)
}
podQ.Max = v[length-1]
}
sum := 0
for _, n := range v {
sum += n
}
podQ.Avg = int(math.Round(float64(sum) / float64(length)))
p.latencyQuantiles = append(p.latencyQuantiles, podQ)
calcSummary := func(name string, inputLatencies []float64) metrics.LatencyQuantiles {
latencySummary := metrics.NewLatencySummary(inputLatencies, name)
latencySummary.UUID = globalCfg.UUID
latencySummary.JobConfig = *factory.jobConfig
latencySummary.Metadata = factory.metadata
latencySummary.MetricName = podLatencyQuantilesMeasurement
return latencySummary
}
for podCondition, latencies := range quantileMap {
p.latencyQuantiles = append(p.latencyQuantiles, calcSummary(string(podCondition), latencies))
}
}

Expand Down
67 changes: 24 additions & 43 deletions pkg/measurements/vmi_latency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ package measurements

import (
"fmt"
"math"
"sort"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -74,7 +72,6 @@ type vmiMetric struct {
VMReadyLatency int `json:"vmReadyLatency"`

MetricName string `json:"metricName"`
JobName string `json:"jobName"`
JobConfig config.Job `json:"jobConfig"`
Metadata interface{} `json:"metadata,omitempty"`
UUID string `json:"uuid"`
Expand Down Expand Up @@ -110,7 +107,6 @@ func (p *vmiLatency) handleCreateVM(obj interface{}) {
Name: vm.Name,
MetricName: vmiLatencyMeasurement,
UUID: globalCfg.UUID,
JobName: factory.jobConfig.Name,
JobConfig: *factory.jobConfig,
Metadata: factory.metadata,
}
Expand Down Expand Up @@ -155,7 +151,6 @@ func (p *vmiLatency) handleCreateVMI(obj interface{}) {
Name: vmi.Name,
MetricName: vmiLatencyMeasurement,
UUID: globalCfg.UUID,
JobName: factory.jobConfig.Name,
}
}
}
Expand Down Expand Up @@ -451,48 +446,34 @@ func (p *vmiLatency) normalizeMetrics() {
}

func (p *vmiLatency) calcQuantiles() {
quantiles := []float64{0.5, 0.95, 0.99}
quantileMap := map[string][]int{}
quantileMap := map[string][]float64{}
for _, normLatency := range p.normLatencies {
if !normLatency.(*vmiMetric).vmReady.IsZero() {
quantileMap["VM"+string(kvv1.VirtualMachineReady)] = append(quantileMap["VM"+string(kvv1.VirtualMachineReady)], normLatency.(*vmiMetric).VMReadyLatency)
quantileMap["VMICreated"] = append(quantileMap["VMICreated"], normLatency.(*vmiMetric).VMICreatedLatency)
quantileMap["VM"+string(kvv1.VirtualMachineReady)] = append(quantileMap["VM"+string(kvv1.VirtualMachineReady)], float64(normLatency.(*vmiMetric).VMReadyLatency))
quantileMap["VMICreated"] = append(quantileMap["VMICreated"], float64(normLatency.(*vmiMetric).VMICreatedLatency))
}

quantileMap["VMI"+string(kvv1.Pending)] = append(quantileMap["VMI"+string(kvv1.Pending)], normLatency.(*vmiMetric).VMIPendingLatency)
quantileMap["VMI"+string(kvv1.Scheduling)] = append(quantileMap["VMI"+string(kvv1.Scheduling)], normLatency.(*vmiMetric).VMISchedulingLatency)
quantileMap["VMI"+string(kvv1.Scheduled)] = append(quantileMap["VMI"+string(kvv1.Scheduled)], normLatency.(*vmiMetric).VMIScheduledLatency)
quantileMap["VMI"+string(kvv1.VirtualMachineInstanceReady)] = append(quantileMap["VMI"+string(kvv1.VirtualMachineInstanceReady)], normLatency.(*vmiMetric).VMIReadyLatency)

quantileMap["PodCreated"] = append(quantileMap["PodCreated"], normLatency.(*vmiMetric).PodCreatedLatency)
quantileMap[string(corev1.PodScheduled)] = append(quantileMap[string(corev1.PodScheduled)], normLatency.(*vmiMetric).PodScheduledLatency)
quantileMap["Pod"+string(corev1.PodInitialized)] = append(quantileMap["Pod"+string(corev1.PodInitialized)], normLatency.(*vmiMetric).PodInitializedLatency)
quantileMap["Pod"+string(corev1.ContainersReady)] = append(quantileMap["Pod"+string(corev1.ContainersReady)], normLatency.(*vmiMetric).PodContainersReadyLatency)
quantileMap["Pod"+string(corev1.PodReady)] = append(quantileMap["Pod"+string(corev1.PodReady)], normLatency.(*vmiMetric).PodReadyLatency)
}
for quantileName, v := range quantileMap {
quantile := metrics.LatencyQuantiles{
QuantileName: quantileName,
UUID: globalCfg.UUID,
Timestamp: time.Now().UTC(),
JobName: factory.jobConfig.Name,
MetricName: vmiLatencyQuantilesMeasurement,
}
sort.Ints(v)
length := len(v)
if length > 1 {
for _, q := range quantiles {
qValue := v[int(math.Ceil(float64(length)*q))-1]
quantile.SetQuantile(q, qValue)
}
quantile.Max = v[length-1]
}
sum := 0
for _, n := range v {
sum += n
}
quantile.Avg = int(math.Round(float64(sum) / float64(length)))
p.latencyQuantiles = append(p.latencyQuantiles, quantile)
quantileMap["VMI"+string(kvv1.Pending)] = append(quantileMap["VMI"+string(kvv1.Pending)], float64(normLatency.(*vmiMetric).VMIPendingLatency))
quantileMap["VMI"+string(kvv1.Scheduling)] = append(quantileMap["VMI"+string(kvv1.Scheduling)], float64(normLatency.(*vmiMetric).VMISchedulingLatency))
quantileMap["VMI"+string(kvv1.Scheduled)] = append(quantileMap["VMI"+string(kvv1.Scheduled)], float64(normLatency.(*vmiMetric).VMIScheduledLatency))
quantileMap["VMI"+string(kvv1.VirtualMachineInstanceReady)] = append(quantileMap["VMI"+string(kvv1.VirtualMachineInstanceReady)], float64(normLatency.(*vmiMetric).VMIReadyLatency))
quantileMap["PodCreated"] = append(quantileMap["PodCreated"], float64(normLatency.(*vmiMetric).PodCreatedLatency))
quantileMap[string(corev1.PodScheduled)] = append(quantileMap[string(corev1.PodScheduled)], float64(normLatency.(*vmiMetric).PodScheduledLatency))
quantileMap["Pod"+string(corev1.PodInitialized)] = append(quantileMap["Pod"+string(corev1.PodInitialized)], float64(normLatency.(*vmiMetric).PodInitializedLatency))
quantileMap["Pod"+string(corev1.ContainersReady)] = append(quantileMap["Pod"+string(corev1.ContainersReady)], float64(normLatency.(*vmiMetric).PodContainersReadyLatency))
quantileMap["Pod"+string(corev1.PodReady)] = append(quantileMap["Pod"+string(corev1.PodReady)], float64(normLatency.(*vmiMetric).PodReadyLatency))

}
calcSummary := func(name string, inputLatencies []float64) metrics.LatencyQuantiles {
latencySummary := metrics.NewLatencySummary(inputLatencies, name)
latencySummary.UUID = globalCfg.UUID
latencySummary.JobConfig = *factory.jobConfig
latencySummary.Metadata = factory.metadata
latencySummary.MetricName = podLatencyQuantilesMeasurement
return latencySummary
}
for podCondition, latencies := range quantileMap {
p.latencyQuantiles = append(p.latencyQuantiles, calcSummary(podCondition, latencies))
}
}

Expand Down

0 comments on commit 7b7a60b

Please sign in to comment.