diff --git a/backend/pkg/commons/metrics/metrics.go b/backend/pkg/commons/metrics/metrics.go index 10fe6b883..302ea5368 100644 --- a/backend/pkg/commons/metrics/metrics.go +++ b/backend/pkg/commons/metrics/metrics.go @@ -66,6 +66,10 @@ var ( Name: "notifications_sent", Help: "Counter of notifications sent with the channel and notification type in the label", }, []string{"channel", "status"}) + State = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "state", + Help: "Gauge for various states", + }, []string{"state"}) ) func init() { diff --git a/backend/pkg/exporter/modules/dashboard_data.go b/backend/pkg/exporter/modules/dashboard_data.go index 9f4eba281..0506799f7 100644 --- a/backend/pkg/exporter/modules/dashboard_data.go +++ b/backend/pkg/exporter/modules/dashboard_data.go @@ -725,6 +725,8 @@ func (d *dashboardData) backfillHeadEpochData(upToEpoch *uint64) (backfillResult } d.log.InfoWithFields(map[string]interface{}{"epoch start": datas[0].Epoch, "epoch end": lastEpoch}, "backfill, aggregated epoch data") } + + metrics.State.WithLabelValues("exporter_v2dash_last_exported_epoch").Set(float64(lastEpoch)) } if lastEpoch%225 < epochFetchParallelism { @@ -845,11 +847,11 @@ func (d *dashboardData) aggregatePerEpoch(updateRollingWindows bool, preventClea err = errGroup.Wait() if err != nil { - metrics.Errors.WithLabelValues("exporter_v2dash_agg_non_roling_fail").Inc() + metrics.Errors.WithLabelValues("exporter_v2dash_agg_non_rolling_fail").Inc() return errors.Wrap(err, "failed to aggregate") } d.log.Infof("[time] all of epoch based aggregation took %v", time.Since(start)) - metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_non_roling").Observe(time.Since(start).Seconds()) + metrics.TaskDuration.WithLabelValues("exporter_v2dash_agg_non_rolling").Observe(time.Since(start).Seconds()) if updateRollingWindows { // todo you could add it to the err group above IF no bootstrap is needed. @@ -874,6 +876,8 @@ func (d *dashboardData) aggregatePerEpoch(updateRollingWindows bool, preventClea } } + metrics.State.WithLabelValues("exporter_v2dash_last_exported_epoch").Set(float64(currentExportedEpoch)) + // clear old hourly aggregated epochs, do not remove epochs from epoch table here as these are needed for Mid aggregation err = d.epochToHour.clearOldHourAggregations(int64(currentExportedEpoch - d.epochToHour.getHourRetentionDurationEpochs())) if err != nil { @@ -963,6 +967,8 @@ func (d *dashboardData) OnFinalizedCheckpoint(_ *constypes.StandardFinalizedChec } } + metrics.State.WithLabelValues("exporter_v2dash_last_finalized_epoch").Set(float64(res.Data.Finalized.Epoch)) + d.headEpochQueue <- res.Data.Finalized.Epoch return nil diff --git a/backend/pkg/exporter/modules/dashboard_data_w_rolling.go b/backend/pkg/exporter/modules/dashboard_data_w_rolling.go index cec5f6097..914fa44b7 100644 --- a/backend/pkg/exporter/modules/dashboard_data_w_rolling.go +++ b/backend/pkg/exporter/modules/dashboard_data_w_rolling.go @@ -109,7 +109,7 @@ func (d *RollingAggregator) aggregateInternal(days int, tableName string, curren } if bootstrap { - metrics.Errors.WithLabelValues(fmt.Sprintf("exporter_v2dash_agg_bootstrap_%dd", days)).Inc() + metrics.Tasks.WithLabelValues(fmt.Sprintf("exporter_v2dash_agg_bootstrap_%dd", days)).Inc() d.log.Infof("rolling %dd bootstraping starting", days) err = d.bootstrap(tx, days, tableName) @@ -198,6 +198,8 @@ func (d *RollingAggregator) aggregateInternal(days int, tableName string, curren } } + metrics.State.WithLabelValues(fmt.Sprintf("exporter_v2dash_rolling_%dd_bounds_end", days)).Set(float64(sanityBounds.EpochEnd)) + err = tx.Commit() if err != nil { return errors.Wrap(err, "failed to commit transaction") @@ -609,7 +611,7 @@ func AddToRollingCustom(tx *sqlx.Tx, custom CustomRolling) error { case <-ctx.Done(): // Query took longer than x minutes, cancel and return error cancel() - + metrics.Errors.WithLabelValues("exporter_v2dash_bandaid").Inc() if debugDeadlockBandaid { _, err := db.AlloyWriter.Exec(`SELECT pg_cancel_backend(pid) FROM pg_stat_activity