From 5f4712004183a800e1ed03ae6d8c087eefe56bfd Mon Sep 17 00:00:00 2001 From: Paula Julve Date: Fri, 12 Jul 2024 14:18:07 +0200 Subject: [PATCH] List EBS Volumes --- pkg/aws/ec2/compute.go | 30 +++++++++++++++++ pkg/aws/ec2/ec2.go | 67 +++++++++++++++++++++++++++++++++---- pkg/aws/services/ec2/ec2.go | 1 + 3 files changed, 91 insertions(+), 7 deletions(-) diff --git a/pkg/aws/ec2/compute.go b/pkg/aws/ec2/compute.go index ea200f86..3f7a2d58 100644 --- a/pkg/aws/ec2/compute.go +++ b/pkg/aws/ec2/compute.go @@ -45,3 +45,33 @@ func ClusterNameFromInstance(instance types.Instance) string { } return "" } + +func ListEBSVolumes(ctx context.Context, client *ec2.EC2) ([]types.Volume, error) { + // Paula: do we want to exclude the snapshots? hmm... + params := &ec22.DescribeVolumesInput{ + Filters: []types.Filter{ + // exclude snapshots + { + Name: aws.String("snapshot-id"), + Values: []string{""}, + }, + }, + } + + pager := ec22.NewDescribeVolumesPaginator(*client, params) + var volumes []types.Volume + + for pager.HasMorePages() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + + volumes = append(volumes, resp.Volumes...) + if resp.NextToken == nil || *resp.NextToken == "" { + break + } + } + + return volumes, nil +} diff --git a/pkg/aws/ec2/ec2.go b/pkg/aws/ec2/ec2.go index 846a3266..3c14997f 100644 --- a/pkg/aws/ec2/ec2.go +++ b/pkg/aws/ec2/ec2.go @@ -48,6 +48,13 @@ var ( []string{"instance", "region", "family", "machine_type", "cluster_name", "price_tier"}, nil, ) + persistentVolumeHourlyCostDesc = prometheus.NewDesc( + prometheus.BuildFQName(cloudcostexporter.MetricPrefix, subsystem, "persistent_volume_usd_per_hour"), + "The cost of an AWS EBS Volume in USD.", + // []string{"cluster_name", "namespace", "persistentvolume", "region", "project", "storage_class", "disk_type"}, + []string{"persistentvolume", "az", "storage_class"}, + nil, + ) ) // Collector is a prometheus collector that collects metrics from AWS EKS clusters. @@ -169,15 +176,20 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) error { c.StorageScrapingInterval = time.Now().Add(c.ScrapeInterval) } - wg := sync.WaitGroup{} - wg.Add(len(c.Regions)) + wgInstances := sync.WaitGroup{} + wgInstances.Add(len(c.Regions)) instanceCh := make(chan []ec2Types.Reservation, len(c.Regions)) + + wgVolumes := sync.WaitGroup{} + wgVolumes.Add(len(c.Regions)) + volumeCh := make(chan []ec2Types.Volume, len(c.Regions)) + for _, region := range c.Regions { go func(region ec2Types.Region) { ctx := context.Background() now := time.Now() c.logger.LogAttrs(ctx, slog.LevelInfo, "Fetching instances", slog.String("region", *region.RegionName)) - defer wg.Done() + defer wgInstances.Done() client := c.ec2RegionClients[*region.RegionName] reservations, err := ListComputeInstances(context.Background(), client) if err != nil { @@ -193,17 +205,44 @@ func (c *Collector) Collect(ch chan<- prometheus.Metric) error { ) instanceCh <- reservations }(region) + + go func(region ec2Types.Region) { + ctx := context.Background() + now := time.Now() + c.logger.LogAttrs(ctx, slog.LevelInfo, "Fetching volumes", slog.String("region", *region.RegionName)) + defer wgVolumes.Done() + client := c.ec2RegionClients[*region.RegionName] + + volumes, err := ListEBSVolumes(context.Background(), &client) + if err != nil { + c.logger.LogAttrs(ctx, slog.LevelError, "Could not list EBS volumes", + slog.String("region", *region.RegionName), + slog.String("message", err.Error())) + return + } + c.logger.LogAttrs(ctx, slog.LevelInfo, "Successfully listed volumes", + slog.String("region", *region.RegionName), + slog.Int("volumes", len(volumes)), + slog.Duration("duration", time.Since(now)), + ) + volumeCh <- volumes + }(region) } go func() { - wg.Wait() + wgInstances.Wait() close(instanceCh) }() - c.emitMetricsFromChannel(instanceCh, ch) + go func() { + wgVolumes.Wait() + close(volumeCh) + }() + c.emitMetricsFromReservationsChannel(instanceCh, ch) + c.emitMetricsFromVolumesChannel(volumeCh, ch) c.logger.LogAttrs(context.TODO(), slog.LevelInfo, "Finished collect", slog.Duration("duration", time.Since(start))) return nil } -func (c *Collector) emitMetricsFromChannel(reservationsCh chan []ec2Types.Reservation, ch chan<- prometheus.Metric) { +func (c *Collector) emitMetricsFromReservationsChannel(reservationsCh chan []ec2Types.Reservation, ch chan<- prometheus.Metric) { for reservations := range reservationsCh { for _, reservation := range reservations { for _, instance := range reservation.Instances { @@ -246,8 +285,22 @@ func (c *Collector) emitMetricsFromChannel(reservationsCh chan []ec2Types.Reserv } } +func (c *Collector) emitMetricsFromVolumesChannel(volumesCh chan []ec2Types.Volume, ch chan<- prometheus.Metric) { + for volumes := range volumesCh { + for _, volume := range volumes { + labelValues := []string{ + *volume.VolumeId, + *volume.AvailabilityZone, + string(volume.VolumeType), + } + + ch <- prometheus.MustNewConstMetric(persistentVolumeHourlyCostDesc, prometheus.GaugeValue, 42, labelValues...) + } + } +} + func (c *Collector) CheckReadiness() bool { - return c.pricingMap.CheckReadiness() + return c.computePricingMap.CheckReadiness() } func (c *Collector) Describe(ch chan<- *prometheus.Desc) error { diff --git a/pkg/aws/services/ec2/ec2.go b/pkg/aws/services/ec2/ec2.go index e0bbf490..3486cb27 100644 --- a/pkg/aws/services/ec2/ec2.go +++ b/pkg/aws/services/ec2/ec2.go @@ -10,4 +10,5 @@ type EC2 interface { DescribeInstances(ctx context.Context, e *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) DescribeRegions(ctx context.Context, e *ec2.DescribeRegionsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeRegionsOutput, error) DescribeSpotPriceHistory(ctx context.Context, input *ec2.DescribeSpotPriceHistoryInput, optFns ...func(*ec2.Options)) (*ec2.DescribeSpotPriceHistoryOutput, error) + DescribeVolumes(context.Context, *ec2.DescribeVolumesInput, ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) }