Merge pull request #47 from shiguredo/feature/new-metrics

Sora 2025.1.0 で追加予定のメトリクスに対応する

CHANGES.md

@@ -9,6 +9,26 @@
 - FIX
   - バグ修正
 
+## develop
+
+- [ADD] 多段リレー（Plumtree） 関連のメトリクスを追加する
+  - Sora 2025.1.0 で Sora API の GetStatsReport API に追加される予定の多段リレー関連の以下のメトリクスを追加する
+    - `cluster_relay_plumtree_sent_gossip_total`
+    - `cluster_relay_plumtree_received_gossip_total`
+    - `cluster_relay_plumtree_sent_ihave_total`
+    - `cluster_relay_plumtree_received_ihave_total`
+    - `cluster_relay_plumtree_sent_graft_total`
+    - `cluster_relay_plumtree_received_graft_total`
+    - `cluster_relay_plumtree_sent_prune_total`
+    - `cluster_relay_plumtree_received_prune_total`
+    - `cluster_relay_plumtree_graft_miss_total`
+    - `cluster_relay_plumtree_skipped_send_total`
+    - `cluster_relay_plumtree_ignored_total`
+  - @sile
+- [ADD] `sora_srtp_sent_sfu_delay_us_total` メトリクスを追加する
+  - Sora 2025.1.0 で Sora API の GetStatsReport API に追加される予定のメトリクス
+  - @sile
+
 ## 2024.7.0
 
 **リリース日**: 2024-12-18

collector/cluster_node.go

@@ -15,6 +15,18 @@ var (
 		clusterRelaySentBytesTotal:       newDescWithLabel("cluster_relay_sent_bytes_total", "The total number of bytes sent by the cluster relay.", []string{"node_name"}),
 		clusterRelayReceivedPacketsTotal: newDescWithLabel("cluster_relay_received_packets_total", "The total number of packets received by the cluster relay.", []string{"node_name"}),
 		clusterRelaySentPacketsTotal:     newDescWithLabel("cluster_relay_sent_packets_total", "The total number of packets sent by the cluster relay.", []string{"node_name"}),
+
+		clusterRelayPlumtreeSentGossipTotal:   newDescWithLabel("cluster_relay_plumtree_sent_gossip_total", "The total number of Plumtree GOSSIP messages sent by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeReceivedGossipTotal:   newDescWithLabel("cluster_relay_plumtree_received_gossip_total", "The total number of Plumtree GOSSIP messages received by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeSentIhaveTotal:   newDescWithLabel("cluster_relay_plumtree_sent_ihave_total", "The total number of Plumtree IHAVE messages sent by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeReceivedIhaveTotal:   newDescWithLabel("cluster_relay_plumtree_received_ihave_total", "The total number of Plumtree IHAVE messages received by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeSentGraftTotal:   newDescWithLabel("cluster_relay_plumtree_sent_graft_total", "The total number of Plumtree GRAFT messages sent by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeReceivedGraftTotal:   newDescWithLabel("cluster_relay_plumtree_received_graft_total", "The total number of Plumtree GRAFT messages received by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeSentPruneTotal:   newDescWithLabel("cluster_relay_plumtree_sent_prune_total", "The total number of Plumtree PRUNE messages sent by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeReceivedPruneTotal:   newDescWithLabel("cluster_relay_plumtree_received_prune_total", "The total number of Plumtree PRUNE messages received by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeGraftMissTotal:   newDescWithLabel("cluster_relay_plumtree_graft_miss_total", "The total number of Plumtree GRAFT messages missed by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeSkippedSendTotal:   newDescWithLabel("cluster_relay_plumtree_skipped_send_total", "The total number of Plumtree messages whose sending was skipped by the cluster relay.", []string{"node_name"}),
+		clusterRelayPlumtreeIgnoredTotal:   newDescWithLabel("cluster_relay_plumtree_ignored_total", "The total number of Plumtree messages received but ignored by the cluster relay.", []string{"node_name"}),
 	}
 )
 
@@ -28,6 +40,18 @@ type SoraClusterMetrics struct {
 	clusterRelaySentBytesTotal       *prometheus.Desc
 	clusterRelayReceivedPacketsTotal *prometheus.Desc
 	clusterRelaySentPacketsTotal     *prometheus.Desc
+
+	clusterRelayPlumtreeSentGossipTotal     *prometheus.Desc
+	clusterRelayPlumtreeReceivedGossipTotal *prometheus.Desc
+	clusterRelayPlumtreeSentIhaveTotal      *prometheus.Desc
+	clusterRelayPlumtreeReceivedIhaveTotal  *prometheus.Desc
+	clusterRelayPlumtreeSentGraftTotal      *prometheus.Desc
+	clusterRelayPlumtreeReceivedGraftTotal  *prometheus.Desc
+	clusterRelayPlumtreeSentPruneTotal      *prometheus.Desc
+	clusterRelayPlumtreeReceivedPruneTotal  *prometheus.Desc
+	clusterRelayPlumtreeGraftMissTotal      *prometheus.Desc
+	clusterRelayPlumtreeSkippedSendTotal    *prometheus.Desc
+	clusterRelayPlumtreeIgnoredTotal        *prometheus.Desc
 }
 
 func (m *SoraClusterMetrics) Describe(ch chan<- *prometheus.Desc) {
@@ -39,6 +63,17 @@ func (m *SoraClusterMetrics) Describe(ch chan<- *prometheus.Desc) {
 	ch <- m.clusterRelaySentBytesTotal
 	ch <- m.clusterRelayReceivedPacketsTotal
 	ch <- m.clusterRelaySentPacketsTotal
+	ch <- m.clusterRelayPlumtreeSentGossipTotal
+	ch <- m.clusterRelayPlumtreeReceivedGossipTotal
+	ch <- m.clusterRelayPlumtreeSentIhaveTotal
+	ch <- m.clusterRelayPlumtreeReceivedIhaveTotal
+	ch <- m.clusterRelayPlumtreeSentGraftTotal
+	ch <- m.clusterRelayPlumtreeReceivedGraftTotal
+	ch <- m.clusterRelayPlumtreeSentPruneTotal
+	ch <- m.clusterRelayPlumtreeReceivedPruneTotal
+	ch <- m.clusterRelayPlumtreeGraftMissTotal
+	ch <- m.clusterRelayPlumtreeSkippedSendTotal
+	ch <- m.clusterRelayPlumtreeIgnoredTotal
 }
 
 func (m *SoraClusterMetrics) Collect(ch chan<- prometheus.Metric, nodeList []soraClusterNode, report soraClusterReport, clusterRelaies []soraClusterRelay) {
@@ -68,5 +103,17 @@ func (m *SoraClusterMetrics) Collect(ch chan<- prometheus.Metric, nodeList []sor
 		ch <- newCounter(m.clusterRelaySentBytesTotal, float64(relayNode.TotalSentByteSize), relayNode.NodeName)
 		ch <- newCounter(m.clusterRelayReceivedPacketsTotal, float64(relayNode.TotalReceived), relayNode.NodeName)
 		ch <- newCounter(m.clusterRelaySentPacketsTotal, float64(relayNode.TotalSent), relayNode.NodeName)
+
+		ch <- newCounter(m.clusterRelayPlumtreeSentGossipTotal, float64(relayNode.Plumtree.TotalSentGossip), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeReceivedGossipTotal, float64(relayNode.Plumtree.TotalReceivedGossip), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeSentIhaveTotal, float64(relayNode.Plumtree.TotalSentIhave), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeReceivedIhaveTotal, float64(relayNode.Plumtree.TotalReceivedIhave), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeSentGraftTotal, float64(relayNode.Plumtree.TotalSentGraft), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeReceivedGraftTotal, float64(relayNode.Plumtree.TotalReceivedGraft), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeSentPruneTotal, float64(relayNode.Plumtree.TotalSentPrune), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeReceivedPruneTotal, float64(relayNode.Plumtree.TotalReceivedPrune), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeGraftMissTotal, float64(relayNode.Plumtree.TotalGraftMiss), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeSkippedSendTotal, float64(relayNode.Plumtree.TotalSkippedSend), relayNode.NodeName)
+		ch <- newCounter(m.clusterRelayPlumtreeIgnoredTotal, float64(relayNode.Plumtree.TotalIgnored), relayNode.NodeName)
 	}
 }

collector/sora_api.go

@@ -51,6 +51,7 @@ type soraSrtpReport struct {
 	TotalReceivedSrtpByteSize  int64 `json:"total_received_srtp_byte_size"`
 	TotalSentSrtp              int64 `json:"total_sent_srtp"`
 	TotalSentSrtpByteSize      int64 `json:"total_sent_srtp_byte_size"`
+	TotalSentSrtpSfuDelayUs    int64 `json:"total_sent_srtp_sfu_delay_us"`
 	TotalDecryptedSrtp         int64 `json:"total_decrypted_srtp"`
 	TotalDecryptedSrtpByteSize int64 `json:"total_decrypted_srtp_byte_size"`
 }
@@ -169,11 +170,26 @@ type soraClusterNode struct {
 }
 
 type soraClusterRelay struct {
-	NodeName              string `json:"node_name"`
-	TotalReceivedByteSize int64  `json:"total_received_byte_size"`
-	TotalSentByteSize     int64  `json:"total_sent_byte_size"`
-	TotalReceived         int64  `json:"total_received"`
-	TotalSent             int64  `json:"total_sent"`
+	NodeName              string                   `json:"node_name"`
+	TotalReceivedByteSize int64                    `json:"total_received_byte_size"`
+	TotalSentByteSize     int64                    `json:"total_sent_byte_size"`
+	TotalReceived         int64                    `json:"total_received"`
+	TotalSent             int64                    `json:"total_sent"`
+	Plumtree              soraClusterRelayPlumtree `json:"plumtree"`
+}
+
+type soraClusterRelayPlumtree struct {
+        TotalSentGossip     int64 `json:"total_sent_gossip"`
+        TotalReceivedGossip int64 `json:"total_received_gossip"`
+        TotalSentIhave      int64 `json:"total_sent_ihave"`
+        TotalReceivedIhave  int64 `json:"total_received_ihave"`
+        TotalSentGraft      int64 `json:"total_sent_graft"`
+        TotalReceivedGraft  int64 `json:"total_received_graft"`
+        TotalSentPrune      int64 `json:"total_sent_prune"`
+        TotalReceivedPrune  int64 `json:"total_received_prune"`
+        TotalGraftMiss      int64 `json:"total_graft_miss"`
+        TotalSkippedSend    int64 `json:"total_skipped_send"`
+        TotalIgnored        int64 `json:"total_ignored"`
 }
 
 type soraLicenseInfo struct {

collector/srtp.go

@@ -8,6 +8,7 @@ var (
 		totalReceivedSrtpByteSize:  newDesc("srtp_received_bytes_total", "The total number of received SRTP bytes."),
 		totalSentSrtp:              newDesc("srtp_sent_packets_total", "The total number of sent SRTP packets."),
 		totalSentSrtpByteSize:      newDesc("srtp_sent_bytes_total", "The total number of sent SRTP bytes."),
+		totalSentSrtpSfuDelayUs:    newDesc("srtp_sent_sfu_delay_us_total", "The total delay introduced by the SFU during the transfer of SRTP packets."),
 		totalDecryptedSrtp:         newDesc("srtp_decrypted_packets_total", "The total number of decrpyted SRTP packets."),
 		totalDecryptedSrtpByteSize: newDesc("srtp_decrpyted_bytes_total", "The total number of decrypted SRTP bytes."),
 	}
@@ -18,6 +19,7 @@ type SrtpMetrics struct {
 	totalReceivedSrtpByteSize  *prometheus.Desc
 	totalSentSrtp              *prometheus.Desc
 	totalSentSrtpByteSize      *prometheus.Desc
+	totalSentSrtpSfuDelayUs    *prometheus.Desc
 	totalDecryptedSrtp         *prometheus.Desc
 	totalDecryptedSrtpByteSize *prometheus.Desc
 }
@@ -27,6 +29,7 @@ func (m *SrtpMetrics) Describe(ch chan<- *prometheus.Desc) {
 	ch <- m.totalReceivedSrtpByteSize
 	ch <- m.totalSentSrtp
 	ch <- m.totalSentSrtpByteSize
+	ch <- m.totalSentSrtpSfuDelayUs
 	ch <- m.totalDecryptedSrtp
 	ch <- m.totalDecryptedSrtpByteSize
 }
@@ -36,6 +39,7 @@ func (m *SrtpMetrics) Collect(ch chan<- prometheus.Metric, report soraSrtpReport
 	ch <- newCounter(m.totalReceivedSrtpByteSize, float64(report.TotalReceivedSrtpByteSize))
 	ch <- newCounter(m.totalSentSrtp, float64(report.TotalSentSrtp))
 	ch <- newCounter(m.totalSentSrtpByteSize, float64(report.TotalSentSrtpByteSize))
+	ch <- newCounter(m.totalSentSrtpSfuDelayUs, float64(report.TotalSentSrtpSfuDelayUs))
 	ch <- newCounter(m.totalDecryptedSrtp, float64(report.TotalDecryptedSrtp))
 	ch <- newCounter(m.totalDecryptedSrtpByteSize, float64(report.TotalDecryptedSrtpByteSize))
 }

main_test.go

@@ -31,14 +31,40 @@ var (
 				"total_received_byte_size": 11,
 				"total_sent_byte_size": 12,
 				"total_received": 13,
-				"total_sent": 14
+				"total_sent": 14,
+				"plumtree": {
+					"total_sent_gossip": 1,
+					"total_received_gossip": 2,
+					"total_sent_ihave": 3,
+					"total_received_ihave": 4,
+					"total_sent_graft": 5,
+					"total_received_graft": 6,
+					"total_sent_prune": 7,
+					"total_received_prune": 8,
+					"total_graft_miss": 9,
+					"total_skipped_send": 10,
+					"total_ignored": 11
+                                }
 			},
 			{
 				"node_name": "node-02",
 				"total_received_byte_size": 21,
 				"total_sent_byte_size": 22,
 				"total_received": 23,
-				"total_sent": 24
+				"total_sent": 24,
+				"plumtree": {
+					"total_sent_gossip": 101,
+					"total_received_gossip": 102,
+					"total_sent_ihave": 103,
+					"total_received_ihave": 104,
+					"total_sent_graft": 105,
+					"total_received_graft": 106,
+					"total_sent_prune": 107,
+					"total_received_prune": 108,
+					"total_graft_miss": 109,
+					"total_skipped_send": 110,
+					"total_ignored": 111
+                                }
 			}
 		],
 		"erlang_vm": {
@@ -166,6 +192,7 @@ var (
 		"total_sent_sctp_byte_size": 111,
 		"total_sent_srtp": 106,
 		"total_sent_srtp_byte_size": 107,
+		"total_sent_srtp_sfu_delay_us": 123456,
 		"total_session_created": 1,
 		"total_session_destroyed": 0,
 		"total_successful_auth_webhook": 96,
@@ -435,6 +462,7 @@ func TestMinimumMetrics(t *testing.T) {
 		"total_sent_sctp_byte_size": 111,
 		"total_sent_srtp": 106,
 		"total_sent_srtp_byte_size": 107,
+		"total_sent_srtp_sfu_delay_us": 123456,
 		"total_session_created": 111,
 		"total_session_destroyed": 222,
 		"total_successful_auth_webhook": 96,

test/maximum.metrics

@@ -38,6 +38,50 @@ sora_cluster_relay_sent_bytes_total{node_name="node-02"} 22
 # TYPE sora_cluster_relay_sent_packets_total counter
 sora_cluster_relay_sent_packets_total{node_name="node-01"} 14
 sora_cluster_relay_sent_packets_total{node_name="node-02"} 24
+# HELP sora_cluster_relay_plumtree_graft_miss_total The total number of Plumtree GRAFT messages missed by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_graft_miss_total counter
+sora_cluster_relay_plumtree_graft_miss_total{node_name="node-01"} 9
+sora_cluster_relay_plumtree_graft_miss_total{node_name="node-02"} 109
+# HELP sora_cluster_relay_plumtree_ignored_total The total number of Plumtree messages received but ignored by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_ignored_total counter
+sora_cluster_relay_plumtree_ignored_total{node_name="node-01"} 11
+sora_cluster_relay_plumtree_ignored_total{node_name="node-02"} 111
+# HELP sora_cluster_relay_plumtree_received_gossip_total The total number of Plumtree GOSSIP messages received by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_received_gossip_total counter
+sora_cluster_relay_plumtree_received_gossip_total{node_name="node-01"} 2
+sora_cluster_relay_plumtree_received_gossip_total{node_name="node-02"} 102
+# HELP sora_cluster_relay_plumtree_received_graft_total The total number of Plumtree GRAFT messages received by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_received_graft_total counter
+sora_cluster_relay_plumtree_received_graft_total{node_name="node-01"} 6
+sora_cluster_relay_plumtree_received_graft_total{node_name="node-02"} 106
+# HELP sora_cluster_relay_plumtree_received_ihave_total The total number of Plumtree IHAVE messages received by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_received_ihave_total counter
+sora_cluster_relay_plumtree_received_ihave_total{node_name="node-01"} 4
+sora_cluster_relay_plumtree_received_ihave_total{node_name="node-02"} 104
+# HELP sora_cluster_relay_plumtree_received_prune_total The total number of Plumtree PRUNE messages received by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_received_prune_total counter
+sora_cluster_relay_plumtree_received_prune_total{node_name="node-01"} 8
+sora_cluster_relay_plumtree_received_prune_total{node_name="node-02"} 108
+# HELP sora_cluster_relay_plumtree_sent_gossip_total The total number of Plumtree GOSSIP messages sent by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_sent_gossip_total counter
+sora_cluster_relay_plumtree_sent_gossip_total{node_name="node-01"} 1
+sora_cluster_relay_plumtree_sent_gossip_total{node_name="node-02"} 101
+# HELP sora_cluster_relay_plumtree_sent_graft_total The total number of Plumtree GRAFT messages sent by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_sent_graft_total counter
+sora_cluster_relay_plumtree_sent_graft_total{node_name="node-01"} 5
+sora_cluster_relay_plumtree_sent_graft_total{node_name="node-02"} 105
+# HELP sora_cluster_relay_plumtree_sent_ihave_total The total number of Plumtree IHAVE messages sent by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_sent_ihave_total counter
+sora_cluster_relay_plumtree_sent_ihave_total{node_name="node-01"} 3
+sora_cluster_relay_plumtree_sent_ihave_total{node_name="node-02"} 103
+# HELP sora_cluster_relay_plumtree_sent_prune_total The total number of Plumtree PRUNE messages sent by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_sent_prune_total counter
+sora_cluster_relay_plumtree_sent_prune_total{node_name="node-01"} 7
+sora_cluster_relay_plumtree_sent_prune_total{node_name="node-02"} 107
+# HELP sora_cluster_relay_plumtree_skipped_send_total The total number of Plumtree messages whose sending was skipped by the cluster relay.
+# TYPE sora_cluster_relay_plumtree_skipped_send_total counter
+sora_cluster_relay_plumtree_skipped_send_total{node_name="node-01"} 10
+sora_cluster_relay_plumtree_skipped_send_total{node_name="node-02"} 110
 # HELP sora_client_type_total The total number of connections by Sora client types
 # TYPE sora_client_type_total counter
 sora_client_type_total{client="android_sdk",state="failed"} 1
@@ -220,6 +264,9 @@ sora_srtp_received_packets_total 108
 # HELP sora_srtp_sent_bytes_total The total number of sent SRTP bytes.
 # TYPE sora_srtp_sent_bytes_total counter
 sora_srtp_sent_bytes_total 107
+# HELP sora_srtp_sent_sfu_delay_us_total The total delay introduced by the SFU during the transfer of SRTP packets.
+# TYPE sora_srtp_sent_sfu_delay_us_total counter
+sora_srtp_sent_sfu_delay_us_total 123456
 # HELP sora_srtp_sent_packets_total The total number of sent SRTP packets.
 # TYPE sora_srtp_sent_packets_total counter
 sora_srtp_sent_packets_total 106

test/minimum.metrics

@@ -74,6 +74,9 @@ sora_srtp_received_packets_total 108
 # HELP sora_srtp_sent_bytes_total The total number of sent SRTP bytes.
 # TYPE sora_srtp_sent_bytes_total counter
 sora_srtp_sent_bytes_total 107
+# HELP sora_srtp_sent_sfu_delay_us_total The total delay introduced by the SFU during the transfer of SRTP packets.
+# TYPE sora_srtp_sent_sfu_delay_us_total counter
+sora_srtp_sent_sfu_delay_us_total 123456
 # HELP sora_srtp_sent_packets_total The total number of sent SRTP packets.
 # TYPE sora_srtp_sent_packets_total counter
 sora_srtp_sent_packets_total 106

test/sora_client_enabled.metrics

@@ -105,6 +105,9 @@ sora_srtp_received_packets_total 108
 # HELP sora_srtp_sent_bytes_total The total number of sent SRTP bytes.
 # TYPE sora_srtp_sent_bytes_total counter
 sora_srtp_sent_bytes_total 107
+# HELP sora_srtp_sent_sfu_delay_us_total The total delay introduced by the SFU during the transfer of SRTP packets.
+# TYPE sora_srtp_sent_sfu_delay_us_total counter
+sora_srtp_sent_sfu_delay_us_total 123456
 # HELP sora_srtp_sent_packets_total The total number of sent SRTP packets.
 # TYPE sora_srtp_sent_packets_total counter
 sora_srtp_sent_packets_total 106