feat: gateway tracking metrics ipld codec and multihash function

packages/gateway/package.json

@@ -20,6 +20,7 @@
   },
   "dependencies": {
     "itty-router": "^2.4.5",
+    "multicodec": "^3.2.1",
     "multiformats": "^9.6.4",
     "nanoid": "^3.1.30",
     "p-any": "^4.0.0",

packages/gateway/src/durable-objects/summary-metrics.js

@@ -1,3 +1,5 @@
+import multicodec from 'multicodec'
+import { CID } from 'multiformats/cid'
 import {
   responseTimeHistogram,
   createResponseTimeHistogramObject,
@@ -11,10 +13,13 @@ import {
  * @property {number} totalCachedResponses total number of cached responses
  * @property {BigInt} totalContentLengthBytes total content length of responses
  * @property {BigInt} totalCachedContentLengthBytes total content length of cached responses
+ * @property {Record<string, number>} totalResponsesByIpldCodec
+ * @property {Record<string, number>} totalResponsesByMultihashFunction
  * @property {Record<string, number>} contentLengthHistogram
  * @property {Record<string, number>} responseTimeHistogram
  *
  * @typedef {Object} FetchStats
+ * @property {string} cid fetched CID
  * @property {number} responseTime number of milliseconds to get response
  * @property {number} contentLength content length header content
  */
@@ -35,6 +40,11 @@ const TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID = 'totalCachedContentLengthBytes'
 const CONTENT_LENGTH_HISTOGRAM_ID = 'contentLengthHistogram'
 // Key to track response time histogram
 const RESPONSE_TIME_HISTOGRAM_ID = 'responseTimeHistogram'
+// Key to track responses by ipld codec
+const TOTAL_RESPONSES_BY_IPLD_CODEC_ID = 'totalResponsesByIpldCodec'
+// Key to track responses by multihash function
+const TOTAL_RESPONSES_BY_MULTIHASH_FUNCTION_ID =
+  'totalResponsesByMultihashFunction'
 
 /**
  * Durable Object for keeping summary metrics of nft.storage Gateway
@@ -65,6 +75,14 @@ export class SummaryMetrics0 {
       this.totalCachedContentLengthBytes =
         (await this.state.storage.get(TOTAL_CACHED_CONTENT_LENGTH_BYTES_ID)) ||
         BigInt(0)
+      /** @type {Record<string, number>} */
+      this.totalResponsesByIpldCodec =
+        (await this.state.storage.get(TOTAL_RESPONSES_BY_IPLD_CODEC_ID)) || {}
+      /** @type {Record<string, number>} */
+      this.totalResponsesByMultihashFunction =
+        (await this.state.storage.get(
+          TOTAL_RESPONSES_BY_MULTIHASH_FUNCTION_ID
+        )) || {}
       // Content length histogram
       this.contentLengthHistogram =
         (await this.state.storage.get(CONTENT_LENGTH_HISTOGRAM_ID)) ||
@@ -94,6 +112,9 @@ export class SummaryMetrics0 {
               totalContentLengthBytes: this.totalContentLengthBytes.toString(),
               totalCachedContentLengthBytes:
                 this.totalCachedContentLengthBytes.toString(),
+              totalResponsesByIpldCodec: this.totalResponsesByIpldCodec,
+              totalResponsesByMultihashFunction:
+                this.totalResponsesByMultihashFunction,
               contentLengthHistogram: this.contentLengthHistogram,
               responseTimeHistogram: this.responseTimeHistogram,
             })
@@ -126,6 +147,7 @@ export class SummaryMetrics0 {
     this.totalCachedResponseTime += stats.responseTime
     this.totalCachedResponses += 1
     this.totalCachedContentLengthBytes += BigInt(stats.contentLength)
+    this._updateCidMetrics(stats)
     this._updateContentLengthMetrics(stats)
     this._updateResponseTimeHistogram(stats)
     // Save updated metrics
@@ -154,6 +176,14 @@ export class SummaryMetrics0 {
         RESPONSE_TIME_HISTOGRAM_ID,
         this.responseTimeHistogram
       ),
+      this.state.storage.put(
+        TOTAL_RESPONSES_BY_IPLD_CODEC_ID,
+        this.totalResponsesByIpldCodec
+      ),
+      this.state.storage.put(
+        TOTAL_RESPONSES_BY_MULTIHASH_FUNCTION_ID,
+        this.totalResponsesByMultihashFunction
+      ),
     ])
   }
 
@@ -164,6 +194,7 @@ export class SummaryMetrics0 {
     // Updated Metrics
     this.totalWinnerResponseTime += stats.responseTime
     this.totalWinnerSuccessfulRequests += 1
+    this._updateCidMetrics(stats)
     this._updateContentLengthMetrics(stats)
     this._updateResponseTimeHistogram(stats)
     // Save updated Metrics
@@ -188,9 +219,44 @@ export class SummaryMetrics0 {
         RESPONSE_TIME_HISTOGRAM_ID,
         this.responseTimeHistogram
       ),
+      this.state.storage.put(
+        TOTAL_RESPONSES_BY_IPLD_CODEC_ID,
+        this.totalResponsesByIpldCodec
+      ),
+      this.state.storage.put(
+        TOTAL_RESPONSES_BY_MULTIHASH_FUNCTION_ID,
+        this.totalResponsesByMultihashFunction
+      ),
     ])
   }
 
+  /**
+   * @param {FetchStats} stats
+   */
+  _updateCidMetrics(stats) {
+    const cid = CID.parse(stats.cid)
+    const cidInspectObj = CID.inspectBytes(cid.bytes.subarray(0, 10))
+    // IPLD codec
+    const ipldCodec = multicodec.getNameFromCode(cidInspectObj.codec)
+    if (this.totalResponsesByIpldCodec[ipldCodec]) {
+      // increment one if exists, otherwise initialize it
+      this.totalResponsesByIpldCodec[ipldCodec] += 1
+    } else {
+      this.totalResponsesByIpldCodec[ipldCodec] = 1
+    }
+
+    // Multihash function
+    const multihashFunction = multicodec.getNameFromCode(
+      cidInspectObj.multihashCode
+    )
+    if (this.totalResponsesByMultihashFunction[multihashFunction]) {
+      // increment one if exists, otherwise initialize it
+      this.totalResponsesByMultihashFunction[multihashFunction] += 1
+    } else {
+      this.totalResponsesByMultihashFunction[multihashFunction] = 1
+    }
+  }
+
   /**
    * @param {FetchStats} stats
    */

packages/gateway/src/gateway.js

@@ -38,19 +38,22 @@ import {
  */
 export async function gatewayGet(request, env, ctx) {
   const startTs = Date.now()
+  const reqUrl = new URL(request.url)
+  const cid = getCidFromSubdomainUrl(reqUrl)
+
   const cache = caches.default
   let res = await cache.match(request.url)
 
   if (res) {
     // Update cache metrics in background
     const responseTime = Date.now() - startTs
 
-    ctx.waitUntil(updateSummaryCacheMetrics(request, env, res, responseTime))
+    ctx.waitUntil(
+      updateSummaryCacheMetrics(request, env, res, responseTime, cid)
+    )
     return res
   }
 
-  const reqUrl = new URL(request.url)
-  const cid = getCidFromSubdomainUrl(reqUrl)
   const pathname = reqUrl.pathname
 
   // Prepare IPFS gateway requests
@@ -91,7 +94,7 @@ export async function gatewayGet(request, env, ctx) {
         )
 
         await Promise.all([
-          storeWinnerGwResponse(request, env, winnerGwResponse),
+          storeWinnerGwResponse(request, env, winnerGwResponse, cid),
           settleGatewayRequests(),
           // Cache request URL in Cloudflare CDN if smaller than CF_CACHE_MAX_OBJECT_SIZE
           contentLengthMb <= CF_CACHE_MAX_OBJECT_SIZE &&
@@ -157,11 +160,12 @@ export async function gatewayGet(request, env, ctx) {
  * @param {Request} request
  * @param {Env} env
  * @param {GatewayResponse} winnerGwResponse
+ * @param {string} cid
  */
-async function storeWinnerGwResponse(request, env, winnerGwResponse) {
+async function storeWinnerGwResponse(request, env, winnerGwResponse, cid) {
   await Promise.all([
     updateGatewayMetrics(request, env, winnerGwResponse, true),
-    updateSummaryWinnerMetrics(request, env, winnerGwResponse),
+    updateSummaryWinnerMetrics(request, env, cid, winnerGwResponse),
   ])
 }
 
@@ -245,14 +249,22 @@ function getHeaders(request) {
  * @param {import('./env').Env} env
  * @param {Response} response
  * @param {number} responseTime
+ * @param {string} cid
  */
-async function updateSummaryCacheMetrics(request, env, response, responseTime) {
+async function updateSummaryCacheMetrics(
+  request,
+  env,
+  response,
+  responseTime,
+  cid
+) {
   // Get durable object for summary
   const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID)
   const stub = env.summaryMetricsDurable.get(id)
 
   /** @type {import('./durable-objects/summary-metrics').FetchStats} */
   const contentLengthStats = {
+    cid,
     contentLength: Number(response.headers.get('content-length')),
     responseTime,
   }
@@ -294,15 +306,17 @@ async function getGatewayRateLimitState(request, env) {
 /**
  * @param {Request} request
  * @param {import('./env').Env} env
+ * @param {string} cid
  * @param {GatewayResponse} gwResponse
  */
-async function updateSummaryWinnerMetrics(request, env, gwResponse) {
+async function updateSummaryWinnerMetrics(request, env, cid, gwResponse) {
   // Get durable object for gateway
   const id = env.summaryMetricsDurable.idFromName(SUMMARY_METRICS_ID)
   const stub = env.summaryMetricsDurable.get(id)
 
   /** @type {import('./durable-objects/summary-metrics').FetchStats} */
   const fetchStats = {
+    cid,
     responseTime: gwResponse.responseTime,
     contentLength: Number(gwResponse.response.headers.get('content-length')),
   }

packages/gateway/src/metrics.js

@@ -267,6 +267,35 @@ export async function metricsGet(request, env, ctx) {
     `# HELP nftgateway_redirect_total Total redirects to gateway.`,
     `# TYPE nftgateway_redirect_total counter`,
     `nftgateway_redirect_total{env="${env.ENV}"} ${metricsCollected.gatewayRedirectCount}`,
+    `# HELP nftgateway_responses_by_ipld_codec_total total of responses by ipld codec.`,
+    `# TYPE nftgateway_responses_by_ipld_codec_total counter`,
+    Object.keys(metricsCollected.summaryMetrics.totalResponsesByIpldCodec)
+      .map(
+        (codec) =>
+          `nftgateway_responses_by_ipld_codec_total{env="${
+            env.ENV
+          }",codec="${codec}"} ${
+            metricsCollected.summaryMetrics.totalResponsesByIpldCodec[codec] ||
+            0
+          }`
+      )
+      .join('\n'),
+    `# HELP nftgateway_responses_by_multihash_function_total total of responses by multihash function.`,
+    `# TYPE nftgateway_responses_by_multihash_function_total counter`,
+    Object.keys(
+      metricsCollected.summaryMetrics.totalResponsesByMultihashFunction
+    )
+      .map(
+        (fn) =>
+          `nftgateway_responses_by_multihash_function_total{env="${
+            env.ENV
+          }",function="${fn}"} ${
+            metricsCollected.summaryMetrics.totalResponsesByMultihashFunction[
+              fn
+            ] || 0
+          }`
+      )
+      .join('\n'),
   ].join('\n')
 
   res = new Response(metrics, {

packages/gateway/test/metrics.spec.js

@@ -123,6 +123,32 @@ test('Gets Metrics content', async (t) => {
   })
 })
 
+test('gets Metrics from content ipld codec and multihash function', async (t) => {
+  const { mf } = t.context
+
+  const p = await mf.dispatchFetch(
+    'http://bafkreidyeivj7adnnac6ljvzj2e3rd5xdw3revw4da7mx2ckrstapoupoq.ipfs.localhost:8787'
+  )
+
+  await p.waitUntil()
+
+  const response = await mf.dispatchFetch('http://localhost:8787/metrics')
+  const metricsResponse = await response.text()
+
+  t.is(
+    metricsResponse.includes(
+      `nftgateway_responses_by_ipld_codec_total{env="test",codec="raw"} 1`
+    ),
+    true
+  )
+  t.is(
+    metricsResponse.includes(
+      `nftgateway_responses_by_multihash_function_total{env="test",function="sha2-256"} 1`
+    ),
+    true
+  )
+})
+
 test('Gets Metrics from faster gateway', async (t) => {
   const { mf } = t.context