diff --git a/beacon-light-client/solidity/tasks/start-publishing.ts b/beacon-light-client/solidity/tasks/start-publishing.ts index f1884e926..fa5414056 100644 --- a/beacon-light-client/solidity/tasks/start-publishing.ts +++ b/beacon-light-client/solidity/tasks/start-publishing.ts @@ -15,6 +15,7 @@ import { accountBalanceGauge, initPrometheusSetup, registerGaugesForStartPublishing, + startResourceMetricsUpdate, } from '@dendreth/utils/ts-utils/prometheus-utils'; const logger = getGenericLogger(); @@ -78,6 +79,7 @@ task('start-publishing', 'Run relayer') initPrometheusSetup(args.prometheusPort, networkName); registerGaugesForStartPublishing(); + startResourceMetricsUpdate(networkName); } const currentConfig = await getNetworkConfig(args.followNetwork); diff --git a/libs/typescript/ts-utils/prometheus-utils.ts b/libs/typescript/ts-utils/prometheus-utils.ts index c53eb9723..f4483e4ab 100644 --- a/libs/typescript/ts-utils/prometheus-utils.ts +++ b/libs/typescript/ts-utils/prometheus-utils.ts @@ -157,3 +157,63 @@ export const numberOfProofPublished = new client.Counter({ help: 'The number of proofs published(since last restart)', labelNames: ['network'], }); + +const gaugeCpuUser = new client.Gauge({ + name: 'process_cpu_user_seconds_total', + help: 'Total user CPU time spent by the process in seconds.', + labelNames: ['label'], // Adding label +}); + +const gaugeCpuSystem = new client.Gauge({ + name: 'process_cpu_system_seconds_total', + help: 'Total system CPU time spent by the process in seconds.', + labelNames: ['label'], // Adding label +}); + +const gaugeMemoryUsage = new client.Gauge({ + name: 'process_memory_bytes', + help: 'Memory usage of the process in bytes.', + labelNames: ['label'], // Adding label +}); + +const gaugeHeapTotal = new client.Gauge({ + name: 'process_heap_total_bytes', + help: 'Total heap memory allocated by the process in bytes.', + labelNames: ['label'], // Adding label +}); + +const gaugeHeapUsed = new client.Gauge({ + name: 'process_heap_used_bytes', + help: 'Heap memory used by the process in bytes.', + labelNames: ['label'], // Adding label +}); + +const gaugeRss = new client.Gauge({ + name: 'process_rss_bytes', + help: 'Resident set size (RSS) memory used by the process in bytes.', + labelNames: ['label'], // Adding label +}); + +export function startResourceMetricsUpdate(label: string) { + register.registerMetric(gaugeCpuUser); + register.registerMetric(gaugeCpuSystem); + register.registerMetric(gaugeMemoryUsage); + register.registerMetric(gaugeHeapTotal); + register.registerMetric(gaugeRss); + + // Set an interval to regularly update the resource metrics + setInterval(() => { + const memoryUsage = process.memoryUsage(); + const cpuUsage = process.cpuUsage(); + + // Update CPU metrics + gaugeCpuUser.labels(label).set(cpuUsage.user / 1e6); // Convert from microseconds to seconds + gaugeCpuSystem.labels(label).set(cpuUsage.system / 1e6); // Convert from microseconds to seconds + + // Update memory metrics + gaugeMemoryUsage.labels(label).set(memoryUsage.rss); // Resident set size (RSS) + gaugeHeapTotal.labels(label).set(memoryUsage.heapTotal); // Total heap allocated + gaugeHeapUsed.labels(label).set(memoryUsage.heapUsed); // Heap memory in use + gaugeRss.labels(label).set(memoryUsage.rss); // RSS memory in bytes + }, 5000); // Update metrics every 5 seconds +} diff --git a/relay/workers/poll-updates/poll-updates-worker.ts b/relay/workers/poll-updates/poll-updates-worker.ts index 2bd386cf8..7510f4309 100644 --- a/relay/workers/poll-updates/poll-updates-worker.ts +++ b/relay/workers/poll-updates/poll-updates-worker.ts @@ -9,10 +9,14 @@ import doUpdate from '@/workers/poll-updates/do_update'; import { getBeaconApi } from '@/implementations/beacon-api'; import { checkConfig } from '@dendreth/utils/ts-utils/common-utils'; import { getGenericLogger } from '@dendreth/utils/ts-utils/logger'; -import { initPrometheusSetup } from '@dendreth/utils/ts-utils/prometheus-utils'; +import { + initPrometheusSetup, + startResourceMetricsUpdate, +} from '@dendreth/utils/ts-utils/prometheus-utils'; const logger = getGenericLogger(); initPrometheusSetup(); +startResourceMetricsUpdate('poll-update-worker'); (async () => { const updatePollingConfig = { diff --git a/relay/workers/prover/prover-worker.ts b/relay/workers/prover/prover-worker.ts index 20b404253..842e53142 100644 --- a/relay/workers/prover/prover-worker.ts +++ b/relay/workers/prover/prover-worker.ts @@ -10,6 +10,7 @@ import yargs from 'yargs'; import { initPrometheusSetup, registerGaugesForProver, + startResourceMetricsUpdate, } from '@dendreth/utils/ts-utils/prometheus-utils'; (async () => { @@ -19,6 +20,7 @@ import { }; initPrometheusSetup(3000); registerGaugesForProver(); + startResourceMetricsUpdate('proofGenerationWorker'); checkConfig(proverConfig);