feat: pricing & metadata caching (#51)

# 🤖 Linear

Closes GIT-218 GIT-219 GIT-220

## Description
We create two repositories for pricing and metadata that will act as
`cache` (note: is not the usual cache for fast retrieval that works in a
Redis way, but instead is a layer that provides a significant
improvement when reindexing so we don't fetch data again)

## Checklist before requesting a review

-   [x] I have conducted a self-review of my code.
-   [x] I have conducted a QA.
-   [x] If it is a core feature, I have included comprehensive tests.

apps/processing/src/services/sharedDependencies.service.ts

@@ -1,7 +1,7 @@
 import { CoreDependencies } from "@grants-stack-indexer/data-flow";
 import { EnvioIndexerClient } from "@grants-stack-indexer/indexer-client";
-import { IpfsProvider } from "@grants-stack-indexer/metadata";
-import { PricingProviderFactory } from "@grants-stack-indexer/pricing";
+import { CachingMetadataProvider, IpfsProvider } from "@grants-stack-indexer/metadata";
+import { CachingPricingProvider, PricingProviderFactory } from "@grants-stack-indexer/pricing";
 import {
     createKyselyDatabase,
     IEventRegistryRepository,
@@ -11,6 +11,8 @@ import {
     KyselyApplicationRepository,
     KyselyDonationRepository,
     KyselyEventRegistryRepository,
+    KyselyMetadataCache,
+    KyselyPricingCache,
     KyselyProjectRepository,
     KyselyRoundRepository,
     KyselyStrategyProcessingCheckpointRepository,
@@ -68,11 +70,23 @@ export class SharedDependenciesService {
             kyselyDatabase,
             env.DATABASE_SCHEMA,
         );
+        const pricingRepository = new KyselyPricingCache(kyselyDatabase, env.DATABASE_SCHEMA);
         const pricingProvider = PricingProviderFactory.create(env, {
             logger,
         });
+        const cachedPricingProvider = new CachingPricingProvider(
+            pricingProvider,
+            pricingRepository,
+            logger,
+        );
 
+        const metadataRepository = new KyselyMetadataCache(kyselyDatabase, env.DATABASE_SCHEMA);
         const metadataProvider = new IpfsProvider(env.IPFS_GATEWAYS_URL, logger);
+        const cachedMetadataProvider = new CachingMetadataProvider(
+            metadataProvider,
+            metadataRepository,
+            logger,
+        );
 
         const eventRegistryRepository = new KyselyEventRegistryRepository(
             kyselyDatabase,
@@ -104,9 +118,9 @@ export class SharedDependenciesService {
                 projectRepository,
                 roundRepository,
                 applicationRepository,
-                pricingProvider,
+                pricingProvider: cachedPricingProvider,
                 donationRepository,
-                metadataProvider,
+                metadataProvider: cachedMetadataProvider,
                 applicationPayoutRepository,
                 transactionManager,
             },

apps/processing/test/unit/sharedDependencies.service.spec.ts

@@ -45,16 +45,20 @@ vi.mock("@grants-stack-indexer/repository", () => ({
     KyselyEventRegistryRepository: vi.fn(),
     KyselyStrategyProcessingCheckpointRepository: vi.fn(),
     KyselyTransactionManager: vi.fn(),
+    KyselyPricingCache: vi.fn(),
+    KyselyMetadataCache: vi.fn(),
 }));
 
 vi.mock("@grants-stack-indexer/pricing", () => ({
     PricingProviderFactory: {
         create: vi.fn(),
     },
+    CachingPricingProvider: vi.fn(),
 }));
 
 vi.mock("@grants-stack-indexer/metadata", () => ({
     IpfsProvider: vi.fn(),
+    CachingMetadataProvider: vi.fn(),
 }));
 
 vi.mock("@grants-stack-indexer/indexer-client", () => ({

packages/metadata/package.json

@@ -28,6 +28,7 @@
         "test:cov": "vitest run --config vitest.config.ts --coverage"
     },
     "dependencies": {
+        "@grants-stack-indexer/repository": "workspace:*",
         "@grants-stack-indexer/shared": "workspace:*",
         "axios": "1.7.7",
         "zod": "3.23.8"

packages/metadata/src/external.ts

@@ -1,4 +1,4 @@
-export { IpfsProvider } from "./internal.js";
+export { IpfsProvider, CachingMetadataProvider } from "./internal.js";
 
 export {
     EmptyGatewaysUrlsException,

packages/metadata/src/providers/cachingProxy.provider.ts

@@ -0,0 +1,53 @@
+import { z } from "zod";
+
+import { ICache } from "@grants-stack-indexer/repository";
+import { ILogger } from "@grants-stack-indexer/shared";
+
+import { IMetadataProvider } from "../internal.js";
+
+/**
+ * A metadata provider that caches metadata lookups from the underlying provider.
+ * When a metadata is requested, it first checks the cache. If found, returns the cached metadata.
+ * If not found in cache, fetches from the underlying provider and caches the result before returning.
+ * Cache failures (both reads and writes) are logged but do not prevent the provider from functioning.
+ */
+export class CachingMetadataProvider implements IMetadataProvider {
+    constructor(
+        private readonly provider: IMetadataProvider,
+        private readonly cache: ICache<string, unknown>,
+        private readonly logger: ILogger,
+    ) {}
+
+    /** @inheritdoc */
+    async getMetadata<T>(
+        ipfsCid: string,
+        validateContent?: z.ZodSchema<T>,
+    ): Promise<T | undefined> {
+        let cachedMetadata: T | undefined = undefined;
+        try {
+            cachedMetadata = (await this.cache.get(ipfsCid)) as T | undefined;
+        } catch (error) {
+            this.logger.debug(`Failed to get cached metadata for IPFS CID ${ipfsCid}`, {
+                error,
+            });
+        }
+
+        if (cachedMetadata) {
+            return cachedMetadata;
+        }
+
+        const metadata = await this.provider.getMetadata<T>(ipfsCid, validateContent);
+
+        if (metadata) {
+            try {
+                await this.cache.set(ipfsCid, metadata);
+            } catch (error) {
+                this.logger.debug(`Failed to cache metadata for IPFS CID ${ipfsCid}`, {
+                    error,
+                });
+            }
+        }
+
+        return metadata;
+    }
+}

packages/metadata/src/providers/index.ts

@@ -1 +1,2 @@
 export * from "./ipfs.provider.js";
+export * from "./cachingProxy.provider.js";

packages/metadata/test/providers/cachingProxy.provider.spec.ts

@@ -0,0 +1,96 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { z } from "zod";
+
+import { ICache } from "@grants-stack-indexer/repository";
+import { ILogger } from "@grants-stack-indexer/shared";
+
+import { IMetadataProvider } from "../../src/internal.js";
+import { CachingMetadataProvider } from "../../src/providers/cachingProxy.provider.js";
+
+describe("CachingMetadataProvider", () => {
+    const mockProvider = {
+        getMetadata: vi.fn(),
+    } as unknown as IMetadataProvider;
+
+    const mockCache = {
+        get: vi.fn(),
+        set: vi.fn(),
+    } as unknown as ICache<string, unknown>;
+
+    const mockLogger = {
+        debug: vi.fn(),
+    } as unknown as ILogger;
+
+    let provider: CachingMetadataProvider;
+
+    beforeEach(() => {
+        vi.clearAllMocks();
+        provider = new CachingMetadataProvider(mockProvider, mockCache, mockLogger);
+    });
+
+    describe("getMetadata", () => {
+        const testCid = "QmTest123";
+        const testData = { foo: "bar" };
+        const testSchema = z.object({ foo: z.string() });
+
+        it("returns cached metadata when available", async () => {
+            vi.spyOn(mockCache, "get").mockResolvedValue(testData);
+
+            const result = await provider.getMetadata(testCid, testSchema);
+
+            expect(result).toEqual(testData);
+            expect(mockCache.get).toHaveBeenCalledWith(testCid);
+            expect(mockProvider.getMetadata).not.toHaveBeenCalled();
+        });
+
+        it("fetches and caches metadata when cache misses", async () => {
+            vi.spyOn(mockCache, "get").mockResolvedValue(undefined);
+            vi.spyOn(mockProvider, "getMetadata").mockResolvedValue(testData);
+
+            const result = await provider.getMetadata(testCid, testSchema);
+
+            expect(result).toEqual(testData);
+            expect(mockCache.get).toHaveBeenCalledWith(testCid);
+            expect(mockProvider.getMetadata).toHaveBeenCalledWith(testCid, testSchema);
+            expect(mockCache.set).toHaveBeenCalledWith(testCid, testData);
+        });
+
+        it("handles cache read failures gracefully", async () => {
+            vi.spyOn(mockCache, "get").mockRejectedValue(new Error("Cache read error"));
+            vi.spyOn(mockProvider, "getMetadata").mockResolvedValue(testData);
+
+            const result = await provider.getMetadata(testCid, testSchema);
+
+            expect(result).toEqual(testData);
+            expect(mockLogger.debug).toHaveBeenCalledWith(
+                `Failed to get cached metadata for IPFS CID ${testCid}`,
+                expect.any(Object),
+            );
+            expect(mockProvider.getMetadata).toHaveBeenCalledWith(testCid, testSchema);
+        });
+
+        it("handles cache write failures gracefully", async () => {
+            vi.spyOn(mockCache, "get").mockResolvedValue(undefined);
+            vi.spyOn(mockCache, "set").mockRejectedValue(new Error("Cache write error"));
+            vi.spyOn(mockProvider, "getMetadata").mockResolvedValue(testData);
+
+            const result = await provider.getMetadata(testCid, testSchema);
+
+            expect(result).toEqual(testData);
+            expect(mockLogger.debug).toHaveBeenCalledWith(
+                `Failed to cache metadata for IPFS CID ${testCid}`,
+                expect.any(Object),
+            );
+        });
+
+        it("returns undefined when metadata is not found", async () => {
+            vi.spyOn(mockCache, "get").mockResolvedValue(undefined);
+            vi.spyOn(mockProvider, "getMetadata").mockResolvedValue(undefined);
+
+            const result = await provider.getMetadata(testCid, testSchema);
+
+            expect(result).toBeUndefined();
+            expect(mockCache.set).not.toHaveBeenCalled();
+        });
+    });
+});

packages/pricing/package.json

@@ -28,6 +28,7 @@
         "test:cov": "vitest run --config vitest.config.ts --coverage"
     },
     "dependencies": {
+        "@grants-stack-indexer/repository": "workspace:*",
         "@grants-stack-indexer/shared": "workspace:*",
         "axios": "1.7.7"
     },

packages/pricing/src/external.ts

@@ -1,6 +1,6 @@
 export type { TokenPrice, IPricingProvider } from "./internal.js";
 
-export { CoingeckoProvider, DummyPricingProvider } from "./internal.js";
+export { CoingeckoProvider, DummyPricingProvider, CachingPricingProvider } from "./internal.js";
 
 export { PricingProviderFactory } from "./internal.js";
 export type {

packages/pricing/src/providers/cachingProxy.provider.ts

@@ -0,0 +1,67 @@
+import { ICache, PriceCacheKey } from "@grants-stack-indexer/repository";
+import { ILogger, TokenCode } from "@grants-stack-indexer/shared";
+
+import { IPricingProvider, TokenPrice } from "../internal.js";
+
+/**
+ * A pricing provider that caches token price lookups from the underlying provider.
+ * When a price is requested, it first checks the cache. If found, returns the cached price.
+ * If not found in cache, fetches from the underlying provider and caches the result before returning.
+ * Cache failures (both reads and writes) are logged but do not prevent the provider from functioning.
+ */
+export class CachingPricingProvider implements IPricingProvider {
+    constructor(
+        private readonly provider: IPricingProvider,
+        private readonly cache: ICache<PriceCacheKey, TokenPrice>,
+        private readonly logger: ILogger,
+    ) {}
+
+    /** @inheritdoc */
+    async getTokenPrice(
+        tokenCode: TokenCode,
+        startTimestampMs: number,
+        endTimestampMs?: number,
+    ): Promise<TokenPrice | undefined> {
+        let cachedPrice: TokenPrice | undefined = undefined;
+        try {
+            cachedPrice = await this.cache.get({
+                tokenCode,
+                timestampMs: startTimestampMs,
+            });
+        } catch (error) {
+            this.logger.debug(
+                `Failed to get cached price for token ${tokenCode} at ${startTimestampMs}`,
+                { error },
+            );
+        }
+
+        if (cachedPrice) {
+            return cachedPrice;
+        }
+
+        const price = await this.provider.getTokenPrice(
+            tokenCode,
+            startTimestampMs,
+            endTimestampMs,
+        );
+
+        if (price) {
+            try {
+                await this.cache.set(
+                    {
+                        tokenCode,
+                        timestampMs: startTimestampMs,
+                    },
+                    price,
+                );
+            } catch (error) {
+                this.logger.debug(
+                    `Failed to cache price for token ${tokenCode} at ${startTimestampMs}`,
+                    { error },
+                );
+            }
+        }
+
+        return price;
+    }
+}

packages/pricing/src/providers/index.ts

@@ -1,2 +1,3 @@
 export * from "./coingecko.provider.js";
 export * from "./dummy.provider.js";
+export * from "./cachingProxy.provider.js";