From 1999ce27770c92a4ef56c2c174c5164c2e45a2fd Mon Sep 17 00:00:00 2001 From: Sandros94 Date: Sun, 2 Feb 2025 16:30:33 +0100 Subject: [PATCH 1/3] feat(driver): new `memory-meta` driver --- docs/2.drivers/memory-meta.md | 50 ++++++++ src/_drivers.ts | 7 +- src/drivers/memory-meta.ts | 197 +++++++++++++++++++++++++++++++ test/drivers/memory-meta.test.ts | 31 +++++ 4 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 docs/2.drivers/memory-meta.md create mode 100644 src/drivers/memory-meta.ts create mode 100644 test/drivers/memory-meta.test.ts diff --git a/docs/2.drivers/memory-meta.md b/docs/2.drivers/memory-meta.md new file mode 100644 index 00000000..5f87434c --- /dev/null +++ b/docs/2.drivers/memory-meta.md @@ -0,0 +1,50 @@ +--- +icon: bi:memory +--- + +# Memory Meta + +> Keep data in memory with support for metadata. + +As per the default `memory` driver it keeps data in memory using [Map](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map). + +## Usage + +**Driver name:** `memory-meta` + +This drive aims to be a more advanced version of the `memory` driver by adding metadata support, while introducing a small overhead. + +By supporting metadata, it allows for features like Time-To-Live (TTL),active by default, and optionally tracking a rough size of the stored data. The TTL can be set per item or globally. + +Each key has its own metadata, that includes: +- `ttl`: remaining Time-To-Live in milliseconds, if set during creation. +- `atime`: last access time. +- `mtime`: last modified time. +- `ctime`: last change time. +- `birthtime`: Creation time. +- `size`: Size in bytes, if enabled. + +::note +For memory efficency all `*time` values are stored in milliseconds since the Unix epoch. But returned as `Date` when called via `getMeta`. +:: + +```js +import { createStorage } from "unstorage"; +import memoryMetaDriver from "unstorage/drivers/memory-meta"; + +const storage = createStorage({ + driver: memoryMetaDriver({ + base: "my-storage", // Optional prefix to use for all keys. + ttl: 1000 * 60 * 60, // default `undefined` + ttlAutoPurge: true, // default `true` + trackSize: true, // default `false` + }), +}); +``` + +**Options:** + +- `base`: Optional prefix to use for all keys. Can be used for namespacing. +- `ttl`: Default Time-To-Live for all items in **milliseconds**. +- `ttlAutoPurge`: Whether to automatically purge expired items. (default: `true`) +- `trackSize`: Whether to track the size of items in bytes. (default: `false`) diff --git a/src/_drivers.ts b/src/_drivers.ts index 1e6e721e..1e6341fa 100644 --- a/src/_drivers.ts +++ b/src/_drivers.ts @@ -20,6 +20,7 @@ import type { HTTPOptions as HttpOptions } from "unstorage/drivers/http"; import type { IDBKeyvalOptions as IndexedbOptions } from "unstorage/drivers/indexedb"; import type { LocalStorageOptions as LocalstorageOptions } from "unstorage/drivers/localstorage"; import type { LRUDriverOptions as LruCacheOptions } from "unstorage/drivers/lru-cache"; +import type { MemoryOptions as MemoryMetaOptions } from "unstorage/drivers/memory-meta"; import type { MongoDbOptions as MongodbOptions } from "unstorage/drivers/mongodb"; import type { NetlifyStoreOptions as NetlifyBlobsOptions } from "unstorage/drivers/netlify-blobs"; import type { OverlayStorageOptions as OverlayOptions } from "unstorage/drivers/overlay"; @@ -32,7 +33,7 @@ import type { UpstashOptions as UpstashOptions } from "unstorage/drivers/upstash import type { VercelBlobOptions as VercelBlobOptions } from "unstorage/drivers/vercel-blob"; import type { VercelKVOptions as VercelKVOptions } from "unstorage/drivers/vercel-kv"; -export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "s3" | "session-storage" | "sessionStorage" | "uploadthing" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV"; +export type BuiltinDriverName = "azure-app-configuration" | "azureAppConfiguration" | "azure-cosmos" | "azureCosmos" | "azure-key-vault" | "azureKeyVault" | "azure-storage-blob" | "azureStorageBlob" | "azure-storage-table" | "azureStorageTable" | "capacitor-preferences" | "capacitorPreferences" | "cloudflare-kv-binding" | "cloudflareKVBinding" | "cloudflare-kv-http" | "cloudflareKVHttp" | "cloudflare-r2-binding" | "cloudflareR2Binding" | "db0" | "deno-kv-node" | "denoKVNode" | "deno-kv" | "denoKV" | "fs-lite" | "fsLite" | "fs" | "github" | "http" | "indexedb" | "localstorage" | "lru-cache" | "lruCache" | "memory-meta" | "memoryMeta" | "memory" | "mongodb" | "netlify-blobs" | "netlifyBlobs" | "null" | "overlay" | "planetscale" | "redis" | "s3" | "session-storage" | "sessionStorage" | "uploadthing" | "upstash" | "vercel-blob" | "vercelBlob" | "vercel-kv" | "vercelKV"; export type BuiltinDriverOptions = { "azure-app-configuration": AzureAppConfigurationOptions; @@ -67,6 +68,8 @@ export type BuiltinDriverOptions = { "localstorage": LocalstorageOptions; "lru-cache": LruCacheOptions; "lruCache": LruCacheOptions; + "memory-meta": MemoryMetaOptions; + "memoryMeta": MemoryMetaOptions; "mongodb": MongodbOptions; "netlify-blobs": NetlifyBlobsOptions; "netlifyBlobs": NetlifyBlobsOptions; @@ -117,6 +120,8 @@ export const builtinDrivers = { "localstorage": "unstorage/drivers/localstorage", "lru-cache": "unstorage/drivers/lru-cache", "lruCache": "unstorage/drivers/lru-cache", + "memory-meta": "unstorage/drivers/memory-meta", + "memoryMeta": "unstorage/drivers/memory-meta", "memory": "unstorage/drivers/memory", "mongodb": "unstorage/drivers/mongodb", "netlify-blobs": "unstorage/drivers/netlify-blobs", diff --git a/src/drivers/memory-meta.ts b/src/drivers/memory-meta.ts new file mode 100644 index 00000000..4422be1c --- /dev/null +++ b/src/drivers/memory-meta.ts @@ -0,0 +1,197 @@ +import type { TransactionOptions, StorageMeta } from "../types"; +import { defineDriver, joinKeys } from "./utils"; + +type MaybePromise = T | Promise; + +export interface MemoryOptions { + /** + * Optional prefix to use for all keys. Can be used for namespacing. + */ + base?: string; + + /** + * Default Time-to-live for items in milliseconds. + */ + ttl?: number; + + /** + * Whether to automatically purge expired items. + * @default true + */ + ttlAutoPurge?: boolean; + + /** + * Whether to track the size of items in bytes. + * @default false + */ + trackSize?: boolean; +} + +export interface MemoryDriver { + data: any; + meta?: { + ttl?: number; + atime?: number; // Access Time + mtime?: number; // Modified Time + ctime?: number; // Change Time + birthtime?: number; // Creation Time + size?: number; // Size in bytes (optional) + timeoutId?: NodeJS.Timeout; // Track timeout for auto-purge + }; +} + +export interface MemoryDriverMeta extends StorageMeta { + ttl?: number; + atime?: Date; + mtime?: Date; + ctime?: Date; + birthtime?: Date; + size?: number; +} + +export interface MemoryDriverInstance { + getInstance: () => Map; + options: MemoryOptions; + has: (key: string) => boolean; + hasItem: (key: string) => boolean; + get: (key: string) => any; + getItem: (key: string) => any; + getItemRaw: (key: string) => any; + set: (key: string, value: any, tOpts?: TransactionOptions) => void; + setItem: (key: string, value: any, tOpts?: TransactionOptions) => void; + setItemRaw: (key: string, value: any, tOpts?: TransactionOptions) => void; + del: (key: string) => void; + remove: (key: string) => void; + removeItem: (key: string) => void; + keys: () => string[]; + getKeys: () => string[]; + getMeta: (key: string) => MaybePromise; + clear: () => void; + dispose: () => void; +} + +const DRIVER_NAME = "memory-meta"; + +export default defineDriver((opts: MemoryOptions) => { + const data = new Map(); + + const base = (opts.base || "").replace(/:$/, ""); + const p = (...keys: string[]) => joinKeys(base, ...keys); + const d = (key: string) => (base ? key.replace(base, "") : key); + + function _get(key: string) { + const item = data.get(p(key)); + if (item) { + item.meta ||= {}; + item.meta.atime = Date.now(); + } + return item?.data ?? null; + } + function _set(key: string, value: any, tOpts?: TransactionOptions) { + const now = Date.now(); + const ttl: number | undefined = tOpts?.ttl || opts.ttl; + const pKey = p(key); + const existing = data.get(pKey); + const meta = { + ...existing?.meta, + mtime: now, + ctime: now, + birthtime: existing?.meta?.birthtime || now, + }; + + if (opts.trackSize) { + meta.size = _calculateSize(value); + } + + if (opts.ttlAutoPurge !== false && ttl) { + if (existing?.meta?.timeoutId) { + clearTimeout(existing.meta.timeoutId); + } + + meta.ttl = now + ttl; + data.set(pKey, { data: value, meta }); + meta.timeoutId = setTimeout(() => data.delete(pKey), ttl); + } else { + data.set(pKey, { data: value, meta }); + } + } + function _removeItem(key: string) { + const pKey = p(key); + const existing = data.get(pKey); + if (existing?.meta?.timeoutId) clearTimeout(existing.meta.timeoutId); + data.delete(pKey); + } + function _calculateSize(value: any): number { + if (value === null || value === undefined) return 0; + if (typeof value === "string") return value.length; + if (typeof value === "number") return 8; + if (typeof value === "boolean") return 4; + if (value instanceof Buffer || value instanceof Uint8Array) + return value.byteLength; + if (value instanceof Blob) return value.size; + // Warning for potentially expensive operations + if (value instanceof Object && Object.keys(value).length > 1000) { + console.warn( + `[${DRIVER_NAME}] Large object detected, size calculation may impact performance` + ); + } + return JSON.stringify(value).length; + } + + return { + name: DRIVER_NAME, + options: opts, + getInstance: () => data, + has(key) { + return data.has(p(key)); + }, + hasItem(key) { + return data.has(p(key)); + }, + get: (key) => _get(key), + getItem: (key) => _get(key), + getItemRaw: (key) => _get(key), + set: (key, value, tOpts) => _set(key, value, tOpts), + setItem: (key, value, tOpts) => _set(key, value, tOpts), + setItemRaw: (key, value, tOpts) => _set(key, value, tOpts), + del: (key) => _removeItem(key), + remove: (key) => _removeItem(key), + removeItem: (key) => _removeItem(key), + keys() { + return [...data.keys()].map((element) => d(element)); + }, + getKeys() { + return [...data.keys()].map((element) => d(element)); + }, + getMeta(key): MaybePromise { + const item = data.get(p(key))?.meta; + if (!item) return null; + + return { + ttl: item.ttl ? item.ttl - Date.now() : undefined, + atime: item.atime ? new Date(item.atime) : undefined, + mtime: item.mtime ? new Date(item.mtime) : undefined, + ctime: item.ctime ? new Date(item.ctime) : undefined, + birthtime: item.birthtime ? new Date(item.birthtime) : undefined, + size: item.size, + }; + }, + clear() { + for (const [key, item] of data) { + if (key.startsWith(base)) { + if (item.meta?.timeoutId) clearTimeout(item.meta.timeoutId); + data.delete(key); + } + } + }, + dispose() { + // Clear all timeouts + for (const [_, item] of data) { + if (item.meta?.timeoutId) { + clearTimeout(item.meta.timeoutId); + } + } + data.clear(); + }, + }; +}); diff --git a/test/drivers/memory-meta.test.ts b/test/drivers/memory-meta.test.ts new file mode 100644 index 00000000..dc571714 --- /dev/null +++ b/test/drivers/memory-meta.test.ts @@ -0,0 +1,31 @@ +import { it, describe, expect } from "vitest"; +import driver from "../../src/drivers/memory-meta"; +import { testDriver } from "./utils"; + +describe("drivers: memory-meta", () => { + testDriver({ + driver: driver({}), + additionalTests(ctx) { + it("should supports meta", async () => { + await ctx.storage.setItem("key", "value"); + const meta = await ctx.storage.getMeta("key"); + + expect(meta.birthtime).toBeInstanceOf(Date); + }); + + it("should set and get a key within ttl", async () => { + await ctx.storage.setItem("key", "value", { ttl: 1000 }); + await new Promise((resolve) => setTimeout(resolve, 500)); + + const meta = await ctx.storage.getMeta("key"); + expect(meta.ttl).toBeDefined(); + expect(meta.ttl).toBeLessThan(1000); + + await new Promise((resolve) => setTimeout(resolve, 501)); + + const dataAfter = await ctx.storage.getItem("key"); + expect(dataAfter).toBeNull(); + }); + }, + }); +}); From 88aa64fa0c8d7f0aa550d8dd8923916bf996ebbf Mon Sep 17 00:00:00 2001 From: Sandros94 Date: Sun, 2 Feb 2025 16:39:40 +0100 Subject: [PATCH 2/3] up --- docs/2.drivers/memory-meta.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/2.drivers/memory-meta.md b/docs/2.drivers/memory-meta.md index 5f87434c..e69924bf 100644 --- a/docs/2.drivers/memory-meta.md +++ b/docs/2.drivers/memory-meta.md @@ -12,7 +12,7 @@ As per the default `memory` driver it keeps data in memory using [Map](https://d **Driver name:** `memory-meta` -This drive aims to be a more advanced version of the `memory` driver by adding metadata support, while introducing a small overhead. +This drive aims to be a more advanced version of the `memory` driver by adding metadata support, while introducing a small overhead and allocation cost. By supporting metadata, it allows for features like Time-To-Live (TTL),active by default, and optionally tracking a rough size of the stored data. The TTL can be set per item or globally. From dfcf48b54d1c275ab5131fe2cd83520deb471d7e Mon Sep 17 00:00:00 2001 From: Sandros94 Date: Tue, 4 Feb 2025 02:35:15 +0100 Subject: [PATCH 3/3] feat: allow clearing specific paths within a namespace --- src/drivers/memory-meta.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/memory-meta.ts b/src/drivers/memory-meta.ts index 4422be1c..cf665cdf 100644 --- a/src/drivers/memory-meta.ts +++ b/src/drivers/memory-meta.ts @@ -176,9 +176,9 @@ export default defineDriver((opts: MemoryOptions) => { size: item.size, }; }, - clear() { + clear(path?: string) { for (const [key, item] of data) { - if (key.startsWith(base)) { + if (key.startsWith(path ? p(path) : base)) { if (item.meta?.timeoutId) clearTimeout(item.meta.timeoutId); data.delete(key); }