diff --git a/site/docs/prefixing.md b/site/docs/prefixing.md new file mode 100644 index 00000000..198b282c --- /dev/null +++ b/site/docs/prefixing.md @@ -0,0 +1,66 @@ +--- +id: prefixing +title: Prefixing +sidebar_label: Prefixing +slug: /prefixing +--- + +## Introduction + +As our app grows, we might want to store our objects in a more organized way. This is where the prefix comes in. + +The prefix is a string that is prepended to the object key. This allows us to organize our objects in a folder-like structure. + +For example, if we have a bucket called `my-bucket` and we want to store our objects in a folder called `my-folder`, we can do that by prepending the prefix `my-folder/` to the object key. + +## Usage + +By default, the prefix is an empty string. This means that the object key is not modified, but if you set a prefix, when you initialize the module, the prefix will be prepended to the object key. + +The default algorithm for prefixing will just prepend the prefix to the object key, but you can also specify a custom algorithm. + +**All services like the `ObjectService` will use the prefix service by default.** + +## Custom prefixing + +In order to use a custom prefixing algorithm, you need to specify the `prefixingAlgorithm` when initializing the module. + +```typescript +class CustomPrefixService implements IPrefixAlgorithm { + prefix(remote: string, prefix: string, bucket?: string): string { + return `${bucket}/${prefix}${remote}`; + } +} + +S3Module.forRoot({ + region: 'region', + accessKeyId: '***', + secretAccessKey: '***', + prefix: 'test/', + prefixAlgorithm: new CustomPrefixService(), +}) +``` + +you can also use injectables + +```typescript + class CustomPrefixWithDIService implements IPrefixAlgorithm { + public constructor(private readonly globalPrefix: string) {} + + prefix(remote: string, prefix: string, bucket?: string): string { + return `${bucket}/${this.globalPrefix}${prefix}${remote}`; + } +} + +S3Module.forRootAsync({ + imports: [SomeModuleThatProvidesTheGlobalPrefix], + prefixAlgorithmInject: ['GLOBAL_PREFIX'], + prefixAlgorithmFactory: (globalPrefix: string) => new CustomPrefixWithDIService(globalPrefix), + useFactory: () => ({ + region: 'region', + accessKeyId: '***', + secretAccessKey: '***', + prefix: 'test/', + }), +}) +``` \ No newline at end of file diff --git a/site/sidebars.js b/site/sidebars.js index d2d46731..296a537e 100644 --- a/site/sidebars.js +++ b/site/sidebars.js @@ -15,7 +15,7 @@ const sidebars = { docsSidebar: { Introduction: ['getting-started'], - Services: ['buckets', 'objects', 'signed-url', 'download-helper', 'deletion-helper'], + Services: ['buckets', 'objects', 'signed-url', 'download-helper', 'prefix', 'deletion-helper'], API: [ { type: 'autogenerated', diff --git a/src/constants.ts b/src/constants.ts index d8e53b88..27c10b3f 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,4 +1,4 @@ export const S3_CONFIG = 's3.config'; export const S3_SERVICE = 's3.service'; - +export const PREFIX_ALGORITHM = 'prefix.algorithm'; export const DEFAULT_EXPIRES_IN = 3600; diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts new file mode 100644 index 00000000..0b57e378 --- /dev/null +++ b/src/interfaces/index.ts @@ -0,0 +1 @@ +export * from './prefix-algorithm.interface'; diff --git a/src/interfaces/prefix-algorithm.interface.ts b/src/interfaces/prefix-algorithm.interface.ts new file mode 100644 index 00000000..9283e592 --- /dev/null +++ b/src/interfaces/prefix-algorithm.interface.ts @@ -0,0 +1,3 @@ +export interface IPrefixAlgorithm { + prefix(remote: string, prefix?: string, bucket?: string): string; +} diff --git a/src/s3.module.ts b/src/s3.module.ts index fc5491f1..7ba74529 100644 --- a/src/s3.module.ts +++ b/src/s3.module.ts @@ -1,8 +1,14 @@ import { HttpModule } from '@nestjs/axios'; import { DynamicModule, Global, Module, Provider } from '@nestjs/common'; -import { S3_CONFIG } from './constants'; +import { PREFIX_ALGORITHM, S3_CONFIG } from './constants'; import { createS3ServiceProvider } from './s3-service.factory'; -import { BucketsService, ObjectsService, PrefixService, SignedUrlService } from './services'; +import { + BucketsService, + DefaultPrefixAlgorithmService, + ObjectsService, + PrefixService, + SignedUrlService, +} from './services'; import { S3AsyncConfig, S3Config } from './types'; import { DeletionService, DownloadService } from './utils'; @@ -21,6 +27,10 @@ const createSharedProviders = (config: S3Config): Provider[] => [ provide: S3_CONFIG, useValue: config, }, + { + provide: PREFIX_ALGORITHM, + useValue: config.prefixAlgorithm ? (config.prefixAlgorithm as any) : new DefaultPrefixAlgorithmService(), + }, ...providers, ]; @@ -30,6 +40,13 @@ const createSharedProvidersAsync = (provider: S3AsyncConfig): Provider[] => [ useFactory: provider.useFactory, inject: provider.inject || [], }, + { + provide: PREFIX_ALGORITHM, + useFactory: provider.prefixAlgorithmFactory + ? provider.prefixAlgorithmFactory + : () => new DefaultPrefixAlgorithmService(), + inject: provider.prefixAlgorithmInject || [], + }, ...providers, ]; diff --git a/src/services/default-prefix-algorithm.service.ts b/src/services/default-prefix-algorithm.service.ts new file mode 100644 index 00000000..d44c1292 --- /dev/null +++ b/src/services/default-prefix-algorithm.service.ts @@ -0,0 +1,11 @@ +import { IPrefixAlgorithm } from '../interfaces'; + +export class DefaultPrefixAlgorithmService implements IPrefixAlgorithm { + prefix(remote: string, prefix?: string, bucket?: string): string { + if (!prefix) { + return remote; + } + + return `${prefix}${remote}`; + } +} diff --git a/src/services/index.ts b/src/services/index.ts index 57a96858..f83f982d 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -2,3 +2,4 @@ export * from './buckers.service'; export * from './objects.service'; export * from './prefix.service'; export * from './signed-url.service'; +export * from './default-prefix-algorithm.service'; diff --git a/src/services/objects.service.ts b/src/services/objects.service.ts index d7315cb2..8c99d2a8 100644 --- a/src/services/objects.service.ts +++ b/src/services/objects.service.ts @@ -46,7 +46,7 @@ export class ObjectsService { new PutObjectCommand({ Bucket: bucket, Body: body, - Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote), + Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote, bucket), ...preparedOptions, }), ); @@ -73,7 +73,7 @@ export class ObjectsService { return this.client.send( new DeleteObjectCommand({ Bucket: bucket, - Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote), + Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote, bucket), ...preparedOptions, }), ); @@ -90,7 +90,7 @@ export class ObjectsService { new DeleteObjectsCommand({ Bucket: bucket, Delete: { - Objects: remotes.map((r) => ({ Key: disableAutoPrefix ? r : this.prefixService.prefix(r) })), + Objects: remotes.map((r) => ({ Key: disableAutoPrefix ? r : this.prefixService.prefix(r, bucket) })), }, ...preparedOptions, }), @@ -103,7 +103,7 @@ export class ObjectsService { return this.client.send( new GetObjectCommand({ Bucket: bucket, - Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote), + Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote, bucket), ...preparedOptions, }), ); diff --git a/src/services/prefix.service.ts b/src/services/prefix.service.ts index 1e983471..19fcf8d3 100644 --- a/src/services/prefix.service.ts +++ b/src/services/prefix.service.ts @@ -1,18 +1,18 @@ import { Inject, Injectable } from '@nestjs/common'; -import { S3_CONFIG } from '../constants'; +import { PREFIX_ALGORITHM, S3_CONFIG } from '../constants'; import { S3Config } from '../types'; +import { IPrefixAlgorithm } from '../interfaces'; @Injectable() export class PrefixService { - public constructor(@Inject(S3_CONFIG) private readonly config: S3Config) {} + public constructor( + @Inject(S3_CONFIG) private readonly config: S3Config, + @Inject(PREFIX_ALGORITHM) private readonly prefixAlgorithm: IPrefixAlgorithm, + ) {} - public prefix(remote: string): string { + public prefix(remote: string, bucket?: string): string { const { prefix } = this.config; - if (!prefix) { - return remote; - } - - return `${prefix}${remote}`; + return this.prefixAlgorithm.prefix(remote, prefix, bucket); } } diff --git a/src/services/signed-url.service.ts b/src/services/signed-url.service.ts index 5a0272ab..4a23f709 100644 --- a/src/services/signed-url.service.ts +++ b/src/services/signed-url.service.ts @@ -26,7 +26,7 @@ export class SignedUrlService { options?: PutObjectOptions, ): Promise { const { disableAutoPrefix, options: preparedOptions } = prepareOptions(options); - const key = disableAutoPrefix ? remote : this.prefixService.prefix(remote); + const key = disableAutoPrefix ? remote : this.prefixService.prefix(remote, bucket); const command = new PutObjectCommand({ Bucket: bucket, @@ -54,7 +54,7 @@ export class SignedUrlService { const command = new GetObjectCommand({ Bucket: bucket, - Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote), + Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote, bucket), ...preparedOptions, }); @@ -73,7 +73,7 @@ export class SignedUrlService { const command = new DeleteObjectCommand({ Bucket: bucket, - Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote), + Key: disableAutoPrefix ? remote : this.prefixService.prefix(remote, bucket), ...preparedOptions, }); @@ -93,7 +93,7 @@ export class SignedUrlService { const command = new DeleteObjectsCommand({ Bucket: bucket, Delete: { - Objects: remotes.map((r) => ({ Key: disableAutoPrefix ? r : this.prefixService.prefix(r) })), + Objects: remotes.map((r) => ({ Key: disableAutoPrefix ? r : this.prefixService.prefix(r, bucket) })), }, ...preparedOptions, }); diff --git a/src/types/index.ts b/src/types/index.ts index 6b3c4e56..2a686270 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -3,3 +3,4 @@ export * from './object-command-options.type'; export * from './s3-config.type'; export * from './signed-url.type'; export * from './disable-auto-prefix.type'; +export * from './prefix-algorithm.type'; diff --git a/src/types/prefix-algorithm.type.ts b/src/types/prefix-algorithm.type.ts new file mode 100644 index 00000000..91dad59d --- /dev/null +++ b/src/types/prefix-algorithm.type.ts @@ -0,0 +1 @@ +export type PrefixAlgorithm = (remote: string, prefix?: string, bucket?: string) => string; diff --git a/src/types/s3-config.type.ts b/src/types/s3-config.type.ts index ecf8f435..d33c48bf 100644 --- a/src/types/s3-config.type.ts +++ b/src/types/s3-config.type.ts @@ -1,5 +1,7 @@ import { Abstract, Type } from '@nestjs/common'; import { ModuleMetadata } from '@nestjs/common/interfaces'; +import { PrefixAlgorithm } from './prefix-algorithm.type'; +import { IPrefixAlgorithm } from '../interfaces'; export type S3Config = { region: string; @@ -7,9 +9,12 @@ export type S3Config = { secretAccessKey: string; prefix?: string; endPoint?: string; + prefixAlgorithm?: IPrefixAlgorithm; }; export type S3AsyncConfig = Pick & { - useFactory: (...args: any[]) => Promise | S3Config; + useFactory: (...args: any[]) => Promise> | Omit; inject?: Array | string | symbol | Abstract>; + prefixAlgorithmFactory?: (...args: any[]) => Promise | IPrefixAlgorithm; + prefixAlgorithmInject?: Array | string | symbol | Abstract>; }; diff --git a/src/utils/deletion.service.ts b/src/utils/deletion.service.ts index 08b322e0..55544af7 100644 --- a/src/utils/deletion.service.ts +++ b/src/utils/deletion.service.ts @@ -1,6 +1,6 @@ import { DeleteObjectOutput, DeleteObjectsCommand, ListObjectsV2Output, S3Client } from '@aws-sdk/client-s3'; import { Inject, Injectable } from '@nestjs/common'; -import { S3_SERVICE } from '../constants'; +import { PREFIX_ALGORITHM, S3_SERVICE } from '../constants'; import { DeleteObjectsOptions, ListObjectsV2Options } from '../types'; import { ObjectsService, PrefixService } from '../services'; import { prepareOptions } from '../helpers'; @@ -10,7 +10,7 @@ export class DeletionService { public constructor( @Inject(S3_SERVICE) private readonly client: S3Client, private readonly objectsService: ObjectsService, - private readonly prefixService: PrefixService, + @Inject(PREFIX_ALGORITHM) private readonly prefixService: PrefixService, ) {} /** @@ -32,7 +32,7 @@ export class DeletionService { do { data = await this.objectsService.listObjectsV2(bucket, { - Prefix: disableAutoPrefix ? prefix : this.prefixService.prefix(prefix), + Prefix: disableAutoPrefix ? prefix : this.prefixService.prefix(prefix, bucket), ContinuationToken: continuationToken, ...listOptions, }); diff --git a/tests/unit-tests/prefix-service.test.ts b/tests/unit-tests/prefix-service.test.ts index c263503c..d11b288e 100644 --- a/tests/unit-tests/prefix-service.test.ts +++ b/tests/unit-tests/prefix-service.test.ts @@ -1,8 +1,9 @@ import { Test, TestingModule } from '@nestjs/testing'; import { S3Module } from '../../src'; import { PrefixService } from '../../src/services'; +import { IPrefixAlgorithm } from '../../src/interfaces'; -describe('Prefix service', () => { +describe('Prefix service default implementation', () => { let testingModule!: TestingModule; let prefixService!: PrefixService; @@ -25,3 +26,76 @@ describe('Prefix service', () => { expect(prefixService.prefix('test.txt')).toEqual('test/test.txt'); }); }); + +describe('Prefix service custom implementation', () => { + let testingModule!: TestingModule; + let prefixService!: PrefixService; + + beforeAll(async () => { + class CustomPrefixService implements IPrefixAlgorithm { + prefix(remote: string, prefix: string, bucket?: string): string { + return `${bucket}/${prefix}${remote}`; + } + } + + testingModule = await Test.createTestingModule({ + imports: [ + S3Module.forRoot({ + region: process.env.AWS_REGION, + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + prefix: 'test/', + prefixAlgorithm: new CustomPrefixService(), + }), + ], + }).compile(); + + prefixService = testingModule.get(PrefixService); + }); + + it('should use the custom implementation', async () => { + expect(prefixService.prefix('test.txt', 'test-bucket')).toEqual('test-bucket/test/test.txt'); + }); +}); + +describe('Prefix service custom implementation with injection', () => { + let testingModule!: TestingModule; + let prefixService!: PrefixService; + + beforeAll(async () => { + class CustomPrefixWithDIService implements IPrefixAlgorithm { + public constructor(private readonly globalPrefix: string) {} + + prefix(remote: string, prefix: string, bucket?: string): string { + return `${bucket}/${this.globalPrefix}${prefix}${remote}`; + } + } + + testingModule = await Test.createTestingModule({ + imports: [ + S3Module.forRootAsync({ + providers: [ + { + provide: 'GLOBAL_PREFIX', + useValue: 'global-prefix/', + }, + ], + prefixAlgorithmInject: ['GLOBAL_PREFIX'], + prefixAlgorithmFactory: (globalPrefix: string) => new CustomPrefixWithDIService(globalPrefix), + useFactory: () => ({ + region: process.env.AWS_REGION, + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + prefix: 'test/', + }), + }), + ], + }).compile(); + + prefixService = testingModule.get(PrefixService); + }); + + it('should use the custom implementation', async () => { + expect(prefixService.prefix('test.txt', 'test-bucket')).toEqual('test-bucket/global-prefix/test/test.txt'); + }); +});