Skip to content

Commit

Permalink
Merge pull request #255 from rr-wfm/feature/az-blob-storage
Browse files Browse the repository at this point in the history
Add Azure Blob Storage as a storage engine for attachments
  • Loading branch information
drodil authored Jan 7, 2025
2 parents 2d5033d + 0f564a7 commit c46f843
Show file tree
Hide file tree
Showing 12 changed files with 439 additions and 114 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -118,3 +118,4 @@ dist
!**/.yarn/versions

dist-types/
app-config.local.yaml
7 changes: 6 additions & 1 deletion docs/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ The allowed configuration values are:

## Storage

- storage.type, string, what kind of storage is used to upload images used in questions. Default is `database`. Available values are 'filesystem', 'database' and 's3'.
- storage.type, string, what kind of storage is used to upload images used in questions. Default is `database`. Available values are 'filesystem', 'database', 's3' and 'azure'.
- storage.maxSizeImage, number, the maximum allowed size of upload files in bytes. Default is `2500000`
- storage.folder, string, what folder is used to storage temporarily images to convert and send to frontend. Default is `/tmp/backstage-qeta-images`
- storage.allowedMimeTypes, string[], A list of allowed upload formats. Default: `png,jpg,jpeg,gif`
Expand All @@ -53,5 +53,10 @@ The allowed configuration values are:
- storage.secretAccessKey, string, secret access key for S3 storage, optional
- storage.region, string, region for S3 storage, optional
- storage.sessionToken, string, AWS session token, optional
- storage.blobStorageAccountName, string, Azure Blob Storage account name, optional
- storage.blobStorageConnectionString, string, Connection String to Azure Blob Storage, optional
- storage.blobStorageContainer, string, Azure Blob Storage container name, optional. Default `backstage-qeta-images`

> Note: For Azure Blob Storage you can either use passwordless authentication by configuring `blobStorageAccountName`. This requires your Backstage backend to run as an Azure Managed Identity. Alternatively, you can use `blobStorageConnectionString` to authenticate with a connection string.

Additionally, there are more config values for the [OpenAI module](ai.md).
5 changes: 4 additions & 1 deletion plugins/qeta-backend/configSchema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ export interface Config {
* @visibility backend
*/
storage?: {
type?: 'database' | 'filesystem' | 's3';
type?: 'database' | 'filesystem' | 's3' | 'azure';
folder?: string;
maxSizeImage?: number;
allowedMimeTypes?: string[];
Expand All @@ -88,6 +88,9 @@ export interface Config {
secretAccessKey?: string;
region?: string;
sessionToken?: string;
blobStorageAccountName?: string;
blobStorageConnectionString?: string;
blobStorageContainer?: string;
};
/**
* Stats config
Expand Down
2 changes: 2 additions & 0 deletions plugins/qeta-backend/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@
},
"dependencies": {
"@aws-sdk/client-s3": "^3.540.0",
"@azure/identity": "^4.5.0",
"@azure/storage-blob": "^12.26.0",
"@backstage/backend-defaults": "backstage:^",
"@backstage/backend-plugin-api": "backstage:^",
"@backstage/catalog-client": "backstage:^",
Expand Down
117 changes: 34 additions & 83 deletions plugins/qeta-backend/src/service/routes/attachments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,15 @@ import multiparty from 'multiparty';
import FilesystemStoreEngine from '../upload/filesystem';
import DatabaseStoreEngine from '../upload/database';
import S3StoreEngine from '../upload/s3';
import AzureBlobStorageEngine from '../upload/azureBlobStorage';
import fs from 'fs';
import FileType from 'file-type';
import { File, RouteOptions } from '../types';
import {
DeleteObjectCommand,
DeleteObjectCommandOutput,
GetObjectCommand,
GetObjectCommandOutput,
} from '@aws-sdk/client-s3';
import { getS3Client } from '../util';
AttachmentStorageEngine,
AttachmentStorageEngineOptions,
} from '../upload/attachmentStorageEngine';
import { getUsername } from '../util';

const DEFAULT_IMAGE_SIZE_LIMIT = 2500000;
const DEFAULT_MIME_TYPES = [
Expand All @@ -23,13 +22,32 @@ const DEFAULT_MIME_TYPES = [
'image/gif',
];

const getStorageEngine = (
storageType: string,
options: AttachmentStorageEngineOptions,
): AttachmentStorageEngine => {
switch (storageType) {
case 'azure':
return AzureBlobStorageEngine(options);
case 's3':
return S3StoreEngine(options);
case 'filesystem':
return FilesystemStoreEngine(options);
case 'database':
default:
return DatabaseStoreEngine(options);
}
};

export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
const { database, config } = options;

// POST /attachments
router.post('/attachments', async (request, response) => {
let attachment: Attachment;

const username = await getUsername(request, options);

const storageType =
config?.getOptionalString('qeta.storage.type') || 'database';
const maxSizeImage =
Expand All @@ -40,9 +58,8 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
DEFAULT_MIME_TYPES;

const form = new multiparty.Form();
const fileSystemEngine = FilesystemStoreEngine(options);
const databaseEngine = DatabaseStoreEngine(options);
const s3Engine = S3StoreEngine(options);

const engine = getStorageEngine(storageType, options);

form.parse(request, async (err, _fields, files) => {
if (err) {
Expand Down Expand Up @@ -84,6 +101,7 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
};

const opts = {
creator: username,
postId: request.query.postId ? Number(request.query.postId) : undefined,
answerId: request.query.answerId
? Number(request.query.answerId)
Expand All @@ -93,18 +111,7 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
: undefined,
};

switch (storageType) {
case 's3':
attachment = await s3Engine.handleFile(file, opts);
break;
case 'filesystem':
attachment = await fileSystemEngine.handleFile(file, opts);
break;
case 'database':
default:
attachment = await databaseEngine.handleFile(file, opts);
break;
}
attachment = await engine.handleFile(file, opts);
response.json(attachment);
});
});
Expand All @@ -119,42 +126,12 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
return;
}

const getS3ImageBuffer = async () => {
const bucket = config.getOptionalString('qeta.storage.bucket');
if (!bucket) {
throw new Error('Bucket name is required for S3 storage');
}
const s3 = getS3Client(config);
const object: GetObjectCommandOutput = await s3.send(
new GetObjectCommand({
Bucket: bucket,
Key: attachment.path,
}),
);

if (!object.Body) {
return undefined;
}
const bytes = await object.Body.transformToByteArray();
return Buffer.from(bytes);
};

let imageBuffer: Buffer | undefined;
switch (attachment.locationType) {
case 's3':
imageBuffer = await getS3ImageBuffer();
break;
case 'filesystem':
imageBuffer = await fs.promises.readFile(attachment.path);
break;
default:
case 'database':
imageBuffer = attachment.binaryImage;
break;
}
const engine = getStorageEngine(attachment.locationType, options);
const imageBuffer = await engine.getAttachmentBuffer(attachment);

if (!imageBuffer) {
response.status(500).send('Attachment buffer is undefined');
response.status(404).end();
return;
}

response.writeHead(200, {
Expand Down Expand Up @@ -186,34 +163,8 @@ export const attachmentsRoutes = (router: Router, options: RouteOptions) => {
return;
}

const deleteS3Image = async () => {
const bucket = config.getOptionalString('qeta.storage.bucket');
if (!bucket) {
throw new Error('Bucket name is required for S3 storage');
}
const s3 = getS3Client(config);
const output: DeleteObjectCommandOutput = await s3.send(
new DeleteObjectCommand({
Bucket: bucket,
Key: attachment.path,
}),
);
if (output.$metadata.httpStatusCode !== 204) {
throw new Error('Failed to delete object');
}
};

switch (attachment.locationType) {
case 's3':
await deleteS3Image();
break;
case 'filesystem':
await fs.promises.rm(attachment.path);
break;
default:
case 'database':
break;
}
const engine = getStorageEngine(attachment.locationType, options);
await engine.deleteAttachment(attachment);

const result = await database.deleteAttachment(uuid);
if (!result) {
Expand Down
23 changes: 23 additions & 0 deletions plugins/qeta-backend/src/service/upload/attachmentStorageEngine.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { Attachment } from '@drodil/backstage-plugin-qeta-common';
import { File } from '../types';
import { Config } from '@backstage/config/index';
import { QetaStore } from '../../database/QetaStore';

export type AttachmentStorageEngineOptions = {
config: Config;
database: QetaStore;
};

export interface AttachmentStorageEngine {
handleFile: (
file: File,
options?: {
creator: string;
postId?: number;
answerId?: number;
collectionId?: number;
},
) => Promise<Attachment>;
getAttachmentBuffer: (attachment: Attachment) => Promise<Buffer | undefined>;
deleteAttachment(attachment: Attachment): Promise<void>;
}
94 changes: 94 additions & 0 deletions plugins/qeta-backend/src/service/upload/azureBlobStorage.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import * as fs from 'fs';
import { v4 as uuidv4 } from 'uuid';
import { Config } from '@backstage/config';
import { QetaStore } from '../../database/QetaStore';
import { Attachment } from '@drodil/backstage-plugin-qeta-common';
import { File } from '../types';
import { getAzureBlobServiceClient } from '../util';
import {
AttachmentStorageEngine,
AttachmentStorageEngineOptions,
} from './attachmentStorageEngine';

class AzureBlobStorageEngine implements AttachmentStorageEngine {
config: Config;
database: QetaStore;
backendBaseUrl: string;
qetaUrl: string;
container: string;

constructor(opts: AttachmentStorageEngineOptions) {
this.config = opts.config;
this.database = opts.database;
this.backendBaseUrl = this.config.getString('backend.baseUrl');
this.qetaUrl = `${this.backendBaseUrl}/api/qeta/attachments`;
this.container =
this.config.getOptionalString('qeta.storage.blobStorageContainer') ||
'backstage-qeta-images';
}

handleFile = async (
file: File,
options?: { postId?: number; answerId?: number; collectionId?: number },
): Promise<Attachment> => {
const imageUuid = uuidv4();
const filename = `image-${imageUuid}-${Date.now()}.${file.ext}`;

const imageURI = `${this.qetaUrl}/${imageUuid}`;
const client = getAzureBlobServiceClient(this.config);
const container = client.getContainerClient(this.container);
if (!(await container.exists())) {
await container.create();
}

await container.uploadBlockBlob(
filename,
fs.createReadStream(file.path),
file.size,
);

return await this.database.postAttachment({
uuid: imageUuid,
locationType: 'azure',
locationUri: imageURI,
extension: file.ext,
mimeType: file.mimeType,
path: filename,
binaryImage: undefined,
...options,
});
};

getAttachmentBuffer = async (attachment: Attachment) => {
const client = getAzureBlobServiceClient(this.config);
const container = client.getContainerClient(this.container);

if (!(await container.exists())) {
return undefined;
}

const blob = container.getBlockBlobClient(attachment.path);
if (await blob.exists()) {
return blob.downloadToBuffer();
}

return undefined;
};

deleteAttachment = async (attachment: Attachment) => {
const client = getAzureBlobServiceClient(this.config);
const container = client.getContainerClient(this.container);
if (!(await container.exists())) {
return;
}

const blob = container.getBlockBlobClient(attachment.path);
if (await blob.exists()) {
await blob.delete();
}
};
}

export default (opts: AttachmentStorageEngineOptions) => {
return new AzureBlobStorageEngine(opts);
};
24 changes: 16 additions & 8 deletions plugins/qeta-backend/src/service/upload/database.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@ import { Config } from '@backstage/config';
import { QetaStore } from '../../database/QetaStore';
import { File } from '../types';
import { v4 as uuidv4 } from 'uuid';
import {
AttachmentStorageEngine,
AttachmentStorageEngineOptions,
} from './attachmentStorageEngine';
import { Attachment } from '@drodil/backstage-plugin-qeta-common';

type Options = {
config: Config;
database: QetaStore;
};

class DatabaseStoreEngine {
class DatabaseStoreEngine implements AttachmentStorageEngine {
config: Config;
database: QetaStore;
backendBaseUrl: string;
qetaUrl: string;

constructor(opts: Options) {
constructor(opts: AttachmentStorageEngineOptions) {
this.config = opts.config;
this.database = opts.database;
this.backendBaseUrl = this.config.getString('backend.baseUrl');
Expand All @@ -38,8 +38,16 @@ class DatabaseStoreEngine {
...options,
});
};

getAttachmentBuffer = async (attachment: Attachment) => {
return attachment.binaryImage;
};

deleteAttachment = async (_attachment: Attachment) => {
// Nothing to do here, since the attachment is stored in the database
};
}

export default (opts: Options) => {
export default (opts: AttachmentStorageEngineOptions) => {
return new DatabaseStoreEngine(opts);
};
Loading

0 comments on commit c46f843

Please sign in to comment.