azblob support

packages/cli-common/package.json

@@ -12,13 +12,15 @@
   ],
   "license": "Apache-2.0",
   "dependencies": {
+    "@inquirer/prompts": "^3.3.0",
     "@oclif/core": "^3.15.1",
     "@preevy/core": "0.0.60",
     "chalk": "^4.1.2",
     "iter-tools-es": "^7.5.3",
     "lodash-es": "^4.17.21"
   },
   "devDependencies": {
+    "@inquirer/type": "^1.2.0",
     "@jest/globals": "29.7.0",
     "@types/lodash-es": "^4.17.12",
     "@types/node": "18",

packages/cli-common/src/index.ts

@@ -8,3 +8,4 @@ export {
 export { formatFlagsToArgs, parseFlags, ParsedFlags } from './lib/flags.js'
 export { initHook } from './hooks/init/load-plugins.js'
 export { default as BaseCommand } from './commands/base-command.js'
+export * as prompts from './prompts.js'

packages/cli-common/src/prompts.ts

@@ -0,0 +1,37 @@
+import * as inquirer from '@inquirer/prompts'
+import chalk from 'chalk'
+
+const nullPrompt = inquirer.createPrompt<boolean, { message: string; value: string }>(
+  (config, done) => {
+    const prefix = inquirer.usePrefix()
+    done(true)
+    return `${prefix} ${chalk.bold(config.message)} ${chalk.cyan(config.value)}`
+  },
+)
+
+export const selectOrSpecify = async ({ message, choices, specifyItem = '(specify)', specifyItemLocation = 'top' }: {
+  message: string
+  choices: { name: string; value: string }[]
+  specifyItem?: string
+  specifyItemLocation?: 'top' | 'bottom'
+}) => {
+  const specify = () => inquirer.input({ message }, { clearPromptOnDone: true })
+  const select = async () => (
+    await inquirer.select({
+      message,
+      choices: specifyItemLocation === 'top' ? [
+        { name: specifyItem, value: undefined },
+        new inquirer.Separator(),
+        ...choices,
+      ] : [
+        ...choices,
+        new inquirer.Separator(),
+        { name: specifyItem, value: undefined },
+      ],
+      loop: false,
+    }, { clearPromptOnDone: true })
+  ) ?? await specify()
+  const result = choices.length ? await select() : await specify()
+  await nullPrompt({ message, value: result })
+  return result
+}

packages/cli/src/fs.ts

@@ -1,8 +1,11 @@
 import { fsTypeFromUrl, localFsFromUrl } from '@preevy/core'
+import { prompts } from '@preevy/cli-common'
 import { googleCloudStorageFs, defaultBucketName as gsDefaultBucketName, defaultProjectId as defaultGceProjectId } from '@preevy/driver-gce'
 import { s3fs, defaultBucketName as s3DefaultBucketName, awsUtils, S3_REGIONS } from '@preevy/driver-lightsail'
 import * as inquirer from '@inquirer/prompts'
+import * as azure from '@preevy/driver-azure'
 import inquirerAutoComplete from 'inquirer-autocomplete-standalone'
+import { asyncFind, asyncTake, asyncToArray } from 'iter-tools-es'
 import { DriverName } from './drivers.js'
 import ambientAwsAccountId = awsUtils.ambientAccountId
 
@@ -21,10 +24,15 @@ export const fsFromUrl = async (url: string, localBaseDir: string) => {
     // eslint-disable-next-line @typescript-eslint/return-await
     return await googleCloudStorageFs(url)
   }
+  if (fsType === 'azblob') {
+    // eslint false positive here on case-sensitive filesystems due to unknown type
+    // eslint-disable-next-line @typescript-eslint/return-await
+    return await azure.fs.azureBlobStorageFs(url)
+  }
   throw new Error(`Unsupported URL type: ${fsType}`)
 }
 
-export const fsTypes = ['local', 's3', 'gs'] as const
+export const fsTypes = ['local', 's3', 'gs', 'azblob'] as const
 export type FsType = typeof fsTypes[number]
 export const isFsType = (s: string): s is FsType => fsTypes.includes(s as FsType)
 
@@ -35,6 +43,9 @@ const defaultFsType = (driver?: string): FsType => {
   if (driver as DriverName === 'gce') {
     return 'gs'
   }
+  if (driver as DriverName === 'azure') {
+    return 'azblob'
+  }
   return 'local'
 }
 
@@ -45,6 +56,7 @@ export const chooseFsType = async ({ driver }: { driver?: string }) => await inq
     { value: 'local', name: 'local file' },
     { value: 's3', name: 'AWS S3' },
     { value: 'gs', name: 'Google Cloud Storage' },
+    { value: 'azblob', name: 'Microsoft Azure Blob Storage' },
   ],
 }) as FsType
 
@@ -100,4 +112,50 @@ export const chooseFs: Record<FsType, FsChooser> = {
 
     return `gs://${bucket}?project=${project}`
   },
+  azblob: async ({ profileAlias, driver }: {
+    profileAlias: string
+    driver?: { name: DriverName; flags: Record<string, unknown> }
+  }) => {
+    const subscriptionId = driver?.name === 'azure'
+      ? driver.flags['subscription-id'] as string
+      : await azure.inquireSubscriptionId().catch(() => undefined)
+
+    const pageSize = 7
+
+    const accounts = subscriptionId
+      ? await asyncToArray(asyncTake(pageSize - 2, azure.fs.listStorageAccounts({ subscriptionId }))).catch(() => [])
+      : []
+
+    const account = await prompts.selectOrSpecify({
+      message: 'Storage account name',
+      choices: accounts.map(({ name }) => ({ value: name, name })),
+      specifyItemLocation: 'bottom',
+    })
+
+    const inquireDomain = () => prompts.selectOrSpecify({
+      message: 'Storage domain',
+      choices: [{ value: azure.fs.DEFAULT_DOMAIN, name: `(default): ${azure.fs.DEFAULT_DOMAIN}` }],
+      specifyItem: '(custom)',
+      specifyItemLocation: 'bottom',
+    })
+
+    const domain = (subscriptionId && accounts.length)
+      ? await (async () => {
+        const foundAccount = accounts.find(a => a.name === account)
+          ?? await asyncFind(({ name }) => name === account, azure.fs.listStorageAccounts({ subscriptionId }))
+        return foundAccount?.blobDomain ?? inquireDomain()
+      })()
+      : await inquireDomain()
+
+    const container = await inquirer.input({
+      message: 'Container name',
+      default: azure.fs.defaultContainerName({ profileAlias }),
+    })
+
+    return azure.fs.toUrl({
+      container,
+      account,
+      domain: domain === azure.fs.DEFAULT_DOMAIN ? undefined : domain,
+    })
+  },
 }

packages/driver-azure/package.json

@@ -16,12 +16,14 @@
     "@azure/arm-compute": "^20.0.0",
     "@azure/arm-network": "^30.2.0",
     "@azure/arm-resources": "^5.2.0",
-    "@azure/arm-storage": "^18.1.0",
+    "@azure/arm-storage": "^18.2.0",
     "@azure/arm-subscriptions": "^5.1.0",
     "@azure/identity": "^3.2.2",
     "@azure/logger": "^1.0.4",
+    "@azure/storage-blob": "^12.17.0",
     "@inquirer/prompts": "^3.3.0",
     "@oclif/core": "^3.15.1",
+    "@preevy/cli-common": "0.0.60",
     "@preevy/core": "0.0.60",
     "inquirer-autocomplete-standalone": "^0.8.1",
     "iter-tools-es": "^7.5.3",

packages/driver-azure/src/driver/index.ts

@@ -1,6 +1,5 @@
 import { Flags, Interfaces } from '@oclif/core'
-import { asyncFirst, asyncMap } from 'iter-tools-es'
-import * as inquirer from '@inquirer/prompts'
+import { asyncMap, asyncToArray } from 'iter-tools-es'
 import inquirerAutoComplete from 'inquirer-autocomplete-standalone'
 import { InferredFlags } from '@oclif/core/lib/interfaces'
 import { Resource, VirtualMachine } from '@azure/arm-compute'
@@ -22,6 +21,7 @@ import {
   Logger,
   machineStatusNodeExporterCommand,
 } from '@preevy/core'
+import { prompts } from '@preevy/cli-common'
 import { pick } from 'lodash-es'
 import { Client, client as createClient, REGIONS } from './client.js'
 import { CUSTOMIZE_BARE_MACHINE } from './scripts.js'
@@ -120,29 +120,41 @@ const flags = {
     required: true,
   }),
   'subscription-id': Flags.string({
-    description: 'Microsoft Azure subscription id',
+    description: 'Microsoft Azure Subscription ID',
     required: true,
   }),
 } as const
 
 type FlagTypes = Omit<Interfaces.InferredFlags<typeof flags>, 'json'>
 
-const inquireFlags = async ({ log: _log }: { log: Logger }) => {
-  const region = await inquirerAutoComplete<string>({
-    message: flags.region.description as string,
-    source: async input => REGIONS.filter(r => !input || r.includes(input.toLowerCase())).map(value => ({ value })),
-    suggestOnly: true,
-    transformer: i => i.toLowerCase(),
+export const inquireSubscriptionId = async (): Promise<string> => {
+  const credential = new DefaultAzureCredential()
+  const subscriptionClient = new SubscriptionClient(credential)
+  const subscriptions = await asyncToArray(subscriptionClient.subscriptions.list()).catch(() => [])
+  return prompts.selectOrSpecify({
+    message: 'Microsoft Azure Subscription ID',
+    choices: subscriptions.map(({ subscriptionId, displayName }) => ({ name: `${displayName} (${subscriptionId})`, value: subscriptionId as string })),
+    specifyItemLocation: 'bottom',
   })
+}
 
+export const inquireRegion = async ({ subscriptionId }: { subscriptionId: string }): Promise<string> => {
   const credential = new DefaultAzureCredential()
   const subscriptionClient = new SubscriptionClient(credential)
-  const defaultSubscriptionId = (await asyncFirst(subscriptionClient.subscriptions.list()))?.subscriptionId
-
-  const subscriptionId = await inquirer.input({
-    message: flags['subscription-id'].description as string,
-    default: defaultSubscriptionId,
+  const regions = await asyncToArray(
+    asyncMap(({ name }) => name as string, subscriptionClient.subscriptions.listLocations(subscriptionId)),
+  ).catch(() => REGIONS)
+  return await inquirerAutoComplete<string>({
+    message: flags.region.description as string,
+    source: async input => regions.filter(r => !input || r.includes(input.toLowerCase())).map(value => ({ value })),
+    suggestOnly: true,
+    transformer: i => i.toLowerCase(),
   })
+}
+
+const inquireFlags = async ({ log: _log }: { log: Logger }) => {
+  const subscriptionId = await inquireSubscriptionId()
+  const region = await inquireRegion({ subscriptionId })
 
   return { region, 'subscription-id': subscriptionId }
 }

packages/driver-azure/src/fs.ts

@@ -0,0 +1,109 @@
+import { DefaultAzureCredential } from '@azure/identity'
+import { BlobServiceClient, RestError } from '@azure/storage-blob'
+import { StorageManagementClient } from '@azure/arm-storage'
+import { VirtualFS } from '@preevy/core'
+import { asyncFilter, asyncMap } from 'iter-tools-es'
+import { join } from 'path'
+
+export const DEFAULT_DOMAIN = 'blob.core.windows.net' as const
+
+export const parseUrl = (url: string, defaults: Partial<{ account: string; domain: string }> = {}) => {
+  const u = new URL(url)
+  if (u.protocol !== 'azblob:') {
+    throw new Error('Azure Blob Storage urls must start with azblob://')
+  }
+
+  const account = u.searchParams.get('storage_account') ?? defaults?.account
+  if (!account) {
+    throw new Error(`Missing storage_account in url and no default storage account provided: ${url}`)
+  }
+
+  return {
+    url: u,
+    container: u.hostname,
+    account,
+    path: u.pathname,
+    domain: u.searchParams.get('domain') ?? defaults?.domain ?? DEFAULT_DOMAIN,
+  }
+}
+
+export const toUrl = (
+  { account, domain, container, path }: { account?: string; domain?: string; container: string; path?: string },
+) => {
+  const u = new URL(`azblob://${container}`)
+  u.pathname = path ?? '/'
+  if (account) {
+    u.searchParams.set('storage_account', account)
+  }
+  if (domain) {
+    u.searchParams.set('domain', domain)
+  }
+  return u.toString() as `azblob://${string}`
+}
+
+export const listContainers = (
+  { account, domain }: { account: string; domain?: string },
+): AsyncIterable<string> => {
+  const client = new BlobServiceClient(`https://${account}.${domain}`, new DefaultAzureCredential())
+  const filtered = asyncFilter(({ deleted }) => !deleted, client.listContainers())
+  return asyncMap(({ name }) => name, filtered)
+}
+
+export const listStorageAccounts = (
+  { subscriptionId }: { subscriptionId: string },
+): AsyncIterable<{ name: string; blobDomain?: string }> => {
+  const client = new StorageManagementClient(new DefaultAzureCredential(), subscriptionId)
+  return asyncMap(
+    ({ name, primaryEndpoints }) => ({
+      name: name as string,
+      blobDomain: /(?<=\.)[^/]+/.exec(primaryEndpoints?.blob ?? '')?.toString() ?? undefined,
+    }),
+    client.storageAccounts.list(),
+  )
+}
+
+const isNotFoundError = (e: unknown): e is RestError => e instanceof RestError && e.statusCode === 404
+const isContainerNotFound = (e: unknown) => isNotFoundError(e) && e.code === 'ContainerNotFound'
+
+const catchNotFoundError = async <T>(fn: () => Promise<T>) => {
+  try {
+    return await fn()
+  } catch (e) {
+    if (isNotFoundError(e)) {
+      return undefined
+    }
+    throw e
+  }
+}
+
+export const containerClient = (url: string) => {
+  const { container, account, path, domain } = parseUrl(url)
+  return {
+    client: new BlobServiceClient(`https://${account}.${domain}`, new DefaultAzureCredential()).getContainerClient(container),
+    path,
+  }
+}
+
+export const azureBlobStorageFs = async (url: string): Promise<VirtualFS> => {
+  const { client, path } = containerClient(url)
+  await client.createIfNotExists()
+
+  return {
+    read: async (filename: string) => {
+      const blob = client.getBlobClient(join(path, filename))
+      return await catchNotFoundError(() => blob.downloadToBuffer())
+    },
+    write: async (filename: string, data: Buffer | string) => {
+      const blob = client.getBlockBlobClient(join(path, filename))
+      await blob.upload(Buffer.isBuffer(data) ? data : Buffer.from(data), data.length)
+    },
+    delete: async (filename: string) => {
+      const blob = client.getBlobClient(join(path, filename))
+      await catchNotFoundError(() => blob.delete())
+    },
+  }
+}
+
+export const defaultContainerName = (
+  { profileAlias }: { profileAlias: string },
+) => ['preevy', profileAlias].join('-')

packages/driver-azure/src/index.ts

@@ -1,3 +1,6 @@
 import azure from './driver/index.js'
 
+export * as fs from './fs.js'
+export { inquireSubscriptionId, inquireRegion } from './driver/index.js'
+
 export default azure

packages/driver-azure/tsconfig.json

@@ -33,5 +33,6 @@
   "references": [
     { "path": "../common" },
     { "path": "../core" },
+    { "path": "../cli-common" },
   ]
 }