feat: nft.storage naive gateway implementation

.github/workflows/gateway.yml

@@ -0,0 +1,41 @@
+name: Gateway
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/gateway/**'
+      - '.github/workflows/gateway.yml'
+  pull_request:
+    paths:
+      - 'packages/gateway/**'
+      - '.github/workflows/gateway.yml'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - run: npx playwright install-deps
+      - run: yarn test:gateway
+  deploy-staging:
+    name: Deploy Staging
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - name: Publish app
+        uses: cloudflare/[email protected]
+        with:
+          apiToken: ${{secrets.CF_API_TOKEN }}
+          workingDirectory: 'packages/gateway'
+          environment: 'staging'

.github/workflows/release.yml

@@ -117,6 +117,13 @@ jobs:
           apiToken: ${{ secrets.CF_API_TOKEN }}
           workingDirectory: 'packages/api'
           environment: 'production'
+      - name: Gateway - Deploy
+        if: ${{ steps.tag-release.outputs.release_created && matrix.package == 'gateway' }}
+        uses: cloudflare/[email protected]
+        with:
+          apiToken: ${{ secrets.CF_API_TOKEN }}
+          workingDirectory: 'packages/gateway'
+          environment: 'production'
       - name: Website - Deploy
         if: ${{ steps.tag-release.outputs.release_created && matrix.package == 'website' }}
         run: ./packages/tools/cli.js deploy-website --email ${{ secrets.CF_EMAIL }} --key ${{secrets.CF_KEY}} --zone ${{ secrets.CF_ZONE }} --account ${{secrets.CF_ACCOUNT}}

package.json

@@ -7,6 +7,7 @@
     "test": "run-s test:*",
     "test:client": "yarn --cwd packages/client test",
     "test:api": "yarn --cwd packages/api test",
+    "test:gateway": "yarn --cwd packages/gateway test",
     "test:website": "yarn --cwd packages/website test",
     "build:client:docs": "yarn --cwd packages/client typedoc",
     "build:website": "yarn --cwd packages/website build",

packages/gateway/README.md

@@ -0,0 +1,31 @@
+# gateway.nft.storage
+
+> The IPFS gateway for nft.storage.
+
+## Getting started
+
+One time set up of your cloudflare worker subdomain for dev:
+
+- `npm install` - Install the project dependencies
+- Sign up to Cloudflare and log in with your default browser.
+- `npm i @cloudflare/wrangler -g` - Install the Cloudflare wrangler CLI
+- `wrangler login` - Authenticate your wrangler cli; it'll open your browser.
+- Copy your cloudflare account id from `wrangler whoami`
+- Update `wrangler.toml` with a new `env`. Set your env name to be the value of `whoami` on your system you can use `npm start` to run the worker in dev mode for you.
+
+  [**wrangler.toml**](./wrangler.toml)
+
+  ```toml
+  [env.bobbytables]
+  workers_dev = true
+  account_id = "<what does the `wrangler whoami` say>"
+  ```
+
+- `npm run publish` - Publish the worker under your env. An alias for `wrangler publish --env $(whoami)`
+- `npm start` - Run the worker in dev mode. An alias for `wrangler dev --env $(whoami)
+
+You only need to `npm start` for subsequent runs. PR your env config to the wrangler.toml, to celebrate 🎉
+
+## API
+
+TODO

packages/gateway/ava.config.js

@@ -0,0 +1,8 @@
+export default {
+  nonSemVerExperiments: {
+    configurableModuleFormat: true,
+  },
+  files: ['test/*.spec.js'],
+  timeout: '5m',
+  nodeArguments: ['--experimental-vm-modules'],
+}

packages/gateway/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "gateway-nft-storage",
+  "version": "0.0.0",
+  "description": "IPFS gateway for nft.storage",
+  "private": true,
+  "type": "module",
+  "module": "./dist/index.mjs",
+  "scripts": {
+    "build": "node scripts/cli.js build",
+    "dev": "miniflare --watch --debug",
+    "deploy": "wrangler publish --env production",
+    "pretest": "npm run build ",
+    "test": "npm-run-all -p -r mock:ipfs.io test:worker",
+    "test:worker": "ava --verbose test/*.spec.js",
+    "mock:ipfs.io": "smoke -p 9081 test/mocks/ipfs.io"
+  },
+  "dependencies": {
+    "multiformats": "^9.5.2"
+  },
+  "devDependencies": {
+    "ava": "^3.15.0",
+    "esbuild": "^0.14.2",
+    "git-rev-sync": "^3.0.1",
+    "miniflare": "^2.0.0-rc.2",
+    "npm-run-all": "^4.1.5",
+    "sade": "^1.7.4",
+    "smoke": "^3.1.1"
+  },
+  "author": "Vasco Santos <[email protected]>",
+  "license": "(Apache-2.0 AND MIT)"
+}

packages/gateway/scripts/cli.js

@@ -0,0 +1,44 @@
+#!/usr/bin/env node
+import fs from 'fs'
+import path from 'path'
+import { fileURLToPath } from 'url'
+import sade from 'sade'
+import { build } from 'esbuild'
+import git from 'git-rev-sync'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const pkg = JSON.parse(
+  fs.readFileSync(path.join(__dirname, '..', 'package.json'), 'utf8')
+)
+
+const prog = sade('gateway')
+
+prog
+  .command('build')
+  .describe('Build the worker.')
+  .option('--env', 'Environment', 'dev')
+  .action(async (opts) => {
+    try {
+      const version = `${pkg.name}@${pkg.version}-${opts.env}+${git.short(
+        __dirname
+      )}`
+
+      await build({
+        entryPoints: [path.join(__dirname, '../src/index.js')],
+        bundle: true,
+        format: 'esm',
+        outfile: 'dist/index.mjs',
+        legalComments: 'external',
+        define: {
+          global: 'globalThis',
+        },
+        minify: opts.env === 'dev' ? false : true,
+        sourcemap: true,
+      })
+    } catch (err) {
+      console.error(err)
+      process.exit(1)
+    }
+  })
+
+prog.parse(process.argv)

packages/gateway/src/cors.js

@@ -0,0 +1,20 @@
+/* eslint-env serviceworker */
+
+/**
+ * @param {Request} request
+ * @param {Response} response
+ * @returns {Response}
+ */
+export function addCorsHeaders(request, response) {
+  // Clone the response so that it's no longer immutable (like if it comes from cache or fetch)
+  response = new Response(response.body, response)
+  const origin = request.headers.get('origin')
+  if (origin) {
+    response.headers.set('Access-Control-Allow-Origin', origin)
+    response.headers.set('Vary', 'Origin')
+  } else {
+    response.headers.set('Access-Control-Allow-Origin', '*')
+  }
+  response.headers.set('Access-Control-Expose-Headers', 'Link')
+  return response
+}

packages/gateway/src/error-handler.js

@@ -0,0 +1,17 @@
+import { JSONResponse } from './utils/json-response.js'
+
+/**
+ * @param {Error & {status?: number;code?: string;}} err
+ */
+export function errorHandler(err) {
+  // TODO: setup sentry
+  console.error(err.stack)
+
+  let error = {
+    code: err.code || 'HTTP_ERROR',
+    message: err.message || 'Server Error',
+  }
+  let status = err.status || 500
+
+  return new JSONResponse(error, { status })
+}

packages/gateway/src/errors.js

@@ -0,0 +1,25 @@
+export class InvalidIpfsPathError extends Error {
+  /**
+   * @param {string} cid
+   */
+  constructor(cid) {
+    super(`invalid ipfs path: invalid path "/ipfs/${cid}/"`)
+    this.name = 'InvalidIpfsPath'
+    this.status = 400
+    this.code = InvalidIpfsPathError.CODE
+  }
+}
+InvalidIpfsPathError.CODE = 'ERROR_INVALID_IPFS_PATH'
+
+export class InvalidUrlError extends Error {
+  /**
+   * @param {string} url
+   */
+  constructor(url) {
+    super(`invalid url: ${url}`)
+    this.name = 'InvalidUrl'
+    this.status = 400
+    this.code = InvalidUrlError.CODE
+  }
+}
+InvalidUrlError.CODE = 'ERROR_INVALID_URL'

packages/gateway/src/index.js

@@ -0,0 +1,43 @@
+import { addCorsHeaders } from './cors.js'
+import { errorHandler } from './error-handler.js'
+import { getCidFromSubdomainUrl } from './utils/cid.js'
+
+/**
+ * @typedef {Object} Env
+ * @property {string} IPFS_GATEWAY
+ */
+
+/**
+ * Handle gateway request
+ * @param {Request} request
+ * @param {Env} env
+ */
+async function handleRequest(request, env) {
+  const publicGatewayUrl = new URL('ipfs', env.IPFS_GATEWAY)
+  const url = new URL(request.url)
+  const cid = getCidFromSubdomainUrl(url.hostname)
+  const response = await fetch(
+    `${publicGatewayUrl.toString()}/${cid}${url.pathname || ''}`
+  )
+
+  // forward gateway response
+  return addCorsHeaders(request, response)
+}
+
+/**
+ * @param {Error} error
+ * @param {Request} request
+ */
+function serverError(error, request) {
+  return addCorsHeaders(request, errorHandler(error))
+}
+
+export default {
+  async fetch(request, env) {
+    try {
+      return await handleRequest(request, env)
+    } catch (error) {
+      return serverError(error, request)
+    }
+  },
+}

packages/gateway/src/utils/cid.js

@@ -0,0 +1,34 @@
+import { CID } from 'multiformats/cid'
+
+import { InvalidIpfsPathError, InvalidUrlError } from '../errors.js'
+
+/**
+ * Parse subdomain URL and return cid
+ *
+ * @param {string} url
+ */
+export function getCidFromSubdomainUrl(url) {
+  // Replace "ipfs-staging" by "ipfs" if needed
+  const nUrl = url.replace('ipfs-staging', 'ipfs')
+  const splitUrl = nUrl.split('.ipfs.')
+
+  if (!splitUrl.length) {
+    throw new InvalidUrlError(url)
+  }
+
+  return normalizeCid(splitUrl[0])
+}
+
+/**
+ * Parse CID and return normalized b32 v1
+ *
+ * @param {string} cid
+ */
+export function normalizeCid(cid) {
+  try {
+    const c = CID.parse(cid)
+    return c.toV1().toString()
+  } catch (err) {
+    throw new InvalidIpfsPathError(cid)
+  }
+}

packages/gateway/src/utils/json-response.js

@@ -0,0 +1,16 @@
+/* eslint-env serviceworker */
+export class JSONResponse extends Response {
+  /**
+   * @param {BodyInit} body
+   * @param {ResponseInit} [init]
+   */
+  constructor(body, init = {}) {
+    init.headers = init.headers || {}
+    init.headers['Content-Type'] = 'application/json;charset=UTF-8'
+    super(JSON.stringify(body), init)
+  }
+}
+
+export function notFound(message = 'Not Found') {
+  return new JSONResponse({ message }, { status: 404 })
+}