feat: nft.storage naive gateway implementation

.github/workflows/gateway.yml

@@ -0,0 +1,41 @@
+name: Gateway
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/gateway/**'
+      - '.github/workflows/gateway.yml'
+  pull_request:
+    paths:
+      - 'packages/gateway/**'
+      - '.github/workflows/gateway.yml'
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - run: npx playwright install-deps
+      - run: yarn test:gateway
+  deploy-staging:
+    name: Deploy Staging
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - uses: bahmutov/npm-install@v1
+      - name: Publish app
+        uses: cloudflare/[email protected]
+        with:
+          apiToken: ${{secrets.CF_API_TOKEN }}
+          workingDirectory: 'packages/gateway'
+          environment: 'staging'

packages/gateway/README.md

@@ -0,0 +1,31 @@
+# gateway.nft.storage
+
+> The IPFS gateway for nft.storage.
+
+## Getting started
+
+One time set up of your cloudflare worker subdomain for dev:
+
+- `npm install` - Install the project dependencies
+- Sign up to Cloudflare and log in with your default browser.
+- `npm i @cloudflare/wrangler -g` - Install the Cloudflare wrangler CLI
+- `wrangler login` - Authenticate your wrangler cli; it'll open your browser.
+- Copy your cloudflare account id from `wrangler whoami`
+- Update `wrangler.toml` with a new `env`. Set your env name to be the value of `whoami` on your system you can use `npm start` to run the worker in dev mode for you.
+
+  [**wrangler.toml**](./wrangler.toml)
+
+  ```toml
+  [env.bobbytables]
+  workers_dev = true
+  account_id = "<what does the `wrangler whoami` say>"
+  ```
+
+- `npm run publish` - Publish the worker under your env. An alias for `wrangler publish --env $(whoami)`
+- `npm start` - Run the worker in dev mode. An alias for `wrangler dev --env $(whoami)
+
+You only need to `npm start` for subsequent runs. PR your env config to the wrangler.toml, to celebrate 🎉
+
+## API
+
+TODO

packages/gateway/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "gateway-nft-storage",
+  "version": "0.0.0",
+  "description": "IPFS gateway for nft.storage",
+  "private": true,
+  "type": "module",
+  "main": "./dist/index.js",
+  "scripts": {
+    "build": "esbuild --bundle --sourcemap --outdir=dist ./src/index.js",
+    "dev": "miniflare --watch --debug",
+    "deploy": "wrangler publish --env production",
+    "pretest": "npm run build ",
+    "test": "npm-run-all -p -r mock:ipfs.io test:worker",
+    "test:worker": "ava --verbose test/*.spec.js",
+    "mock:ipfs.io": "smoke -p 9081 test/mocks/ipfs.io"
+  },
+  "dependencies": {
+    "multiformats": "^9.5.2"
+  },
+  "devDependencies": {
+    "ava": "^3.15.0",
+    "esbuild": "^0.14.2",
+    "miniflare": "^2.0.0-rc.2",
+    "npm-run-all": "^4.1.5",
+    "smoke": "^3.1.1"
+  },
+  "author": "Vasco Santos <[email protected]>",
+  "license": "(Apache-2.0 AND MIT)"
+}

packages/gateway/src/cors.js

@@ -0,0 +1,20 @@
+/* eslint-env serviceworker */
+
+/**
+ * @param {Request} request
+ * @param {Response} response
+ * @returns {Response}
+ */
+export function addCorsHeaders(request, response) {
+  // Clone the response so that it's no longer immutable (like if it comes from cache or fetch)
+  response = new Response(response.body, response)
+  const origin = request.headers.get('origin')
+  if (origin) {
+    response.headers.set('Access-Control-Allow-Origin', origin)
+    response.headers.set('Vary', 'Origin')
+  } else {
+    response.headers.set('Access-Control-Allow-Origin', '*')
+  }
+  response.headers.set('Access-Control-Expose-Headers', 'Link')
+  return response
+}

packages/gateway/src/error-handler.js

@@ -0,0 +1,17 @@
+import { JSONResponse } from './utils/json-response.js'
+
+/**
+ * @param {Error & {status?: number;code?: string;}} err
+ */
+export function errorHandler(err) {
+  // TODO: setup sentry
+  console.error(err.stack)
+
+  let error = {
+    code: err.code || 'HTTP_ERROR',
+    message: err.message || 'Server Error',
+  }
+  let status = err.status || 500
+
+  return new JSONResponse(error, { status })
+}

packages/gateway/src/index.js

@@ -0,0 +1,32 @@
+import { addCorsHeaders } from './cors.js'
+import { errorHandler } from './error-handler.js'
+import { getCidFromSubdomainUrl } from './utils/cid.js'
+
+/**
+ * Handle gateway request
+ * @param {Request} request
+ */
+async function handleRequest(request) {
+  const publicGatewayUrl = new URL('ipfs', IPFS_GATEWAY)
+  const cid = getCidFromSubdomainUrl(request.url)
+  const response = await fetch(`${publicGatewayUrl.toString()}/${cid}`)
+
+  // forward gateway response
+  return addCorsHeaders(request, response)
+}
+
+/**
+ * @param {Error} error
+ * @param {Request} request
+ */
+function serverError(error, request) {
+  return addCorsHeaders(request, errorHandler(error))
+}
+
+addEventListener('fetch', (event) => {
+  event.respondWith(
+    handleRequest(event.request).catch((error) =>
+      serverError(error, event.request)
+    )
+  )
+})

packages/gateway/src/utils/cid.js

@@ -0,0 +1,31 @@
+import { CID } from 'multiformats/cid'
+
+/**
+ * Parse subdomain URL and return cid
+ *
+ * @param {string} url
+ */
+export function getCidFromSubdomainUrl(url) {
+  const nUrl = url.replace(/https?:\/\//, '')
+  const splitUrl = nUrl.split('.ipfs.')
+
+  if (!splitUrl.length) {
+    throw new Error(`invalid url: ${url}`)
+  }
+
+  return normalizeCid(splitUrl[0])
+}
+
+/**
+ * Parse CID and return normalized b32 v1
+ *
+ * @param {string} cid
+ */
+export function normalizeCid(cid) {
+  try {
+    const c = CID.parse(cid)
+    return c.toV1().toString()
+  } catch (err) {
+    throw new Error(`invalid ipfs path: invalid path "/ipfs/${cid}/": ${err}`)
+  }
+}

packages/gateway/src/utils/json-response.js

@@ -0,0 +1,16 @@
+/* eslint-env serviceworker */
+export class JSONResponse extends Response {
+  /**
+   * @param {BodyInit} body
+   * @param {ResponseInit} [init]
+   */
+  constructor(body, init = {}) {
+    init.headers = init.headers || {}
+    init.headers['Content-Type'] = 'application/json;charset=UTF-8'
+    super(JSON.stringify(body), init)
+  }
+}
+
+export function notFound(message = 'Not Found') {
+  return new JSONResponse({ message }, { status: 404 })
+}

packages/gateway/test/index.spec.js

@@ -0,0 +1,45 @@
+import test from 'ava'
+import { Miniflare } from 'miniflare'
+
+test.beforeEach((t) => {
+  // Create a new Miniflare environment for each test
+  const mf = new Miniflare({
+    // Autoload configuration from `.env`, `package.json` and `wrangler.toml`
+    envPath: true,
+    packagePath: true,
+    wranglerConfigPath: true,
+    // We don't want to rebuild our worker for each test, we're already doing
+    // it once before we run all tests in package.json, so disable it here.
+    // This will override the option in wrangler.toml.
+    buildCommand: undefined,
+    wranglerConfigEnv: 'test',
+  })
+
+  t.context = {
+    mf,
+  }
+})
+
+test('Fails when invalid cid is provided', async (t) => {
+  const { mf } = t.context
+
+  const invalidCid = 'bafy'
+  const response = await mf.dispatchFetch(`${invalidCid}.ipfs.localhost:8787`)
+  t.is(response.status, 500)
+
+  const jsonResponse = await response.json()
+  t.is(jsonResponse.code, 'HTTP_ERROR')
+  t.is(
+    jsonResponse.message,
+    `invalid ipfs path: invalid path "/ipfs/${invalidCid}/": SyntaxError: Unexpected end of data`
+  )
+})
+
+test('Gets content', async (t) => {
+  const { mf } = t.context
+
+  const response = await mf.dispatchFetch(
+    'bafkreidchi5c4c3kwr5rpkvvwnjz3lh44xi2y2lnbldehwmpplgynigidm.ipfs.localhost:8787'
+  )
+  t.is(await response.text(), 'Hello gateway.nft.storage!')
+})

packages/gateway/test/mocks/ipfs.io/get_ipfs#@cid.js

@@ -0,0 +1,10 @@
+/**
+ * https://github.com/sinedied/smoke#javascript-mocks
+ */
+module.exports = () => {
+  return {
+    statusCode: 200,
+    headers: { 'Content-Type': 'text/plain' },
+    body: 'Hello gateway.nft.storage!',
+  }
+}

packages/gateway/test/mocks/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "mocks",
+  "version": "1.0.0",
+  "description": "just here to fix cjs loading"
+}

packages/gateway/wrangler.toml

@@ -0,0 +1,39 @@
+# gateway.nft.storage wrangler config.
+name = "gateway-nft-storage"
+
+# `javascript` means our we'll send pre-built javascript code. 
+#  ...as opposed to `webpack` where wrangler builds our code for us.
+type = "javascript"
+
+account_id = ""
+watch_dir = "src"
+compatibility_date = "2021-12-03"
+
+[build]
+command = "npm run build"
+[build.upload]
+format = "service-worker"
+
+# PROD!
+[env.production]
+# name = "gateway-nft-storage-production"
+account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account
+zone_id = "fc6cb51dbc2d0b9a729eae6a302a49c9"    # nft.storage zone
+route = "*gateway.nft.storage/*"
+vars = { IPFS_GATEWAY = "https://ipfs.io" }
+
+[env.staging]
+# name = "gateway-nft-storage-staging"
+account_id = "fffa4b4363a7e5250af8357087263b3a" # Protocol Labs CF account
+zone_id = "fc6cb51dbc2d0b9a729eae6a302a49c9"    # nft.storage zone
+route = "*gateway-staging.nft.storage/*"
+vars = { IPFS_GATEWAY = "https://ipfs.io" }
+
+[env.test]
+workers_dev = true
+vars = { IPFS_GATEWAY = "http://localhost:9081" }
+
+[env.vsantos]
+workers_dev = true
+account_id = "7ec0b7cf2ec201b2580374e53ba5f37b"
+vars = { IPFS_GATEWAY = "https://ipfs.io" }