feat: generate results with gateway-conformance

.github/aggregate.ts

@@ -0,0 +1,132 @@
+import fs  from 'fs'
+import process from 'process'
+import path from 'path'
+import { z } from 'zod'
+
+interface ReportOutput {
+  metadata: {
+    time: string
+    version?: string
+    job_url?: string
+    gateway_url: string
+  }
+  results: {
+    [key: string]: {
+      pass: number
+      fail: number
+      skip: number
+    }
+  }
+}
+
+type GatewayURL = string
+
+// At the moment test2json is likely to miss some outputs.
+// We'll accept "Unknown" as a valid outcome for now.
+// Related: https://github.com/golang/go/issues/61767
+const Outcome = z.enum(['pass', 'fail', 'skip', 'Unknown'])
+
+const ReportFileInput = z.intersection(
+  z.record(z.object({
+    path: z.array(z.string()),
+    time: z.string(),
+    outcome: Outcome,
+    output: z.string().optional(),
+    meta: z.object({
+      group: z.string().optional(),
+    }).optional(),
+  })),
+  z.object({
+    TestMetadata: z.object({
+      time: z.string(),
+      meta: z.object({
+        version: z.string().optional(),
+        job_url: z.string().optional(),
+        gateway_url: z.string(),
+      })
+    }).optional(),
+  })
+)
+
+/**
+ * Processes a report from a given filePath and extracts important data.
+ */
+const processReport = (filePath: string): [GatewayURL, ReportOutput] => {
+  const resolvedPath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath);
+  const reportContent = ReportFileInput.parse(JSON.parse(fs.readFileSync(resolvedPath, 'utf8')))
+
+  // extract the TestMetadata
+  const { TestMetadata, ...allOtherTests } = reportContent
+
+  if (!TestMetadata) {
+    throw new Error(`No TestMetadata found in ${resolvedPath}`)
+  }
+
+  const { time, meta } = TestMetadata
+  const { version, job_url, gateway_url } = meta
+
+  // Then extract the test results we care about.
+  const groups = Object.entries(allOtherTests)
+    .filter(([_, value]) => value.path.length === 1) // keep only the tests at the root
+    .map(([_key, value]) => {
+      // keep only the outcomes and groups
+      return {
+        outcome: value.outcome,
+        group: value.meta?.group ?? 'Others',
+      }
+    })
+    .reduce((acc, value) => {
+      // then group by "group" value and sum their outcomes
+      const { group } = value
+      const outcome = value.outcome === 'Unknown' ? 'fail' : value.outcome
+
+      if (!acc[group]) {
+        acc[group] = {
+          pass: 0,
+          fail: 0,
+          skip: 0,
+        }
+      }
+
+      acc[group][outcome] += 1
+
+      return acc
+    }, {} as { [key: string]: { pass: number, fail: number, skip: number } })
+
+  return [
+    gateway_url,
+    {
+      metadata: {
+        time, version, job_url, gateway_url,
+      },
+      results: groups,
+    },
+  ]
+}
+
+/**
+ * Main function to process all input files and write the results to standard output.
+ */
+const main = async (): Promise<void> => {
+  const inputs: string[] = process.argv.slice(2) // List of json reports to aggregate.
+
+  const results: {[key: string]: ReportOutput} = {}
+
+  inputs.forEach((filePath) => {
+    try {
+      const [name, report] = processReport(filePath)
+      results[name] = report
+    } catch (err) {
+      console.error(`Error processing ${filePath}`, err)
+    }
+  })
+
+  fs.writeFileSync(1, JSON.stringify(results, null, 2) + '\n')
+}
+
+main()
+  .then(() => process.exit(0))
+  .catch((err) => {
+    console.error(err)
+    process.exit(1)
+  })

.github/workflows/gateway-conformance.yml

@@ -0,0 +1,138 @@
+name: Gateway Conformance Dashboard
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  # Enable this when we're ready to generate the dashboard
+  # schedule:
+  #   - cron: "0 */6 * * *" # every six hours
+
+concurrency:
+  group: "conformance"
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  configure:
+    runs-on: ubuntu-latest
+    outputs:
+      gateways: ${{ steps.set-matrix.outputs.gateways }}
+    steps:
+     - name: Checkout repository
+       uses: actions/checkout@v3
+     - name: Set matrix data
+       id: set-matrix
+       run: |
+        jq --raw-input --slurp --compact-output 'split("\n")[:-1]' gateways.txt | \
+          tr -d '\n' | \
+          xargs --null --max-chars=2000000 -I {} echo "gateways={}" | \
+          tee -a "$GITHUB_OUTPUT"
+  conformance:
+    runs-on: ubuntu-latest
+    needs: configure
+    strategy:
+      matrix:
+          gateway_url: ${{ fromJson(needs.configure.outputs.gateways) }}
+      fail-fast: false
+    steps:
+      # 1. Generate the slug used for reporting
+      - name: Generate slug
+        id: slug
+        env:
+          GATEWAY_URL: ${{ matrix.gateway_url }}
+        run: |
+          slug=$(echo "${GATEWAY_URL}" |
+              sed -e 's/http[s]\?:\/\///' \
+              -e 's/[:/@.]/-/g' \
+              -e 's/[^A-Za-z0-9\-]/-/g' |
+              tr "[:upper:]" "[:lower:]")
+          echo "slug=$slug" >> $GITHUB_OUTPUT
+
+      # 2. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0
+        with:
+          gateway-url: ${{ matrix.gateway_url }}
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          report: report.json
+          accept-test-failure: true
+
+      # 3. Upload the results
+      - name: Upload MD summary
+        # TODO: generate a minimal output.md in the action
+        run: cat output.md | sed '/Failures\/Errors/,$d' >> $GITHUB_STEP_SUMMARY
+      - name: Upload JSON output
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ steps.slug.outputs.slug }}.json
+          path: |
+            ./output.json
+            ./output.html
+            ./report.json
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ steps.slug.outputs.slug }}.html
+          path: output.html
+
+  aggregate:
+    permissions:
+      contents: write
+    runs-on: "ubuntu-latest"
+    needs: [conformance]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Aggregate reports
+        run: |
+          mkdir ./reports
+
+          # download-artifact downloads artifacts in a directory named after the artifact
+          # details: https://github.com/actions/download-artifact#download-all-artifacts
+          for folder in ./artifacts/conformance-*.json; do
+            file="${folder}/report.json"
+            new_name="${folder#.\/artifacts\/conformance-}" # drop path prefix "./artifacts/conformance-"
+            new_file="./reports/${new_name}"
+            cp "${file}" "${new_file}"
+          done
+      - name: Upload Data Aggregates
+        # This will be useful for local debugging
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: dashboard-reports
+          path: ./reports
+      - name: Generate final report
+        run: |
+          npm ci --include=dev
+          npx ts-node ./.github/aggregate.ts ./reports/*.json > ./src/report.json
+      - name: Upload Report
+        # This will be useful for local debugging
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: dashboard-report
+          path: ./src/report.json
+      - name: Configure git
+        run: |
+          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com>"
+          git config --global user.name "${GITHUB_ACTOR}"
+      - name: Push
+        run: |
+          git add src/report.json
+          git commit -m "chore: update conformance results"
+          git push

README.md

@@ -15,7 +15,7 @@ View the Public Gateway Checker in action
 
 ## Adding a new public gateway
 
-If you'd like to add a new public gateway, please edit `./src/gateways.ts`:
+If you'd like to add a new public gateway, please edit `./gateways.txt`:
 
 1. Add the gateway's address to the bottom of the list
 2. Make sure the final item in the list does **not** have a comma at the end, but all preceding items do

gateways.txt

@@ -0,0 +1,82 @@
+https://ipfs.io
+https://dweb.link
+https://gateway.ipfs.io
+https://ninetailed.ninja
+https://via0.com
+https://ipfs.eternum.io
+https://hardbin.com
+https://cloudflare-ipfs.com
+https://astyanax.io
+https://cf-ipfs.com
+https://gateway.originprotocol.com
+https://gateway.pinata.cloud
+https://ipfs.sloppyta.co
+https://ipfs.busy.org
+https://ipfs.greyh.at
+https://gateway.serph.network
+https://gw3.io
+https://jorropo.net
+https://ipfs.fooock.com
+https://cdn.cwinfo.net
+https://aragon.ventures
+https://permaweb.io
+https://ipfs.best-practice.se
+https://storjipfs-gateway.com
+https://ipfs.runfission.com
+https://ipfs.trusti.id
+https://ipfs.overpi.com
+https://ipfs.ink
+https://ipfsgateway.makersplace.com
+https://ipfs.funnychain.co
+https://ipfs.telos.miami
+https://ipfs.mttk.net
+https://ipfs.fleek.co
+https://ipfs.jbb.one
+https://ipfs.yt
+https://hashnews.k1ic.com
+https://ipfs.drink.cafe
+https://ipfs.kavin.rocks
+https://ipfs.denarius.io
+https://crustwebsites.net
+https://ipfs0.sjc.cloudsigma.com
+http://ipfs.genenetwork.org
+https://ipfs.eth.aragon.network
+https://ipfs.smartholdem.io
+https://ipfs.xoqq.ch
+http://natoboram.mynetgear.com:8080
+https://video.oneloveipfs.com
+https://ipfs.scalaproject.io
+https://search.ipfsgate.com
+https://ipfs.decoo.io
+https://alexdav.id
+https://ipfs.uploads.nu
+https://hub.textile.io
+https://ipfs1.pixura.io
+https://ravencoinipfs-gateway.com
+https://konubinix.eu
+https://ipfs.tubby.cloud
+https://ipfs.lain.la
+https://ipfs.kaleido.art
+https://ipfs.slang.cx
+https://ipfs.arching-kaos.com
+https://storry.tv
+https://ipfs.1-2.dev
+https://dweb.eu.org
+https://permaweb.eu.org
+https://ipfs.namebase.io
+https://ipfs.tribecap.co
+https://ipfs.kinematiks.com
+https://nftstorage.link
+https://gravity.jup.io
+http://fzdqwfb5ml56oadins5jpuhe6ki6bk33umri35p5kt2tue4fpws5efid.onion
+https://tth-ipfs.com
+https://ipfs.chisdealhd.co.uk
+https://ipfs.alloyxuast.tk
+https://4everland.io
+https://ipfs-gateway.cloud
+https://w3s.link
+https://cthd.icu
+https://ipfs.tayfundogdas.me
+https://ipfs.jpu.jp
+https://ipfs.soul-network.com
+https://twdragon.net