fetch (#51)

* ci macos * remove old get * test with fetch provider * allow-net for deno * remove ci on node 16.x * cli * fetch on ci * remove log * test-install-net * fetch doc
datablockset · Nov 7, 2023 · 5dbf8bf · 5dbf8bf
1 parent 5a90eba
commit 5dbf8bf
Show file tree

Hide file tree

Showing 6 changed files with 85 additions and 144 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,7 +10,7 @@ on:
     branches: [ "main" ]
 
 env:
-  VERSION: 0.1.1
+  VERSION: 0.2.0
 
 jobs:
   node:
@@ -19,7 +19,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x, 20.x, 21.x]
+        node-version: [18.x, 20.x, 21.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
@@ -34,9 +34,13 @@ jobs:
     - run: npm run blockset-prepare
     - run: npm test
 
-  node-windows:
+  node-os:
 
-    runs-on: windows-latest
+    strategy:
+      matrix:
+        os: [windows-latest, macos-13]
+
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v3
@@ -57,7 +61,7 @@ jobs:
     - run: cargo install blockset
     - run: npm run blockset-prepare
     # - run: deno run --quiet --allow-read --allow-env --allow-net --allow-hrtime ./test.mjs
-    - run: deno run --quiet --allow-read --allow-write ./test.mjs
+    - run: deno run --quiet --allow-read --allow-write --allow-net --allow-hrtime ./test.mjs
 
   bun:
 
@@ -82,7 +86,7 @@ jobs:
         path: blockset-js-${{ env.VERSION }}.tgz
         name: blockset-js.tgz
 
-  test-install:
+  test-install-local:
 
     needs: pack
     runs-on: ubuntu-latest
@@ -95,4 +99,16 @@ jobs:
     - run: echo "Hello World !" > _example.txt
     - run: blockset add ./_example.txt
     - run: npx blockset-js hsdfs1zs8fnf810v8f9k6pjdkz7b4y95sembmswac5sjt _out.txt
-    - run: blockset add ./_out.txt
+    - run: blockset add ./_out.txt
+
+  test-install-net:
+
+    needs: pack
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/download-artifact@v3
+      with:
+        name: blockset-js.tgz
+    - run: npm install -g blockset-js-${{ env.VERSION }}.tgz
+    - run: npx blockset-js awt9x8564999k276wap2e5b7n10575ffy946kencva4ve _out.txt 410f5a49.blockset-js-test.pages.dev
+    - run: less ./_out.txt
diff --git a/README.md b/README.md
@@ -26,5 +26,19 @@ npm uninstall -g blockset-js
 Get a file by address:
 
 ```console
-npx blockset-js hsdfs1zs8fnf810v8f9k6pjdkz7b4y95sembmswac5sjt out.txt
+npx blockset-js <address> <outputFile> [hostName]
+```
+
+### Examples
+
+Get a file from remote storage:
+
+```console
+npx blockset-js awt9x8564999k276wap2e5b7n10575ffy946kencva4ve out.txt 410f5a49.blockset-js-test.pages.dev
 ```
+
+Get a file from local storage:
+
+```console
+npx blockset-js hsdfs1zs8fnf810v8f9k6pjdkz7b4y95sembmswac5sjt out.txt
+```
diff --git a/cli.mjs b/cli.mjs
@@ -1,15 +1,16 @@
 #!/usr/bin/env node
 
 import index from './index.mjs'
-const { getAsync } = index
+const { get, asyncFileProvider, fetchProvider } = index
 
 var args = process.argv.slice(2)
-console.log(`args = ${args}`);
 
-if (args.length !== 2) {
-  console.log("Warning: Requires 2 arguments");
-  console.log("npx blockset-js [address] [outputFile]");
+if (args.length < 2) {
+  console.log("Warning: Requires 2 or more arguments");
+  console.log("npx blockset-js <address> <outputFile> [hostName]");
   process.exit();
 }
 
-getAsync([args[0], args[1]])
+const hostName = args[2]
+const provider = hostName === undefined ? asyncFileProvider : fetchProvider(hostName)
+get(provider)([args[0], args[1]])
diff --git a/index.mjs b/index.mjs
@@ -140,13 +140,12 @@ const nextState = state => {
   }
 }
 
-
-/** @type {Provider} */
-const fetchProvider = {
-  read: address => fetch(`https://410f5a49.blockset-js-test.pages.dev/${getPath(address)}`)
+/** @type {(hostName: string) => Provider} */
+const fetchProvider = hostName => ({
+  read: address => fetch(`https://${hostName}/${getPath(address)}`)
     .then(async (resp) => resp.arrayBuffer().then(buffer => new Uint8Array(buffer))),
     write: path => buffer => fsPromises.appendFile(path, buffer)
-}
+})
 
 /** @type {Provider} */
 const asyncFileProvider = {
@@ -169,7 +168,7 @@ const syncFileProvider = {
 // }
 
 /** @type {(provider: Provider) => (root: [string, string]) => Promise<number>} */
-const getAsyncWithProvider = ({ read, write }) => async ([root, file]) => {
+const get = ({ read, write }) => async ([root, file]) => {
   const tempFile = `_temp_${root}`
   /** @type {State} */
   let state = [[[root, true], null]]
@@ -223,48 +222,9 @@ const getAsyncWithProvider = ({ read, write }) => async ([root, file]) => {
   }
 }
 
-
-/** @type {(root: [string, string]) => Promise<number>} */
-const getAsync = getAsyncWithProvider(asyncFileProvider)
-
-/** @type {(root: [string, string]) => Promise<number>} */
-const getSync = getAsyncWithProvider(syncFileProvider)
-
-/** @type {(root: string) => (file: string) => number} */
-const get = root => file => {
-  /** @type {State} */
-  let state = [[[root, true], null]]
-  let buffer = new Uint8Array()
-  try {
-    while (true) {
-      const blockLast = state.at(-1)
-      if (blockLast === undefined) {
-        fs.writeFileSync(file, buffer)
-        return 0
-      }
-
-      const address = blockLast[0]
-      const path = getPath(address)
-      const data = fs.readFileSync(path)
-      insertBlock(state)([address, data])
-      const next = nextState(state)
-      if (next[0] === 'error') {
-        console.error(`${next[1]}`)
-        return -1
-      }
-
-      for (let w of next[1]) {
-        buffer = new Uint8Array([...buffer, ...w]);
-      }
-    }
-  } catch (err) {
-    console.error(err);
-    return -1
-  }
-}
-
 export default {
   get,
-  getAsync,
-  getSync
+  syncFileProvider,
+  asyncFileProvider,
+  fetchProvider
 }
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "blockset-js",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "BLOCKSET on JavaScript",
   "keywords": ["blockset", "content-addressable", "storage", "cdt", "content-dependent-tree", "hash"],
   "main": "index.mjs",

diff --git a/test.mjs b/test.mjs
@@ -8,12 +8,13 @@ import fs from 'node:fs'
 import fsPromises from 'node:fs/promises'
 /** @typedef {import('./subtree.mjs').State} StateSubTree */
 /** @typedef {import('./tree.mjs').State} StateTree */
+/** @typedef {import('./index.mjs').Provider} Provider */
 const { toAddress, getParityBit } = base32
 const { compress } = sha224
 const { merge, byteToDigest, len } = digest256
 const { highestOne256, height, push: pushSubTree } = subtree
 const { push: pushTree, end: endTree } = tree
-const { get, getAsync, getSync } = index
+const { get, syncFileProvider, asyncFileProvider, fetchProvider } = index
 
 console.log(`test start`)
 
@@ -187,10 +188,10 @@ console.log(`test start`)
       if (result !== null) { throw result }
       result = pushSubTree(state)(b)
       if (result !== null) { throw result }
-      if (state.length !== 2 ) { throw state.length }
+      if (state.length !== 2) { throw state.length }
       result = pushSubTree(state)(a)
       if (result !== null) { throw result }
-      if (state.length !== 2 ) { throw state.length }
+      if (state.length !== 2) { throw state.length }
       result = pushSubTree(state)(a)
       const mergeCB_AA = merge(merge(c)(b))(merge(a)(a))
       if (result != mergeCB_AA) { throw result }
@@ -214,7 +215,7 @@ console.log(`test start`)
 const readExample = file => {
   let buffer = fs.readFileSync(file)
   let temp = []
-  for(let ch of buffer) {
+  for (let ch of buffer) {
     if (ch !== 13) {
       temp.push(ch)
     }
@@ -223,108 +224,57 @@ const readExample = file => {
 }
 
 /** @type {(f: () => Promise<void> | void) => Promise<void>} */
-const runTest = async(f) => {
+const runTest = async (f) => {
   const t0 = performance.now();
   await f()
   const t1 = performance.now();
   console.log(`Call to ${f.name} took ${t1 - t0} milliseconds.`);
 }
 
-const testGetSyncOld1 = () => {
-  const exitCode = get('vqra44skpkefw4bq9k96xt9ks84221dmk1pzaym86cqd6')('_out_list1')
-  if (exitCode !== 0) { throw exitCode }
-
-  const bufferIn = readExample(`examples/list.txt`)
-  const bufferOut = fs.readFileSync(`_out_list1`)
-  if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
-}
-
-const testGetSyncOld2 = () =>
-{
-  const exitCode = get('awt9x8564999k276wap2e5b7n10575ffy946kencva4ve')('examples/_out_list2')
-  if (exitCode !== 0) { throw exitCode }
-
-  const bufferIn = readExample(`examples/list2.txt`)
-  const bufferOut = fs.readFileSync(`examples/_out_list2`)
-  if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
-}
-
-const testGetSyncOldRepeat = () =>
-{
-  const exitCode = get('d963x31mwgb8svqe0jmkxh8ar1f8p2dawebnan4aj6hvd')('_out_repeat')
-  if (exitCode !== 0) { throw exitCode }
-
-  const bufferIn = readExample(`examples/repeat.txt`)
-  const bufferOut = fs.readFileSync(`_out_repeat`)
-  if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
-}
-
-const testGetSync1 = async() => {
-  const exitCode = await getSync(['vqra44skpkefw4bq9k96xt9ks84221dmk1pzaym86cqd6', '_out_list1_async'])
-  if (exitCode !== 0) { throw exitCode }
-
-  const bufferIn = readExample(`examples/list.txt`)
-  const bufferOut = await fsPromises.readFile(`_out_list1_async`)
-  if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
-}
-
-const testGetSync2 = async() => {
-  const exitCode = await getSync(['awt9x8564999k276wap2e5b7n10575ffy946kencva4ve', 'examples/_out_list2_async'])
-  if (exitCode !== 0) { throw exitCode }
-
-  const bufferIn = readExample(`examples/list2.txt`)
-  const bufferOut = await fsPromises.readFile(`examples/_out_list2_async`)
-  if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
-}
-
-const testGetSyncRepeat = async() => {
-  const exitCode = await getSync(['d963x31mwgb8svqe0jmkxh8ar1f8p2dawebnan4aj6hvd', '_out_repeat_async'])
-  if (exitCode !== 0) { throw exitCode }
-
-  const bufferIn = readExample(`examples/repeat.txt`)
-  const bufferOut = await fsPromises.readFile(`_out_repeat_async`)
-  if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
-}
-
-{
-  const testGetAsync1 = async() => {
-    const exitCode = await getAsync(['vqra44skpkefw4bq9k96xt9ks84221dmk1pzaym86cqd6', '_out_list1_async'])
+/** @type {(provider: Provider) => Promise<void>} */
+const runTestsGet = async (provider) => {
+  const getWithProvider = get(provider)
+  const testGet1 = async () => {
+    const exitCode = await getWithProvider(['vqra44skpkefw4bq9k96xt9ks84221dmk1pzaym86cqd6', '_out_list1_async'])
     if (exitCode !== 0) { throw exitCode }
 
     const bufferIn = readExample(`examples/list.txt`)
     const bufferOut = await fsPromises.readFile(`_out_list1_async`)
     if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
   }
 
-  const testGetAsync2 = async() => {
-    const exitCode = await getAsync(['awt9x8564999k276wap2e5b7n10575ffy946kencva4ve', 'examples/_out_list2_async'])
+  const testGet2 = async () => {
+    const exitCode = await getWithProvider(['awt9x8564999k276wap2e5b7n10575ffy946kencva4ve', 'examples/_out_list2_async'])
     if (exitCode !== 0) { throw exitCode }
 
     const bufferIn = readExample(`examples/list2.txt`)
     const bufferOut = await fsPromises.readFile(`examples/_out_list2_async`)
     if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
   }
 
-  const testGetAsyncRepeat = async() => {
-    const exitCode = await getAsync(['d963x31mwgb8svqe0jmkxh8ar1f8p2dawebnan4aj6hvd', '_out_repeat_async'])
+  const testGetRepeat = async () => {
+    const exitCode = await getWithProvider(['d963x31mwgb8svqe0jmkxh8ar1f8p2dawebnan4aj6hvd', '_out_repeat_async'])
     if (exitCode !== 0) { throw exitCode }
 
     const bufferIn = readExample(`examples/repeat.txt`)
     const bufferOut = await fsPromises.readFile(`_out_repeat_async`)
     if (!bufferOut.equals(bufferIn)) { throw 'files are different' }
   }
 
-  const mainTestAsync = async() => {
-    await runTest(testGetSyncOld1)
-    await runTest(testGetSyncOld2)
-    await runTest(testGetSyncOldRepeat)
-    await runTest(testGetSync1)
-    await runTest(testGetSync2)
-    await runTest(testGetSyncRepeat)
-    await runTest(testGetAsync1)
-    await runTest(testGetAsync2)
-    await runTest(testGetAsyncRepeat)
-  }
+  await runTest(testGet1)
+  await runTest(testGet2)
+  await runTest(testGetRepeat)
+}
+
+const testFetchProvider = fetchProvider('410f5a49.blockset-js-test.pages.dev')
+
+const mainTestAsync = async () => {
+  console.log('sync provider')
+  await runTestsGet(syncFileProvider)
+  console.log('async provider')
+  await runTestsGet(asyncFileProvider)
+  console.log('fetch provider')
+  await runTestsGet(testFetchProvider)
+}
 
-  mainTestAsync()
-}
+mainTestAsync()