diff --git a/README.md b/README.md index f5e46902..43b80b73 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,13 @@ Get block. * `cid` is the content id of [type CID](https://github.com/ipld/js-cid#readme). +#### `Promise> repo.blocks.query (query, reconstructsCids)` + +Query what blocks are available in blockstore. + +* `query` is a object as specified in [interface-datastore](https://github.com/ipfs/interface-datastore#query). +* `reconstructsCids` a flag defining if the block's key is a reconstructed CID (eq. CIDv1 with RAW IPLD codec) or multihash + Datastore: #### `repo.datastore` diff --git a/src/blockstore-utils.js b/src/blockstore-utils.js index b1d00f74..fa2650c7 100644 --- a/src/blockstore-utils.js +++ b/src/blockstore-utils.js @@ -12,18 +12,53 @@ const CID = require('cids') */ exports.cidToKey = cid => { const enc = new base32.Encoder() - return new Key('/' + enc.write(cid.buffer).finalize(), false) + return new Key('/' + enc.write(cid.multihash).finalize(), false) } /** * Transform a datastore Key instance to a CID + * As Key is a multihash of the CID, it is reconstructed using IPLD's RAW codec. + * Hence it is highly probable that stored CID will differ from a CID retrieved from blockstore. * * @param {Key} key * @returns {CID} */ -exports.keyToCid = key => { + +function keyToCid (key) { + // Block key is of the form / + const decoder = new base32.Decoder() + const buff = decoder.write(key.toString().slice(1)).finalize() + return new CID(1, 'raw', Buffer.from(buff)) +} + +exports.keyToCid = keyToCid + +/** + * Transform a datastore Key instance to a multihash instance. + * + * @param {Key} key + * @returns {Buffer} + */ + +function keyToMultihash (key) { // Block key is of the form / const decoder = new base32.Decoder() const buff = decoder.write(key.toString().slice(1)).finalize() - return new CID(Buffer.from(buff)) + return Buffer.from(buff) +} + +exports.keyToMultihash = keyToMultihash + +/** + * Transforms a datastore Key containing multihash to a Key that contains reconstructed CID + * + * @param {Key} key + * @returns {CID} + */ +function keyToCidKey (key) { + const cid = keyToCid(key) + const enc = new base32.Encoder() + return new Key('/' + enc.write(cid.buffer).finalize(), false) } + +exports.keyToCidKey = keyToCidKey diff --git a/src/blockstore.js b/src/blockstore.js index 27fc5415..aacc4508 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -5,7 +5,7 @@ const ShardingStore = core.ShardingDatastore const Block = require('ipfs-block') const CID = require('cids') const errcode = require('err-code') -const { cidToKey } = require('./blockstore-utils') +const { cidToKey, keyToCidKey } = require('./blockstore-utils') module.exports = async (filestore, options) => { const store = await maybeWithSharding(filestore, options) @@ -26,10 +26,15 @@ function createBaseStore (store) { * Query the store. * * @param {object} query + * @param {boolean} reconstructsCids - Defines if Keys are converted to a reconstructed CID using IPLD_RAW codec * @return {Iterable} */ - async * query (query) { + async * query (query, reconstructsCids = false) { for await (const block of store.query(query)) { + if (reconstructsCids) { + block.key = keyToCidKey(block.key) + } + yield block } }, @@ -44,26 +49,8 @@ function createBaseStore (store) { throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID') } const key = cidToKey(cid) - let blockData - try { - blockData = await store.get(key) - return new Block(blockData, cid) - } catch (err) { - if (err.code === 'ERR_NOT_FOUND') { - const otherCid = cidToOtherVersion(cid) - - if (!otherCid) { - throw err - } - - const otherKey = cidToKey(otherCid) - const blockData = await store.get(otherKey) - await store.put(key, blockData) - return new Block(blockData, cid) - } - - throw err - } + const blockData = await store.get(key) + return new Block(blockData, cid) }, /** * Write a single block to the store. @@ -112,18 +99,14 @@ function createBaseStore (store) { * Does the store contain block with this cid? * * @param {CID} cid - * @returns {Promise} + * @returns {Promise} */ - async has (cid) { + has (cid) { if (!CID.isCID(cid)) { throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID') } - const exists = await store.has(cidToKey(cid)) - if (exists) return exists - const otherCid = cidToOtherVersion(cid) - if (!otherCid) return false - return store.has(cidToKey(otherCid)) + return store.has(cidToKey(cid)) }, /** * Delete a block from the store @@ -147,11 +130,3 @@ function createBaseStore (store) { } } } - -function cidToOtherVersion (cid) { - try { - return cid.version === 0 ? cid.toV1() : cid.toV0() - } catch (err) { - return null - } -} diff --git a/src/index.js b/src/index.js index 5b203be2..6efbd5cc 100644 --- a/src/index.js +++ b/src/index.js @@ -277,11 +277,11 @@ class IpfsRepo { let count = new Big(0) let size = new Big(0) - for await (const block of this.blocks.query({})) { + for await (const block of this.blocks.query({}, false)) { count = count.plus(1) size = size .plus(block.value.byteLength) - .plus(block.key._buf.byteLength) + .plus(block.key.toBuffer().byteLength) } return { count, size } @@ -292,7 +292,7 @@ async function getSize (queryFn) { let sum = new Big(0) for await (const block of queryFn.query({})) { sum.plus(block.value.byteLength) - .plus(block.key._buf.byteLength) + .plus(block.key.toBuffer().byteLength) } return sum } diff --git a/test/blockstore-test.js b/test/blockstore-test.js index 07ca44e6..128b4ce8 100644 --- a/test/blockstore-test.js +++ b/test/blockstore-test.js @@ -85,9 +85,11 @@ module.exports = (repo) => { close () { } + has () { return true } + batch () { return { put () { @@ -217,6 +219,7 @@ module.exports = (repo) => { close () { } + get (c) { if (c.toString() === key.toString()) { throw err diff --git a/test/blockstore-utils-test.js b/test/blockstore-utils-test.js index 3bea8544..913ad3f3 100644 --- a/test/blockstore-utils-test.js +++ b/test/blockstore-utils-test.js @@ -11,7 +11,8 @@ const Repo = require('../src') module.exports = () => { describe('blockstore utils', () => { it('converts a CID to a datastore Key and back', () => { - const originalCid = new CID('Qme6KJdKcp85TYbLxuLV7oQzMiLremD7HMoXLZEmgo6Rnh') + // CIDv1 in base32 with IPLD raw codec + const originalCid = new CID('bafkreihkb3vrxxex5zvzkr3s3a6noe223r7jka4ofjy2nkzu27kueg76ii') const key = Repo.utils.blockstore.cidToKey(originalCid) expect(key instanceof Key).to.be.true() const cid = Repo.utils.blockstore.keyToCid(key)