diff --git a/README.md b/README.md index e7fc9499..a2507994 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,20 @@ const config = require('ssb-config') const caps = require('ssb-caps') const stack = SecretStack({ caps }) - .use(require('ssb-db')) // << required - .use(require('ssb-backlinks')) // << required index - .use(require('ssb-query')) // << required index - .use(require('ssb-tribes')) - .use(require('ssb-private1')) // if you want to support old decryption - // *order matters*, load tribes first + .use(require('ssb-db2/core')) + .use(require('ssb-classic')) + .use(require('ssb-db2/compat')) + .use(require('ssb-db2/compat/feedstate')) + .use(require('ssb-box2')) .use(...) -const ssb = stack(config) +const ssb = stack({ + ...config, + box2: { + ...config.box2, + legacyMode: true + } +}) ``` @@ -34,7 +39,7 @@ ssb.tribes.create({}, (err, info) => { test: 'kia ora, e te whānau', recps: [groupId] // <<< you can now put a groupId in the recps } - ssb.publish(content, (err, msg) => { + ssb.tribes.publish(content, (err, msg) => { // tada msg is encrypted to group! const cookie = '@YXkE3TikkY4GFMX3lzXUllRkNTbj5E+604AkaO1xbz8=.ed25519' @@ -53,7 +58,7 @@ ssb.tribes.create({}, (err, info) => { This plugin provides functions for creating groups and administering things about them, but it also provides a bunch of "automatic" behviour. -1. **When you publish a message with `recps` it will auto-encrypt** the content when: +1. **When you publish a message with `recps` using `ssb.tribes.publish` it will auto-encrypt** the content when: - there are 1-16 FeedIds (direct mesaaging) - there is 1 GroupId (private group messaging) - there is 1 GroupId followed by 1-15 FeedId @@ -78,14 +83,22 @@ This plugin provides functions for creating groups and administering things abou ## Requirements A Secret-Stack server running the plugins: -- `ssb-db` >= 20.3.0 +- `ssb-db2/core` >= 7.1.1 +- `ssb-classic` - `ssb-tribes` -- `ssb-backlinks` >= 2.1.1 - used for adding group tangle meta data to messages -- `ssb-query` >= 2.4.5 - used for listing groups linked to your feedId, or subgroup linked to groups +- `ssb-db2/compat` +- `ssb-db2/compat/feedstate` +- `ssb-box2` >= 7.2.0 - `ssb-replicate` - (optional) used to auto-replicate people who you're in groups with +The secret stack option `config.box2.legacyMode` also needs to be `true`. + ## API +### `ssb.tribes.publish(content, cb)` + +A wrapper around `ssb.db.create` that makes sure you have correct tangles (if relevant) in your message. Mutates `content`. You need to put recipients in `content.recps` if you want it to be encrypted. + ### `ssb.tribes.create(opts, cb)` Mint a new private group. @@ -325,4 +338,3 @@ Find subGroups which have linked with a groupId (see `ssb.tribes.link.createSubG ## TODO - [ ] add the latest known "sequence" at time of add-member, so we know if we need to reindex! - diff --git a/envelope.js b/envelope.js deleted file mode 100644 index 4a124953..00000000 --- a/envelope.js +++ /dev/null @@ -1,148 +0,0 @@ -/* eslint-disable camelcase */ - -const { isFeed, isCloakedMsg: isGroup } = require('ssb-ref') -const { box, unboxKey, unboxBody } = require('envelope-js') -const { SecretKey, poBoxKey, DiffieHellmanKeys, DHKeys } = require('ssb-private-group-keys') -const isPoBox = require('ssb-private-group-keys/lib/is-po-box') // TODO find better home -const bfe = require('ssb-bfe') - -const { isValidRecps } = require('./lib') - -function isEnvelope (ciphertext) { - return ciphertext.endsWith('.box2') - // && base64? -} - -module.exports = function Envelope (keystore, state) { - const easyPoBoxKey = poBoxKey.easy(state.keys) - - function addDMPairSync (myKeys, theirId) { - const myId = myKeys.id - const myDhKeys = new DHKeys(myKeys, { fromEd25519: true }) - const theirKeys = { public: bfe.encode(theirId).slice(2) } - const theirDhKeys = new DHKeys(theirKeys, { fromEd25519: true }) - return keystore.dm.add(myId, theirId, myDhKeys, theirDhKeys, (err) => { - if (err) console.error(err) - }) - } - - function getDmKey (theirId) { - if (!keystore.dm.has(state.keys.id, theirId)) addDMPairSync(state.keys, theirId) - return keystore.dm.get(state.keys.id, theirId) - } - - function boxer (content, previousFeedState) { - const recps = [...content.recps] - // NOTE avoid mutating the original recps - if (process.env.NODE_ENV !== 'test') { - // slip my own_key into a slot if there's space - // we disable in tests because it makes checking unboxing really hard! - if (recps.indexOf(state.keys.id) < 0) recps.push(state.keys.id) - } - - if (!isValidRecps(recps)) throw isValidRecps.error - - const recipentKeys = recps.map(recp => { - if (isGroup(recp)) { - const keyInfo = keystore.group.get(recp) - if (!keyInfo) throw new Error(`unknown groupId ${recp}, cannot encrypt message`) - return keyInfo.writeKey - } - if (isFeed(recp)) { - if (recp === state.keys.id) return keystore.self.get() // use a special key for your own feedId - else return getDmKey(recp) - } - if (isPoBox(recp)) return easyPoBoxKey(recp) - - // isValidRecps should guard against hitting this - throw new Error('did now how to map recp > keyInfo') - }) - - const plaintext = Buffer.from(JSON.stringify(content), 'utf8') - const msgKey = new SecretKey().toBuffer() - - const previousMessageId = bfe.encode(previousFeedState.id) - - const envelope = box(plaintext, state.feedId, previousMessageId, msgKey, recipentKeys) - return envelope.toString('base64') + '.box2' - } - - /* unboxer components */ - function key (ciphertext, { author, previous }) { - if (!isEnvelope(ciphertext)) return null - - const envelope = Buffer.from(ciphertext.replace('.box2', ''), 'base64') - const feed_id = bfe.encode(author) - const prev_msg_id = bfe.encode(previous) - - let readKey - - /* check my DM keys (self, other) */ - if (author === state.keys.id) { - const trial_own_keys = [keystore.self.get()] - readKey = unboxKey(envelope, feed_id, prev_msg_id, trial_own_keys, { maxAttempts: 16 }) - if (readKey) return readKey - } else { - const trial_dm_keys = [getDmKey(author)] - - readKey = unboxKey(envelope, feed_id, prev_msg_id, trial_dm_keys, { maxAttempts: 16 }) - if (readKey) return readKey - } - - /* check my group keys */ - const trial_group_keys = keystore.group.listSync().map(groupId => keystore.group.get(groupId).readKeys).flat() - // NOTE we naively try *every* group key. Several optimizations are possible to improve this (if needed) - // 1. keep "try all" approach, but bubble successful keys to the front (frequently active groups get quicker decrypts) - // 2. try only groups this message (given author) - cache group membership, and use this to inform keys tried - readKey = unboxKey(envelope, feed_id, prev_msg_id, trial_group_keys, { maxAttempts: 1 }) - // NOTE the group recp is only allowed in the first slot, - // so we only test group keys in that slot (maxAttempts: 1) - if (readKey) return readKey - - /* check my poBox keys */ - // TODO - consider how to reduce redundent computation + memory use here - const trial_poBox_keys = keystore.poBox.list() - .map(poBoxId => { - const data = keystore.poBox.get(poBoxId) - - const poBox_dh_secret = Buffer.concat([ - bfe.toTF('encryption-key', 'box2-pobox-dh'), - data.key - ]) - - const poBox_id = bfe.encode(poBoxId) - const poBox_dh_public = Buffer.concat([ - bfe.toTF('encryption-key', 'box2-pobox-dh'), - poBox_id.slice(2) - ]) - - const author_id = bfe.encode(author) - const author_dh_public = new DiffieHellmanKeys({ public: author }, { fromEd25519: true }) - .toBFE().public - - return poBoxKey(poBox_dh_secret, poBox_dh_public, poBox_id, author_dh_public, author_id) - }) - - return unboxKey(envelope, feed_id, prev_msg_id, trial_poBox_keys, { maxAttempts: 16 }) - } - - function value (ciphertext, { author, previous }, read_key) { - if (!isEnvelope(ciphertext)) return null - - // TODO change unboxer signature to allow us to optionally pass variables - // from key() down here to save computation - const envelope = Buffer.from(ciphertext.replace('.box2', ''), 'base64') - const feed_id = bfe.encode(author) - const prev_msg_id = bfe.encode(previous) - - const plaintext = unboxBody(envelope, feed_id, prev_msg_id, read_key) - if (!plaintext) return - - return JSON.parse(plaintext.toString('utf8')) - } - - return { - boxer, - unboxer: { key, value } - } -} diff --git a/index.js b/index.js index 72934dfc..c8cd1886 100644 --- a/index.js +++ b/index.js @@ -1,18 +1,13 @@ -const { join } = require('path') const set = require('lodash.set') const { isFeed, isCloakedMsg: isGroup } = require('ssb-ref') -const KeyRing = require('ssb-keyring') const bfe = require('ssb-bfe') -const Obz = require('obz') const pull = require('pull-stream') const paraMap = require('pull-paramap') -const Envelope = require('./envelope') const listen = require('./listen') const { GetGroupTangle, tanglePrune, groupId: buildGroupId, poBoxKeys } = require('./lib') const Method = require('./method') -const RebuildManager = require('./rebuild-manager') module.exports = { name: 'tribes', @@ -53,50 +48,22 @@ module.exports = { } function init (ssb, config) { + if (!(config.box2 && config.box2.legacyMode)) throw Error('ssb-tribes error: config.box2.legacyMode needs to be `true`') + const state = { keys: ssb.keys, feedId: bfe.encode(ssb.id), - loading: { - keystore: Obz() - }, newAuthorListeners: [], closed: false } - /* secret keys store / helper */ - const keystore = {} // HACK we create an Object so we have a reference to merge into - KeyRing(join(config.path, 'tribes/keystore'), (err, api) => { - if (err) throw err - - api.signing.addNamed(ssb.keys.id, ssb.keys, (err) => { - if (err) throw err - - Object.assign(keystore, api) // merging into existing reference - state.loading.keystore.set(false) - }) - }) ssb.close.hook(function (close, args) { - const next = () => close.apply(this, args) - onKeystoreReady(() => keystore.close(next)) - state.closed = true // NOTE must be after onKeystoreReady call + state.closed = true + close.apply(this, args) }) - /* register the boxer / unboxer */ - const { boxer, unboxer } = Envelope(keystore, state) - ssb.addBoxer({ init: onKeystoreReady, value: boxer }) - ssb.addUnboxer({ init: onKeystoreReady, ...unboxer }) - - function onKeystoreReady (done) { - if (state.closed === true) return - if (state.loading.keystore.value === false) return done() - - state.loading.keystore.once(done) - } - - /* start listeners */ - const rebuildManager = new RebuildManager(ssb) const processedNewAuthors = {} function processAuthors (groupId, authors, adder, cb) { @@ -111,13 +78,26 @@ function init (ssb, config) { state.newAuthorListeners.forEach(fn => fn({ groupId, newAuthors: [...newAuthors] })) // we don't rebuild if we're the person who added them if (adder !== ssb.id) { - const reason = ['add-member', ...newAuthors].join('+') - rebuildManager.rebuild(reason) + ssb.db.reindexEncrypted((err) => { + if (err) console.error('error reindexing encrypted after new members found', err) + }) } newAuthors.forEach(author => processedNewAuthors[groupId].add(author)) cb() } + /* start listeners */ + + /* We care about group/add-member messages others have posted which: + * 1. add us to a new group + * 2. add other people to a group we're already in + * + * In (2) we may be able to skip re-indexing if they haven't published + * any brand new private messages since they were added. + * This would require knowing their feed seq at time they were entrusted with key + * (because they can't post messages to the group before then) + */ + pull( listen.addMember(ssb), pull.asyncMap((m, cb) => { @@ -132,16 +112,19 @@ function init (ssb, config) { ...m.value.content.recps.filter(isFeed) ]) - const record = keystore.group.get(groupId) - // if we haven't been in the group since before, register the group - if (record == null) { - return keystore.group.add(groupId, { key: groupKey, root }, (err) => { - if (err) return cb(err) + ssb.box2.getGroupInfo(groupId, (err, record) => { + if (err) return cb(Error("Couldn't get group info when add-member msg was found", { cause: err })) + + // if we haven't been in the group since before, register the group + if (record == null) { + return ssb.box2.addGroupInfo(groupId, { key: groupKey, root }, (err) => { + if (err) return cb(err) + processAuthors(groupId, authors, m.value.author, cb) + }) + } else { processAuthors(groupId, authors, m.value.author, cb) - }) - } else { - processAuthors(groupId, authors, m.value.author, cb) - } + } + }) }) }), pull.drain(() => {}, (err) => { @@ -156,7 +139,7 @@ function init (ssb, config) { const groupId = msg.value.content.recps[0] if (excludes.includes(ssb.id)) { - keystore.group.exclude(groupId) + ssb.box2.excludeGroupInfo(groupId) } }, err => { if (err) console.error('Listening for new excludeMembers errored:', err) @@ -165,11 +148,12 @@ function init (ssb, config) { listen.poBox(ssb, m => { const { poBoxId, key: poBoxKey } = m.value.content.keys.set - keystore.poBox.add(poBoxId, { key: poBoxKey }, (err) => { + ssb.box2.addPoBox(poBoxId, { key: poBoxKey }, (err) => { if (err) throw err - const reason = ['po-box', poBoxId].join() - rebuildManager.rebuild(reason) + ssb.db.reindexEncrypted((err) => { + if (err) console.error('error reindexing encrypted after pobox found', err) + }) }) }) @@ -181,52 +165,53 @@ function init (ssb, config) { .forEach(id => ssb.replicate.request({ id, replicate: true })) }) - state.loading.keystore.once(() => { - pull( - pull.values(keystore.group.listSync()), - paraMap( - (groupId, cb) => scuttle.group.listAuthors(groupId, (err, feedIds) => { - if (err) return cb(new Error('error listing authors to replicate on start')) - cb(null, feedIds) - }), - 5 - ), - pull.flatten(), - pull.unique(), - pull.drain(feedId => { - if (feedId === ssb.id) return - ssb.replicate.request({ id: feedId, replicate: true }) - }, (err) => { - if (err) console.error('error on initializing replication of group members') - }) - ) - }) + pull( + ssb.box2.listGroupIds(), + paraMap( + (groupId, cb) => scuttle.group.listAuthors(groupId, (err, feedIds) => { + if (err) return cb(new Error('error listing authors to replicate on start')) + cb(null, feedIds) + }), + 5 + ), + pull.flatten(), + pull.unique(), + pull.drain(feedId => { + if (feedId === ssb.id) return + ssb.replicate.request({ id: feedId, replicate: true }) + }, (err) => { + if (err) console.error('error on initializing replication of group members') + }) + ) } }) - /* We care about group/add-member messages others have posted which: - * 1. add us to a new group - * 2. add other people to a group we're already in - * - * In (2) we may be able to skip re-indexing if they haven't published - * any brand new private messages since they were added. - * This would require knowing their feed seq at time they were entrusted with key - * (because they can't post messages to the group before then) - */ + /* API */ const isMemberType = (type) => type === 'group/add-member' || type === 'group/exclude-member' /* Tangle: auto-add tangles.group info to all private-group messages */ - const getGroupTangle = GetGroupTangle(ssb, keystore, 'group') - const getMembersTangle = GetGroupTangle(ssb, keystore, 'members') - ssb.publish.hook(function (publish, args) { - const [content, cb] = args - if (!content.recps) return publish.apply(this, args) + const getGroupTangle = GetGroupTangle(ssb, null, 'group') + const getMembersTangle = GetGroupTangle(ssb, null, 'members') + + const tribePublish = (content, cb) => { + if (!content.recps) { + return ssb.db.create({ + content + }, cb) + } + + if (!isGroup(content.recps[0])) { + return ssb.db.create({ + content, + encryptionFormat: 'box2' + }, cb) + } - if (!isGroup(content.recps[0])) return publish.apply(this, args) + ssb.box2.getGroupInfo(content.recps[0], (err, groupInfo) => { + if (err) return cb(Error('error on getting group info in publish', { cause: err })) - onKeystoreReady(() => { - if (!keystore.group.has(content.recps[0])) return cb(Error('unknown groupId')) + if (!groupInfo) return cb(Error('unknown groupId')) getGroupTangle(content.recps[0], (err, groupTangle) => { if (err) return cb(Error("Couldn't get group tangle", { cause: err })) @@ -236,7 +221,10 @@ function init (ssb, config) { // we only want to have to calculate the members tangle if it's gonna be used if (!isMemberType(content.type)) { - return publish.call(this, content, cb) + return ssb.db.create({ + content, + encryptionFormat: 'box2' + }, cb) } getMembersTangle(content.recps[0], (err, membersTangle) => { @@ -245,49 +233,42 @@ function init (ssb, config) { set(content, 'tangles.members', membersTangle) tanglePrune(content, 'members') - publish.call(this, content, cb) + ssb.db.create({ + content, + encryptionFormat: 'box2' + }, cb) }) }) }) - }) + } - /* API */ - const scuttle = Method(ssb, keystore, state) // ssb db methods + const scuttle = Method(ssb) // ssb db methods const tribeCreate = (opts, cb) => { opts = opts || {} // NOTE this catches opts = null, leave it like this - onKeystoreReady(() => { - scuttle.group.init((err, data) => { - if (err) return cb(err) - // NOTE this checks out group/init message was encrypted with the right `previous`. - // There is a potential race condition where the init method calls `ssb.getFeedState` to - // access `previous` but while encrypting the `group/init` message content another - // message is pushed into the queue, making our enveloping invalid. - const initValue = data.groupInitMsg.value - const readKey = unboxer.key(initValue.content, initValue) - if (!readKey) return cb(new Error('tribes.group.init failed, please try again while not publishing other messages')) + scuttle.group.init((err, data) => { + if (err) return cb(err) - // addMember the admin - scuttle.group.addMember(data.groupId, [ssb.id], {}, (err) => { - if (err) return cb(err) + // addMember the admin + scuttle.group.addMember(data.groupId, [ssb.id], {}, (err) => { + if (err) return cb(err) - // add a P.O. Box to the group (maybe) - if (!opts.addPOBox) return cb(null, data) - else { - scuttle.group.addPOBox(data.groupId, (err, poBoxId) => { - if (err) cb(err) - cb(null, { ...data, poBoxId }) - }) - } - }) + // add a P.O. Box to the group (maybe) + if (!opts.addPOBox) return cb(null, data) + else { + scuttle.group.addPOBox(data.groupId, (err, poBoxId) => { + if (err) cb(err) + cb(null, { ...data, poBoxId }) + }) + } }) }) } const tribeGet = (id, cb) => { - onKeystoreReady(() => { - const data = keystore.group.get(id) + ssb.box2.getGroupInfo(id, (err, data) => { + if (err) return cb(err) if (!data) return cb(new Error(`unknown groupId ${id})`)) scuttle.link.findParentGroupLinks(id, (err, parentGroupLinks) => { @@ -310,22 +291,21 @@ function init (ssb, config) { function tribeList (opts, cb) { if (typeof opts === 'function') return tribeList({}, opts) - onKeystoreReady(() => { - pull( - pull.values(keystore.group.listSync()), - paraMap(tribeGet, 4), - opts.subtribes - ? null - : pull.filter(tribe => tribe.parentGroupId === undefined), - pull.map(tribe => tribe.groupId), - pull.collect(cb) - ) - }) + pull( + ssb.box2.listGroupIds(), + paraMap(tribeGet, 4), + opts.subtribes + ? null + : pull.filter(tribe => tribe.parentGroupId === undefined), + pull.map(tribe => tribe.groupId), + pull.collect(cb) + ) } return { + publish: tribePublish, register (groupId, info, cb) { - keystore.group.add(groupId, info, cb) + ssb.box2.addGroupInfo(groupId, info, cb) }, create: tribeCreate, list: tribeList, @@ -384,30 +364,31 @@ function init (ssb, config) { create (opts, cb) { const { id: poBoxId, secret } = poBoxKeys.generate() - onKeystoreReady(() => { - keystore.poBox.add(poBoxId, { key: secret }, (err) => { - if (err) return cb(err) + ssb.box2.addPoBox(poBoxId, { key: secret }, (err) => { + if (err) return cb(err) - cb(null, { poBoxId, poBoxKey: secret }) - }) + cb(null, { poBoxId, poBoxKey: secret }) }) }, get: scuttle.group.getPOBox - }, + } // for internal use - ssb-ahau uses this for backups - ownKeys: { - list (cb) { - onKeystoreReady(() => { - cb(null, [keystore.self.get()]) - }) - }, - register (key, cb) { - onKeystoreReady(() => { - keystore.self.set(key, cb) - }) - } - } + // TODO: does ahau use this for backups though? + // i couldn't find a self.get function in box2 so we might have to add that if this is needed + // ownKeys: { + // list (cb) { + // onKeystoreReady(() => { + // cb(null, [keystore.self.get()]) + // }) + // }, + // register (key, cb) { + // onKeystoreReady(() => { + // //ssb.box2.setOwnDMKey(key) + // keystore.self.set(key, cb) + // }) + // } + // } } } diff --git a/lib/get-group-tangle.js b/lib/get-group-tangle.js index 2d3442f3..1cf26804 100644 --- a/lib/get-group-tangle.js +++ b/lib/get-group-tangle.js @@ -2,106 +2,70 @@ const { isCloakedMsg: isGroup } = require('ssb-ref') const pull = require('pull-stream') const Reduce = require('@tangle/reduce') const Strategy = require('@tangle/strategy') +const { allocAndEncode, seekKey2 } = require('bipf') +const { where, equal, toPullStream } = require('ssb-db2/operators') // for figuring out what "previous" should be for the group -module.exports = function GetGroupTangle (server, keystore, tangle = 'group') { - const strategy = new Strategy({}) - const cache = new Map([]) // groupId > new Reduce (tangleTips) +const B_CONTENT = allocAndEncode('content') +const B_TANGLES = allocAndEncode('tangles') +const B_ROOT = allocAndEncode('root') - // LISTEN - // listen to all new messages that come in - // if a new message comes in which has msg.value.content.tangles.group.root that is in cache - // update the value in the cache (this will use our new addNodes functionality) +module.exports = function GetGroupTangle (server, _, tangle = 'group') { + const strategy = new Strategy({}) - pull( - server.createLogStream({ live: true, old: false, private: true }), - // we need to have this here because rebuilds cause the stream to start again and updateCache isn't idempotent - pull.unique('key'), - pull.drain(updateCache) - ) - // server.post(m => server.get({ id: m.key, private: true, meta: true }, (err, msg) => { - // updateCache(msg) - // })) + const B_TANGLE = allocAndEncode(tangle) - function updateCache (msg) { - const { recps, tangles } = msg.value.content - // If the message has recipients get the group ID, which may be the first slot. - const msgGroupId = recps && recps[0] - // Check if msg is part of a cached group - if (cache.has(msgGroupId)) { - // Add message to Reduce - if (tangles && tangles.group && tangles.group.previous) { - cache.get(msgGroupId).addNodes([{ - key: msg.key, - previous: tangles.group.previous - }]) // Get key and previous from msg - } - } + function seekTanglesTangleRoot (buffer, start, pValue) { + if (pValue < 0) return -1 + const pValueContent = seekKey2(buffer, pValue, B_CONTENT, 0) + if (pValueContent < 0) return -1 + const pValueContentTangles = seekKey2(buffer, pValueContent, B_TANGLES, 0) + if (pValueContentTangles < 0) return -1 + const pValueContentTanglesTangle = seekKey2(buffer, pValueContentTangles, B_TANGLE, 0) + if (pValueContentTanglesTangle < 0) return -1 + return seekKey2(buffer, pValueContentTanglesTangle, B_ROOT, 0) } return function getGroupTangle (groupId, cb) { if (!isGroup(groupId)) return cb(new Error(`get-group-tangle expects valid groupId, got: ${groupId}`)) - const info = keystore.group.get(groupId) - if (!info) return cb(new Error(`get-group-tangle: unknown groupId ${groupId}`)) + server.box2.getGroupInfo(groupId, (err, info) => { + if (err) return cb(Error("Couldn't get group info for group tangle", { cause: err })) + if (!info) return cb(new Error(`get-group-tangle: unknown groupId ${groupId}`)) - // if it's in the cache, then get the cached value, then callback - if (cache.has(groupId)) { - // this timeout seems to help for some reason. in some cases messages were posted too fast with tangles 'in parallel', e.g. 2 messages both just having previous: [rootMsgId] - return setTimeout(() => { - return cb(null, { - root: info.root, - previous: Object.keys(cache.get(groupId).state) - }) - }, 0) - } - // if not in cache, compute it and add to the cache - - const query = [ - { - $filter: { - dest: info.root, - value: { - content: { - tangles: { - [tangle]: { root: info.root } - } - } - } - } - }, - { - $map: { - key: ['key'], - previous: ['value', 'content', 'tangles', tangle, 'previous'] - } - } - ] - // NOTE: a query which gets all update messages to the root message + // NOTE: a query which gets all update messages to the root message - pull( - server.backlinks.read({ query }), - pull.collect((err, nodes) => { - if (err) return cb(err) + pull( + server.db.query( + where( + equal(seekTanglesTangleRoot, info.root, { indexType: `tangles${tangle}Update`, prefix: true }) + ), + toPullStream() + ), + pull.map(msg => ({ + key: msg.key, + previous: msg.value.content.tangles[tangle].previous + })), + pull.collect((err, nodes) => { + if (err) return cb(err) - // NOTE: backlink query does not get root node - nodes.push({ key: info.root, previous: null }) + // NOTE: db query does not get root node + nodes.push({ key: info.root, previous: null }) - // Create a Reduce using the message contents - // NOTE - do NOT store the whole msg (node) - // we're not doing any reducing of transformations, we care only about - // reducing the graph to find the tips - // each node should be pruned down to e.g. { key: '%D', previous: ['%B', '%C'] } + // Create a Reduce using the message contents + // NOTE - do NOT store the whole msg (node) + // we're not doing any reducing of transformations, we care only about + // reducing the graph to find the tips + // each node should be pruned down to e.g. { key: '%D', previous: ['%B', '%C'] } - const reduce = new Reduce(strategy, { nodes }) - // Store Reduce in the cache to use/update it later. - cache.set(groupId, reduce) - cb(null, { - root: info.root, - previous: Object.keys(reduce.state) + const reduce = new Reduce(strategy, { nodes }) + cb(null, { + root: info.root, + previous: Object.keys(reduce.state) + }) }) - }) - ) + ) + }) } } diff --git a/lib/group-id.js b/lib/group-id.js index 87566057..a79fc94b 100644 --- a/lib/group-id.js +++ b/lib/group-id.js @@ -42,7 +42,8 @@ function fromMsgKey (msg, msgKey) { } function fromGroupKey (msg, groupKey) { - const { author, previous, content } = msg.value + const { author, previous } = msg.value + const content = (msg.meta && msg.meta.originalContent) || msg.value.content const envelope = Buffer.from(content.replace('.box2', ''), 'base64') const feed_id = bfe.encode(author) diff --git a/listen.js b/listen.js index b76dfc5a..f6ea3c56 100644 --- a/listen.js +++ b/listen.js @@ -1,18 +1,30 @@ // const flumeView = require('flumeview-reduce') const pull = require('pull-stream') +const pullMany = require('pull-many') const CRUT = require('ssb-crut') +const { where, and, type, isDecrypted, live: dbLive, toPullStream } = require('ssb-db2/operators') const { isValid: isAddMember } = require('./spec/group/add-member') const { isValid: isExcludeMember } = require('./spec/group/exclude-member') const poBoxSpec = require('./spec/group/po-box') -const mockSSB = { backlinks: true, query: true } -const { isUpdate: isPOBox } = new CRUT(mockSSB, poBoxSpec).spec module.exports = { addMember (ssb) { return pull( - ssb.messagesByType({ type: 'group/add-member', private: true, live: true }), + pullMany([ + ssb.db.query( + where( + and( + isDecrypted('box2'), + type('group/add-member') + ) + ), + dbLive({ old: true }), + toPullStream() + ), + ssb.db.reindexed() + ]), // NOTE this will run through all messages on each startup, which will help guarentee // all messages have been emitted AND processed // (same not true if we used a dummy flume-view) @@ -20,20 +32,49 @@ module.exports = { pull.filter(isAddMember), pull.unique('key') // NOTE we DO NOT filter our own messages out - // this is important for rebuilding indexes and keystore state if we have to restore our feed + // this is important for rebuilding indexes and keyring state if we have to restore our feed ) }, excludeMember (ssb) { return pull( - ssb.messagesByType({ type: 'group/exclude-member', private: true, live: true }), + pullMany([ + ssb.db.query( + where( + and( + isDecrypted('box2'), + type('group/exclude-member') + ) + ), + dbLive({ old: true }), + toPullStream() + ), + ssb.db.reindexed() + ]), pull.filter(m => m.sync !== true), pull.filter(isExcludeMember), pull.unique('key') ) }, poBox (ssb, emit) { + const { isUpdate: isPOBox } = new CRUT(ssb, poBoxSpec).spec + pull( - ssb.messagesByType({ type: 'group/po-box', private: true, live: true }), + pullMany([ + ssb.db.query( + where( + and( + isDecrypted('box2'), + type('group/po-box') + ) + ), + dbLive({ old: true }), + toPullStream() + ), + pull( + ssb.db.reindexed(), + pull.filter(msg => msg.value.content.type === 'group/po-box') + ) + ]), // NOTE this will run through all messages on each startup, which will help guarentee // all messages have been emitted AND processed // (same not true if we used a dummy flume-view) @@ -41,7 +82,7 @@ module.exports = { pull.filter(isPOBox), pull.unique('key'), // NOTE we DO NOT filter our own messages out - // this is important for rebuilding indexes and keystore state if we have to restore our feed + // this is important for rebuilding indexes and keyring state if we have to restore our feed pull.drain(emit) ) } diff --git a/method/group.js b/method/group.js index 38f480ae..13572789 100644 --- a/method/group.js +++ b/method/group.js @@ -1,9 +1,8 @@ -const { box } = require('envelope-js') const { keySchemes } = require('private-group-spec') const { SecretKey } = require('ssb-private-group-keys') -const bfe = require('ssb-bfe') const Crut = require('ssb-crut') const pull = require('pull-stream') +const { where, type: dbType, toPullStream } = require('ssb-db2/operators') const { groupId: buildGroupId, poBoxKeys } = require('../lib') const initSpec = require('../spec/group/init') @@ -11,8 +10,15 @@ const addMemberSpec = require('../spec/group/add-member') const excludeMemberSpec = require('../spec/group/exclude-member') const groupPoBoxSpec = require('../spec/group/po-box') -module.exports = function GroupMethods (ssb, keystore, state) { - const groupPoBoxCrut = new Crut(ssb, groupPoBoxSpec) +module.exports = function GroupMethods (ssb) { + const groupPoBoxCrut = new Crut( + ssb, + groupPoBoxSpec, + { + publish: (...args) => ssb.tribes.publish(...args), + feedId: ssb.id + } + ) return { init (cb) { @@ -26,39 +32,39 @@ module.exports = function GroupMethods (ssb, keystore, state) { if (!initSpec.isValid(content)) return cb(new Error(initSpec.isValid.errorsString)) /* enveloping */ - // we have to do it manually this one time, because the auto-boxing checks for a known groupId + // we have to do it differently this one time, because the auto-boxing checks for a known groupId // but the groupId is derived from the messageId of this message (which does not exist yet - const plain = Buffer.from(JSON.stringify(content), 'utf8') - const msgKey = new SecretKey().toBuffer() - const recipientKeys = [ + const recps = [ { key: groupKey.toBuffer(), scheme: keySchemes.private_group }, - keystore.self.get() // sneak this in so can decrypt it ourselves without rebuild! + ssb.id // sneak this in so can decrypt it ourselves without rebuild! ] // TODO - // consider making sure creator can always open the group (even if lose keystore) + // consider making sure creator can always open the group (even if lose keyring) // would also require adding groupKey to this message - ssb.getFeedState(ssb.id, (err, previousFeedState) => { + ssb.db.create({ + content, + recps, + encryptionFormat: 'box2' + }, (err, initMsg) => { if (err) return cb(err) - const previousMessageId = bfe.encode(previousFeedState.id) - - const envelope = box(plain, state.feedId, previousMessageId, msgKey, recipientKeys) - const ciphertext = envelope.toString('base64') + '.box2' - - ssb.publish(ciphertext, (err, groupInitMsg) => { + ssb.get({ id: initMsg.key, meta: true }, (err, groupInitMsg) => { if (err) return cb(err) const data = { - groupId: buildGroupId({ groupInitMsg, msgKey }), + groupId: buildGroupId({ + groupInitMsg, + groupKey: groupKey.toBuffer() + }), groupKey: groupKey.toBuffer(), root: groupInitMsg.key, groupInitMsg } - keystore.group.add(data.groupId, { key: data.groupKey, root: data.root }, (err) => { + ssb.box2.addGroupInfo(data.groupId, { key: data.groupKey, root: data.root }, (err) => { if (err) return cb(err) cb(null, data) }) @@ -67,62 +73,59 @@ module.exports = function GroupMethods (ssb, keystore, state) { }, addMember (groupId, authorIds, opts = {}, cb) { - const { writeKey, root } = keystore.group.get(groupId) + ssb.box2.getGroupInfo(groupId, (err, groupInfo) => { + if (err) return cb(err) - const content = { - type: 'group/add-member', - version: 'v1', - groupKey: writeKey.key.toString('base64'), - root, - tangles: { - members: { - root, - previous: [root] // TODO calculate previous for members tangle + const { writeKey, root } = groupInfo + + const content = { + type: 'group/add-member', + version: 'v1', + groupKey: writeKey.key.toString('base64'), + root, + tangles: { + group: { root, previous: [root] }, + members: { root, previous: [root] } + // NOTE: these are dummy entries which are over-written in the publish function }, + recps: [groupId, ...authorIds] + } - group: { root, previous: [root] } - // NOTE: this is a dummy entry which is over-written in publish hook - }, - recps: [groupId, ...authorIds] - } - - if (opts.text) content.text = opts.text + if (opts.text) content.text = opts.text - if (!addMemberSpec.isValid(content)) return cb(new Error(addMemberSpec.isValid.errorsString)) + if (!addMemberSpec.isValid(content)) return cb(new Error(addMemberSpec.isValid.errorsString)) - ssb.publish(content, cb) + ssb.tribes.publish(content, cb) + }) }, excludeMembers (groupId, authorIds, cb) { - const { root } = keystore.group.get(groupId) - - const content = { - type: 'group/exclude-member', - excludes: authorIds, - tangles: { - members: { root, previous: [root] }, - group: { root, previous: [root] } - // NOTE: these are dummy entries which are over-written in the publish hook - }, - recps: [groupId] - } + ssb.box2.getGroupInfo(groupId, (err, groupInfo) => { + if (err) return cb(Error("Couldn't get group info when excluding", { cause: err })) + + const { root } = groupInfo + + const content = { + type: 'group/exclude-member', + excludes: authorIds, + tangles: { + group: { root, previous: [root] }, + members: { root, previous: [root] } + // NOTE: these are dummy entries which are over-written in the publish function + }, + recps: [groupId] + } - if (!excludeMemberSpec.isValid(content)) return cb(new Error(excludeMemberSpec.isValid.errorsString)) + if (!excludeMemberSpec.isValid(content)) return cb(new Error(excludeMemberSpec.isValid.errorsString)) - ssb.publish(content, cb) + ssb.tribes.publish(content, cb) + }) }, listAuthors (groupId, cb) { - const query = [{ - $filter: { - value: { - content: { - type: 'group/add-member' - } - } - } - }] - pull( - ssb.query.read({ query }), + ssb.db.query( + where(dbType('group/add-member')), + toPullStream() + ), pull.filter(addMemberSpec.isValid), pull.filter(msg => groupId === msg.value.content.recps[0] @@ -133,18 +136,11 @@ module.exports = function GroupMethods (ssb, keystore, state) { pull.collect((err, addedMembers) => { if (err) return cb(err) - const excludedQuery = [{ - $filter: { - value: { - content: { - type: 'group/exclude-member' - } - } - } - }] - pull( - ssb.query.read({ query: excludedQuery }), + ssb.db.query( + where(dbType('group/exclude-member')), + toPullStream() + ), pull.filter(excludeMemberSpec.isValid), pull.filter(msg => groupId === msg.value.content.recps[0] @@ -166,13 +162,13 @@ module.exports = function GroupMethods (ssb, keystore, state) { ) }, addPOBox (groupId, cb) { - const info = keystore.group.get(groupId) + const info = ssb.box2.getGroupInfo(groupId) if (!info) return cb(new Error('unknown groupId: ' + groupId)) const { id: poBoxId, secret } = poBoxKeys.generate() - keystore.poBox.add(poBoxId, { key: secret }, (err) => { - if (err) return cb(err) + ssb.box2.addPoBox(poBoxId, { key: secret }, (err) => { + if (err) return cb(Error("Couldn't add pbox to box2 when adding pobox", { cause: err })) const props = { keys: { diff --git a/method/index.js b/method/index.js index d7e1fe92..fce6d7dd 100644 --- a/method/index.js +++ b/method/index.js @@ -1,30 +1,14 @@ const Link = require('./link') const Group = require('./group') -module.exports = function Method (ssb, keystore, state) { +module.exports = function Method (ssb) { const link = Link(ssb) - const group = Group(ssb, keystore, state) + const group = Group(ssb) return { - group: { - init: patient(group.init), - addMember: patient(group.addMember), - excludeMembers: patient(group.excludeMembers), - listAuthors: group.listAuthors, - addPOBox: patient(group.addPOBox), - getPOBox: group.getPOBox - }, + group, link // create createSubGroupLink findGroupByFeedId findParentGroupLinks findSubGroupLinks } - - function patient (fn) { - // for functions that need keystore to be ready - return function (...args) { - if (state.loading.keystore.value === false) return fn(...args) - - state.loading.keystore.once(() => fn(...args)) - } - } } diff --git a/method/link.js b/method/link.js index 8e630db9..0928915c 100644 --- a/method/link.js +++ b/method/link.js @@ -1,12 +1,72 @@ const Crut = require('ssb-crut') const { isFeed, isCloakedMsg: isGroup } = require('ssb-ref') +const { allocAndEncode, seekKey2 } = require('bipf') const pull = require('pull-stream') +const { where, and, type: dbType, author, equal, toPullStream } = require('ssb-db2/operators') const FeedGroupLink = require('../spec/link/feed-group') const GroupSubGroupLink = require('../spec/link/group-subgroup') module.exports = function Link (ssb) { - const feedGroupLink = new Crut(ssb, FeedGroupLink) - const groupSubGroupLink = new Crut(ssb, GroupSubGroupLink) + const feedGroupLink = new Crut( + ssb, + FeedGroupLink, + { + publish: (...args) => ssb.tribes.publish(...args), + feedId: ssb.id + } + ) + const groupSubGroupLink = new Crut( + ssb, + GroupSubGroupLink, + { + publish: (...args) => ssb.tribes.publish(...args), + feedId: ssb.id + } + ) + + const B_CONTENT = allocAndEncode('content') + const B_PARENT = allocAndEncode('parent') + const B_CHILD = allocAndEncode('child') + const B_TANGLES = allocAndEncode('tangles') + const B_LINK = allocAndEncode('link') + const B_ROOT = allocAndEncode('root') + const B_PREVIOUS = allocAndEncode('previous') + + function seekParent (buffer, start, pValue) { + if (pValue < 0) return -1 + const pValueContent = seekKey2(buffer, pValue, B_CONTENT, 0) + if (pValueContent < 0) return -1 + return seekKey2(buffer, pValueContent, B_PARENT, 0) + } + + function seekChild (buffer, start, pValue) { + if (pValue < 0) return -1 + const pValueContent = seekKey2(buffer, pValue, B_CONTENT, 0) + if (pValueContent < 0) return -1 + return seekKey2(buffer, pValueContent, B_CHILD, 0) + } + + function seekTanglesLinkRoot (buffer, start, pValue) { + if (pValue < 0) return -1 + const pValueContent = seekKey2(buffer, pValue, B_CONTENT, 0) + if (pValueContent < 0) return -1 + const pValueContentTangles = seekKey2(buffer, pValueContent, B_TANGLES, 0) + if (pValueContentTangles < 0) return -1 + const pValueContentTanglesLink = seekKey2(buffer, pValueContentTangles, B_LINK, 0) + if (pValueContentTanglesLink < 0) return -1 + return seekKey2(buffer, pValueContentTanglesLink, B_ROOT, 0) + } + + function seekTanglesLinkPrevious (buffer, start, pValue) { + if (pValue < 0) return -1 + const pValueContent = seekKey2(buffer, pValue, B_CONTENT, 0) + if (pValueContent < 0) return -1 + const pValueContentTangles = seekKey2(buffer, pValueContent, B_TANGLES, 0) + if (pValueContentTangles < 0) return -1 + const pValueContentTanglesLink = seekKey2(buffer, pValueContentTangles, B_LINK, 0) + if (pValueContentTanglesLink < 0) return -1 + return seekKey2(buffer, pValueContentTanglesLink, B_PREVIOUS, 0) + } // NOTE this is not generalised to all links, it's about group links function findLinks (type, opts = {}, cb) { @@ -15,25 +75,19 @@ module.exports = function Link (ssb) { if (child && !isGroup(child)) return cb(new Error(`findLinks expected a groupId for child, got ${child} instead.`)) if (!parent && !child) return cb(new Error('findLinks expects a parent or child id to be given')) - const query = [{ - $filter: { - value: { - content: { - type, - ...opts, - tangles: { - link: { root: null, previous: null } - } - } - } - } - }] - pull( - // NOTE: using ssb-query instead of ssb-backlinks - // because the backlinks query will get EVERY message which contains the groupId in it, which will be a LOT for a group - // then filters that massive amount down to the ones which have the dest in the right place - ssb.query.read({ query }), + ssb.db.query( + where( + and( + dbType(type), + parent && equal(seekParent, parent, { indexType: 'linkParent', prefix: true }), + child && equal(seekChild, child, { indexType: 'linkChild', prefix: true }), + equal(seekTanglesLinkRoot, null, { indexType: 'tanglesLinkRoot', prefix: true }), + equal(seekTanglesLinkPrevious, null, { indexType: 'tanglesLinkPrevious', prefix: true }) + ) + ), + toPullStream() + ), pull.unique(link => { if (parent) return link.value.content.child else return link.value.content.parent @@ -89,26 +143,19 @@ module.exports = function Link (ssb) { findGroupByFeedId (feedId, cb) { if (!isFeed(feedId)) return cb(new Error('requires a valid feedId')) - const query = [{ - $filter: { - value: { - author: feedId, // link published by this person - content: { - type: 'link/feed-group', - parent: feedId, // and linking from that feedId - tangles: { - link: { root: null, previous: null } - } - } - } - } - }] - pull( - // NOTE: using ssb-query instead of ssb-backlinks - // because the backlinks query will get EVERY message which contains the groupId in it, which will be a LOT for a group - // then filters that massive amount down to the ones which have the dest in the right place - ssb.query.read({ query }), + ssb.db.query( + where( + and( + author(feedId), + dbType('link/feed-group'), + equal(seekParent, feedId, { indexType: 'parent', prefix: true }), + equal(seekTanglesLinkRoot, null, { indexType: 'tanglesLinkRoot', prefix: true }), + equal(seekTanglesLinkPrevious, null, { indexType: 'tanglesLinkPrevious', prefix: true }) + ) + ), + toPullStream() + ), pull.filter(feedGroupLink.spec.isRoot), pull.filter(link => { return link.value.content.child === link.value.content.recps[0] diff --git a/package-lock.json b/package-lock.json index c6fab065..069df3e6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,24 +13,19 @@ "@tangle/overwrite": "^3.0.1", "@tangle/reduce": "^5.0.5", "@tangle/strategy": "^4.1.2", - "charwise": "^3.0.1", "envelope-js": "^1.3.2", "envelope-spec": "^1.1.1", - "futoin-hkdf": "^1.5.2", "is-my-json-valid": "^2.20.6", "is-my-ssb-valid": "^1.2.2", - "level": "^6.0.1", "lodash.set": "^4.3.2", - "mkdirp": "^1.0.4", - "obz": "^1.1.0", "private-group-spec": "^8.0.0", "pull-level": "^2.0.4", + "pull-many": "^1.0.9", "pull-paramap": "^1.2.2", "pull-stream": "^3.7.0", "sodium-native": "^3.4.1", "ssb-bfe": "^3.7.0", - "ssb-crut": "^4.6.2", - "ssb-keyring": "^5.6.0", + "ssb-crut": "^5.1.0", "ssb-keys": "^8.5.0", "ssb-private-group-keys": "^0.4.1", "ssb-ref": "^2.16.0", @@ -40,8 +35,8 @@ "cross-env": "^7.0.3", "is-canonical-base64": "^1.1.1", "scuttle-testbot": "^2.1.0", - "ssb-backlinks": "^2.1.1", - "ssb-query": "^2.4.5", + "ssb-box2": "^7.2.0", + "ssb-db2": "^7.1.1", "ssb-replicate": "^1.3.3", "standard": "^17.1.0", "tap-arc": "^0.4.0", @@ -296,6 +291,7 @@ "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "dev": true, "dependencies": { "buffer": "^5.5.0", "immediate": "^3.2.3", @@ -643,6 +639,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, "funding": [ { "type": "github", @@ -658,21 +655,6 @@ } ] }, - "node_modules/base64-url": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/base64-url/-/base64-url-2.3.3.tgz", - "integrity": "sha512-dLMhIsK7OplcDauDH/tZLvK7JmUZK3A7KiQpjNzsBrM6Etw7hzNI1tLEywqJk9NnwkgWuFKSlx/IUO7vF6Mo8Q==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/binary-search": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", - "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", - "dev": true - }, "node_modules/binary-search-bounds": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/binary-search-bounds/-/binary-search-bounds-2.0.5.tgz", @@ -720,6 +702,7 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, "funding": [ { "type": "github", @@ -848,7 +831,8 @@ "node_modules/charwise": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/charwise/-/charwise-3.0.1.tgz", - "integrity": "sha512-RcdumNsM6fJZ5HHbYunqj2bpurVRGsXour3OR+SlLEHFhG6ALm54i6Osnh+OvO7kEoSBzwExpblYFH8zKQiEPw==" + "integrity": "sha512-RcdumNsM6fJZ5HHbYunqj2bpurVRGsXour3OR+SlLEHFhG6ALm54i6Osnh+OvO7kEoSBzwExpblYFH8zKQiEPw==", + "dev": true }, "node_modules/chloride": { "version": "2.4.1", @@ -909,64 +893,6 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, - "node_modules/compare-at-paths": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/compare-at-paths/-/compare-at-paths-1.0.0.tgz", - "integrity": "sha512-Ke1ejo/RZ+Hzku4gcW34uPMOR4Cpq87MAotELgV9mwiAzDN726cu+eWo0zWg1vRIfyf6yK5bW9uIW+c/SksQ5w==", - "dev": true, - "dependencies": { - "libnested": "^1.3.2", - "tape": "^4.9.1", - "typewiselite": "^1.0.0" - } - }, - "node_modules/compare-at-paths/node_modules/deep-equal": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz", - "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==", - "dev": true, - "dependencies": { - "is-arguments": "^1.0.4", - "is-date-object": "^1.0.1", - "is-regex": "^1.0.4", - "object-is": "^1.0.1", - "object-keys": "^1.1.1", - "regexp.prototype.flags": "^1.2.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/compare-at-paths/node_modules/tape": { - "version": "4.17.0", - "resolved": "https://registry.npmjs.org/tape/-/tape-4.17.0.tgz", - "integrity": "sha512-KCuXjYxCZ3ru40dmND+oCLsXyuA8hoseu2SS404Px5ouyS0A99v8X/mdiLqsR5MTAyamMBN7PRwt2Dv3+xGIxw==", - "dev": true, - "dependencies": { - "@ljharb/resumer": "~0.0.1", - "@ljharb/through": "~2.3.9", - "call-bind": "~1.0.2", - "deep-equal": "~1.1.1", - "defined": "~1.0.1", - "dotignore": "~0.1.2", - "for-each": "~0.3.3", - "glob": "~7.2.3", - "has": "~1.0.3", - "inherits": "~2.0.4", - "is-regex": "~1.1.4", - "minimist": "~1.2.8", - "mock-property": "~1.0.0", - "object-inspect": "~1.12.3", - "resolve": "~1.22.6", - "string.prototype.trim": "~1.2.8" - }, - "bin": { - "tape": "bin/tape" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1142,6 +1068,7 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "dev": true, "dependencies": { "abstract-leveldown": "~6.2.1", "inherits": "^2.0.3" @@ -1240,6 +1167,7 @@ "version": "6.3.0", "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "dev": true, "dependencies": { "abstract-leveldown": "^6.2.1", "inherits": "^2.0.3", @@ -1283,6 +1211,7 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, "dependencies": { "prr": "~1.0.1" }, @@ -2104,51 +2033,6 @@ "pull-write": "^1.1.1" } }, - "node_modules/flumeview-links": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/flumeview-links/-/flumeview-links-2.1.0.tgz", - "integrity": "sha512-VzqbNJ9qYE9eopCrhWXlPSoIoW4xanMy6YFyVSaYAwQIPoIZ+F1s1D3vmzOLXGhPesgHiXAlxmFSzXM5C9DKbQ==", - "dev": true, - "dependencies": { - "deep-equal": "^2.0.3", - "flumeview-level": "^4.0.3", - "map-filter-reduce": "^3.2.2", - "pull-flatmap": "0.0.1", - "pull-stream": "^3.6.14" - } - }, - "node_modules/flumeview-query": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/flumeview-query/-/flumeview-query-8.0.0.tgz", - "integrity": "sha512-uPTT5I26ePMc6Xhjebu1aiaHAd7P3EqyE9SZB6B9ZIvXtMXhFYNk7iO1yzh1ZXp3aYzdYmrI9k8mSz9urZ9gNQ==", - "dev": true, - "dependencies": { - "deep-equal": "^1.0.1", - "flumeview-level": "^4.0.3", - "map-filter-reduce": "^3.2.0", - "pull-flatmap": "0.0.1", - "pull-paramap": "^1.1.3", - "pull-sink-through": "0.0.0", - "pull-stream": "^3.4.0" - } - }, - "node_modules/flumeview-query/node_modules/deep-equal": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz", - "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==", - "dev": true, - "dependencies": { - "is-arguments": "^1.0.4", - "is-date-object": "^1.0.1", - "is-regex": "^1.0.4", - "object-is": "^1.0.1", - "object-keys": "^1.1.1", - "regexp.prototype.flags": "^1.2.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/flumeview-reduce": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/flumeview-reduce/-/flumeview-reduce-1.4.0.tgz", @@ -2515,6 +2399,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, "funding": [ { "type": "github", @@ -2542,7 +2427,8 @@ "node_modules/immediate": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.3.0.tgz", - "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==" + "integrity": "sha512-HR7EVodfFUdQCTIeySw+WDRFJlPcLOJbXfwwZ7Oom6tjsvZ3bOkCDJHehQC3nxJrv7+f9XecwazynjU8e4Vw3Q==", + "dev": true }, "node_modules/import-fresh": { "version": "3.3.0", @@ -3167,6 +3053,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", + "dev": true, "dependencies": { "level-js": "^5.0.0", "level-packager": "^5.1.0", @@ -3184,6 +3071,7 @@ "version": "9.0.2", "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "dev": true, "dependencies": { "buffer": "^5.6.0" }, @@ -3195,6 +3083,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==", + "dev": true, "engines": { "node": ">=6" } @@ -3203,6 +3092,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "dev": true, "dependencies": { "errno": "~0.1.1" }, @@ -3214,6 +3104,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "dev": true, "dependencies": { "inherits": "^2.0.4", "readable-stream": "^3.4.0", @@ -3227,6 +3118,7 @@ "version": "5.0.2", "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "dev": true, "dependencies": { "abstract-leveldown": "~6.2.3", "buffer": "^5.5.0", @@ -3238,6 +3130,7 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "dev": true, "dependencies": { "encoding-down": "^6.3.0", "levelup": "^4.3.2" @@ -3258,6 +3151,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", + "dev": true, "dependencies": { "xtend": "^4.0.2" }, @@ -3269,6 +3163,7 @@ "version": "5.6.0", "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "dev": true, "hasInstallScript": true, "dependencies": { "abstract-leveldown": "~6.2.1", @@ -3283,6 +3178,7 @@ "version": "4.4.0", "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", + "dev": true, "dependencies": { "deferred-leveldown": "~5.3.0", "level-errors": "~2.0.0", @@ -3307,12 +3203,6 @@ "node": ">= 0.8.0" } }, - "node_modules/libnested": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/libnested/-/libnested-1.5.2.tgz", - "integrity": "sha512-DbiwHL8454goYRp5Xn9vUA5XU6x8rNh8BmZ7ywSTUhVBIiDS7ev/FT6+AwU2/ZKW2jEOC7WKhpkJfExaQwosRA==", - "dev": true - }, "node_modules/libsodium": { "version": "0.7.13", "resolved": "https://registry.npmjs.org/libsodium/-/libsodium-0.7.13.tgz", @@ -3386,7 +3276,8 @@ "node_modules/lodash.find": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/lodash.find/-/lodash.find-4.6.0.tgz", - "integrity": "sha512-yaRZoAV3Xq28F1iafWN1+a0rflOej93l1DQUejs3SZ41h2O9UJBoS9aueGjPDgAl4B6tPC0NuuchLKaDQQ3Isg==" + "integrity": "sha512-yaRZoAV3Xq28F1iafWN1+a0rflOej93l1DQUejs3SZ41h2O9UJBoS9aueGjPDgAl4B6tPC0NuuchLKaDQQ3Isg==", + "dev": true }, "node_modules/lodash.isequal": { "version": "4.5.0", @@ -3437,20 +3328,6 @@ "resolved": "https://registry.npmjs.org/ltgt/-/ltgt-2.2.1.tgz", "integrity": "sha512-AI2r85+4MquTw9ZYqabu4nMwy9Oftlfa/e/52t9IjtfG+mGBbTNdAoZ3RQKLHR6r0wQnwZnPIEh/Ya6XTWAKNA==" }, - "node_modules/map-filter-reduce": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/map-filter-reduce/-/map-filter-reduce-3.2.2.tgz", - "integrity": "sha512-p+NIGQbEBxlw/qWwG+NME98G/9kjOQI70hmaH8QEZtIWfTmfMYLKQW4PJChP4izPHNAxlOfv/qefP0+2ZXn84A==", - "dev": true, - "dependencies": { - "binary-search": "^1.2.0", - "compare-at-paths": "^1.0.0", - "pull-sink-through": "0.0.0", - "pull-sort": "^1.0.1", - "pull-stream": "^3.4.3", - "typewiselite": "^1.0.0" - } - }, "node_modules/map-merge": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/map-merge/-/map-merge-1.1.0.tgz", @@ -3493,6 +3370,7 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, "bin": { "mkdirp": "bin/cmd.js" }, @@ -3622,7 +3500,8 @@ "node_modules/napi-macros": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-macros/-/napi-macros-2.0.0.tgz", - "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==" + "integrity": "sha512-A0xLykHtARfueITVDernsAWdtIMbOJgKgcluwENp3AlsKN/PloyO10HtmoqnFAQAcxPkgZN7wdfPfEd0zNGxbg==", + "dev": true }, "node_modules/natural-compare": { "version": "1.4.0", @@ -3661,6 +3540,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==", + "dev": true, "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", @@ -3837,7 +3717,8 @@ "node_modules/obz": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/obz/-/obz-1.1.0.tgz", - "integrity": "sha512-cG6v76kgWh48urwdsFSkxQlKWCKFYkxZJMhOIG9Aj1uPKTnNW9Hvo/ROyBfGzqaZD3K75K3jhsanKssRPkNKYA==" + "integrity": "sha512-cG6v76kgWh48urwdsFSkxQlKWCKFYkxZJMhOIG9Aj1uPKTnNW9Hvo/ROyBfGzqaZD3K75K3jhsanKssRPkNKYA==", + "dev": true }, "node_modules/on-change-network-strict": { "version": "1.0.0", @@ -4166,7 +4047,8 @@ "node_modules/prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==" + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true }, "node_modules/pull-abortable": { "version": "4.1.1", @@ -4241,12 +4123,6 @@ "pull-pause": "~0.0.2" } }, - "node_modules/pull-flatmap": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/pull-flatmap/-/pull-flatmap-0.0.1.tgz", - "integrity": "sha512-9BgwZPZ6J22kPf9ExoVI3m2yMHdCK8uPf58p6L63t36IgH7NrCX+p3QV8cQ4JmYjwvXDZzimVuJ7IW7iLxm7cA==", - "dev": true - }, "node_modules/pull-goodbye": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/pull-goodbye/-/pull-goodbye-0.0.3.tgz", @@ -4310,6 +4186,14 @@ "looper": "^4.0.0" } }, + "node_modules/pull-many": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/pull-many/-/pull-many-1.0.9.tgz", + "integrity": "sha512-+jUydDVlj/HsvtDqxWMSsiRq3B0HVo7RhBV4C0p2nZRS3mFTUEu9SPEBN+B5PMaW8KTnblYhTIaKg7oXgGnj4Q==", + "dependencies": { + "pull-stream": "^3.4.5" + } + }, "node_modules/pull-next": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/pull-next/-/pull-next-1.0.1.tgz", @@ -4376,22 +4260,6 @@ "integrity": "sha512-CBkejkE5nX50SiSEzu0Qoz4POTJMS/mw8G6aj3h3M/RJoKgggLxyF0IyTZ0mmpXFlXRcLmLmIEW4xeYn7AeDYw==", "dev": true }, - "node_modules/pull-sink-through": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/pull-sink-through/-/pull-sink-through-0.0.0.tgz", - "integrity": "sha512-XjpF/3+WRsWfw+XhPgj3I/6WO15VYOKLsXAtWt1sRKOqA6N7bnarut5zuE2GcxbAzspaQxKmMcbUYZ8QorPxcQ==", - "dev": true - }, - "node_modules/pull-sort": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/pull-sort/-/pull-sort-1.0.2.tgz", - "integrity": "sha512-jGcAHMP+0Le+bEIhSODlbNNd3jW+S6XrXOlhVzfcKU5HQZjP92OzQSgHHSlwvWRsiTWi+UGgbFpL/5gGgmFoVQ==", - "dev": true, - "dependencies": { - "pull-defer": "^0.2.3", - "pull-stream": "^3.6.9" - } - }, "node_modules/pull-stream": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/pull-stream/-/pull-stream-3.7.0.tgz", @@ -4634,6 +4502,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -5171,18 +5040,6 @@ "ssb-db2": ">=3.4.0" } }, - "node_modules/ssb-backlinks": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ssb-backlinks/-/ssb-backlinks-2.1.1.tgz", - "integrity": "sha512-iv/B5EyjvNiKOeT3RkTuBRyU14GJ1sf5wnr07JOWeVI3OyNzedZ8z229yzZSaGHTYpbiSOcs9Z2CR8CkLQupQQ==", - "dev": true, - "dependencies": { - "base64-url": "^2.3.3", - "flumeview-links": "^2.1.0", - "pull-stream": "^3.6.14", - "ssb-ref": "^2.14.0" - } - }, "node_modules/ssb-bfe": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/ssb-bfe/-/ssb-bfe-3.7.0.tgz", @@ -5215,47 +5072,59 @@ } }, "node_modules/ssb-box2": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ssb-box2/-/ssb-box2-3.0.1.tgz", - "integrity": "sha512-jQhrsEyrpqnUciEb1qzqc/SJCAx3hTm48BMzMy1bE//xQthWRahigTffflOM7pdRieGlxwGlHc8PpagBBZOkhA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/ssb-box2/-/ssb-box2-7.2.0.tgz", + "integrity": "sha512-LOIgzULj8wNPNk89zmLTRcc35Xq35uOuG/4Rq74ae4pPVY50oYEir7mdRNRiAjuIPTxbVmY6YwXVUehBEkxk1w==", "dev": true, "dependencies": { - "envelope-js": "^1.3.0", - "private-group-spec": "^1.2.0", + "envelope-js": "^1.3.2", + "private-group-spec": "^2.1.1", + "pull-defer": "^0.2.3", + "pull-stream": "^3.6.14", "ssb-bfe": "^3.7.0", - "ssb-keyring": "^2.2.0", - "ssb-private-group-keys": "^0.4.1", + "ssb-keyring": "^7.0.0", + "ssb-private-group-keys": "^1.1.1", "ssb-ref": "^2.16.0", - "ssb-uri2": "^2.4.0" + "ssb-uri2": "^2.4.1" } }, "node_modules/ssb-box2/node_modules/private-group-spec": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-1.2.0.tgz", - "integrity": "sha512-O7SfG+vZIZgqDXy/wjsuTRI5LaozW4rxaZBpGmwlcDfjIvxvYWNboyNm1PoQUU6j4dQ02V1tOQVLDq9u2RzolA==", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-2.1.1.tgz", + "integrity": "sha512-j8mL15mfUWZVrEjlTzCgvJ75dDUNBzUt760d42CFE1yjxAmNBLxtTu7ef/8nLBMmTjLEsnsAsnPjeI+NSpdlug==", "dev": true, "dependencies": { "is-my-ssb-valid": "^1.2.0" } }, "node_modules/ssb-box2/node_modules/ssb-keyring": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ssb-keyring/-/ssb-keyring-2.2.0.tgz", - "integrity": "sha512-pXNzYsQbtdWoWAjEPT5lVrZs/Px4ypJU2m4S34tTN6KivZukHPcflj1/IbZSDj7kHJGQhJRcyqGQ3MxgQdFBrA==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ssb-keyring/-/ssb-keyring-7.0.0.tgz", + "integrity": "sha512-HBeAFCdjqSLCiors10s+AHfaFm78ac97oqeWgDg9Es4gK2LUs/ZQ1DjoYH+BS4EnTh5yE+Y9E8ska4WWc0p4Dw==", "dev": true, "dependencies": { "charwise": "^3.0.1", "level": "^6.0.1", + "lodash.find": "^4.6.0", "mkdirp": "^1.0.4", - "private-group-spec": "^1.1.3", + "private-group-spec": "^8.0.0", "pull-level": "^2.0.4", - "pull-stream": "^3.6.14", - "ssb-private-group-keys": "^1.0.0", + "pull-stream": "^3.7.0", + "ssb-private-group-keys": "^1.1.2", "ssb-ref": "^2.16.0", - "ssb-uri2": "^2.4.0" + "ssb-uri2": "^2.4.1" + } + }, + "node_modules/ssb-box2/node_modules/ssb-keyring/node_modules/private-group-spec": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-8.0.0.tgz", + "integrity": "sha512-wahEIhxe27fIJjFs52XyekLc4CQyHyu7FjYZJbiSCMxTYxb9cXluS1uRtkPfPwkwIPTDtxloO8Rf/3LvcppwNQ==", + "dev": true, + "dependencies": { + "is-my-ssb-valid": "^1.2.0" } }, - "node_modules/ssb-box2/node_modules/ssb-keyring/node_modules/ssb-private-group-keys": { + "node_modules/ssb-box2/node_modules/ssb-private-group-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/ssb-private-group-keys/-/ssb-private-group-keys-1.1.2.tgz", "integrity": "sha512-0UPPmxy61qmbDmP71J2vhX6UPfCtXa/CNehxYTgk2+AaLXsnA0perGZAiOWm9niGEU50TYYC5/jsIfjz4IiD9A==", @@ -5268,6 +5137,15 @@ "ssb-bfe": "^3.5.0" } }, + "node_modules/ssb-box2/node_modules/ssb-private-group-keys/node_modules/private-group-spec": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-1.2.0.tgz", + "integrity": "sha512-O7SfG+vZIZgqDXy/wjsuTRI5LaozW4rxaZBpGmwlcDfjIvxvYWNboyNm1PoQUU6j4dQ02V1tOQVLDq9u2RzolA==", + "dev": true, + "dependencies": { + "is-my-ssb-valid": "^1.2.0" + } + }, "node_modules/ssb-classic": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/ssb-classic/-/ssb-classic-1.1.0.tgz", @@ -5367,10 +5245,11 @@ } }, "node_modules/ssb-crut": { - "version": "4.6.3", - "resolved": "https://registry.npmjs.org/ssb-crut/-/ssb-crut-4.6.3.tgz", - "integrity": "sha512-PZ95Rq8D1u4mBp4zI+uT8C/cEhiw82D3s8FUUJYsV0xrP+PI/uUmOaRiE0T9n/Z7KNkn0fyCYFK9IJUwkDN5MA==", + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ssb-crut/-/ssb-crut-5.1.0.tgz", + "integrity": "sha512-tgvUWJwOUMAfEk3B7++fhxKmlE7/h+kNoKWTNoPNUWIo0TdoS/2n9zsaYdULwHfxZsZcozoUvONf7xy5BNgZRg==", "dependencies": { + "@tangle/graph": "^3.2.0", "@tangle/reduce": "^5.0.4", "@tangle/strategy": "^4.1.2", "fast-json-stable-stringify": "^2.1.0", @@ -5494,27 +5373,52 @@ "integrity": "sha512-J437PvCMZZKNT88+VRh6dkmh1ndZzwGwDzb5ZZl3QEsl+U9wIlt8hG+Y1gXVOhH75gf8JyceKGiG6WFUBbxyGQ==", "dev": true }, - "node_modules/ssb-keyring": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/ssb-keyring/-/ssb-keyring-5.6.0.tgz", - "integrity": "sha512-6pLwJkcHzJcnmWgiAScYTwUbPYvMxGem4X25Y00rIukbVElhKZ2cx6bmeOpKdlVkCTdM+qc2/q3XLLGKsZoWwA==", + "node_modules/ssb-db2/node_modules/private-group-spec": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-1.2.0.tgz", + "integrity": "sha512-O7SfG+vZIZgqDXy/wjsuTRI5LaozW4rxaZBpGmwlcDfjIvxvYWNboyNm1PoQUU6j4dQ02V1tOQVLDq9u2RzolA==", + "dev": true, + "dependencies": { + "is-my-ssb-valid": "^1.2.0" + } + }, + "node_modules/ssb-db2/node_modules/ssb-box2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ssb-box2/-/ssb-box2-3.0.1.tgz", + "integrity": "sha512-jQhrsEyrpqnUciEb1qzqc/SJCAx3hTm48BMzMy1bE//xQthWRahigTffflOM7pdRieGlxwGlHc8PpagBBZOkhA==", + "dev": true, + "dependencies": { + "envelope-js": "^1.3.0", + "private-group-spec": "^1.2.0", + "ssb-bfe": "^3.7.0", + "ssb-keyring": "^2.2.0", + "ssb-private-group-keys": "^0.4.1", + "ssb-ref": "^2.16.0", + "ssb-uri2": "^2.4.0" + } + }, + "node_modules/ssb-db2/node_modules/ssb-keyring": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ssb-keyring/-/ssb-keyring-2.2.0.tgz", + "integrity": "sha512-pXNzYsQbtdWoWAjEPT5lVrZs/Px4ypJU2m4S34tTN6KivZukHPcflj1/IbZSDj7kHJGQhJRcyqGQ3MxgQdFBrA==", + "dev": true, "dependencies": { "charwise": "^3.0.1", "level": "^6.0.1", - "lodash.find": "^4.6.0", "mkdirp": "^1.0.4", - "private-group-spec": "^8.0.0", + "private-group-spec": "^1.1.3", "pull-level": "^2.0.4", - "pull-stream": "^3.7.0", - "ssb-private-group-keys": "^1.1.2", + "pull-stream": "^3.6.14", + "ssb-private-group-keys": "^1.0.0", "ssb-ref": "^2.16.0", - "ssb-uri2": "^2.4.1" + "ssb-uri2": "^2.4.0" } }, - "node_modules/ssb-keyring/node_modules/ssb-private-group-keys": { + "node_modules/ssb-db2/node_modules/ssb-keyring/node_modules/ssb-private-group-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/ssb-private-group-keys/-/ssb-private-group-keys-1.1.2.tgz", "integrity": "sha512-0UPPmxy61qmbDmP71J2vhX6UPfCtXa/CNehxYTgk2+AaLXsnA0perGZAiOWm9niGEU50TYYC5/jsIfjz4IiD9A==", + "dev": true, "dependencies": { "envelope-js": "^1.3.2", "futoin-hkdf": "^1.5.1", @@ -5523,14 +5427,6 @@ "ssb-bfe": "^3.5.0" } }, - "node_modules/ssb-keyring/node_modules/ssb-private-group-keys/node_modules/private-group-spec": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/private-group-spec/-/private-group-spec-1.2.0.tgz", - "integrity": "sha512-O7SfG+vZIZgqDXy/wjsuTRI5LaozW4rxaZBpGmwlcDfjIvxvYWNboyNm1PoQUU6j4dQ02V1tOQVLDq9u2RzolA==", - "dependencies": { - "is-my-ssb-valid": "^1.2.0" - } - }, "node_modules/ssb-keys": { "version": "8.5.0", "resolved": "https://registry.npmjs.org/ssb-keys/-/ssb-keys-8.5.0.tgz", @@ -5590,17 +5486,6 @@ "is-my-ssb-valid": "^1.2.0" } }, - "node_modules/ssb-query": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/ssb-query/-/ssb-query-2.4.5.tgz", - "integrity": "sha512-/QX6+DJkghqq1ZTbgYpOvaI+gx2O7ee1TRUM9yiOlVjh1XAQBevcBj0zO+W3TsNllX86urqBrySd/AEfFfUpIw==", - "dev": true, - "dependencies": { - "explain-error": "^1.0.1", - "flumeview-query": "^8.0.0", - "pull-stream": "^3.6.2" - } - }, "node_modules/ssb-ref": { "version": "2.16.0", "resolved": "https://registry.npmjs.org/ssb-ref/-/ssb-ref-2.16.0.tgz", @@ -5786,6 +5671,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -6241,12 +6127,6 @@ "integrity": "sha512-B2B+wo5gC7VRAqcFEiUCjS6CJ1vmeYZ7uzY3Jsu6UNStHiF+W0vkhZAmQaj5m9sC2KVrpyHGRzGuhz3M6+n/8A==", "dev": true }, - "node_modules/typewiselite": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/typewiselite/-/typewiselite-1.0.0.tgz", - "integrity": "sha512-J9alhjVHupW3Wfz6qFRGgQw0N3gr8hOkw6zm7FZ6UR1Cse/oD9/JVok7DNE9TT9IbciDHX2Ex9+ksE6cRmtymw==", - "dev": true - }, "node_modules/uint48be": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/uint48be/-/uint48be-2.0.1.tgz", @@ -6295,7 +6175,8 @@ "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true }, "node_modules/version-guard": { "version": "1.1.1", diff --git a/package.json b/package.json index 2269e2d1..23827538 100644 --- a/package.json +++ b/package.json @@ -21,24 +21,19 @@ "@tangle/overwrite": "^3.0.1", "@tangle/reduce": "^5.0.5", "@tangle/strategy": "^4.1.2", - "charwise": "^3.0.1", "envelope-js": "^1.3.2", "envelope-spec": "^1.1.1", - "futoin-hkdf": "^1.5.2", "is-my-json-valid": "^2.20.6", "is-my-ssb-valid": "^1.2.2", - "level": "^6.0.1", "lodash.set": "^4.3.2", - "mkdirp": "^1.0.4", - "obz": "^1.1.0", "private-group-spec": "^8.0.0", "pull-level": "^2.0.4", + "pull-many": "^1.0.9", "pull-paramap": "^1.2.2", "pull-stream": "^3.7.0", "sodium-native": "^3.4.1", "ssb-bfe": "^3.7.0", - "ssb-crut": "^4.6.2", - "ssb-keyring": "^5.6.0", + "ssb-crut": "^5.1.0", "ssb-keys": "^8.5.0", "ssb-private-group-keys": "^0.4.1", "ssb-ref": "^2.16.0", @@ -48,8 +43,8 @@ "cross-env": "^7.0.3", "is-canonical-base64": "^1.1.1", "scuttle-testbot": "^2.1.0", - "ssb-backlinks": "^2.1.1", - "ssb-query": "^2.4.5", + "ssb-box2": "^7.2.0", + "ssb-db2": "^7.1.1", "ssb-replicate": "^1.3.3", "standard": "^17.1.0", "tap-arc": "^0.4.0", diff --git a/rebuild-manager.js b/rebuild-manager.js deleted file mode 100644 index f481d8ba..00000000 --- a/rebuild-manager.js +++ /dev/null @@ -1,117 +0,0 @@ -const Obz = require('obz') - -/* - * We want the RebuildManager to field requests for a db rebuild to be done. - * It can receive many of these, but will only start the rebuild once indexing is complete - * The reason for this is to minimise the number of times a rebuild is required - we don't want to - * have to do one for each time we discover a new group member... - * - * After the rebuild is complete the RebuildManager ensures all callbacks passed to it are then run - */ - -const log = (...str) => console.info('db.rebuild', ...str) - -module.exports = class RebuildManager { - constructor (ssb) { - this.ssb = ssb - // isIndexing - this.isInitializing = false - this.isRebuilding = false - - this.requests = new Requests() - this.nextRequests = new Requests() - - ssb.rebuild.hook((rebuild, [cb]) => { - log('(ﾉ´ヮ´)ﾉ*:･ﾟ✧') - // log('reasons', JSON.stringify(this.requests.reasons, null, 2)) - this.isRebuilding = true - - rebuild(err => { - log('finished', ssb.id) - // rebuild done - this.requests.callback(err) - - this.requests = this.nextRequests - this.nextRequests = new Requests() - this.isRebuilding = false - - if (typeof cb === 'function') cb(err) - else err && console.error(err) - - // if there are outstanding rebuild requests, start a new rebuild - if (this.requests.hasRequests()) { - this.initiateRebuild() - } - }) - }) - } - - rebuild (reason, cb) { - if (this.isRebuilding) { - // if the current rebuild was already kicked off by a reason we're not registering, it's safe to - // add it to the current rebuild process (so long as the "reasons" was specific enough) - if (this.requests.has(reason)) this.requests.add(reason, cb) - // otherwise, queue it up for the next round of rebuilds - this is because we may have added new things to - // the keystore which would justify re-trying decrypting all messages - else this.nextRequests.add(reason, cb) - - return - } - - this.requests.add(reason, cb) - this.initiateRebuild() - } - - initiateRebuild () { - if (this.isInitializing) return - - this.isInitializing = true - const interval = setInterval( - () => { - if (this.isIndexing) return - - this.ssb.rebuild() - this.isInitializing = false - - clearInterval(interval) - }, - 100 - ) - } - - get isIndexing () { - return this.ssb.status().sync.sync !== true - } -} - -function Requests () { - const reasons = new Set([]) - const callbacks = Obz() - - return { - add (reason, cb) { - if (!reason) throw new Error('rebuild requests request a reason') - reasons.add(reason) - - if (cb === undefined) return - - if (typeof cb === 'function') callbacks.once(cb) - else throw new Error(`expected cb to be function, got ${cb}`) - }, - has (reason) { - return reasons.has(reason) - }, - hasRequests () { - return reasons.size > 0 - }, - callback (err) { - callbacks.set(err) - }, - get reasons () { - return Array.from(reasons) - }, - get size () { - return reasons.size - } - } -} diff --git a/test/api/add-po-box.test.js b/test/api/add-po-box.test.js index 150946a8..93767821 100644 --- a/test/api/add-po-box.test.js +++ b/test/api/add-po-box.test.js @@ -1,6 +1,7 @@ const test = require('tape') const pull = require('pull-stream') const isPoBox = require('ssb-private-group-keys/lib/is-po-box') // TODO find better home +const { where, and, author, isDecrypted, descending, toPullStream } = require('ssb-db2/operators') const { Server } = require('../helpers') @@ -17,7 +18,15 @@ test('tribes.addPOBox', t => { t.true(isPoBox(poBoxId), 'adding P.O. Box to group returns poBoxId') pull( - server.createUserStream({ id: server.id, reverse: true, limit: 1, private: true }), + server.db.query( + where(and( + author(server.id), + isDecrypted('box2') + )), + descending(), + toPullStream() + ), + pull.take(1), pull.drain(m => { const { type, recps, keys } = m.value.content t.equal(type, 'group/po-box', 'po-box type msg') @@ -27,6 +36,8 @@ test('tribes.addPOBox', t => { server.close() t.end() + }, err => { + if (err) t.fail(err) }) ) }) diff --git a/test/api/create.test.js b/test/api/create.test.js index 4678e083..69874614 100644 --- a/test/api/create.test.js +++ b/test/api/create.test.js @@ -2,6 +2,7 @@ const test = require('tape') const { isCloakedMsg: isGroup } = require('ssb-ref') const isPoBox = require('ssb-private-group-keys/lib/is-po-box') // TODO find better home const pull = require('pull-stream') +const { where, and, author, isDecrypted, toPullStream, descending } = require('ssb-db2/operators') const { Server } = require('../helpers') @@ -15,7 +16,7 @@ test('tribes.create', t => { const { groupId, groupKey, groupInitMsg } = data t.true(isGroup(groupId), 'returns group identifier - groupId') t.true(Buffer.isBuffer(groupKey) && groupKey.length === 32, 'returns group symmetric key - groupKey') - t.match(groupInitMsg.value.content, /^[a-zA-Z0-9/+]+=*\.box2$/, 'encrypted init msg') + t.match(groupInitMsg.meta.originalContent, /^[a-zA-Z0-9/+]+=*\.box2$/, 'encrypted init msg') server.get({ id: groupInitMsg.key, private: true }, (err, value) => { if (err) throw err @@ -33,7 +34,16 @@ test('tribes.create', t => { // check I published a group/add-member to myself pull( - server.createUserStream({ id: server.id, private: true, reverse: true }), + server.db.query( + where( + and( + isDecrypted('box2'), + author(server.id) + ) + ), + descending(), + toPullStream() + ), pull.map(msg => msg.value.content), pull.collect((err, msgContents) => { if (err) throw err diff --git a/test/api/invite.test.js b/test/api/invite.test.js index bed35ec4..4072a89f 100644 --- a/test/api/invite.test.js +++ b/test/api/invite.test.js @@ -1,3 +1,5 @@ +const pull = require('pull-stream') +const { where, author, toPromise, descending } = require('ssb-db2/operators') const { test, p, Run, Server, replicate, FeedId } = require('../helpers') test('tribes.invite', async t => { @@ -5,10 +7,21 @@ test('tribes.invite', async t => { const kaitiaki = Server({ name: 'kaitiaki' }) const newPerson = Server({ name: 'new-person', debug: !true }) - const { groupId, groupKey, groupInitMsg } = await p(kaitiaki.tribes.create)({}) + const { groupId, groupKey, groupInitMsg } = await p(kaitiaki.tribes.create)({}).catch(err => { + console.error(err) + t.fail(err) + }) t.true(groupId, 'creates group') - const selfAdd = await p(kaitiaki.getLatest)(kaitiaki.id) + const selfAdd = (await pull( + kaitiaki.db.query( + where(author(kaitiaki.id)), + descending(), + toPromise() + ) + ))[0] + + t.true(selfAdd, 'got addition of self') const authorIds = [ newPerson.id, @@ -44,36 +57,14 @@ test('tribes.invite', async t => { } const { key: greetingKey } = await run( 'kaitiaki published message', - p(kaitiaki.publish)(greetingContent) + p(kaitiaki.tribes.publish)(greetingContent) ) - let numberRebuilds = 0 - const rebuildPromise = new Promise((resolve, reject) => { - // hook ssb.rebuild - this way we can piggyback the "done" callback of that - // and know when the get requests "should" work by - newPerson.rebuild.hook(function (rebuild, args) { - const [cb] = args - rebuild.call(this, (err) => { - numberRebuilds++ - if (typeof cb === 'function') cb(err) - if (numberRebuilds === 1) resolve() - }) - }) - }) - await run( 'replicated', p(replicate)({ from: kaitiaki, to: newPerson, live: false }) ) - await rebuildPromise - - // const pull = require('pull-stream') - // const msgs = await pull( - // newPerson.createLogStream({ private: true }), - // pull.map(m => m.value.content), - // pull.collectAsPromise() - // ) - // console.log(msgs) + await p(setTimeout)(500) const greetingMsg = await run( 'new-person can get message', @@ -87,7 +78,7 @@ test('tribes.invite', async t => { text: 'Thank you kaitiaki', recps: [groupId] } - const { key: replyKey } = await p(newPerson.publish)(replyContent) + const { key: replyKey } = await p(newPerson.tribes.publish)(replyContent) await p(replicate)({ from: newPerson, to: kaitiaki, live: false }) const replyMsg = await Getter(kaitiaki)(replyKey) t.deepEqual(replyMsg.value.content, replyContent, 'kaitiaki can read things from new person') diff --git a/test/api/subtribe/create.test.js b/test/api/subtribe/create.test.js index 0e9ebb3c..54dfd508 100644 --- a/test/api/subtribe/create.test.js +++ b/test/api/subtribe/create.test.js @@ -2,6 +2,7 @@ const test = require('tape') const { isCloakedMsg: isGroup } = require('ssb-ref') const isPoBox = require('ssb-private-group-keys/lib/is-po-box') // TODO find better home const pull = require('pull-stream') +const { where, type, toPullStream } = require('ssb-db2/operators') const { Server } = require('../../helpers') @@ -62,18 +63,11 @@ test('tribes.subtribe.create', t => { }) function getLink (cb) { - const query = [{ - $filter: { - value: { - content: { - type: 'link/group-group/subgroup' - } - } - } - }] - pull( - server.query.read({ query }), + server.db.query( + where(type('link/group-group/subgroup')), + toPullStream() + ), pull.collect((err, msgs) => err ? cb(err) : cb(null, msgs[0])) ) } diff --git a/test/generate/group-id.js b/test/generate/group-id.js index 4279f922..f443be7c 100644 --- a/test/generate/group-id.js +++ b/test/generate/group-id.js @@ -4,7 +4,7 @@ const generators = [ (i) => { const server = Server() - server.publish({ type: 'first' }, (err, msg) => { + server.tribes.publish({ type: 'first' }, (err, msg) => { if (err) throw err server.tribes.create('3 musketeers', (err, data) => { diff --git a/test/generate/unbox.js b/test/generate/unbox.js index 60657b73..1d4fd3ec 100644 --- a/test/generate/unbox.js +++ b/test/generate/unbox.js @@ -20,7 +20,7 @@ const generators = [ recps: [groupId] } - server.publish(content, (err, msg) => { + server.tribes.publish(content, (err, msg) => { if (err) throw err server.close() @@ -50,7 +50,7 @@ const generators = [ const content1 = { type: 'first' } - server.publish(content1, (_, firstMsg) => { + server.tribes.publish(content1, (_, firstMsg) => { const groupId = GroupId() const groupKey = GroupKey() const root = MsgId() @@ -62,7 +62,7 @@ const generators = [ recps: [groupId] } - server.publish(content2, (err, msg) => { + server.tribes.publish(content2, (err, msg) => { if (err) throw err server.close() @@ -99,7 +99,7 @@ const generators = [ body: { tick: Date.now() } } - server.publish(content1, (_, firstMsg) => { + server.tribes.publish(content1, (_, firstMsg) => { const groupId = GroupId() const groupKey = GroupKey() const root = MsgId() @@ -113,7 +113,7 @@ const generators = [ recps: [groupId, friendId.toSSB()] } - server.publish(content2, (err, msg) => { + server.tribes.publish(content2, (err, msg) => { if (err) throw err const key = directMessageKey.easy(server.keys)(friendId.toSSB()) diff --git a/test/helpers/test-bot.js b/test/helpers/test-bot.js index ee6b2fdb..7db2d4ff 100644 --- a/test/helpers/test-bot.js +++ b/test/helpers/test-bot.js @@ -12,8 +12,14 @@ module.exports = function TestBot (opts = {}) { // } let stack = Server // eslint-disable-line - .use(require('ssb-backlinks')) - .use(require('ssb-query')) + // .use(require('ssb-backlinks')) + // .use(require('ssb-query')) + .use(require('ssb-db2/core')) + .use(require('ssb-classic')) + .use(require('ssb-db2/compat')) + .use(require('ssb-db2/compat/feedstate')) + // .use(require("ssb-db2/compat/publish")) + .use(require('ssb-box2')) .use(require('../..')) // ssb-tribes - NOTE load it after ssb-backlinks if (opts.installReplicate === true) { @@ -23,6 +29,11 @@ module.exports = function TestBot (opts = {}) { if (opts.name) opts.name = 'ssb-tribes/' + opts.name const ssb = stack({ + box2: { + legacyMode: true, + ...opts.box2 + }, + // we don't want testbot to import db2 for us, we want more granularity and control of dep versions db1: true, ...opts }) @@ -40,11 +51,5 @@ module.exports = function TestBot (opts = {}) { ssb.tribes.application = Application(ssb) } - ssb.close.hook((close, args) => { - return setTimeout(() => { - close(...args) - }, 10 * 1000) - }) - return ssb } diff --git a/test/indexes-check.test.js b/test/indexes-check.test.js deleted file mode 100644 index 70ae853d..00000000 --- a/test/indexes-check.test.js +++ /dev/null @@ -1,145 +0,0 @@ -// we were seeing weird behaviour in ssb-backlinks -// this test is designed to pin that down - -const test = require('tape') -const pull = require('pull-stream') - -const { Server } = require('./helpers') - -const PROFILE = 'profile/person' -const LINK = 'link/feed-profile' - -function createRecords (server, t, cb) { - // this function creates a group and published a handful of messages to that group - // it calls back with the groupId and the list of published messages - - /* state */ - const published = [] // contents of messages we published - let lastMsgId - - server.tribes.create({}, (err, { groupId } = {}) => { - t.error(err, 'creates group') - - const recps = [groupId] - // recps = null // passes all tests - - let count = 0 - pull( - pull.values([PROFILE, LINK, PROFILE, LINK]), - // pull.values([PROFILE, LINK, PROFILE, LINK, PROFILE]), // ✓ checkIndex - // pull.values([PROFILE, LINK, PROFILE, LINK, LINK]), // x checkIndex - pull.map(type => { - return (type === PROFILE) - ? { type, count: count++, recps } - : { type, count: count++, parent: server.id, child: lastMsgId, recps } - }), - pull.asyncMap((content, cb) => { - server.publish(content, (err, msg) => { - if (err) return cb(err) - const label = typeof msg.value.content === 'string' ? 'encrypted' : 'public' - t.ok(true, `${label} (${content.count}, ${content.type})`) - lastMsgId = msg.key - published.push(content) - cb(null, msg) - }) - }), - pull.collect((err) => { - t.error(err, 'all published') - - cb(null, { groupId, published }) - }) - ) - }) -} - -function testSuite (indexName, createSource) { - function checkIndex (server, t, published, cb) { - pull( - createSource(server), - pull.collect((err, results) => { - t.error(err, `${indexName}.read`) - - const _published = published.filter(i => i.type === LINK) - // t.equal(_published.length, results.length, 'finds all messages') - - const outputs = results.map(m => m.value.content) - - _published.forEach((link, i) => { - t.deepEqual(outputs[i], link, `${indexName} finds (${link.count}, ${link.type})`) - }) - cb(null) - }) - ) - } - - test(indexName, t => { - const name = `${indexName}-be-good-${Date.now()}` - let server = Server({ name }) - const keys = server.keys - - createRecords(server, t, (_, { groupId, published }) => { - t.comment(`> check ${indexName} results`) - checkIndex(server, t, published, () => { - t.comment('> check again (after server restart)') - server.close(err => { - if (err) throw err - server = Server({ name, keys, startUnclean: true }) - t.ok(server.whoami(), 'server restart') - - checkIndex(server, t, published, () => { - t.comment('> check AGAIN (after a publish + restart)') - const anything = { - type: 'doop', - recps: [groupId] - } - server.publish(anything, (err) => { - t.error(err, 'publish anything') - const keys = server.keys - server.close(err => { - if (err) throw err - server = Server({ name, keys, startUnclean: true }) - t.ok(server.whoami(), 'server restart') - - checkIndex(server, t, published, () => { - server.close(t.end) - }) - }) - }) - }) - }) - }) - }) - }) -} - -testSuite('backlinks', (server) => { - const query = [{ - $filter: { - dest: server.id, - value: { - content: { - type: LINK, - parent: server.id - } - } - } - }] - - return server.backlinks.read({ query }) -}) - -testSuite('query', (server) => { - const query = [{ - $filter: { - value: { - author: server.id, - content: { - type: LINK, - parent: server.id - } - } - } - }] - - return server.query.read({ query }) -}) diff --git a/test/lib/get-group-tangle.test.js b/test/lib/get-group-tangle.test.js index 9385027b..6b5b975f 100644 --- a/test/lib/get-group-tangle.test.js +++ b/test/lib/get-group-tangle.test.js @@ -4,6 +4,7 @@ const { Server, replicate } = require('../helpers') const pull = require('pull-stream') const paraMap = require('pull-paramap') const { GetGroupTangle } = require('../../lib') +const { where, author, descending, toPullStream, toCallback } = require('ssb-db2/operators') test('get-group-tangle unit test', t => { const name = `get-group-tangle-${Date.now()}` @@ -12,18 +13,9 @@ test('get-group-tangle unit test', t => { // - creating a group and publishing messages (ssb-tribes) server.tribes.create(null, (err, data) => { if (err) throw err - const keystore = { - group: { - get (groupId) { - return { ...data, root: data.groupInitMsg.key } // rootMsgId - } - } - } - // NOTE: Tribes create callback with different data than keystore.group.get :( - // Somebody should probably fix that // NOTE: Publishing has a queue which means if you publish many things in a row there is a delay before those values are in indexes to be queried. - const _getGroupTangle = GetGroupTangle(server, keystore) + const _getGroupTangle = GetGroupTangle(server) const getGroupTangle = (id, cb) => { setTimeout(() => _getGroupTangle(id, cb), 300) } @@ -33,7 +25,11 @@ test('get-group-tangle unit test', t => { const rootKey = data.groupInitMsg.key pull( - server.createUserStream({ id: server.id, reverse: true }), + server.db.query( + where(author(server.id)), + descending(), + toPullStream() + ), pull.map(m => m.key), pull.take(1), pull.collect((err, keys) => { @@ -53,14 +49,14 @@ test('get-group-tangle unit test', t => { recps: [data.groupId] }) - server.publish(content(), (err, msg) => { + server.tribes.publish(content(), (err, msg) => { if (err) throw err getGroupTangle(data.groupId, (err, { root, previous }) => { if (err) throw err t.deepEqual({ root, previous }, { root: data.groupInitMsg.key, previous: [msg.key] }, 'adding message to root') - server.publish(content(), (err, msg) => { + server.tribes.publish(content(), (err, msg) => { if (err) throw err getGroupTangle(data.groupId, (err, { root, previous }) => { if (err) throw err @@ -77,34 +73,6 @@ test('get-group-tangle unit test', t => { }) }) -test('get-group-tangle (cache)', t => { - const name = `get-group-tangle-cache-${Date.now()}` - const server = Server({ name }) - - let queryCalls = 0 - server.backlinks.read.hook(function (read, args) { - queryCalls += 1 - - return read(...args) - }) - server.tribes.create(null, (err, data) => { - if (err) throw err - - // 1 for group tangle, 1 for members tangle - t.equal(queryCalls, 2, 'no cache for publishing of group/add-member, a backlink query was run') - const content = { type: 'memo', recps: [data.groupId] } - - server.publish(content, (err, msg) => { - if (err) throw err - - t.equal(queryCalls, 2, 'cache used for publishing next message') - - server.close() - t.end() - }) - }) -}) - const n = 100 test(`get-group-tangle-${n}-publishes`, t => { const publishArray = new Array(n).fill().map((item, i) => i) @@ -117,7 +85,7 @@ test(`get-group-tangle-${n}-publishes`, t => { pull( pull.values(publishArray), paraMap( - (value, cb) => server.publish({ type: 'memo', value, recps: [groupId] }, cb), + (value, cb) => server.tribes.publish({ type: 'memo', value, recps: [groupId] }, cb), 4 ), paraMap( @@ -129,7 +97,6 @@ test(`get-group-tangle-${n}-publishes`, t => { count += (m.value.content.tangles.group.previous.length) }, () => { - // t.equal(count, n, 'We expect there to be no branches in our groupTangle') t.true(count < n * 8, 'We expect bounded branching with fast publishing') server.close() @@ -146,39 +113,43 @@ test('get-group-tangle', t => { plan: 5, test: (t) => { const DESCRIPTION = 'auto adds group tangle' - // this is an integration test, as we've hooked get-group-tangle into ssb.publish + // this is an integration test, as we've hooked get-group-tangle into ssb.tribes.publish const ssb = Server() ssb.tribes.create(null, (err, data) => { t.error(err, 'create group') - ssb.getLatest(ssb.id, (err, selfAdd) => { - t.error(err, 'get self invite') + ssb.db.query( + where(author(ssb.id)), + descending(), + toCallback((err, [selfAdd]) => { + t.error(err, 'got self add') - const groupRoot = data.groupInitMsg.key - const groupId = data.groupId + const groupRoot = data.groupInitMsg.key + const groupId = data.groupId - const content = { - type: 'yep', - recps: [groupId] - } + const content = { + type: 'yep', + recps: [groupId] + } - ssb.publish(content, (err, msg) => { - t.error(err, 'publish a message') + ssb.tribes.publish(content, (err, msg) => { + t.error(err, 'publish a message') - ssb.get({ id: msg.key, private: true }, (err, A) => { - t.error(err, 'get that message back') + ssb.get({ id: msg.key, private: true }, (err, A) => { + t.error(err, 'get that message back') - t.deepEqual( - A.content.tangles.group, // actual - { root: groupRoot, previous: [selfAdd.key] }, // expected - DESCRIPTION + ' (auto added tangles.group)' - ) + t.deepEqual( + A.content.tangles.group, // actual + { root: groupRoot, previous: [selfAdd.key] }, // expected + DESCRIPTION + ' (auto added tangles.group)' + ) - ssb.close() + ssb.close() + }) }) }) - }) + ) }) } } @@ -206,20 +177,14 @@ test('get-group-tangle with branch', t => { // Alice creates a group alice.tribes.create(null, (err, data) => { if (err) throw err + // Prepare to get Alice's group tangle from both servers - const keystore = { - group: { - get (groupId) { - return { ...data, root: data.groupInitMsg.key } // rootMsgId - } - } - } const DELAY = 200 - const _getAliceGroupTangle = GetGroupTangle(alice, keystore) + const _getAliceGroupTangle = GetGroupTangle(alice) const getAliceGroupTangle = (id, cb) => { setTimeout(() => _getAliceGroupTangle(id, cb), DELAY) } - const _getBobGroupTangle = GetGroupTangle(bob, keystore) + const _getBobGroupTangle = GetGroupTangle(bob) const getBobGroupTangle = (id, cb) => { setTimeout(() => _getBobGroupTangle(id, cb), DELAY) } @@ -251,13 +216,13 @@ test('get-group-tangle with branch', t => { recps: [data.groupId] }) - alice.publish(content(), (err, msg) => { + alice.tribes.publish(content(), (err, msg) => { t.error(err, 'alice publishes a new message') // NOTE With the content.recps we are adding we are asking Bob to know about a group before he's // found out about it for himself whenBobHasGroup(data.groupId, () => { - bob.publish(content(), (err, msg) => { + bob.tribes.publish(content(), (err, msg) => { if (err) throw err // Then Bob shares his message with Alice replicate({ from: bob, to: alice, name }, (err) => { @@ -301,10 +266,8 @@ test('members tangle', async t => { await p(setTimeout)(300) const bobInvite = await p(alice.tribes.invite)(groupId, [bob.id], {}) - const keystore = { group: { get: () => ({ root }) } } - - const _getGroupTangle = p(GetGroupTangle(alice, keystore, 'group')) - const _getMembersTangle = p(GetGroupTangle(alice, keystore, 'members')) + const _getGroupTangle = p(GetGroupTangle(alice, null, 'group')) + const _getMembersTangle = p(GetGroupTangle(alice, null, 'members')) const getGroupTangle = p((id, cb) => { setTimeout(() => _getGroupTangle(id, cb), 300) }) diff --git a/test/lib/tangle-prune.test.js b/test/lib/tangle-prune.test.js index 2bcc0d2a..0101d5d2 100644 --- a/test/lib/tangle-prune.test.js +++ b/test/lib/tangle-prune.test.js @@ -24,7 +24,7 @@ test('lib/tangle-prune', async t => { } return new Promise((resolve, reject) => { - ssb.publish(content, (err, msg) => { + ssb.tribes.publish(content, (err, msg) => { if (err) return resolve(false) ssb.get({ id: msg.key, private: true }, (err, msgVal) => { diff --git a/test/publish.test.js b/test/publish.test.js index 2c664955..dc877e10 100644 --- a/test/publish.test.js +++ b/test/publish.test.js @@ -16,12 +16,8 @@ test('publish (to groupId)', t => { text: 'summer has arrived in wellington!', recps: [groupId] } - /* NOTE with this we confirm that content.recps isn't mutated while sneakin our own_key in! */ - process.env.NODE_ENV = 'production' - console.log('NODE_ENV', process.env.NODE_ENV) - Object.freeze(content.recps) - server.publish(content, (err, msg) => { + server.tribes.publish(content, (err, msg) => { t.error(err, 'msg published to group') t.true(msg.value.content.endsWith('.box2'), 'publishes envelope cipherstring') const cipherTextSize = Buffer.from(msg.value.content.replace('.box2', ''), 'base64').length @@ -31,14 +27,10 @@ test('publish (to groupId)', t => { t.deepEqual(msg.value.content, content, 'can open envelope!') const plainTextSize = Buffer.from(JSON.stringify(msg.value.content)).length - const expectedSize = 32 + (msg.value.content.recps.length + 1) * 32 + (plainTextSize + 16) + const expectedSize = 32 + msg.value.content.recps.length * 32 + (plainTextSize + 16) // header + (recp key slots) + (content + HMAC) - // NOTE - we sneak ourselves in as a key_slot when we can - t.equal(cipherTextSize, expectedSize, 'cipherTextSize overhead correct') // 112 bytes + t.equal(cipherTextSize, expectedSize, 'cipherTextSize overhead correct') - /* return ENV to testing */ - process.env.NODE_ENV = 'test' - console.log('NODE_ENV', process.env.NODE_ENV) server.close(t.end) }) }) @@ -55,7 +47,7 @@ test('publish (to groupId we dont have key for)', t => { recps: [groupId] } - server.publish(content, (err) => { + server.tribes.publish(content, (err) => { t.match(err.message, /unknown groupId/, 'returns error') server.close(t.end) }) @@ -76,7 +68,7 @@ test('publish (group + feedId)', t => { recps: [groupId, feedId] } - server.publish(content, (err, msg) => { + server.tribes.publish(content, (err, msg) => { t.error(err) t.true(msg.value.content.endsWith('.box2'), 'publishes envelope cipherstring') @@ -106,8 +98,8 @@ test('publish (DMs: myFeedId + feedId)', async t => { } try { - const msg = await p(alice.publish)(content) - await p(alice.publish)({ type: 'doop' }) + const msg = await p(alice.tribes.publish)(content) + await p(alice.tribes.publish)({ type: 'doop' }) t.true(msg.value.content.endsWith('.box2'), 'publishes envelope cipherstring') const aliceGet = await p(alice.get)({ id: msg.key, private: true, meta: true }) @@ -134,7 +126,7 @@ test('publish (bulk)', t => { .map(() => ({ type: 'test', recps: [groupId] })) bulk.forEach((content, i) => { - server.publish(content, (err, msg) => { + server.tribes.publish(content, (err, msg) => { if (err) t.error(err, `${i + 1} published`) if (typeof msg.value.content !== 'string') t.fail(`${i + 1} encrypted`) @@ -152,7 +144,7 @@ test('publish (bulk)', t => { /* works fine */ // pull( // pull.values(bulk), - // pull.asyncMap(server.publish), + // pull.asyncMap(server.tribes.publish), // pull.drain( // () => process.stdout.write('✓'), // (err) => { diff --git a/test/rebuild-manager.test.js b/test/rebuild-manager.test.js deleted file mode 100644 index 500080ea..00000000 --- a/test/rebuild-manager.test.js +++ /dev/null @@ -1,111 +0,0 @@ -const test = require('tape') -const { promisify: p } = require('util') -const Server = require('scuttle-testbot') -// NOTE we do not use the helpers/test-bot as that has a RebuildManager installed with ssb-tribes! - -const Manager = require('../rebuild-manager') - -async function setup () { - const ssb = Server({ - db1: true - }) - // ssb.post(console.log) - - // NOTE - We cannot rebuild an empty DB ?! - await p(ssb.publish)({ type: 'filler', text: new Array(1000).fill('dog').join() }) - await p(ssb.publish)({ type: 'filler', text: new Array(1000).fill('cat').join() }) - - // we fake indexing taking some time to be done - const TIME_TILL_INDEX_DONE = 1000 - setTimeout(() => { ssb._isIndexingDone = true }, TIME_TILL_INDEX_DONE) - - ssb.status.hook(status => { - const current = status() - if (ssb._isIndexingDone) return current - - // current.sync.plugins.links = 5000 - current.sync.sync = false - return current - }) - - return ssb -} - -test('rebuild-manager', t => { - t.plan(3) - setup().then(ssb => { - // we wrap ssb.rebuild once to know exactly what goes through to the db - // we expect only one call to come through from the rebuildManager - // AFTER indexing is complete - let rebuildCount = 0 - ssb.rebuild.hook((rebuild, [cb]) => { - t.true(ssb.status().sync.sync, 'rebuild only gets called once indexing is done') - rebuildCount++ - rebuild(cb) - }) - - // this wraps ssb.rebuild again - const manager = new Manager(ssb) - - manager.rebuild('my dog') - manager.rebuild('my cat', () => t.pass('callback in the middle')) - manager.rebuild('my fish', () => { - t.equal(rebuildCount, 1, 'db rebuild only gets called once') - - ssb.close() - }) - }) -}) - -test('rebuild-manager (rebuild called during rebuild with EXISTING reason)', t => { - t.plan(4) - setup().then(ssb => { - let rebuildCount = 0 - ssb.rebuild.hook((rebuild, [cb]) => { - t.true(ssb.status().sync.sync, 'rebuild only gets called once indexing is done') - rebuildCount++ - rebuild.call(ssb, cb) - manager.rebuild('my cat', () => { // << 'my cat' already cited as EXISTING reason for rebuild - t.equal(rebuildCount, 1, 'existing reason and cb folded into current run') - }) - }) - - const manager = new Manager(ssb) - - manager.rebuild('my fish') - manager.rebuild('my dog', () => t.pass('callback in the middle')) - manager.rebuild('my cat', () => { - t.equal(rebuildCount, 1, 'db rebuild only gets called once') - - ssb.close() - }) - }) -}) - -test('rebuild-manager (rebuild called during rebuild with NEW reason)', t => { - t.plan(5) - setup().then(ssb => { - let rebuildCount = 0 - ssb.rebuild.hook((rebuild, [cb]) => { - t.true(ssb.status().sync.sync, 'rebuild only gets called once indexing is done') // see this twice - rebuildCount++ - rebuild.call(ssb, cb) - - if (rebuildCount === 1) { - manager.rebuild('my pig', () => { // << 'my pig' is a NEW reason for rebuilding! - t.equal(rebuildCount, 2, 'rebuild called second time') - - ssb.close() - }) - } - }) - - const manager = new Manager(ssb) - - manager.rebuild('my fish') - manager.rebuild('my dog', () => t.pass('callback in the middle')) - manager.rebuild('my cat', () => { - t.equal(rebuildCount, 1, 'first rebuild completed') - }) - }) -}) diff --git a/test/rebuild.test.js b/test/rebuild.test.js index 0e1c4bbf..38b2da61 100644 --- a/test/rebuild.test.js +++ b/test/rebuild.test.js @@ -1,7 +1,10 @@ const test = require('tape') const pull = require('pull-stream') +const { promisify: p } = require('util') +const { where, and, author, isDecrypted, toPullStream } = require('ssb-db2/operators') const { Server, replicate, FeedId } = require('./helpers') +const listen = require('../listen') function nMessages (n, { type = 'post', recps } = {}) { return new Array(20).fill(type).map((val, i) => { @@ -11,7 +14,7 @@ function nMessages (n, { type = 'post', recps } = {}) { }) } -test('rebuild (I am added to a group)', t => { +test('rebuild (I am added to a group)', async t => { const admin = Server() const me = Server() const name = (id) => { @@ -21,47 +24,42 @@ test('rebuild (I am added to a group)', t => { } } - replicate({ from: admin, to: me, name, live: true }) - - /* set up listener */ - me.rebuild.hook(function (rebuild, [cb]) { - t.pass('I automatically call a rebuild') - - rebuild(() => { - cb && cb() - t.true(me.status().sync.sync, 'all indexes updated') // 2 - - pull( - me.createUserStream({ id: admin.id, private: true }), - pull.drain( - m => { - t.equal(typeof m.value.content, 'object', `I auto-unbox msg: ${m.value.content.type}`) - }, - (err) => { - if (err) throw err - admin.close() - me.close() - t.end() - } - ) - ) - }) - t.false(me.status().sync.sync, 'all indexes updating') // 1 - }) - /* kick off the process */ - admin.tribes.create({}, (err, data) => { - if (err) throw err + const data = await p(admin.tribes.create)({}) + + await p(admin.tribes.invite)(data.groupId, [me.id], { text: 'ahoy' }) + + await replicate({ from: admin, to: me, name }) + + // time for rebuilding + await p(setTimeout)(500) + + t.true(me.status().sync.sync, 'all indexes updated') + + const msgs = await pull( + me.db.query( + where(and( + author(admin.id), + isDecrypted('box2') + )), + toPullStream() + ), + pull.map(m => { + t.equal(typeof m.value.content, 'object', `I auto-unbox msg: ${m.value.content.type}`) + return m + }), + pull.collectAsPromise() + ) - admin.tribes.invite(data.groupId, [me.id], { text: 'ahoy' }, (err, invite) => { - t.error(err, 'admin adds me to group') - if (err) throw err - }) - }) + t.equal(msgs.length, 3, 'we got 3 messages to auto unbox') + + await Promise.all([ + p(admin.close)(), + p(me.close)() + ]) }) test('rebuild (I am added to a group, then someone else is added)', t => { - // t.plan(9) const admin = Server() const me = Server() const bob = Server() @@ -76,19 +74,14 @@ test('rebuild (I am added to a group, then someone else is added)', t => { } let groupId - // Setting up listeners ready for initial action - - replicate({ from: admin, to: me, name, live: true }) - replicate({ from: admin, to: bob, name, live: true }) - - admin.rebuild.hook(function (rebuild, [cb]) { + admin.db.reindexEncrypted.hook(function (rebuild, [cb]) { t.fail('admin should not rebuild') throw new Error('stop') }) /* after I am added, admin adds bob */ let myRebuildCount = 0 - me.rebuild.hook(function (rebuild, [cb]) { + me.db.reindexEncrypted.hook(function (rebuild, [cb]) { myRebuildCount++ if (myRebuildCount > 2) throw new Error('I should only rebuild twice!') // 1st time - I realise I've been added to a group, and re-index @@ -101,9 +94,10 @@ test('rebuild (I am added to a group, then someone else is added)', t => { if (myRebuildCount === 2) { // I publish 20 messages to the group pull( - pull.values(nMessages(20, { type: 'me', recps: [groupId] })), - pull.asyncMap(me.publish), + pull.values(nMessages(20, { type: 'itsame', recps: [groupId] })), + pull.asyncMap(me.tribes.publish), pull.collect((err) => { + if (err) console.error('publish 20 err', err) t.error(err, 'I publish 20 messages to the group') replicate({ from: me, to: bob, name, live: false }, (err) => { @@ -115,6 +109,7 @@ test('rebuild (I am added to a group, then someone else is added)', t => { t.error(err, 'I shut down') admin.tribes.invite(groupId, [bob.id], { text: 'hi!' }, (err) => { t.error(err, 'admin adds bob to the group') + replicate({ from: admin, to: bob, name, live: false }) }) }) }) @@ -125,7 +120,7 @@ test('rebuild (I am added to a group, then someone else is added)', t => { }) let rebuildCount = 0 - bob.rebuild.hook(function (rebuild, [cb]) { + bob.db.reindexEncrypted.hook(function (rebuild, [cb]) { const _count = ++rebuildCount switch (_count) { @@ -135,18 +130,24 @@ test('rebuild (I am added to a group, then someone else is added)', t => { case 2: t.pass('bob calls rebuild (realises me + zelf are in group)') break + case 3: + t.pass('bob calls rebuild again i guess') + break default: t.fail(`rebuild called too many times: ${_count}`) } rebuild(() => { cb && cb() - if (_count !== 2) return + if (_count !== 3) return /* check can see all of my group messages */ let seenMine = 0 pull( - bob.createLogStream({ private: true }), + bob.db.query( + where(isDecrypted('box2')), + toPullStream() + ), pull.map(m => m.value.content), pull.drain( ({ type, count, recps }) => { @@ -155,7 +156,7 @@ test('rebuild (I am added to a group, then someone else is added)', t => { if (type === 'group/add-member') { comment += `: ${recps.filter(r => r[0] === '@').map(name)}` } - if (type === 'me') { + if (type === 'itsame') { seenMine++ if (count === 0 || count === 19) comment += `(${count})` else if (count === 1) comment += '...' @@ -166,9 +167,11 @@ test('rebuild (I am added to a group, then someone else is added)', t => { (err) => { if (seenMine === 20) t.equal(seenMine, 20, 'bob saw 20 messages from me') if (err) throw err - bob.close() - admin.close() - t.end() + + Promise.all([ + p(bob.close)(), + p(admin.close)() + ]).then(() => t.end()) } ) ) @@ -178,51 +181,51 @@ test('rebuild (I am added to a group, then someone else is added)', t => { // Action which kicks everthing off starts here /* admin adds alice + zelf to a group */ - admin.tribes.create({}, (err, data) => { - if (err) throw err - + Promise.resolve().then(async () => { + const data = await p(admin.tribes.create)({}) groupId = data.groupId - admin.tribes.invite(groupId, [me.id], { text: 'ahoy' }, (err) => { - t.error(err, 'admin adds alice to group') - - setTimeout(() => { - // we do this seperately to test if rebuild gets called 2 or 3 times - // should wait till indexing done before rebuilding again - admin.tribes.invite(groupId, [zelfId], { text: 'ahoy' }, (err) => { - t.error(err, 'admin adds zelf to group') - if (err) throw err - }) - }, 1000) - }) + + await replicate({ from: admin, to: me, name }) + await replicate({ from: admin, to: bob, name }) + + await p(admin.tribes.invite)(groupId, [me.id], { text: 'ahoy' }) + + await replicate({ from: admin, to: me, name }) + await replicate({ from: admin, to: bob, name }) + + await p(setTimeout)(1000) + + // we do this seperately to test if rebuild gets called 2 or 3 times + // should wait till indexing done before rebuilding again + await p(admin.tribes.invite)(groupId, [zelfId], { text: 'ahoy' }) + + await replicate({ from: admin, to: me, name }) + await replicate({ from: admin, to: bob, name }) }) }) -test('rebuild (not called when I invite another member)', t => { +test('rebuild (not called when I invite another member)', async t => { const server = Server() let rebuildCalled = false - server.rebuild.hook(function (rebuild, args) { + server.db.reindexEncrypted.hook(function (rebuild, args) { rebuildCalled = true rebuild(...args) }) - server.tribes.create(null, (err, data) => { - t.error(err, 'I create a group') + const data = await p(server.tribes.create)(null) + const { groupId } = data - const { groupId } = data - const feedId = FeedId() + const feedId = FeedId() - server.tribes.invite(groupId, [feedId], {}, (err) => { - t.error(err, 'I add someone to the group') + await p(server.tribes.invite)(groupId, [feedId], {}) - setTimeout(() => { - t.false(rebuildCalled, 'I did not rebuild my indexes') - server.close() - t.end() - }, 1e3) - }) - }) + await p(setTimeout)(1000) + + t.false(rebuildCalled, 'I did not rebuild my indexes') + + await p(server.close)() }) test('rebuild from listen.addMember', t => { @@ -247,7 +250,7 @@ test('rebuild from listen.addMember', t => { setTimeout(() => checkRebuildDone(done), 500) } pull( - A.messagesByType({ type: 'group/add-member', private: true, live: true }), + listen.addMember(A), pull.filter(m => !m.sync), pull.drain(m => { t.equal(m.value.content.root, root, `listened + heard the group/add-member: ${++heardCount}`) @@ -295,11 +298,10 @@ test('rebuild (I learn about a new PO Box)', t => { } let groupId - replicate({ from: admin, to: me, name, live: true }) /* set up listener */ let rebuildCount = 0 - me.rebuild.hook((rebuild, [cb]) => { + me.db.reindexEncrypted.hook((rebuild, [cb]) => { rebuildCount++ const run = rebuildCount if (rebuildCount === 1) t.pass('rebuild started (group/add-member)') @@ -311,7 +313,10 @@ test('rebuild (I learn about a new PO Box)', t => { if (run === 1) { t.error(err, 'rebuild finished (group/add-member)') - admin.tribes.addPOBox(groupId, (err) => t.error(err, 'admin adds po-box')) + admin.tribes.addPOBox(groupId, (err) => { + t.error(err, 'admin adds po-box') + replicate({ from: admin, to: me, name }) + }) } // eslint-disable-line else if (run === 2) { t.error(err, 'rebuild finished (group/po-box)') @@ -327,9 +332,15 @@ test('rebuild (I learn about a new PO Box)', t => { if (err) throw err groupId = data.groupId - admin.tribes.invite(data.groupId, [me.id], { text: 'ahoy' }, (err, invite) => { - t.error(err, 'admin adds me to group') - if (err) throw err + replicate({ from: admin, to: me, name }, (err) => { + t.error(err, 'replicated after group create') + + admin.tribes.invite(data.groupId, [me.id], { text: 'ahoy' }, (err, invite) => { + t.error(err, 'admin adds me to group') + if (err) throw err + + replicate({ from: admin, to: me, name }) + }) }) }) }) @@ -346,7 +357,7 @@ test('rebuild (added to group with poBox)', t => { /* set up listener */ let rebuildCount = 0 - me.rebuild.hook((rebuild, [cb]) => { + me.db.reindexEncrypted.hook((rebuild, [cb]) => { rebuildCount++ if (rebuildCount === 1) t.pass('rebuild started (group/add-member)') else if (rebuildCount === 2) t.pass('rebuild started (group/po-box)') @@ -372,11 +383,15 @@ test('rebuild (added to group with poBox)', t => { admin.tribes.create({ addPOBox: true }, (err, data) => { if (err) throw err - replicate({ from: admin, to: me, name, live: true }) - - admin.tribes.invite(data.groupId, [me.id], { text: 'ahoy' }, (err, invite) => { - t.error(err, 'admin adds me to group') + replicate({ from: admin, to: me, name }, (err) => { if (err) throw err + + admin.tribes.invite(data.groupId, [me.id], { text: 'ahoy' }, (err, invite) => { + t.error(err, 'admin adds me to group') + if (err) throw err + + replicate({ from: admin, to: me, name }) + }) }) }) }) diff --git a/test/unbox.test.js b/test/unbox.test.js index 0d58d6cc..819e3e9f 100644 --- a/test/unbox.test.js +++ b/test/unbox.test.js @@ -3,14 +3,8 @@ const test = require('tape') const pull = require('pull-stream') const { promisify: p } = require('util') -const { decodeLeaves, Server, Run } = require('./helpers') - -const envelope = require('../envelope') - -const vectors = [ - require('./vectors/unbox1.json'), - require('./vectors/unbox2.json') -].map(decodeLeaves) +const { where, and, type, slowEqual, toPullStream } = require('ssb-db2/operators') +const { Server, Run } = require('./helpers') test('unbox', async t => { const run = Run(t) @@ -27,7 +21,7 @@ test('unbox', async t => { } const msg = await run( `publish with recps: ${recps}`, - p(ssb.publish)(content) + p(ssb.tribes.publish)(content) ) t.true(msg.value.content.endsWith('.box2'), 'box') @@ -48,18 +42,15 @@ test('unbox', async t => { const backlinks = await run( 'get backlinks', pull( - ssb.query.read({ - query: [{ - $filter: { - value: { - content: { - type: 'dummy', - groupRef: groupInitMsg.key - } - } - } - }] - }), + ssb.db.query( + where( + and( + type('dummy'), + slowEqual('value.content.groupRef', groupInitMsg.key) + ) + ), + toPullStream() + ), pull.map(m => m.value.content.recps[0]), // just pull the recps on each one pull.collectAsPromise() @@ -70,45 +61,3 @@ test('unbox', async t => { await run('close ssb', p(ssb.close)(true)) t.end() }) - -test('unbox - test vectors', async t => { - console.log('vectors:', vectors.length) - - vectors.forEach(vector => { - const { msgs, trial_keys } = vector.input - - const mockKeyStore = { - group: { - listSync: () => ['a'], - get: () => ({ - readKeys: trial_keys - }) - }, - dm: { - has: () => true, - get: () => - ({ key: Buffer.alloc(32), scheme: 'junk' }) // just here to stop code choking - } - } - - const mockState = { - keys: { - curve: 'ed25519', - public: '3Wr/Swdt8vTC4NdOWJKaIX2hU2qcarSSdFpV4eyJKVw=.ed25519', - private: 'rlg3ciKZRCcYLjbDfy4eymsvNzCHRXDfWw65PvttqiXdav9LB23y9MLg105YkpohfaFTapxqtJJ0WlXh7IkpXA==.ed25519', - id: '@3Wr/Swdt8vTC4NdOWJKaIX2hU2qcarSSdFpV4eyJKVw=.ed25519' - } - } - - const { unboxer } = envelope(mockKeyStore, mockState) - const ciphertext = msgs[0].value.content - - const read_key = unboxer.key(ciphertext, msgs[0].value) - const body = unboxer.value(ciphertext, msgs[0].value, read_key) - t.deepEqual(body, vector.output.msgsContent[0], vector.description) - }) - - console.log('DONE') - - t.end() -}) diff --git a/test/vectors/unbox1.json b/test/vectors/unbox1.json deleted file mode 100644 index fe7c647b..00000000 --- a/test/vectors/unbox1.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "type": "unbox", - "description": "open envelope (group_keys + previous === null)", - "input": { - "msgs": [ - { - "key": "%iPTskfm08k9sfg/i8aXwXbdefCzBuUeaNey507slX/I=.sha256", - "value": { - "previous": null, - "sequence": 1, - "author": "@GU3nw+rEjXOEKEXFxqf1WeVUZX42bHrJRUJfwrhW+bg=.ed25519", - "timestamp": 1592534932480, - "hash": "sha256", - "content": "PGHe5NpkJJgv71rhqa+tLNGy2K/cvnWpPVFpKJEllWweHfXvq4+rbdwbw3e1ax5RLMP+708zpulqy/Y/TvlYe6D4cT7cqiLhRD8NzdM0W28t9YGNJWbJpjW2NccdSRBPD0ZIuwAkXyshosUrxi8CrbmW+4XfFmGPXu8DzHTqpN+j56kyKFd//SfVd8Dy5GwZLpKZPYf62dpInTQlJkPb6NcJQ3+lUe1hhi97iOtIhsB/8jtOPjF/V6qUXrSu/nCs/g==.box2", - "signature": "416FiLEtLG+CvbqE1OlrF9Py18WA/sVMgbN2Od2jRIiReqowzOTEp8wmiN+1wCDx7k+kU+ZRb5/nzaxiwDbKBg==.sig.ed25519" - }, - "timestamp": 1592534932480.001 - } - ], - "trial_keys": [ - { - "key": "tXD0fgOUVKj4qnhYdgB/B2wjq/9YM7tBydIBYzY8wl8=", - "scheme": "envelope-large-symmetric-group" - }, - { - "key": "Is0U1U121chtepf0QcnF0JREhp9NNUgqsPE/1ODGV4Q=", - "scheme": "envelope-large-symmetric-group" - } - ] - }, - "output": { - "msgsContent": [ - { - "type": "alert", - "text": "get ready to scuttle!", - "recps": [ - "%VTmwWB0ou53Tj99NPyu2iomZuzhVwp0CIiovdeZCKRg=.cloaked" - ] - } - ] - } -} \ No newline at end of file diff --git a/test/vectors/unbox2.json b/test/vectors/unbox2.json deleted file mode 100644 index 2ff7f2c9..00000000 --- a/test/vectors/unbox2.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "type": "unbox", - "description": "open envelope (group_keys + previous !== null)", - "input": { - "msgs": [ - { - "key": "%1Splppmh9AyVZiKnqXcpYKEic5n1dEMZ90c80QQbjxg=.sha256", - "value": { - "previous": "%735w71E4jLhYLDcdM3zRBDbeOVXm9p+Q54napNZP518=.sha256", - "sequence": 2, - "author": "@4IXio7MZcoBl4LGlAa894kCvFAvpqOEUPwPOiLbuagY=.ed25519", - "timestamp": 1592534932594, - "hash": "sha256", - "content": "nRNJ2jLRoHItfcNrWsHgY6KqX3UxW9atpjua36UINPjVMj3drgR/nEYyZbSACg5YzLUTU9xX2o+CWmTnQL4AgYI6GnxwT87pBXr/hMFtEZ3PvTvZR2VJPDGnHeCNjvZ7nRJkfFxB8XlSy0KmTGyU2+TnprHOWYgRkLA4RwwS3lvIqyuyVfU3EhEbtRCD84xBrNORznXzrugywVNyF9h3EterDjdvHsHt64MTWZxqPZdD0atxJRfIkLwJDe9JK8WlQQ==.box2", - "signature": "wfcsLX59vmELQfYybdtHZjVtvas4X9ffC1Xa95/vL8ax9xgnIgwjh4EA2Tg1rk3CEmh8iTNPyN3R5PBlVCx5AA==.sig.ed25519" - }, - "timestamp": 1592534932594.001 - } - ], - "trial_keys": [ - { - "key": "SAoMdSI7THVRSr+dPzYdgMIM5f1NFcpqdtn741hHOq0=", - "scheme": "envelope-large-symmetric-group" - }, - { - "key": "A7lRB1lxh+vz+FIwDfl+GbtztIM3z0xbiYz7anCp4o8=", - "scheme": "envelope-large-symmetric-group" - } - ] - }, - "output": { - "msgsContent": [ - { - "type": "alert", - "text": "get ready to scuttle!", - "recps": [ - "%93kpQXoHNYFeCP5OzDIHVTCCK8qJzCl+m08zHlN14oU=.cloaked" - ] - } - ] - } -} \ No newline at end of file