From 6e22c84a463fe748229e63f86458a0f4a826e53f Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Mon, 17 Apr 2023 14:55:17 +0200 Subject: [PATCH 01/11] Start adding added-to-old-epoch test --- test/exclude-members.test.js | 87 ++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/test/exclude-members.test.js b/test/exclude-members.test.js index 1fa020a..0b36b60 100644 --- a/test/exclude-members.test.js +++ b/test/exclude-members.test.js @@ -417,3 +417,90 @@ test("If you're not the excluder nor the excludee then you should still be in th await p(bob.close)(true) await p(carol.close)(true) }) + +test('Get added to an old epoch but still find newer epochs', async (t) => { + const alice = Testbot({ + keys: ssbKeys.generate(null, 'alice'), + mfSeed: Buffer.from( + '000000000000000000000000000000000000000000000000000000000000a1ce', + 'hex' + ), + }) + const bob = Testbot({ + keys: ssbKeys.generate(null, 'bob'), + mfSeed: Buffer.from( + '0000000000000000000000000000000000000000000000000000000000000b0b', + 'hex' + ), + }) + const carol = Testbot({ + keys: ssbKeys.generate(null, 'carol'), + mfSeed: Buffer.from( + '00000000000000000000000000000000000000000000000000000000000ca201', + 'hex' + ), + }) + + await alice.tribes2.start() + await bob.tribes2.start() + await carol.tribes2.start() + t.pass('tribes2 started for everyone') + + await p(alice.metafeeds.findOrCreate)() + const bobRoot = await p(bob.metafeeds.findOrCreate)() + const carolRoot = await p(carol.metafeeds.findOrCreate)() + + await replicate(alice, bob) + await replicate(alice, carol) + await replicate(bob, carol) + t.pass('everyone replicates their trees') + + const { id: groupId } = await alice.tribes2 + .create() + .catch((err) => t.error(err, 'alice failed to create group')) + + const { key: firstPostId } = await alice.tribes2 + .publish({ + type: 'test', + text: 'first post', + recps: [groupId], + }) + .catch(t.fail) + + // replicate the first group messages to bob when he can't decrypt them + await replicate(alice, bob).catch(t.error) + + await alice.tribes2 + .addMembers(groupId, [bobRoot.id, carolRoot.id]) + .then(() => t.pass('added bob and carol')) + .catch((err) => t.error(err, 'add bob and carol fail')) + + await alice.tribes2 + .excludeMembers(groupId, [carolRoot.id]) + .then((res) => { + t.pass('alice excluded carol') + return res + }) + .catch((err) => t.error(err, 'remove member fail')) + + const { key: secondPostId } = await alice.tribes2 + .publish({ + type: 'test', + text: 'second post', + recps: [groupId], + }) + .catch(t.fail) + + // only replicate bob's invite to him once we're already on the new epoch + await replicate(alice, bob).catch(t.error) + + // TODO: bob list invites + + // TODO: bob use invite + + // TODO: bob can read first and second alice post + + await p(alice.close)(true) + await p(bob.close)(true) + await p(carol.close)(true) +}) From e2a97d058b3b1dda8b2f0f6bd03c32d1a4bb947f Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Mon, 17 Apr 2023 16:51:34 +0200 Subject: [PATCH 02/11] Adapt listInvites to being added to multiple epochs --- README.md | 2 +- index.js | 33 +++++++++++++++++++++++++++++---- package.json | 1 + test/exclude-members.test.js | 25 ++++++++++++++++++++----- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 0ffbc1f..99ef812 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ Returns a pull stream source listing every known member of the group with id `gr ### `ssb.tribes2.listInvites() => source` -Returns a pull stream source listing invites (another user sent you one with `addMembers`) that you haven't accepted yet. The invites are on the same format as that of #create. +Returns a pull stream source listing groupIds of groups that you're invited to (another user sent you an invite with `addMembers`) that you haven't accepted yet. The invites are on the same format as that of #create. ### `ssb.tribes2.acceptInvite(groupId, cb)` diff --git a/index.js b/index.js index 8d28f45..485288f 100644 --- a/index.js +++ b/index.js @@ -7,6 +7,7 @@ const pull = require('pull-stream') const paraMap = require('pull-paramap') const pullMany = require('pull-many') const lodashGet = require('lodash.get') +const uniqBy = require('lodash.uniqby') const clarify = require('clarify-error') const { where, @@ -362,7 +363,8 @@ module.exports = { // prettier-ignore if (err) return cb(clarify(err, 'Failed to list group IDs when listing invites')) - const source = pull( + let acc = {} + pull( ssb.db.query( where(and(isDecrypted('box2'), type('group/add-member'))), toPullStream() @@ -392,10 +394,32 @@ module.exports = { readKeys: [{ key, scheme }], root: lodashGet(msg, 'value.content.root'), } - }) - ) + }), + // aggregate multiple readKeys from invites to different epochs into one invite object + pull.drain( + (invite) => { + // TODO: which writeKey should be picked?? + // or is that maybe not that important to decide here, maybe for acceptInvite? + if (acc[invite.id]) { + // readKeys from both invites + acc[invite.id].readKeys.push(...invite.readKeys) + // but no duplicates + acc[invite.id].readKeys = uniqBy( + acc[invite.id].readKeys, + (readKey) => readKey.key.toString('base64') + ) + } else { + acc[invite.id] = invite + } + }, + (err) => { + if (err) return cb('todo') - return cb(null, source) + const inviteArray = Object.values(acc) + return cb(null, pull.values(inviteArray)) + } + ) + ) }) ) }) @@ -415,6 +439,7 @@ module.exports = { pull.drain( (groupInfo) => { foundInvite = true + // TODO: loop over all the readKeys we've found and add them all ssb.box2.addGroupInfo( groupInfo.id, { diff --git a/package.json b/package.json index f8944a2..2236fea 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "is-canonical-base64": "^1.1.1", "lodash.get": "^4.4.2", "lodash.set": "^4.3.2", + "lodash.uniqby": "^4.7.0", "private-group-spec": "^6.0.0", "pull-paramap": "^1.2.2", "pull-stream": "^3.7.0", diff --git a/test/exclude-members.test.js b/test/exclude-members.test.js index 0b36b60..f2d25f5 100644 --- a/test/exclude-members.test.js +++ b/test/exclude-members.test.js @@ -477,10 +477,7 @@ test('Get added to an old epoch but still find newer epochs', async (t) => { await alice.tribes2 .excludeMembers(groupId, [carolRoot.id]) - .then((res) => { - t.pass('alice excluded carol') - return res - }) + .then(() => t.pass('alice excluded carol')) .catch((err) => t.error(err, 'remove member fail')) const { key: secondPostId } = await alice.tribes2 @@ -494,7 +491,25 @@ test('Get added to an old epoch but still find newer epochs', async (t) => { // only replicate bob's invite to him once we're already on the new epoch await replicate(alice, bob).catch(t.error) - // TODO: bob list invites + const bobInvites = await pull( + bob.tribes2.listInvites(), + pull.collectAsPromise() + ).catch(t.fail) + t.deepEquals( + bobInvites.map((invite) => invite.id), + [groupId], + 'bob has an invite to the group' + ) + t.equals( + bobInvites[0].readKeys.length, + 2, + 'there are 2 readKeys in the invite' + ) + t.notEquals( + bobInvites[0].readKeys[0].key.toString('base64'), + bobInvites[0].readKeys[1].key.toString('base64'), + 'the two readKeys are different' + ) // TODO: bob use invite From 7a4bcbae321c38995310d6828ebb5cc875e28d79 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Mon, 17 Apr 2023 16:58:12 +0200 Subject: [PATCH 03/11] Revert listInvites readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 99ef812..0ffbc1f 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ Returns a pull stream source listing every known member of the group with id `gr ### `ssb.tribes2.listInvites() => source` -Returns a pull stream source listing groupIds of groups that you're invited to (another user sent you an invite with `addMembers`) that you haven't accepted yet. The invites are on the same format as that of #create. +Returns a pull stream source listing invites (another user sent you one with `addMembers`) that you haven't accepted yet. The invites are on the same format as that of #create. ### `ssb.tribes2.acceptInvite(groupId, cb)` From 171626b276115c9a2d076942ad79de4f9d6fd2d8 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Tue, 18 Apr 2023 14:22:48 +0200 Subject: [PATCH 04/11] Add all epoch keys when accepting an invite --- index.js | 37 +++++++++++++++++++++--------------- package.json | 1 + test/exclude-members.test.js | 18 ++++++++++++++++-- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/index.js b/index.js index 485288f..eefa2e7 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,7 @@ const { promisify } = require('util') const pull = require('pull-stream') const paraMap = require('pull-paramap') const pullMany = require('pull-many') +const multicb = require('multicb') const lodashGet = require('lodash.get') const uniqBy = require('lodash.uniqby') const clarify = require('clarify-error') @@ -439,23 +440,29 @@ module.exports = { pull.drain( (groupInfo) => { foundInvite = true - // TODO: loop over all the readKeys we've found and add them all - ssb.box2.addGroupInfo( - groupInfo.id, - { - key: groupInfo.writeKey.key, - root: groupInfo.root, - }, - (err) => { + + const done = multicb() + + groupInfo.readKeys.forEach((readKey) => { + ssb.box2.addGroupInfo( + groupInfo.id, + { + key: readKey.key, + root: groupInfo.root, + }, + done() + ) + }) + + done((err) => { + // prettier-ignore + if (err) return cb(clarify(err, 'Failed to add group info when accepting an invite')) + ssb.db.reindexEncrypted((err) => { // prettier-ignore - if (err) return cb(clarify(err, 'Failed to add group info when accepting an invite')) - ssb.db.reindexEncrypted((err) => { - // prettier-ignore - if (err) cb(clarify(err, 'Failed to reindex encrypted messages when accepting an invite')) + if (err) cb(clarify(err, 'Failed to reindex encrypted messages when accepting an invite')) else cb(null, groupInfo) - }) - } - ) + }) + }) }, (err) => { // prettier-ignore diff --git a/package.json b/package.json index 2236fea..a1dece3 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "lodash.get": "^4.4.2", "lodash.set": "^4.3.2", "lodash.uniqby": "^4.7.0", + "multicb": "^1.2.2", "private-group-spec": "^6.0.0", "pull-paramap": "^1.2.2", "pull-stream": "^3.7.0", diff --git a/test/exclude-members.test.js b/test/exclude-members.test.js index f2d25f5..5f486a5 100644 --- a/test/exclude-members.test.js +++ b/test/exclude-members.test.js @@ -511,9 +511,23 @@ test('Get added to an old epoch but still find newer epochs', async (t) => { 'the two readKeys are different' ) - // TODO: bob use invite + await bob.tribes2.acceptInvite(groupId).catch(t.fail) - // TODO: bob can read first and second alice post + await p(setTimeout)(5000) + + const bobGotFirstMsg = await p(bob.db.get)(firstPostId) + t.notEquals( + typeof bobGotFirstMsg.content, + 'string', + "bob managed to decrypt alice's first message" + ) + + const bobGotSecondMsg = await p(bob.db.get)(secondPostId) + t.notEquals( + typeof bobGotSecondMsg.content, + 'string', + "bob managed to decrypt alice's second message" + ) await p(alice.close)(true) await p(bob.close)(true) From 1e454efceb62724ac1e292c7be9b73972fb8b6dd Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Tue, 18 Apr 2023 14:29:25 +0200 Subject: [PATCH 05/11] Remove timeout and adjust comments --- index.js | 5 ++--- test/exclude-members.test.js | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index eefa2e7..536204b 100644 --- a/index.js +++ b/index.js @@ -399,8 +399,6 @@ module.exports = { // aggregate multiple readKeys from invites to different epochs into one invite object pull.drain( (invite) => { - // TODO: which writeKey should be picked?? - // or is that maybe not that important to decide here, maybe for acceptInvite? if (acc[invite.id]) { // readKeys from both invites acc[invite.id].readKeys.push(...invite.readKeys) @@ -443,6 +441,8 @@ module.exports = { const done = multicb() + // TODO: which writeKey should be picked?? + // this will essentially pick a random write key groupInfo.readKeys.forEach((readKey) => { ssb.box2.addGroupInfo( groupInfo.id, @@ -514,7 +514,6 @@ module.exports = { // look for new epochs that we're added to pull( ssb.db.query( - // TODO: does this output new stuff if we accept an invite to an old epoch and then find additions to newer epochs? where(and(isDecrypted('box2'), type('group/add-member'))), live({ old: true }), toPullStream() diff --git a/test/exclude-members.test.js b/test/exclude-members.test.js index 5f486a5..abb249d 100644 --- a/test/exclude-members.test.js +++ b/test/exclude-members.test.js @@ -418,7 +418,7 @@ test("If you're not the excluder nor the excludee then you should still be in th await p(carol.close)(true) }) -test('Get added to an old epoch but still find newer epochs', async (t) => { +test.only('Get added to an old epoch but still find newer epochs', async (t) => { const alice = Testbot({ keys: ssbKeys.generate(null, 'alice'), mfSeed: Buffer.from( @@ -513,8 +513,6 @@ test('Get added to an old epoch but still find newer epochs', async (t) => { await bob.tribes2.acceptInvite(groupId).catch(t.fail) - await p(setTimeout)(5000) - const bobGotFirstMsg = await p(bob.db.get)(firstPostId) t.notEquals( typeof bobGotFirstMsg.content, From 460573888f9ece108beec3772ee6c2ed05f68cf8 Mon Sep 17 00:00:00 2001 From: Jacob Karlsson Date: Tue, 18 Apr 2023 14:29:40 +0200 Subject: [PATCH 06/11] Remove .only --- test/exclude-members.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/exclude-members.test.js b/test/exclude-members.test.js index abb249d..c497c5c 100644 --- a/test/exclude-members.test.js +++ b/test/exclude-members.test.js @@ -418,7 +418,7 @@ test("If you're not the excluder nor the excludee then you should still be in th await p(carol.close)(true) }) -test.only('Get added to an old epoch but still find newer epochs', async (t) => { +test('Get added to an old epoch but still find newer epochs', async (t) => { const alice = Testbot({ keys: ssbKeys.generate(null, 'alice'), mfSeed: Buffer.from( From c57cdf8157e42a71001f8fed7b5258e8744d885e Mon Sep 17 00:00:00 2001 From: mix irving Date: Wed, 19 Apr 2023 14:57:24 +1200 Subject: [PATCH 07/11] refactor listInvites, add isGroup operator --- index.js | 244 ++++++++++++++++++++----------------- lib/operators.js | 35 ++++++ package.json | 1 + test/invites.test.js | 7 +- test/lib/epochs.test.js | 2 +- test/lib/operators.test.js | 61 ++++++++++ 6 files changed, 230 insertions(+), 120 deletions(-) create mode 100644 lib/operators.js create mode 100644 test/lib/operators.test.js diff --git a/index.js b/index.js index 536204b..c1a09a4 100644 --- a/index.js +++ b/index.js @@ -6,9 +6,9 @@ const { promisify } = require('util') const pull = require('pull-stream') const paraMap = require('pull-paramap') const pullMany = require('pull-many') +const pullDefer = require('pull-defer') const multicb = require('multicb') const lodashGet = require('lodash.get') -const uniqBy = require('lodash.uniqby') const clarify = require('clarify-error') const { where, @@ -35,6 +35,7 @@ const buildGroupId = require('./lib/build-group-id') const addTangles = require('./lib/tangles/add-tangles') const publishAndPrune = require('./lib/prune-publish') const MetaFeedHelpers = require('./lib/meta-feed-helpers') +const { isGroup } = require('./lib/operators') // const Epochs = require('./lib/epochs') module.exports = { @@ -312,119 +313,135 @@ module.exports = { } function listMembers(groupId, opts = {}) { - return pull( - pull.values([0]), - pull.asyncMap((n, cb) => { - get(groupId, (err, group) => { - // prettier-ignore - if (err) return cb(clarify(err, 'Failed to get group info when listing members')) + const deferedSource = pullDefer.source() - if (group.excluded) { - return cb( - new Error("We're excluded from this group, can't list members") - ) - } else { - const source = pull( - ssb.db.query( - where(and(isDecrypted('box2'), type('group/add-member'))), - opts.live ? live({ old: true }) : null, - toPullStream() - ), - pull.map((msg) => lodashGet(msg, 'value.content.recps', [])), - pull.filter( - (recps) => recps.length > 1 && recps[0] === groupId - ), - pull.map((recps) => recps.slice(1)), - pull.flatten(), - pull.unique() + get(groupId, (err, group) => { + // prettier-ignore + if (err) + return deferedSource.abort( + clarify(err, 'Failed to get group info when listing members') + ) + if (group.excluded) + return deferedSource.abort( + new Error("We're excluded from this group, can't list members") + ) + + const source = pull( + ssb.db.query( + where( + and( + isDecrypted('box2'), + type('group/add-member'), + isGroup(groupId) ) - return cb(null, source) - } - }) - }), - pull.flatten() - ) + ), + opts.live ? live({ old: true }) : null, + toPullStream() + ), + pull.map((msg) => msg.value.content.recps[0]), + pull.unique() + ) + + deferedSource.resolve(source) + }) + + return deferedSource } function listInvites() { - return pull( - pull.values([0]), // dummy value used to kickstart the stream - pull.asyncMap((n, cb) => { - ssb.metafeeds.findOrCreate((err, myRoot) => { - // prettier-ignore - if (err) return cb(clarify(err, 'Failed to get root metafeed when listing invites')) + const deferedSource = pullDefer.source() - pull( - pullMany([ - ssb.box2.listGroupIds(), - ssb.box2.listGroupIds({ excluded: true }), - ]), - pull.flatten(), - pull.collect((err, groupIds) => { - // prettier-ignore - if (err) return cb(clarify(err, 'Failed to list group IDs when listing invites')) - - let acc = {} - pull( - ssb.db.query( - where(and(isDecrypted('box2'), type('group/add-member'))), - toPullStream() - ), - pull.filter((msg) => - // it's an addition of us - lodashGet(msg, 'value.content.recps', []).includes( - myRoot.id - ) - ), - pull.filter( - (msg) => - // we haven't already accepted the addition - !groupIds.includes( - lodashGet(msg, 'value.content.recps[0]') - ) - ), - pull.map((msg) => { - const key = Buffer.from( - lodashGet(msg, 'value.content.groupKey'), - 'base64' - ) - const scheme = keySchemes.private_group - return { - id: lodashGet(msg, 'value.content.recps[0]'), - writeKey: { key, scheme }, - readKeys: [{ key, scheme }], - root: lodashGet(msg, 'value.content.root'), - } - }), - // aggregate multiple readKeys from invites to different epochs into one invite object - pull.drain( - (invite) => { - if (acc[invite.id]) { - // readKeys from both invites - acc[invite.id].readKeys.push(...invite.readKeys) - // but no duplicates - acc[invite.id].readKeys = uniqBy( - acc[invite.id].readKeys, - (readKey) => readKey.key.toString('base64') - ) - } else { - acc[invite.id] = invite - } - }, - (err) => { - if (err) return cb('todo') - - const inviteArray = Object.values(acc) - return cb(null, pull.values(inviteArray)) - } - ) - ) - }) - ) - }) - }), - pull.flatten() - ) + ssb.metafeeds.findOrCreate((err, myRoot) => { + // prettier-ignore + if (err) return deferedSource.abort(clarify(err, 'Failed to get root metafeed when listing invites')) + + getMyGroups((err, myGroups) => { + // prettier-ignore + if (err) return deferedSource.abort(clarify(err, 'Failed to list group IDs when listing invites')) + + const source = pull( + // get all the groupIds we've heard of from invites + ssb.db.query( + where(and(isDecrypted('box2'), type('group/add-member'))), + toPullStream() + ), + pull.filter((msg) => isAddMember(msg)), + pull.map((msg) => msg.value.content.recps[0]), + pull.unique(), + + // drop those we're a part of already + pull.filter((groupId) => !myGroups.has(groupId)), + + // gather all the secrets shared for each group + pull.asyncMap(getGroupInviteData) + ) + + deferedSource.resolve(source) + }) + }) + + return deferedSource + + // listInvites helpers + + function getMyGroups(cb) { + const myGroups = new Set() + + pull( + pullMany([ + ssb.box2.listGroupIds(), + ssb.box2.listGroupIds({ excluded: true }), + ]), + pull.flatten(), + pull.drain( + (groupId) => myGroups.add(groupId), + (err) => { + if (err) return cb(err) + return cb(null, myGroups) + } + ) + ) + } + + function getGroupInviteData(groupId, cb) { + let root + const secrets = new Set() + + pull( + ssb.db.query( + where( + and( + isDecrypted('box2'), + type('group/add-member'), + isGroup(groupId) + ) + ), + toPullStream() + ), + pull.filter((msg) => isAddMember(msg)), + pull.drain( + (msg) => { + root ||= msg.value.content.root + secrets.add(msg.value.content.groupKey) + }, + (err) => { + if (err) return cb(err) + + const readKeys = [...secrets].map((secret) => ({ + key: Buffer.from(secret, 'base64'), + scheme: keySchemes.private_group, + })) + const invite = { + id: groupId, + root, + readKeys, + // writeKey :shrug: + } + return cb(null, invite) + } + ) + ) + } } function acceptInvite(groupId, cb) { @@ -527,15 +544,14 @@ module.exports = { paraMap((msg, cb) => { pull( ssb.box2.listGroupIds(), + pull.filter((groupId) => + groupId.includes(msg.value.content.recps[0]) + ), + pull.take(1), pull.collect((err, groupIds) => { // prettier-ignore if (err) return cb(clarify(err, "Error getting groups we're already in when looking for new epochs")) - - if (groupIds.includes(msg.value.content.recps[0])) { - return cb(null, msg) - } else { - return cb() - } + cb(null, groupIds.length ? msg : null) }) ) }, 4), diff --git a/lib/operators.js b/lib/operators.js new file mode 100644 index 0000000..753379b --- /dev/null +++ b/lib/operators.js @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2023 Mix Irving +// +// SPDX-License-Identifier: LGPL-3.0-only + +const { allocAndEncode, seekKey2, iterate } = require('bipf') +const jitdbOperators = require('jitdb/operators') +const { equal } = jitdbOperators + +const BIPF_CONTENT = allocAndEncode('content') +const BIPF_RECPS = allocAndEncode('recps') + +function seekFirstRecp (buffer, start, pValue) { + if (pValue < 0) return -1 + const pValueContent = seekKey2(buffer, pValue, BIPF_CONTENT, 0) + if (pValueContent < 0) return -1 + const pValueRecps = seekKey2(buffer, pValueContent, BIPF_RECPS, 0) + if (pValueRecps < 0) return -1 + + let pValueFirstRecp + const error = iterate(buffer, pValueRecps, (_, pointer, key) => { + pValueFirstRecp = pointer + return true + }) + if (error === -1) return -1 + return pValueFirstRecp +} + +module.exports = { + isGroup(value) { + return equal(seekFirstRecp, value, { + prefix: 32, + indexType: 'value_content_recps_0', + }) + } +} diff --git a/package.json b/package.json index a1dece3..be7051c 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "lodash.uniqby": "^4.7.0", "multicb": "^1.2.2", "private-group-spec": "^6.0.0", + "pull-defer": "^0.2.3", "pull-paramap": "^1.2.2", "pull-stream": "^3.7.0", "ssb-bfe": "^3.7.0", diff --git a/test/invites.test.js b/test/invites.test.js index 2a28eba..478b230 100644 --- a/test/invites.test.js +++ b/test/invites.test.js @@ -46,15 +46,12 @@ test('lists correct group invite and accepting actually does something', async ( t.pass('alice and bob replicate') const invites = await pull(bob.tribes2.listInvites(), pull.collectAsPromise()) + .catch(t.error) t.equal(invites.length, 1, 'bob has 1 invite') const invite = invites[0] t.equal(invite.id, group.id, 'correct group id in invite') - t.true(invite.writeKey.key.equals(group.writeKey.key), 'correct writeKey') - t.true( - invite.readKeys[0].key.equals(group.readKeys[0].key), - 'correct readKey' - ) + t.deepEquals(invite.readKeys, group.readKeys, 'correct readKey') t.equal(invite.root, group.root, 'correct root') const msgEnc = await p(bob.db.get)(group.root).catch(t.fail) diff --git a/test/lib/epochs.test.js b/test/lib/epochs.test.js index fb3373b..ca9e76a 100644 --- a/test/lib/epochs.test.js +++ b/test/lib/epochs.test.js @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2022 Mix Irving +// SPDX-FileCopyrightText: 2023 Mix Irving // // SPDX-License-Identifier: CC0-1.0 diff --git a/test/lib/operators.test.js b/test/lib/operators.test.js new file mode 100644 index 0000000..a346bc8 --- /dev/null +++ b/test/lib/operators.test.js @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2023 Mix Irving +// +// SPDX-License-Identifier: CC0-1.0 + +const test = require('tape') +const { promisify } = require('util') + +const Server = require('../helpers/testbot') +const { isGroup } = require('../../lib/operators') + +test('lib/operators - isGroup', async t => { + const alice = Server() + await alice.tribes2.start() + + const [group, group2] = await Promise.all([ + alice.tribes2.create(), + alice.tribes2.create() + ]) + + const content = { + type: 'post', + text: 'hey hey', + recps: [group.id] + } // NOTE this gets mutates with tangles.group + + await Promise.all([ + alice.tribes2.publish(content), + alice.tribes2.publish({ + type: 'post', + text: 'ho ho', + recps: [group2.id] + }), + promisify(alice.db.create)({ + content: { + type: 'post', + text: 'he he' + } + }) + ]) + + const { where, and, type, toPromise } = alice.db.operators + const results = await alice.db.query( + where( + and( + type('post'), + isGroup(group.id) + ) + ), + toPromise() + ) + + t.deepEqual( + results.map(m => m.value.content), + [content], + 'finds the message in the group' + ) + + alice.close() + + t.end() +}) From ec8559a59f02bb69e355a22ff60585215c501cd6 Mon Sep 17 00:00:00 2001 From: mix irving Date: Wed, 19 Apr 2023 15:09:19 +1200 Subject: [PATCH 08/11] remove un-needed deps --- index.js | 100 ++++++++++++++++++++------------------------------- package.json | 3 -- 2 files changed, 38 insertions(+), 65 deletions(-) diff --git a/index.js b/index.js index c1a09a4..31a19fd 100644 --- a/index.js +++ b/index.js @@ -7,8 +7,6 @@ const pull = require('pull-stream') const paraMap = require('pull-paramap') const pullMany = require('pull-many') const pullDefer = require('pull-defer') -const multicb = require('multicb') -const lodashGet = require('lodash.get') const clarify = require('clarify-error') const { where, @@ -317,14 +315,9 @@ module.exports = { get(groupId, (err, group) => { // prettier-ignore - if (err) - return deferedSource.abort( - clarify(err, 'Failed to get group info when listing members') - ) - if (group.excluded) - return deferedSource.abort( - new Error("We're excluded from this group, can't list members") - ) + if (err) return deferedSource.abort(clarify(err, 'Failed to get group info when listing members')) + // prettier-ignore + if (group.excluded) return deferedSource.abort( new Error("We're excluded from this group, can't list members")) const source = pull( ssb.db.query( @@ -351,33 +344,28 @@ module.exports = { function listInvites() { const deferedSource = pullDefer.source() - ssb.metafeeds.findOrCreate((err, myRoot) => { + getMyGroups((err, myGroups) => { // prettier-ignore - if (err) return deferedSource.abort(clarify(err, 'Failed to get root metafeed when listing invites')) + if (err) return deferedSource.abort(clarify(err, 'Failed to list group IDs when listing invites')) - getMyGroups((err, myGroups) => { - // prettier-ignore - if (err) return deferedSource.abort(clarify(err, 'Failed to list group IDs when listing invites')) - - const source = pull( - // get all the groupIds we've heard of from invites - ssb.db.query( - where(and(isDecrypted('box2'), type('group/add-member'))), - toPullStream() - ), - pull.filter((msg) => isAddMember(msg)), - pull.map((msg) => msg.value.content.recps[0]), - pull.unique(), + const source = pull( + // get all the groupIds we've heard of from invites + ssb.db.query( + where(and(isDecrypted('box2'), type('group/add-member'))), + toPullStream() + ), + pull.filter((msg) => isAddMember(msg)), + pull.map((msg) => msg.value.content.recps[0]), + pull.unique(), - // drop those we're a part of already - pull.filter((groupId) => !myGroups.has(groupId)), + // drop those we're a part of already + pull.filter((groupId) => !myGroups.has(groupId)), - // gather all the secrets shared for each group - pull.asyncMap(getGroupInviteData) - ) + // gather all the data required for each group-invite + pull.asyncMap(getGroupInviteData) + ) - deferedSource.resolve(source) - }) + deferedSource.resolve(source) }) return deferedSource @@ -447,47 +435,35 @@ module.exports = { function acceptInvite(groupId, cb) { if (cb === undefined) return promisify(acceptInvite)(groupId) - let foundInvite = false pull( listInvites(), - pull.filter((groupInfo) => groupInfo.id === groupId), + pull.filter((inviteInfo) => inviteInfo.id === groupId), pull.take(1), - pull.drain( - (groupInfo) => { - foundInvite = true - - const done = multicb() - - // TODO: which writeKey should be picked?? - // this will essentially pick a random write key - groupInfo.readKeys.forEach((readKey) => { - ssb.box2.addGroupInfo( - groupInfo.id, - { - key: readKey.key, - root: groupInfo.root, - }, - done() - ) - }) + pull.collect((err, inviteInfos) => { + // prettier-ignore + if (err) return cb(clarify(err, 'Failed to list invites when accepting an invite')) + // prettier-ignore + if (!inviteInfos.length) return cb(new Error("Didn't find invite for that group id")) - done((err) => { + // TODO: which writeKey should be picked?? + // this will essentially pick a random write key + const { id, root, readKeys } = inviteInfos[0] + pull( + pull.values(readKeys), + pull.asyncMap((readKey, cb) => { + ssb.box2.addGroupInfo(id, { key: readKey.key, root }, cb) + }), + pull.collect((err) => { // prettier-ignore if (err) return cb(clarify(err, 'Failed to add group info when accepting an invite')) ssb.db.reindexEncrypted((err) => { // prettier-ignore if (err) cb(clarify(err, 'Failed to reindex encrypted messages when accepting an invite')) - else cb(null, groupInfo) + else cb(null, inviteInfos[0]) }) }) - }, - (err) => { - // prettier-ignore - if (err) return cb(clarify(err, 'Failed to list invites when accepting an invite')) - // prettier-ignore - if (!foundInvite) return cb(new Error("Didn't find invite for that group id")) - } - ) + ) + }) ) } diff --git a/package.json b/package.json index be7051c..d8e1f9b 100644 --- a/package.json +++ b/package.json @@ -40,10 +40,7 @@ "envelope-spec": "^1.1.1", "fast-deep-equal": "^3.1.3", "is-canonical-base64": "^1.1.1", - "lodash.get": "^4.4.2", "lodash.set": "^4.3.2", - "lodash.uniqby": "^4.7.0", - "multicb": "^1.2.2", "private-group-spec": "^6.0.0", "pull-defer": "^0.2.3", "pull-paramap": "^1.2.2", From 9a23261885aca72537c636b151679ef75f2443a6 Mon Sep 17 00:00:00 2001 From: mix irving Date: Wed, 19 Apr 2023 16:34:13 +1200 Subject: [PATCH 09/11] collect groupRecp into operators --- index.js | 9 +++--- lib/operators.js | 45 ++++++++++++++++++++++---- lib/tangles/get-tangle-updates.js | 29 ++--------------- test/lib/operators.test.js | 54 +++++++++++++++++++++++++++++-- test/list-members.test.js | 26 +++++++++------ 5 files changed, 113 insertions(+), 50 deletions(-) diff --git a/index.js b/index.js index 31a19fd..61b1161 100644 --- a/index.js +++ b/index.js @@ -33,7 +33,7 @@ const buildGroupId = require('./lib/build-group-id') const addTangles = require('./lib/tangles/add-tangles') const publishAndPrune = require('./lib/prune-publish') const MetaFeedHelpers = require('./lib/meta-feed-helpers') -const { isGroup } = require('./lib/operators') +const { groupRecp } = require('./lib/operators') // const Epochs = require('./lib/epochs') module.exports = { @@ -325,13 +325,14 @@ module.exports = { and( isDecrypted('box2'), type('group/add-member'), - isGroup(groupId) + groupRecp(groupId) ) ), opts.live ? live({ old: true }) : null, toPullStream() ), - pull.map((msg) => msg.value.content.recps[0]), + pull.map((msg) => msg.value.content.recps.slice(1)), + pull.flatten(), pull.unique() ) @@ -401,7 +402,7 @@ module.exports = { and( isDecrypted('box2'), type('group/add-member'), - isGroup(groupId) + groupRecp(groupId) ) ), toPullStream() diff --git a/lib/operators.js b/lib/operators.js index 753379b..f26029d 100644 --- a/lib/operators.js +++ b/lib/operators.js @@ -3,17 +3,18 @@ // SPDX-License-Identifier: LGPL-3.0-only const { allocAndEncode, seekKey2, iterate } = require('bipf') -const jitdbOperators = require('jitdb/operators') -const { equal } = jitdbOperators +const { equal } = require('jitdb/operators') -const BIPF_CONTENT = allocAndEncode('content') -const BIPF_RECPS = allocAndEncode('recps') +const B_CONTENT = allocAndEncode('content') +const B_TANGLES = allocAndEncode('tangles') +const B_ROOT = allocAndEncode('root') +const B_RECPS = allocAndEncode('recps') function seekFirstRecp (buffer, start, pValue) { if (pValue < 0) return -1 - const pValueContent = seekKey2(buffer, pValue, BIPF_CONTENT, 0) + const pValueContent = seekKey2(buffer, pValue, B_CONTENT, 0) if (pValueContent < 0) return -1 - const pValueRecps = seekKey2(buffer, pValueContent, BIPF_RECPS, 0) + const pValueRecps = seekKey2(buffer, pValueContent, B_RECPS, 0) if (pValueRecps < 0) return -1 let pValueFirstRecp @@ -25,8 +26,38 @@ function seekFirstRecp (buffer, start, pValue) { return pValueFirstRecp } +const B_TANGLE_MAP = new Map() +// tangle: StrinG => B_TANGLE: Buffer + +function SeekTangleRoot (tangle) { + if (!B_TANGLE_MAP.has(tangle)) { + B_TANGLE_MAP.set(tangle, allocAndEncode(tangle)) + } + + const B_TANGLE = B_TANGLE_MAP.get(tangle) + + return function seekTangleRoot (buffer, start, p) { + if (p < 0) return -1 + p = seekKey2(buffer, p, B_CONTENT, 0) + if (p < 0) return -1 + p = seekKey2(buffer, p, B_TANGLES, 0) + if (p < 0) return -1 + p = seekKey2(buffer, p, B_TANGLE, 0) + if (p < 0) return -1 + return seekKey2(buffer, p, B_ROOT, 0) + } +} + module.exports = { - isGroup(value) { + tangleRoot(tangle, value) { + const seekTangleRoot = SeekTangleRoot(tangle) + return equal(seekTangleRoot, value, { + prefix: 32, + indexType: `value.content.tangles.${tangle}.root`, + }) + }, + + groupRecp(value) { return equal(seekFirstRecp, value, { prefix: 32, indexType: 'value_content_recps_0', diff --git a/lib/tangles/get-tangle-updates.js b/lib/tangles/get-tangle-updates.js index bea1f4c..2258172 100644 --- a/lib/tangles/get-tangle-updates.js +++ b/lib/tangles/get-tangle-updates.js @@ -1,13 +1,10 @@ // SPDX-FileCopyrightText: 2022 Mix Irving // // SPDX-License-Identifier: LGPL-3.0-only + const pull = require('pull-stream') -const { seekKey } = require('bipf') -const B_VALUE = Buffer.from('value') -const B_CONTENT = Buffer.from('content') -const B_TANGLES = Buffer.from('tangles') -const B_ROOT = Buffer.from('root') +const { tangleRoot } = require('../../lib/operators') function getTangleUpdates(ssb, tangle, root, cb) { pull( @@ -19,30 +16,10 @@ getTangleUpdates.stream = tangleUpdateStream module.exports = getTangleUpdates function tangleUpdateStream (ssb, tangle, root) { - const B_TANGLE = Buffer.from(tangle) - - function tangleRoot(value) { - return ssb.db.operators.equal(seekTangleRoot, value, { - indexType: `value.content.tangles.${tangle}.root`, - }) - } - function seekTangleRoot(buffer) { - let p = 0 // note you pass in p! - p = seekKey(buffer, p, B_VALUE) - if (p < 0) return - p = seekKey(buffer, p, B_CONTENT) - if (p < 0) return - p = seekKey(buffer, p, B_TANGLES) - if (p < 0) return - p = seekKey(buffer, p, B_TANGLE) - if (p < 0) return - return seekKey(buffer, p, B_ROOT) - } - const { where, and, toPullStream } = ssb.db.operators return pull( - ssb.db.query(where(and(tangleRoot(root))), toPullStream()), + ssb.db.query(where(and(tangleRoot(tangle, root))), toPullStream()), pull.filter(m => ( Array.isArray(m.value.content.tangles[tangle].previous) && m.value.content.tangles[tangle].previous.length > 0 diff --git a/test/lib/operators.test.js b/test/lib/operators.test.js index a346bc8..9d85624 100644 --- a/test/lib/operators.test.js +++ b/test/lib/operators.test.js @@ -6,9 +6,9 @@ const test = require('tape') const { promisify } = require('util') const Server = require('../helpers/testbot') -const { isGroup } = require('../../lib/operators') +const { groupRecp, tangleRoot } = require('../../lib/operators') -test('lib/operators - isGroup', async t => { +test('lib/operators - groupRecp', async t => { const alice = Server() await alice.tribes2.start() @@ -43,7 +43,7 @@ test('lib/operators - isGroup', async t => { where( and( type('post'), - isGroup(group.id) + groupRecp(group.id) ) ), toPromise() @@ -59,3 +59,51 @@ test('lib/operators - isGroup', async t => { t.end() }) + +test('lib/operators - tangleRoot', async t => { + const alice = Server() + + const rootId = await promisify(alice.db.create)({ + content: { + type: 'slime', + tangles: { + slime: { root: null, previous: null } + } + } + }) + + const updateContent1 = { + type: 'slime', + count: 1, + tangles: { + slime: { root: rootId, previous: [rootId] } + } + } + const updateContent2 = { + type: 'slime', + count: 2, + tangles: { + slime: { root: rootId, previous: [rootId] } + } + } + await promisify(alice.db.create)({ content: updateContent1 }) + await promisify(alice.db.create)({ content: updateContent2 }) + + const { where, toPromise } = alice.db.operators + const results = await alice.db.query( + where( + tangleRoot('slime', rootId) + ), + toPromise() + ) + + t.deepEqual( + results.map(m => m.value.content), + [updateContent1, updateContent2], + 'finds the message in the tangle' + ) + + alice.close() + + t.end() +}) diff --git a/test/list-members.test.js b/test/list-members.test.js index fa65974..040f084 100644 --- a/test/list-members.test.js +++ b/test/list-members.test.js @@ -34,13 +34,17 @@ test('list members', async (t) => { ), }) - await alice.tribes2.start() - await bob.tribes2.start() - await carol.tribes2.start() - - const aliceRoot = await p(alice.metafeeds.findOrCreate)() - const bobRoot = await p(bob.metafeeds.findOrCreate)() - const carolRoot = await p(carol.metafeeds.findOrCreate)() + await Promise.all([ + alice.tribes2.start(), + bob.tribes2.start(), + carol.tribes2.start(), + ]) + + const [ aliceRoot, bobRoot, carolRoot ] = await Promise.all([ + p(alice.metafeeds.findOrCreate)(), + p(bob.metafeeds.findOrCreate)(), + p(carol.metafeeds.findOrCreate)(), + ]) await replicate(alice, bob) t.pass('alice and bob replicated their trees') @@ -74,9 +78,11 @@ test('list members', async (t) => { ) }) - await p(alice.close)(true) - await p(bob.close)(true) - await p(carol.close)(true) + await Promise.all([ + p(alice.close)(true), + p(bob.close)(true), + p(carol.close)(true), + ]) }) test('live list members', async (t) => { From cf13384e2237793fd803ed976f35e99ca6a23e30 Mon Sep 17 00:00:00 2001 From: mix irving Date: Thu, 20 Apr 2023 13:47:31 +1200 Subject: [PATCH 10/11] Update index.js --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 61b1161..d6447ca 100644 --- a/index.js +++ b/index.js @@ -522,7 +522,7 @@ module.exports = { pull( ssb.box2.listGroupIds(), pull.filter((groupId) => - groupId.includes(msg.value.content.recps[0]) + groupId === msg.value.content.recps[0] ), pull.take(1), pull.collect((err, groupIds) => { From 042158a731337c1c469373a43ac5b66096335d4e Mon Sep 17 00:00:00 2001 From: mix irving Date: Thu, 20 Apr 2023 13:50:57 +1200 Subject: [PATCH 11/11] Update index.js --- index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/index.js b/index.js index d6447ca..fbb45fd 100644 --- a/index.js +++ b/index.js @@ -424,7 +424,6 @@ module.exports = { id: groupId, root, readKeys, - // writeKey :shrug: } return cb(null, invite) }