diff --git a/README.md b/README.md index 6255c62..4a3761d 100644 --- a/README.md +++ b/README.md @@ -110,11 +110,11 @@ NOTE: If `create` finds an empty (i.e. seemingly unused) group feed, it will sta - `id` _GroupUri_ - an SSB URI that's safe to use publicly to name the group, and is used in `recps` to trigger encrypting messages to that group - `subfeed` _Keys_ - the keys of the subfeed you should publish group data to - - `writeKey` _GroupKey_ - the current key used for publishing new messages to the group. It is one of the `readKeys`. - - `readKeys` _[GroupKey]_ - an array of all keys used to read messages for this group. + - `writeKey` _GroupSecret_ - the current key used for publishing new messages to the group. It is one of the `readKeys`. + - `readKeys` _[GroupSecret]_ - an array of all keys used to read messages for this group. - `root` _MessagedId_ - the MessageId of the `group/init` message of the group, encoded as an ssb-uri. - where _GroupKey_ is an object of the format + where _GroupSecret_ is an object of the format - `key` _Buffer_ - the symmetric key used by the group for encryption - `scheme` _String_ - the scheme for this key @@ -139,7 +139,7 @@ Publish `group/add-member` messages to a group of peers, which gives them all th - `feedIds` _[FeedId]_ - an Array of 1-15 different ids for peers (accepts ssb-uri or sigil feed ids) - `opts` _Object_ - with the options: - `text` _String_ - A piece of text attached to the addition. Visible to the whole group and the newly added people. -- `cb` _Function_ - a callback of signature `(err, msg)` +- `cb` _Function_ - a callback of signature `(err, Array)` ### `ssb.tribes2.excludeMembers(groupId, feedIds, opts, cb) diff --git a/index.js b/index.js index 7f3ecac..da1fc5b 100644 --- a/index.js +++ b/index.js @@ -14,7 +14,6 @@ const clarify = require('clarify-error') const { where, and, - or, isDecrypted, type, toPullStream, @@ -64,7 +63,12 @@ module.exports = { findOrCreateGroupWithoutMembers, getRootFeedIdFromMsgId, } = MetaFeedHelpers(ssb) - const { getPreferredEpoch, getMembers } = Epochs(ssb) + const { + getTipEpochs, + getPredecessorEpochs, + getPreferredEpoch, + getMembers, + } = Epochs(ssb) function create(opts = {}, cb) { if (cb === undefined) return promisify(create)(opts) @@ -80,7 +84,7 @@ module.exports = { const data = { id: buildGroupId({ groupInitMsg, - groupKey: secret.toBuffer(), + secret: secret.toBuffer(), }), writeKey: { key: secret.toBuffer(), @@ -136,6 +140,8 @@ module.exports = { function addMembers(groupId, feedIds, opts = {}, cb) { if (cb === undefined) return promisify(addMembers)(groupId, feedIds, opts) + opts.oldSecrets ??= true + if (!feedIds || feedIds.length === 0) { return cb(new Error('No feedIds provided to addMembers')) } @@ -147,7 +153,7 @@ module.exports = { return cb(new Error('addMembers only supports bendybutt-v1 feed IDs')) } - get(groupId, (err, { writeKey, root }) => { + get(groupId, (err, { root }) => { // prettier-ignore if (err) return cb(clarify(err, `Failed to get group details when adding members`)) @@ -155,34 +161,58 @@ module.exports = { // prettier-ignore if (err) return cb(clarify(err, "couldn't get root id of author of root msg")) - const content = { - type: 'group/add-member', - version: 'v2', - groupKey: writeKey.key.toString('base64'), - root, - creator: rootAuthorId, - recps: [groupId, ...feedIds], - } + getTipEpochs(groupId, (err, tipEpochs) => { + // prettier-ignore + if (err) return cb(clarify(err, "Couldn't get tip epochs when adding members")) - if (opts.text) content.text = opts.text + const getFeed = opts?._feedKeys + ? (cb) => cb(null, { keys: opts._feedKeys }) + : findOrCreateAdditionsFeed - const getFeed = opts?._feedKeys - ? (cb) => cb(null, { keys: opts._feedKeys }) - : findOrCreateAdditionsFeed + getFeed((err, additionsFeed) => { + // prettier-ignore + if (err) return cb(clarify(err, 'Failed to find or create additions feed when adding members')) - getFeed((err, additionsFeed) => { - // prettier-ignore - if (err) return cb(clarify(err, 'Failed to find or create additions feed when adding members')) + const options = { + isValid: isAddMember, + tangles: ['members'], + feedKeys: additionsFeed.keys, + } + pull( + pull.values(tipEpochs), + pull.asyncMap((tipEpoch, cb) => { + getPredecessorEpochs( + groupId, + tipEpoch.id, + (err, predecessors) => { + // prettier-ignore + if (err) return cb(clarify(err, "Couldn't get predecessor epochs when adding members")) + + const oldSecrets = predecessors.map((pred) => + pred.secret.toString('base64') + ) + const content = { + type: 'group/add-member', + version: 'v2', + secret: tipEpoch.secret.toString('base64'), + oldSecrets: opts.oldSecrets ? oldSecrets : undefined, + root, + creator: rootAuthorId, + text: opts?.text, + recps: [groupId, ...feedIds], + } + return cb(null, content) + } + ) + }), + pull.asyncMap((content, cb) => publish(content, options, cb)), + pull.collect((err, msgs) => { + // prettier-ignore + if (err) return cb(clarify(err, 'Failed to publish add-member message(s)')) - const options = { - isValid: isAddMember, - tangles: ['members'], - feedKeys: additionsFeed.keys, - } - publish(content, options, (err, msg) => { - // prettier-ignore - if (err) return cb(clarify(err, 'Failed to publish add-member message')) - return cb(null, msg) + return cb(null, msgs) + }) + ) }) }) }) @@ -221,15 +251,15 @@ module.exports = { const remainingMembers = beforeMembers.filter( (member) => !feedIds.includes(member) ) - const newGroupKey = new SecretKey() - const addInfo = { key: newGroupKey.toBuffer() } + const newSecret = new SecretKey() + const addInfo = { key: newSecret.toBuffer() } ssb.box2.addGroupInfo(groupId, addInfo, (err) => { // prettier-ignore if (err) return cb(clarify(err, "Couldn't store new key when excluding members")) const newKey = { - key: newGroupKey.toBuffer(), + key: newSecret.toBuffer(), scheme: keySchemes.private_group, } ssb.box2.pickGroupWriteKey(groupId, newKey, (err) => { @@ -239,7 +269,7 @@ module.exports = { const newEpochContent = { type: 'group/init', version: 'v2', - groupKey: newGroupKey.toString('base64'), + secret: newSecret.toString('base64'), tangles: { members: { root: null, previous: null }, }, @@ -256,7 +286,12 @@ module.exports = { pull( pull.values(chunk(remainingMembers, 15)), pull.asyncMap((membersToAdd, cb) => - addMembers(groupId, membersToAdd, {}, cb) + addMembers( + groupId, + membersToAdd, + { oldSecrets: false }, + cb + ) ), pull.collect((err) => { // prettier-ignore @@ -420,6 +455,7 @@ module.exports = { function getGroupInviteData(groupId, cb) { let root const secrets = new Set() + let writeSecret = null pull( ssb.db.query( @@ -436,7 +472,13 @@ module.exports = { pull.drain( (msg) => { root ||= msg.value.content.root - secrets.add(msg.value.content.groupKey) + const oldSecrets = msg.value.content.oldSecrets + if (oldSecrets) { + oldSecrets.forEach((oldSecret) => secrets.add(oldSecret)) + } + const latestSecret = msg.value.content.secret + secrets.add(latestSecret) + writeSecret = latestSecret }, (err) => { if (err) return cb(err) @@ -449,6 +491,10 @@ module.exports = { id: groupId, root, readKeys, + writeKey: { + key: Buffer.from(writeSecret, 'base64'), + scheme: keySchemes.private_group, + }, } return cb(null, invite) } @@ -471,8 +517,8 @@ module.exports = { if (!inviteInfos.length) return cb(new Error("Didn't find invite for that group id")) // TODO: which writeKey should be picked?? - // this will essentially pick a random write key - const { id, root, readKeys } = inviteInfos[0] + // this will essentially pick a random write key from the current epoch tips + const { id, root, readKeys, writeKey } = inviteInfos[0] pull( pull.values(readKeys), pull.asyncMap((readKey, cb) => { @@ -481,10 +527,15 @@ module.exports = { pull.collect((err) => { // prettier-ignore if (err) return cb(clarify(err, 'Failed to add group info when accepting an invite')) - ssb.db.reindexEncrypted((err) => { + ssb.box2.pickGroupWriteKey(id, writeKey, (err) => { // prettier-ignore - if (err) cb(clarify(err, 'Failed to reindex encrypted messages when accepting an invite')) - else cb(null, inviteInfos[0]) + if (err) return cb(clarify(err, 'Failed to pick a write key when accepting invite to a group')) + + ssb.db.reindexEncrypted((err) => { + // prettier-ignore + if (err) cb(clarify(err, 'Failed to reindex encrypted messages when accepting an invite')) + else cb(null, inviteInfos[0]) + }) }) }) ) diff --git a/lib/build-group-id.js b/lib/build-group-id.js index a21f2cd..ce469b9 100644 --- a/lib/build-group-id.js +++ b/lib/build-group-id.js @@ -15,7 +15,7 @@ module.exports = function buildGroupId({ groupInitMsg, readKey, msgKey, - groupKey, + secret, }) { const msgId = bfe.encode(groupInitMsg.key) @@ -25,10 +25,10 @@ module.exports = function buildGroupId({ readKey = toBuffer(groupInitMsg.value.meta.unbox) } else if (msgKey) { readKey = fromMsgKey(groupInitMsg, msgKey) - } else if (groupKey) { + } else if (secret) { // when we've just heard a group/add-member message, we need to calculate // groupId and only have these two things - readKey = fromGroupKey(groupInitMsg, groupKey) + readKey = fromSecret(groupInitMsg, secret) } else { throw new Error('Read key must be defined???') } @@ -52,7 +52,7 @@ function fromMsgKey(msg, msgKey) { return derive(msgKey, [LABELS.read_key]) } -function fromGroupKey(msg, groupKey) { +function fromSecret(msg, secret) { const { author, previous, content } = msg.value const envelope = Buffer.from(content.replace('.box2', ''), 'base64') @@ -60,7 +60,7 @@ function fromGroupKey(msg, groupKey) { const prev_msg_id = bfe.encode(previous) const group_key = { - key: toBuffer(groupKey), + key: toBuffer(secret), scheme: keySchemes.private_group, } return unboxKey(envelope, feed_id, prev_msg_id, [group_key], { diff --git a/lib/epochs.js b/lib/epochs.js index 7387686..c08cb70 100644 --- a/lib/epochs.js +++ b/lib/epochs.js @@ -76,7 +76,7 @@ function Epochs(ssb) { ssb.metafeeds.findRootFeedId(epochRoot.value.author, cb) }, secret(epochRoot, cb) { - cb(null, epochRoot.value.content.groupKey) + cb(null, epochRoot.value.content.secret) }, members(epochRoot, cb) { getMembers(epochRoot.key, cb) @@ -105,6 +105,55 @@ function Epochs(ssb) { }) } + function getTipEpochs(groupId, cb) { + if (cb === undefined) return p(this.getTipEpochs).call(this, groupId) + + const opts = { getters: pluck(allGetters, ['author', 'secret']) } + epochsReduce(groupId, opts, (err, reduce) => { + // prettier-ignore + if (err) return cb(clarify(err, 'Failed to resolve epoch @tangle/reduce')) + + const tips = Object.keys(reduce.state).map((id) => { + const info = { + id, + previous: reduce.graph.getNode(id).previous, + ...reduce.state[id][id], + } + info.secret = Buffer.from(info.secret, 'base64') + return info + }) + + return cb(null, tips) + }) + } + + /** Gets all epochs leading up to the specified epoch, all the way back to the root. + * Does not include the specified epoch nor epochs in epoch forks. + */ + function getPredecessorEpochs(groupId, epochRootId, cb) { + if (cb === undefined) + return p(this.getPredecessorEpochs).call(this, groupId, epochRootId) + + const opts = { getters: pluck(allGetters, ['author', 'secret']) } + epochsReduce(groupId, opts, (err, reduce) => { + // prettier-ignore + if (err) return cb(clarify(err, "Couldn't get epoch reducer when getting predecessors")) + + const predecessors = reduce.graph.getHistory(epochRootId).map((id) => { + const node = reduce.graph.getNode(id) + const data = node.data[id] + + return { + id, + previous: node.previous, + secret: Buffer.from(data.secret, 'base64'), + } + }) + + return cb(null, predecessors) + }) + } + function getMembers(epochRootId, cb) { if (cb === undefined) return p(getMembers)(epochRootId) @@ -344,7 +393,6 @@ function Epochs(ssb) { pull( epochNodeStream(ssb, groupId, { getters, live }), pull.map((node) => { - console.log('adding epoch', node.id) reduce.addNodes([node]) return true }) @@ -384,6 +432,8 @@ function Epochs(ssb) { return { getEpochs, + getTipEpochs, + getPredecessorEpochs, getMembers, getPreferredEpoch, getMissingMembers, diff --git a/lib/meta-feed-helpers.js b/lib/meta-feed-helpers.js index 9b535fc..4cac8d0 100644 --- a/lib/meta-feed-helpers.js +++ b/lib/meta-feed-helpers.js @@ -144,7 +144,7 @@ module.exports = (ssb) => { const content = { type: 'group/init', version: 'v2', - groupKey: secret.toString('base64'), + secret: secret.toString('base64'), tangles: { group: { root: null, previous: null }, epoch: { root: null, previous: null }, diff --git a/listeners.js b/listeners.js index c77a6cc..17b83e9 100644 --- a/listeners.js +++ b/listeners.js @@ -94,7 +94,7 @@ module.exports = function startListeners(ssb, getPreferredEpoch, onError) { (msg) => { const groupId = msg.value.content.recps[0] - const secret = Buffer.from(msg.value.content.groupKey, 'base64') + const secret = Buffer.from(msg.value.content.secret, 'base64') ssb.box2.addGroupInfo(groupId, { key: secret }, (err) => { // prettier-ignore if (err && !isClosed) return onError(clarify(err, 'Cannot add new epoch key that we found')) diff --git a/package.json b/package.json index d6af624..17ca281 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ssb-tribes2", - "version": "0.4.0", + "version": "1.0.1", "description": "SSB private groups with ssb-db2", "repository": { "type": "git", @@ -25,7 +25,7 @@ "files": [ "package.json.license", "LICENSES/*", - "lib/*.js", + "lib/**/*.js", "*.js" ], "dependencies": { @@ -43,7 +43,7 @@ "jitdb": "^7.0.7", "lodash.chunk": "^4.2.0", "lodash.set": "^4.3.2", - "private-group-spec": "^7.0.0", + "private-group-spec": "^8.0.0", "pull-abortable": "^4.1.1", "pull-defer": "^0.2.3", "pull-flat-merge": "^2.0.3", @@ -52,7 +52,7 @@ "set.prototype.difference": "^1.0.2", "ssb-bfe": "^3.7.0", "ssb-box2": "^7.1.0", - "ssb-db2": "^7.1.0", + "ssb-db2": "^7.1.1", "ssb-meta-feeds": "^0.39.0", "ssb-private-group-keys": "^1.1.2", "ssb-ref": "^2.16.0", @@ -68,7 +68,6 @@ "rimraf": "^3.0.2", "secret-stack": "^6.4.1", "ssb-bendy-butt": "^1.0.2", - "ssb-caps": "^1.1.0", "ssb-classic": "^1.1.0", "ssb-ebt": "^9.1.2", "ssb-keys": "^8.5.0", diff --git a/test/add-member.test.js b/test/add-member.test.js index dba7171..9e19555 100644 --- a/test/add-member.test.js +++ b/test/add-member.test.js @@ -6,10 +6,11 @@ const test = require('tape') const pull = require('pull-stream') const { promisify: p } = require('util') const ssbKeys = require('ssb-keys') -const Testbot = require('./helpers/testbot') -const replicate = require('./helpers/replicate') +const { Testbot, replicate, Run } = require('./helpers') test('get added to a group', async (t) => { + const run = Run(t) + const alice = Testbot({ keys: ssbKeys.generate(null, 'alice'), mfSeed: Buffer.from( @@ -25,63 +26,41 @@ test('get added to a group', async (t) => { ), }) - await Promise.all([alice.tribes2.start(), bob.tribes2.start()]) - t.pass('tribes2 started for both alice and bob') + await run( + 'tribes2 started for both alice and bob', + Promise.all([alice.tribes2.start(), bob.tribes2.start()]) + ) await p(alice.metafeeds.findOrCreate)() const bobRoot = await p(bob.metafeeds.findOrCreate)() - await replicate(alice, bob) - t.pass('alice and bob replicate their trees') + await run('alice and bob replicate their trees', replicate(alice, bob)) const { id: groupId, writeKey, root, - } = await alice.tribes2.create().catch((err) => { - console.error('alice failed to create group', err) - t.fail(err) - }) - t.pass('alice created a group') + } = await run('alice created a group', alice.tribes2.create()) - await alice.tribes2.addMembers(groupId, [bobRoot.id]).catch((err) => { - console.error('add member fail', err) - t.fail(err) - }) - t.pass('alice added bob to the group') + await run( + 'alice added bob to the group', + alice.tribes2.addMembers(groupId, [bobRoot.id]) + ) - await replicate(alice, bob) - .then(() => - t.pass('alice and bob replicate after bob getting added to the group') - ) - .catch((err) => { - console.error( - 'failed to replicate after alice added bob to the group', - err - ) - t.error(err) - }) + await run( + 'alice and bob replicate after bob getting added to the group', + replicate(alice, bob) + ) - await bob.tribes2.acceptInvite(groupId).catch((err) => { - console.error('failed to accept invite', err) - t.fail(err) - }) + await run('bob accepted invite', bob.tribes2.acceptInvite(groupId)) - t.pass('bob accepted invite') - - await new Promise((res) => - pull( - bob.tribes2.list(), - pull.collect((err, bobList) => { - t.equal(bobList.length, 1, 'bob is a member of a group now') - const group = bobList[0] - t.equal(group.id, groupId, 'group id is correct') - t.true(group.writeKey.key.equals(writeKey.key)) - t.equal(group.root, root) - res() - }) - ) - ) + const bobList = await pull(bob.tribes2.list(), pull.collectAsPromise()) + + t.equal(bobList.length, 1, 'bob is a member of a group now') + const group = bobList[0] + t.equal(group.id, groupId, 'group id is correct') + t.true(group.writeKey.key.equals(writeKey.key)) + t.equal(group.root, root) await Promise.all([p(alice.close)(true), p(bob.close)(true)]) }) @@ -116,7 +95,7 @@ test('add member', async (t) => { const newMembers = [newPersonRoot.id] - const encryptedInvite = await kaitiaki.tribes2.addMembers( + const [encryptedInvite] = await kaitiaki.tribes2.addMembers( group.id, newMembers, { @@ -129,7 +108,8 @@ test('add member', async (t) => { const expected = { type: 'group/add-member', version: 'v2', - groupKey: group.writeKey.key.toString('base64'), + secret: group.writeKey.key.toString('base64'), + oldSecrets: [], root: group.root, creator: kaitiakiRoot.id, @@ -272,3 +252,196 @@ test('addMembers too many members', async (t) => { await p(alice.close)(true) }) + +test('addMembers adds to all the tip epochs and gives keys to all the old epochs as well', async (t) => { + // alice adds bob and carol + // alice and bob remove carol at the same time, creating forked epochs + // everyone still replicates and sees the fork + // alice adds david to the group, and he should see both forks and the original epoch + + const run = Run(t) + + const alice = Testbot({ name: 'alice' }) + const bob = Testbot({ name: 'bob' }) + const carol = Testbot({ name: 'carol' }) + const david = Testbot({ name: 'david' }) + + await run( + 'clients started', + Promise.all([ + alice.tribes2.start(), + bob.tribes2.start(), + carol.tribes2.start(), + david.tribes2.start(), + ]) + ) + + const [, bobRootId, carolRootId, davidRootId] = ( + await run( + 'got peer roots', + Promise.all( + [alice, bob, carol, david].map((peer) => + p(peer.metafeeds.findOrCreate)() + ) + ) + ) + ).map((root) => root.id) + + async function replicateAll() { + await p(setTimeout)(2000) + + await replicate(alice, bob, carol, david) + .then(() => t.pass('replicated all')) + .catch((err) => t.error(err, 'replicated all')) + + await p(setTimeout)(2000) + } + + await replicateAll() + + const { id: groupId, writeKey: firstEpochKey } = await run( + 'alice created group', + alice.tribes2.create() + ) + const firstEpochSecret = firstEpochKey.key.toString('base64') + + const { key: firstEpochPostId } = await run( + 'alice published in first epoch', + alice.tribes2.publish({ + type: 'test', + text: 'first post', + recps: [groupId], + }) + ) + + await run( + 'alice added bob and carol', + alice.tribes2.addMembers(groupId, [bobRootId, carolRootId]) + ) + + await replicateAll() + + await run('bob accepted invite', bob.tribes2.acceptInvite(groupId)) + + await run( + 'alice and bob excluded carol', + Promise.all([ + alice.tribes2.excludeMembers(groupId, [carolRootId]), + bob.tribes2.excludeMembers(groupId, [carolRootId]), + ]) + ) + + const { key: aliceForkPostId } = await run( + 'alice published in her fork', + alice.tribes2.publish({ + type: 'test', + text: 'alice fork post', + recps: [groupId], + }) + ) + + const { writeKey: aliceForkKey } = await run( + 'alice got info on her fork', + alice.tribes2.get(groupId) + ) + const aliceForkSecret = aliceForkKey.key.toString('base64') + + const { key: bobForkPostId } = await run( + 'bob posted in his fork', + bob.tribes2.publish({ + type: 'test', + text: 'bob fork post', + recps: [groupId], + }) + ) + + const { writeKey: bobForkKey } = await run( + 'bob got info on his fork', + bob.tribes2.get(groupId) + ) + const bobForkSecret = bobForkKey.key.toString('base64') + + await replicateAll() + + const addDavid = await run( + 'david got added to the group by alice', + alice.tribes2.addMembers(groupId, [davidRootId]) + ) + t.equal(addDavid.length, 2, 'David got added to both forks') + + const adds = await run( + 'alice got her additions of david', + Promise.all(addDavid.map((add) => p(alice.db.get)(add.key))) + ) + const addContents = adds.map((add) => add.content) + const addAliceFork = addContents.find( + (content) => content.secret === aliceForkSecret + ) + t.equal( + addAliceFork.secret, + aliceForkSecret, + "gave david the secret to alice's fork" + ) + t.deepEqual( + addAliceFork.oldSecrets, + [firstEpochSecret], + "gave david the secret to the initial epoch, in the addition to alice's fork" + ) + + const addBobFork = addContents.find( + (content) => content.secret === bobForkSecret + ) + t.equal( + addBobFork.secret, + bobForkSecret, + "gave david the secret to bob's fork" + ) + t.deepEqual( + addBobFork.oldSecrets, + [firstEpochSecret], + "gave david the secret to the initial epoch, in the addition to bob's fork" + ) + + await run('replicated', replicate(alice, david)) + + await run('david accepted invite', david.tribes2.acceptInvite(groupId)) + + const bobForkMsg = await run( + "david got bob's post in his fork", + p(david.db.get)(bobForkPostId) + ) + t.notEquals( + typeof bobForkMsg.content, + 'string', + "david decrypted the msg in bob's fork" + ) + + const aliceForkMsg = await run( + "david got alice's post in her fork", + p(david.db.get)(aliceForkPostId) + ) + t.notEquals( + typeof aliceForkMsg.content, + 'string', + "david decrypted the msg in alice's fork" + ) + + const firstEpochMsg = await run( + "david got alice's post in the initial epoch", + p(david.db.get)(firstEpochPostId) + ) + t.notEquals( + typeof firstEpochMsg.content, + 'string', + 'david decrypted the msg in the first epoch' + ) + + await Promise.all([ + p(alice.close)(true), + p(bob.close)(true), + p(carol.close)(true), + p(david.close)(true), + ]) + .then(() => t.pass('clients close')) + .catch((err) => t.error(err)) +}) diff --git a/test/create.test.js b/test/create.test.js index b4e2800..433033e 100644 --- a/test/create.test.js +++ b/test/create.test.js @@ -39,7 +39,7 @@ test('create more', async (t) => { t.true(isIdentityGroupSSBURI(group.id), 'returns group identifier - groupId') t.true( Buffer.isBuffer(group.writeKey.key) && group.writeKey.key.length === 32, - 'returns group symmetric key - groupKey' + 'returns group symmetric key - groupSecret' ) const msgVal = await p(ssb.db.get)(group.root).catch(t.fail) @@ -51,7 +51,7 @@ test('create more', async (t) => { { type: 'group/init', version: 'v2', - groupKey: group.writeKey.key.toString('base64'), + secret: group.writeKey.key.toString('base64'), tangles: { group: { root: null, previous: null }, epoch: { root: null, previous: null }, @@ -71,7 +71,8 @@ test('create more', async (t) => { { type: 'group/add-member', version: 'v2', - groupKey: group.writeKey.key.toString('base64'), + secret: group.writeKey.key.toString('base64'), + oldSecrets: [], creator: rootFeed.id, root: group.root, recps: [group.id, root.id], // me being added to the group @@ -198,15 +199,19 @@ test("create reuses a group feed that hasn't had members yet (because of an earl createEmptyGroupFeed({ server, root }, (err, groupFeed) => { if (err) t.fail(err) + const secret = new SecretKey( + Buffer.from(groupFeed.purpose, 'base64') + ) const content = { type: 'group/init', + version: 'v2', + secret: groupFeed.purpose, tangles: { group: { root: null, previous: null }, + members: { root: null, previous: null }, + epoch: { root: null, previous: null }, }, } - const secret = new SecretKey( - Buffer.from(groupFeed.purpose, 'base64') - ) const recps = [ { key: secret.toBuffer(), scheme: keySchemes.private_group }, root.id, diff --git a/test/exclude-members.test.js b/test/exclude-members.test.js index 16e8314..912b700 100644 --- a/test/exclude-members.test.js +++ b/test/exclude-members.test.js @@ -55,7 +55,7 @@ test('add and exclude a person, post on the new feed', async (t) => { .create() .catch((err) => t.error(err, 'alice failed to create group')) - const addBobMsg = await alice.tribes2 + const [addBobMsg] = await alice.tribes2 .addMembers(groupId, [bobId]) .catch((err) => t.error(err, 'add member fail')) @@ -113,7 +113,7 @@ test('add and exclude a person, post on the new feed', async (t) => { const firstInit = firstContents[0] t.equal(firstInit.type, 'group/init') - t.equal(firstInit.groupKey, writeKey1.key.toString('base64')) + t.equal(firstInit.secret, writeKey1.key.toString('base64')) const excludeMsg = firstContents[1] @@ -138,7 +138,7 @@ test('add and exclude a person, post on the new feed', async (t) => { t.equal(secondInit.type, 'group/init') t.equal(secondInit.version, 'v2') - t.equal(secondInit.groupKey, writeKey2.key.toString('base64')) + t.equal(secondInit.secret, writeKey2.key.toString('base64')) t.deepEqual(secondInit.tangles.members, { root: null, previous: null }) t.deepEqual( secondInit.tangles.epoch, @@ -165,6 +165,16 @@ test('add and exclude a person, post on the new feed', async (t) => { t.equal(reinviteMsg.type, 'group/add-member') t.deepEqual(reinviteMsg.recps, [groupId, aliceId]) + t.equal( + reinviteMsg.secret, + writeKey2.key.toString('base64'), + 're-addition gives secret to new epoch' + ) + t.equal( + reinviteMsg.oldSecrets, + undefined, + "re-addition doesn't send secrets to old epochs" + ) const secondInitKey = fromMessageSigil(msgsFromSecond[0].key) t.deepEqual( @@ -610,21 +620,8 @@ test('Can exclude a person in a group with a lot of members', async (t) => { }) test("restarting the client doesn't make us rejoin old stuff", async (t) => { - const alice = Testbot({ - keys: ssbKeys.generate(null, 'alice'), - mfSeed: Buffer.from( - '000000000000000000000000000000000000000000000000000000000000a1ce', - 'hex' - ), - }) - let bob = Testbot({ - name: 'bobrestart', - keys: ssbKeys.generate(null, 'bob'), - mfSeed: Buffer.from( - '0000000000000000000000000000000000000000000000000000000000000b0b', - 'hex' - ), - }) + const alice = Testbot({ name: 'alice' }) + let bob = Testbot({ name: 'bob' }) await Promise.all([alice.tribes2.start(), bob.tribes2.start()]) @@ -664,15 +661,7 @@ test("restarting the client doesn't make us rejoin old stuff", async (t) => { await p(bob.close)(true).then(() => t.pass("bob's client was closed")) await p(setTimeout)(500) - bob = Testbot({ - rimraf: false, - name: 'bobrestart', - keys: ssbKeys.generate(null, 'bob'), - mfSeed: Buffer.from( - '0000000000000000000000000000000000000000000000000000000000000b0b', - 'hex' - ), - }) + bob = Testbot({ name: 'bob', rimraf: false }) t.pass('bob got a new client') await bob.tribes2.start().then(() => t.pass('bob restarted')) diff --git a/test/helpers/index.js b/test/helpers/index.js new file mode 100644 index 0000000..f8c4792 --- /dev/null +++ b/test/helpers/index.js @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2022 Mix Irving +// +// SPDX-License-Identifier: Unlicense + +module.exports = { + countGroupFeeds: require('./count-group-feeds'), + replicate: require('./replicate'), + Run: require('./run'), + Testbot: require('./testbot'), +} diff --git a/test/helpers/replicate.js b/test/helpers/replicate.js index 5479557..bcd2911 100644 --- a/test/helpers/replicate.js +++ b/test/helpers/replicate.js @@ -8,27 +8,74 @@ const pullMany = require('pull-many') const deepEqual = require('fast-deep-equal') /** - * Fully replicates person1's metafeed tree to person2 and vice versa + * Fully replicates between two or more peers + * + * Known bug: If you're e.g. created a group and posted in it but not invited anyone before replicating, then the group creator has to be the first peer listed. This is because we use branchStream (which only lists feeds you can decrypt the metafeed tree reference to) to figure out what to replicate, but we use getVectorClock (which lists *every* feed you have) to figure out when replication is done. So doing bob<->alice (where alice created the group) then bob<->carol fails, because bob can't pass along the group feed that alice posted on. */ -module.exports = async function replicate(person1, person2) { +module.exports = async function replicate(...peers) { + if (peers.length === 1 && Array.isArray(peers[0])) peers = peers[0] + if (peers.length === 2) return replicatePair(...peers) + + return pull( + pull.values(peers), + pull.asyncMap((person1, cb) => { + pull( + pull.values(peers), + pull.asyncMap((person2, cb) => { + if (person1.id === person2.id) return cb(null, true) + + replicatePair(person1, person2) + .then(() => cb(null, true)) + .catch((err) => cb(err)) + }), + pull.collect(cb) + ) + }), + pull.collectAsPromise() + ) +} + +async function replicatePair(person1, person2) { + // const start = Date.now() + // const ID = [person1, person2] + // .map(p => p.name || p.id.slice(0, 10)) + // .join('-') + // Establish a network connection const conn = await p(person1.connect)(person2.getAddress()) + const isSync = await ebtReplicate(person1, person2).catch((err) => + console.error('Error with ebtReplicate:\n', err) + ) + if (!isSync) { + console.error('EBT failed to replicate! Final state:') + console.error(person1.id, await p(person1.getVectorClock)()) + console.error(person2.id, await p(person2.getVectorClock)()) + } + + await p(conn.close)(true).catch(console.error) + // const time = Date.now() - start + // const length = Math.max(Math.round(time / 100), 1) + // console.log(ID, Array(length).fill('▨').join(''), time + 'ms') +} + +async function ebtReplicate(person1, person2) { // ensure persons are replicating all the trees in their forests, // from top to bottom const stream = setupFeedRequests(person1, person2) // Wait until both have replicated all feeds in full (are in sync) - await retryUntil(async () => { + const isSync = async () => { const clocks = await Promise.all([ p(person1.getVectorClock)(), p(person2.getVectorClock)(), ]) return deepEqual(...clocks) - }) + } + const isSuccess = await retryUntil(isSync) stream.abort() - await p(conn.close)(true) + return isSuccess } function setupFeedRequests(person1, person2) { @@ -60,13 +107,16 @@ function setupFeedRequests(person1, person2) { return drain } +// try an async task up to 100 times till it returns true +// if success retryUntil returns true, otherwise false async function retryUntil(checkIsDone) { let isDone = false for (let i = 0; i < 100; i++) { isDone = await checkIsDone() - if (isDone) return + if (isDone) return true await p(setTimeout)(100) } - if (!isDone) throw new Error('retryUntil timed out') + + return false } diff --git a/test/helpers/run.js b/test/helpers/run.js new file mode 100644 index 0000000..ebb7794 --- /dev/null +++ b/test/helpers/run.js @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2022 Mix Irving +// +// SPDX-License-Identifier: Unlicense + +module.exports = function Run(t) { + // this function takes care of running a promise and logging + // (or testing) that it happens and any errors are handled + return async function run(label, promise, opts = {}) { + const { isTest = true, timer = false, logError = false } = opts + + if (timer) console.time('> ' + label) + return promise + .then((res) => { + if (isTest) t.pass(label) + return res + }) + .catch((err) => { + t.error(err, label) + if (logError) console.error(err) + }) + .finally(() => timer && console.timeEnd('> ' + label)) + } +} diff --git a/test/helpers/testbot.js b/test/helpers/testbot.js index 0f1007b..ce87bd7 100644 --- a/test/helpers/testbot.js +++ b/test/helpers/testbot.js @@ -7,22 +7,30 @@ const ssbKeys = require('ssb-keys') const bendyButtFormat = require('ssb-ebt/formats/bendy-butt') const path = require('path') const rimraf = require('rimraf') -const caps = require('ssb-caps') +const crypto = require('crypto') + +const shs = crypto.randomBytes(32) +// ensure this is unique per run so peers can connect with one another but NOT +// the same as the main-net ssb-caps (so this doesn't try gossiping with e.g. +// manyverse instances) let count = 0 -/** opts.path (optional) - * opts.name (optional) - convenience method for deterministic opts.path - * opts.keys (optional) - * opts.rimraf (optional) - clear the directory before start (default: true) +/** opts.path (optional) + * opts.name (optional) - convenience method for deterministic opts.path + * opts.keys (optional) + * opts.rimraf (optional) - clear the directory before start (default: true) */ module.exports = function createSbot(opts = {}) { const dir = opts.path || `/tmp/ssb-tribes2-tests-${opts.name || count++}` if (opts.rimraf !== false) rimraf.sync(dir) - const keys = opts.keys || ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) + const keys = + opts.keys || opts.name + ? ssbKeys.generate(null, opts.name) + : ssbKeys.loadOrCreateSync(path.join(dir, 'secret')) - const stack = SecretStack({ appKey: caps.shs }) + const stack = SecretStack({ caps: { shs } }) .use(require('ssb-db2/core')) .use(require('ssb-classic')) .use(require('ssb-bendy-butt')) @@ -30,6 +38,8 @@ module.exports = function createSbot(opts = {}) { .use(require('ssb-box2')) .use(require('ssb-db2/compat/feedstate')) .use(require('ssb-db2/compat/ebt')) + .use(require('ssb-db2/compat/db')) // for legacy replicate + .use(require('ssb-db2/compat/history-stream')) // for legacy replicate .use(require('ssb-ebt')) .use(require('../..')) @@ -44,11 +54,47 @@ module.exports = function createSbot(opts = {}) { writeTimeout: 10, }, metafeeds: { - seed: opts.mfSeed, + seed: opts.mfSeed || mfSeedFromName(opts.name), }, }) + sbot.name = opts.name sbot.ebt.registerFormat(bendyButtFormat) return sbot } + +function mfSeedFromName(name) { + if (!name) return + + switch (name) { + case 'alice': + return Buffer.from( + '000000000000000000000000000000000000000000000000000000000000a1ce', + 'hex' + ) + case 'bob': + return Buffer.from( + '0000000000000000000000000000000000000000000000000000000000000b0b', + 'hex' + ) + case 'carol': + return Buffer.from( + '00000000000000000000000000000000000000000000000000000000000ca201', + 'hex' + ) + case 'david': + return Buffer.from( + '00000000000000000000000000000000000000000000000000000000000da71d', + 'hex' + ) + case 'oscar': + return Buffer.from( + '0000000000000000000000000000000000000000000000000000000000005ca4', + 'hex' + ) + + default: + throw new Error('no mfSeed set up for ' + name) + } +} diff --git a/test/lib/tangles/get-tangle-data.test.js b/test/lib/tangles/get-tangle-data.test.js index 19e6504..34ded0e 100644 --- a/test/lib/tangles/get-tangle-data.test.js +++ b/test/lib/tangles/get-tangle-data.test.js @@ -20,8 +20,7 @@ const Testbot = require('../../helpers/testbot') const replicate = require('../../helpers/replicate') test('get-tangle-data unit test', (t) => { - const name = `get-group-tangle-${Date.now()}` - const server = Testbot({ name }) + const server = Testbot() server.metafeeds.findOrCreate( { purpose: 'group/additions' }, @@ -208,7 +207,7 @@ test('get-tangle with branch', async (t) => { const group = await p(alice.tribes2.create)(null).catch(t.fail) t.pass('alice created a group') - const invite = await p(alice.tribes2.addMembers)(group.id, [bobRoot.id], { + const [invite] = await p(alice.tribes2.addMembers)(group.id, [bobRoot.id], { text: 'ahoy', }).catch(t.fail) t.pass('alice invited bob') @@ -281,9 +280,13 @@ test('members tangle works', async (t) => { const group = await alice.tribes2.create().catch(t.fail) t.pass('alice created a group') - const bobInvite = await p(alice.tribes2.addMembers)(group.id, [bobRoot.id], { - text: 'ahoy', - }).catch(t.fail) + const [bobInvite] = await p(alice.tribes2.addMembers)( + group.id, + [bobRoot.id], + { + text: 'ahoy', + } + ).catch(t.fail) await replicate(alice, bob) await bob.tribes2.acceptInvite(group.id) const bobPost = await bob.tribes2.publish({ @@ -312,7 +315,7 @@ test('members tangle works', async (t) => { 'members tangle is correct' ) - const carolInviteEnc = await p(alice.tribes2.addMembers)( + const [carolInviteEnc] = await p(alice.tribes2.addMembers)( group.id, [carolRoot.id], { diff --git a/test/list-and-get.test.js b/test/list-and-get.test.js index 4d615fc..b96eb20 100644 --- a/test/list-and-get.test.js +++ b/test/list-and-get.test.js @@ -11,9 +11,7 @@ const Testbot = require('./helpers/testbot') const replicate = require('./helpers/replicate') test('tribes.list + tribes.get', (t) => { - const name = `list-and-get-groups-${Date.now()}` - let server = Testbot({ name }) - const keys = server.keys + let server = Testbot({ name: 'alice' }) server.tribes2.create(null, (err, group) => { t.error(err, 'create group') @@ -40,7 +38,7 @@ test('tribes.list + tribes.get', (t) => { server.close(true, (err) => { t.error(err, 'closes server') - server = Testbot({ name, rimraf: false, keys }) + server = Testbot({ name: 'alice', rimraf: false }) pull( server.tribes2.list(), pull.collect((err, newList) => { diff --git a/test/prune-publish.test.js b/test/prune-publish.test.js index 8827100..7f14088 100644 --- a/test/prune-publish.test.js +++ b/test/prune-publish.test.js @@ -59,28 +59,34 @@ test('prune a message with way too big `previous`', async (t) => { t.end() }) -test('publish many messages that might need pruning', async (t) => { - const n = 5000 - const ssb = Testbot() - - const group = await p(ssb.tribes2.create)(null) - - const start = Date.now() - let count = 0 - await Promise.all( - Array.from({ length: n }, (_, i) => { - const content = { type: 'potato', count: i, recps: [group.id] } - return ssb.tribes2.publish(content, null).then(() => { - count++ - if (count % 500 === 0) t.pass(count) +test( + 'publish many messages that might need pruning', + { timeout: 60 * 1000 }, + async (t) => { + const n = 5000 + const ssb = Testbot({ db2: {} }) + + const group = await p(ssb.tribes2.create)(null) + + const start = Date.now() + let count = 0 + await Promise.all( + Array.from({ length: n }, (_, i) => { + const content = { type: 'potato', count: i, recps: [group.id] } + return ssb.tribes2.publish(content, null).then(() => { + count++ + if (count % 1000 === 0) t.pass(count) + }) }) - }) - ) - .then(() => { - t.pass(`published ${n} messages in ${Date.now() - start}ms`) - }) - .catch(t.error) + ) + .then(() => { + t.pass(`published ${n} messages in ${Date.now() - start}ms`) + }) + .catch(t.error) - await p(ssb.close)(true) - t.end() -}) + await p(setTimeout)(1000) + await p(ssb.close)(true) + + t.end() + } +)