Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update members tangle calculation for exclusion spec #75

Merged
merged 7 commits into from
Mar 30, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const {
const { SecretKey } = require('ssb-private-group-keys')
const { fromMessageSigil, isBendyButtV1FeedSSBURI } = require('ssb-uri2')
const buildGroupId = require('./lib/build-group-id')
const AddTangles = require('./lib/add-tangles')
const addTangles = require('./lib/add-tangles')
const publishAndPrune = require('./lib/prune-publish')
const MetaFeedHelpers = require('./lib/meta-feed-helpers')

Expand All @@ -39,7 +39,6 @@ module.exports = {
},
// eslint-disable-next-line no-unused-vars
init(ssb, config) {
const addTangles = AddTangles(ssb)
const {
secretKeyFromString,
findOrCreateAdditionsFeed,
Expand Down Expand Up @@ -276,7 +275,7 @@ module.exports = {
}
const groupId = recps[0]

addTangles(content, tangles, (err, content) => {
addTangles(ssb, content, tangles, (err, content) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to add group tangle when publishing to a group'))

Expand Down
37 changes: 15 additions & 22 deletions lib/add-tangles.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,19 @@
const set = require('lodash.set')
const clarify = require('clarify-error')
const { isIdentityGroupSSBURI } = require('ssb-uri2')
const GetTangle = require('./get-tangle')

module.exports = function AddTangles(server) {
const getTangle = {
group: GetTangle(server, 'group'),
members: GetTangle(server, 'members'),
epoch: GetTangle(server, 'epoch'),
}
const getTangle = require('./get-tangle')

/**
* Note that this mutates `content`
* `tangles` is an array like ["group", "members"]
*/
module.exports = function addTangles(server, content, tangles, cb) {
function addSomeTangles(content, tangles, cb) {
if (tangles.length === 0) return cb(null, content)

const currTangle = tangles[0]

getTangle[currTangle](content.recps[0], (err, generatedTangle) => {
getTangle(server, currTangle, content.recps[0], (err, generatedTangle) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to get group tangle when adding group tangle to message'))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (err) return cb(clarify(err, 'Failed to get group tangle when adding group tangle to message'))
if (err) return cb(clarify(err, `Failed to get ${currTangle} tangle when adding group tangle to message`))


Expand All @@ -35,20 +33,15 @@ module.exports = function AddTangles(server) {
})
}

/**
* Note that this mutates `content`
* `tangles` is an array like ["group", "members"]
*/
return function addTangles(content, tangles, cb) {
if (!content.recps) return cb(null, content)
if (!content.recps) return cb(new Error('Missing recps when adding tangles'))

if (!isIdentityGroupSSBURI(content.recps[0])) return cb(null, content)
if (!isIdentityGroupSSBURI(content.recps[0]))
return cb(new Error('recps[0] is not a group id when adding tangles'))

addSomeTangles(content, tangles, (err, content) => {
// prettier-ignore
if (err) return cb(clarify(err, 'failed to add tangles to content'))
addSomeTangles(content, tangles, (err, content) => {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is sufficient:

  addSomeTangles(content, tangles, cb)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know but I wanted more clarify since this gets run a lot

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And new Error('CLARIFICATION', {cause: err}) is the new clarify(err, 'CLARIFICATION')

// prettier-ignore
if (err) return cb(clarify(err, 'failed to add tangles to content'))

return cb(null, content)
})
}
return cb(null, content)
})
}
129 changes: 82 additions & 47 deletions lib/get-tangle.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ const Reduce = require('@tangle/reduce')
const Strategy = require('@tangle/strategy')
const clarify = require('clarify-error')
const { isIdentityGroupSSBURI, fromMessageSigil } = require('ssb-uri2')

// for figuring out what "previous" should be for the group
const { where, author, toPullStream } = require('ssb-db2/operators')

const strategy = new Strategy({})

Expand All @@ -17,57 +16,95 @@ function toUri(link) {
return link.startsWith('%') ? fromMessageSigil(link) : link
}

/** `server` is the ssb server you're using. `tangle` is the name of the tangle in the group you're looking for, e.g. "group" or "members" */
module.exports = function GetTangle(server, tangle) {
const getUpdates = GetUpdates(server, tangle)
function getTangleRoot(server, groupId, tangle, cb) {
server.box2.getGroupInfo(groupId, (err, info) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to get group info when getting a tangle'))

return function getTangle(groupId, cb) {
if (!isIdentityGroupSSBURI(groupId)) {
// prettier-ignore
return cb(new Error(`get-tangle expects valid groupId, got: ${groupId}`))
if (!info) {
return cb(new Error(`get-tangle: unknown groupId ${groupId}`))
}

server.box2.getGroupInfo(groupId, (err, info) => {
if (tangle === 'members') {
// we find the id of the init msg for the current epoch
pull(
server.metafeeds.branchStream({ old: true, live: false }),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Question: does this return all feeds or just your meta-feed feeds? (I can't remember and the docs don't say!)

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you have to specify opts.root to only get from one person

// get all leaf feeds
pull.filter((branch) => branch.length === 4),
pull.map((branch) => branch[3]),
// get all feeds for this epoch
pull.filter(
(feed) => feed.purpose === info.writeKey.key.toString('base64')
),
pull.map((feed) =>
pull(
server.db.query(where(author(feed.id)), toPullStream()),
// get all first messages, since that's where the init would be
pull.take(1)
)
),
pull.flatten(),
// find the init
pull.filter((msg) => msg.value?.content?.type === 'group/init'),
Comment on lines +39 to +48
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
pull.map((feed) =>
pull(
server.db.query(where(author(feed.id)), toPullStream()),
// get all first messages, since that's where the init would be
pull.take(1)
)
),
pull.flatten(),
// find the init
pull.filter((msg) => msg.value?.content?.type === 'group/init'),
pull.map((feed) =>
pull(
server.db.query(
where(and(author(feed.id), type('group/init'))),
toPullStream()
),
pull.filter(isInit)
pull.take(1)
)
),
pull.flatten(),

will require type and isInit to be imported

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is faster because we already have an index on type

pull.take(1),
pull.drain(
(msg) => cb(null, fromMessageSigil(msg.key)),
(err) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to find init msg for current epoch when trying to get members tangle'))
}
)
)
} else {
return cb(null, info.root)
}
})
}

/** for figuring out what "previous" should be for the group. `server` is the ssb server you're using. `tangle` is the name of the tangle in the group you're looking for, e.g. "group" or "members" */
function getTangle(server, tangle, groupId, cb) {
if (!isIdentityGroupSSBURI(groupId)) {
// prettier-ignore
return cb(new Error(`get-tangle expects valid groupId, got: ${groupId}`))
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In ssb-tribes I put a cache over the groups tangle https://github.com/ssbc/ssb-tribes/blob/master/lib/get-group-tangle.js#L12

I will perhaps look at adding that in another PR in the future.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The jitdb cache seems really fast after the first query

getTangleRoot(server, groupId, tangle, (err, root) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to get tangle root when getting tangle'))

getUpdates(server, tangle, root, (err, msgs) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to get group key info when getting a tangle'))

if (!info) {
return cb(new Error(`get-tangle: unknown groupId ${groupId}`))
}

getUpdates(info.root, (err, msgs) => {
// prettier-ignore
if (err) return cb(clarify(err, 'Failed to read updates when getting tangle'))

const nodes = msgs.map((msg) => ({
key: toUri(msg.key),
previous: msg.value.content.tangles[tangle].previous,
}))
// NOTE: getUpdates query does not get root node
nodes.push({ key: info.root, previous: null })

// Create a Reduce using the message contents
// NOTE - do NOT store the whole msg (node)
// we're not doing any reducing of transformations, we care only about
// reducing the graph to find the tips
// each node should be pruned down to e.g. { key: '%D', previous: ['%B', '%C'] }

const reduce = new Reduce(strategy, { nodes })
cb(null, {
root: info.root,
previous: Object.keys(reduce.state),
})
if (err) return cb(clarify(err, 'Failed to read updates when getting tangle'))

const nodes = msgs.map((msg) => ({
key: toUri(msg.key),
previous: msg.value.content.tangles[tangle].previous,
}))
// NOTE: getUpdates query does not get root node
nodes.push({ key: root, previous: null })

// Create a Reduce using the message contents
// NOTE - do NOT store the whole msg (node)
// we're not doing any reducing of transformations, we care only about
// reducing the graph to find the tips
// each node should be pruned down to e.g. { key: '%D', previous: ['%B', '%C'] }

const reduce = new Reduce(strategy, { nodes })
cb(null, {
root,
previous: Object.keys(reduce.state),
})
})
}
})
}
module.exports = getTangle

const B_VALUE = Buffer.from('value')
const B_CONTENT = Buffer.from('content')
const B_TANGLES = Buffer.from('tangles')
const B_ROOT = Buffer.from('root')

function GetUpdates(ssb, tangle) {
function getUpdates(ssb, tangle, root, cb) {
const { seekKey } = require('bipf')
const B_TANGLE = Buffer.from(tangle)

Expand All @@ -89,11 +126,9 @@ function GetUpdates(ssb, tangle) {
return seekKey(buffer, p, B_ROOT)
}

return function getUpdates(id, cb) {
const { where, and, toPullStream } = ssb.db.operators
pull(
ssb.db.query(where(and(tangleRoot(id))), toPullStream()),
pull.collect(cb)
)
}
const { where, and, toPullStream } = ssb.db.operators
pull(
ssb.db.query(where(and(tangleRoot(root))), toPullStream()),
pull.collect(cb)
)
Comment on lines +129 to +133
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
const { where, and, toPullStream } = ssb.db.operators
pull(
ssb.db.query(where(and(tangleRoot(root))), toPullStream()),
pull.collect(cb)
)
const { where, toCallback } = ssb.db.operators
ssb.db.query(where(tangleRoot(root)), toCallback(cb))

}
11 changes: 10 additions & 1 deletion test/exclude-members.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -124,13 +124,22 @@ test('add and remove a person, post on the new feed', async (t) => {

t.equal(reinviteMsg.type, 'group/add-member')
t.deepEqual(reinviteMsg.recps, [groupId, aliceRoot.id])
// TODO: check members tangle

const msgsFromSecond = await alice.db.query(
where(author(secondFeedId)),
toPromise()
)

const secondInitKey = fromMessageSigil(msgsFromSecond[0].key)
t.deepEqual(
reinviteMsg.tangles.members,
{
root: secondInitKey,
previous: [secondInitKey],
},
'members tangle resets after new epoch'
)

const secondContents = msgsFromSecond.map((msg) => msg.value.content)

t.equal(secondContents.length, 2, '2 messages on second (new) feed')
Expand Down
Loading