From ad10af255f80182a4e4f70ded747554eca7fcce7 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 21 Sep 2021 16:38:23 +0300 Subject: [PATCH 1/4] new API branchStream() --- api.js | 5 +++ feeds-lookup.js | 105 ++++++++++++++++++++++++++++++++++++++++++++---- package.json | 2 + test/api.js | 29 +++++++++++++ 4 files changed, 134 insertions(+), 7 deletions(-) diff --git a/api.js b/api.js index 10eb87b..357dbf8 100644 --- a/api.js +++ b/api.js @@ -86,6 +86,10 @@ exports.init = function (sbot, config) { return sbot.metafeeds.lookup.findByIdSync(feedId) } + function branchStream(opts) { + return sbot.metafeeds.lookup.branchStream(opts) + } + function filterTombstoned(metafeed, maybeVisit, cb) { if (!metafeed || typeof metafeed === 'function') { cb(new Error('filterTombstoned() requires a valid metafeed argument')) @@ -250,6 +254,7 @@ exports.init = function (sbot, config) { ensureLoaded, create, findOrCreate, + branchStream, filterTombstoned, findTombstoned, } diff --git a/feeds-lookup.js b/feeds-lookup.js index 1b7c754..9a32eac 100644 --- a/feeds-lookup.js +++ b/feeds-lookup.js @@ -1,5 +1,7 @@ const { seekKey } = require('bipf') const pull = require('pull-stream') +const cat = require('pull-cat') +const Notify = require('pull-notify') const SSBURI = require('ssb-uri2') const DeferredPromise = require('p-defer') const { @@ -46,7 +48,10 @@ exports.init = function (sbot, config) { const stateLoadedP = DeferredPromise() let loadStateRequested = false let liveDrainer = null - const lookup = new Map() // feedId => details + let notifyNewBranch = null + const detailsLookup = new Map() // feedId => details + const childrenLookup = new Map() // feedId => Set + const roots = new Set() const ensureQueue = { _map: new Map(), // feedId => Array add(feedId, cb) { @@ -106,14 +111,45 @@ exports.init = function (sbot, config) { } function updateLookup(msg) { - const { type, subfeed } = msg.value.content + const { type, subfeed, metafeed } = msg.value.content + + // Update roots + if (!detailsLookup.has(metafeed)) { + detailsLookup.set(metafeed, null) + roots.add(metafeed) + } + + // Update children + if (childrenLookup.has(metafeed)) { + const subfeeds = childrenLookup.get(metafeed) + if (type.startsWith('metafeed/add/')) { + subfeeds.add(subfeed) + } else if (type === 'metafeed/tombstone') { + subfeeds.delete(subfeed) + if (subfeeds.size === 0) { + childrenLookup.delete(metafeed) + } + } + } else { + if (type.startsWith('metafeed/add/')) { + const subfeeds = new Set() + subfeeds.add(subfeed) + childrenLookup.set(metafeed, subfeeds) + } + } + + // Update details if (type.startsWith('metafeed/add/')) { - lookup.set(subfeed, msgToDetails(msg)) + detailsLookup.set(subfeed, msgToDetails(msg)) + roots.delete(subfeed) ensureQueue.flush(subfeed) } else if (type === 'metafeed/tombstone') { - lookup.delete(subfeed) + detailsLookup.delete(subfeed) + roots.delete(subfeed) ensureQueue.flush(subfeed) } + + if (notifyNewBranch) notifyNewBranch(makeBranch(subfeed)) } function loadState() { @@ -128,9 +164,11 @@ exports.init = function (sbot, config) { stateLoaded = true stateLoadedP.resolve() + notifyNewBranch = Notify() sbot.close.hook(function (fn, args) { - if (liveDrainer) liveDrainer.abort() + if (liveDrainer) liveDrainer.abort(true) + if (notifyNewBranch) notifyNewBranch.abort(true) fn.apply(this, args) }) @@ -147,6 +185,38 @@ exports.init = function (sbot, config) { ) } + function makeBranch(subfeed) { + const details = detailsLookup.get(subfeed) + const branch = [[subfeed, details]] + while (branch[0][1]) { + const metafeedId = branch[0][1].metafeed + const details = detailsLookup.get(metafeedId) || null + branch.unshift([metafeedId, details]) + } + return branch + } + + function traverseBranchesUnder(feedId, previousBranch, visit) { + const details = detailsLookup.get(feedId) || null + const branch = [...previousBranch, [feedId, details]] + visit(branch) + if (childrenLookup.has(feedId)) { + for (const childFeedId of childrenLookup.get(feedId)) { + traverseBranchesUnder(childFeedId, branch, visit) + } + } + } + + function branchStreamOld() { + const branches = [] + for (const rootMetafeedId of roots) { + traverseBranchesUnder(rootMetafeedId, [], (branch) => { + branches.push(branch) + }) + } + return pull.values(branches) + } + return { loadState(cb) { if (!loadStateRequested) { @@ -159,7 +229,7 @@ exports.init = function (sbot, config) { ensureLoaded(feedId, cb) { if (!loadStateRequested) loadState() - if (lookup.has(feedId)) cb() + if (detailsLookup.has(feedId)) cb() else ensureQueue.add(feedId, cb) }, @@ -169,7 +239,7 @@ exports.init = function (sbot, config) { } assertFeedId(feedId) - return lookup.get(feedId) + return detailsLookup.get(feedId) }, findById(feedId, cb) { @@ -201,5 +271,26 @@ exports.init = function (sbot, config) { }) ) }, + + branchStream(opts) { + if (!notifyNewBranch) return pull.empty() + const { live = true, old = false, root = null } = opts || {} + const filterFn = root + ? (branch) => branch.length > 0 && branch[0][0] === root + : () => true + + if (old && live) { + return pull( + cat([branchStreamOld(), notifyNewBranch.listen()]), + pull.filter(filterFn) + ) + } else if (live) { + return pull(notifyNewBranch.listen(), pull.filter(filterFn)) + } else if (old) { + return pull(branchStreamOld(), pull.filter(filterFn)) + } else { + return pull.empty() + } + }, } } diff --git a/package.json b/package.json index 088c895..8b8d788 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,8 @@ "is-canonical-base64": "^1.1.1", "p-defer": "^3.0.0", "promisify-tuple": "^1.2.0", + "pull-cat": "^1.1.11", + "pull-notify": "^0.1.1", "pull-stream": "^3.6.14", "ssb-bendy-butt": "~0.12.3", "ssb-bfe": "^3.1.1", diff --git a/test/api.js b/test/api.js index ab212da..4ec4633 100644 --- a/test/api.js +++ b/test/api.js @@ -2,6 +2,7 @@ const test = require('tape') const ssbKeys = require('ssb-keys') const path = require('path') const rimraf = require('rimraf') +const pull = require('pull-stream') const SecretStack = require('secret-stack') const caps = require('ssb-caps') const { author, where, toCallback } = require('ssb-db2/operators') @@ -233,6 +234,34 @@ test('findById and findByIdSync', (t) => { }) }) +test('branchStream', (t) => { + pull( + sbot.metafeeds.branchStream({ old: true, live: false }), + pull.collect((err, branches) => { + t.error(err, 'no err') + t.equal(branches.length, 5, '5 branches') + + t.equal(branches[0].length, 1, 'root mf alone') + t.equal(typeof branches[0][0][0], 'string', 'root mf alone') + t.equal(branches[0][0][1], null, 'root mf alone') + + t.equal(branches[1].length, 2, 'main branch') + t.equal(branches[1][1][1].feedpurpose, 'main', 'main branch') + + t.equal(branches[2].length, 2, 'chess branch') + t.equal(branches[2][1][1].feedpurpose, 'chess', 'chess branch') + + t.equal(branches[3].length, 2, 'indexes branch') + t.equal(branches[3][1][1].feedpurpose, 'indexes', 'indexes branch') + + t.equal(branches[4].length, 3, 'index branch') + t.equal(branches[4][2][1].feedpurpose, 'index', 'indexes branch') + + t.end() + }) + ) +}) + test('restart sbot', (t) => { sbot.close(true, () => { sbot = SecretStack({ appKey: caps.shs }) From 71da362ad2ed2bfa657df124ba41ed14ca8a879d Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 22 Sep 2021 12:35:04 +0300 Subject: [PATCH 2/4] fix initialization state for branchStream --- feeds-lookup.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/feeds-lookup.js b/feeds-lookup.js index 9a32eac..a4f78ea 100644 --- a/feeds-lookup.js +++ b/feeds-lookup.js @@ -153,6 +153,9 @@ exports.init = function (sbot, config) { } function loadState() { + loadStateRequested = true + notifyNewBranch = Notify() + pull( sbot.db.query( where(and(authorIsBendyButtV1(), isPublic())), @@ -164,7 +167,6 @@ exports.init = function (sbot, config) { stateLoaded = true stateLoadedP.resolve() - notifyNewBranch = Notify() sbot.close.hook(function (fn, args) { if (liveDrainer) liveDrainer.abort(true) @@ -219,10 +221,8 @@ exports.init = function (sbot, config) { return { loadState(cb) { - if (!loadStateRequested) { - loadStateRequested = true - loadState() - } + if (!loadStateRequested) loadState() + if (cb) stateLoadedP.promise.then(cb) }, @@ -273,7 +273,7 @@ exports.init = function (sbot, config) { }, branchStream(opts) { - if (!notifyNewBranch) return pull.empty() + if (!loadStateRequested) loadState() const { live = true, old = false, root = null } = opts || {} const filterFn = root ? (branch) => branch.length > 0 && branch[0][0] === root From db453958a7ab37a17719187a5b04f530ea1b73f1 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 12:51:40 +0300 Subject: [PATCH 3/4] update README and deprecate some APIs --- README.md | 174 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 99 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 18e1a8d..bc3ced6 100644 --- a/README.md +++ b/README.md @@ -30,21 +30,21 @@ It lives alongside your existing "classical" feed, which we'll refer to as **main feed**. ```js -sbot.metafeeds.create((err, metafeed) => { +sbot.metafeeds.findOrCreate((err, metafeed) => { console.log(metafeed) // { seed, keys } }) ``` Now this has created the `seed`, which in turn is used to generate an [ssb-keys] `keys` object. The `seed` is actually also published on the _main_ feed as a -private message to yourself, to allow recovering it in the future. The first two -arguments above are null when we're creating the _root_ meta feed, but not in -other use cases of `create`. +private message to yourself, to allow recovering it in the future. -There can only be one _root_ meta feed, so even if you call `create` many times, -it will not create duplicates, it will just load the meta feed `{ seed, keys }`. +There can only be one _root_ meta feed, so even if you call `findOrCreate` many +times, it will not create duplicates, it will just load the meta feed +`{ seed, keys }`. -Now you can create subfeeds _under_ that root meta feed like this: +Now you can create subfeeds _under_ that root meta feed by passing two arguments +to `findOrCreate`, before the callback, like this: ```js const details = { @@ -56,7 +56,7 @@ const details = { }, } -sbot.metafeeds.create(metafeed, details, (err, subfeed) => { +sbot.metafeeds.findOrCreate(metafeed, details, (err, subfeed) => { console.log(subfeed) // { // feedpurpose: 'mygame', @@ -76,22 +76,7 @@ application-specific messages (such as for a game). The `details` argument always needs `feedpurpose` and `feedformat` (supports `classic` for ed25519 normal SSB feeds, and `bendy butt`). -To look up sub feeds belonging to the root meta feed, we can use `filter` and -`find`. These are similar to Array [filter](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter) -and [find](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/find): - -```js -sbot.metafeeds.filter( - metafeed, - (f) => f.feedpurpose === 'mygame', - (err, feeds) => { - console.log(feeds) - // [ - // { feedpurpose, subfeed, keys } - // ] - } -) -``` +To look up sub feeds belonging to the root meta feed, we can use `find`. ```js sbot.metafeeds.find( @@ -112,31 +97,6 @@ with the fields: - `keys`: the [ssb-keys] compatible `{ curve, public, private, id }` object - `metadata`: the same object used when creating the subfeed -Finally, there are many cases where you want to create a subfeed **only if** it -doesn't yet exist. For those purposes, use `findOrCreate`, which is a mix of -`find` and `create`. It literally will internally call `find`, and only call -`create` if `find` did not find a subfeed. For instance: - -```js -const details = { - feedpurpose: 'mygame', - feedformat: 'classic', - metadata: { - score: 0, - whateverElse: true, - }, -} - -sbot.metafeeds.findOrCreate( - metafeed, - (f) => f.feedpurpose === 'mygame', - details, // only used if the "find" part fails - (err, feed) => { - console.log(feed) - } -) -``` - ## API Some of these APIs can only be used on feeds that **you own** (🌻) while other @@ -163,6 +123,64 @@ You can also call `sbot.metafeeds.ensureLoaded(feedId, cb)` on an individual basis to make sure that `findByIdSync` will operate at the correct time when the `feedId`'s metadata has been processed in the local database. +### 🌷 `sbot.metafeeds.branchStream(opts)` + +Returns a **[pull-stream] source** of all "branches" in the meta feed trees. + +A "branch" is an array where the first item is the root meta feed and the +subsequent items are the children and grandchildren of the root. A branch looks +like this: + +```js +[ + [rootMetafeedId, null], + [childMetafeedId, childDetails], + [grandchildMetafeedId, grandchildDetails], + [grandgrandchildSubfeedId, grandgrandChildDetails], +] +``` + +Or in general, an `Array<[FeedId, Details | null]>`. **details** if the object +with `{feedpurpose, feedformat, metafeed, metadata}` like in `findById`. + +`branchStream` will emit all possible branches, which means sub-branches are +included. For instance, in the example above, `branchStream` would also emit: + +```js +[ + [rootMetafeedId, null] +] +``` + +and + +```js +[ + [rootMetafeedId, null], + [childMetafeedId, childDetails], +] +``` + +and + +```js +[ + [rootMetafeedId, null], + [childMetafeedId, childDetails], + [grandchildMetafeedId, grandchildDetails], +] +``` + +The `opts` argument can have the following properties: + +- `opts.root` _String_ - a feed ID for a root meta feed, only branches that have + this root would appear in the pull-stream source, otherwise all branches from + all possible root meta feeds will be included. (Default: `null`) +- `opts.old` _Boolean_ - whether or not to include currently loaded (by + `loadState`) trees. (Default: `false`) +- `opts.live` _Boolean_ - whether or not to include subsequent meta feed trees + during the execution of your program. (Default: `true`) + ### 🌻 `sbot.metafeeds.find(metafeed, visit, cb)` _Looks for the first subfeed of `metafeed` that satisfies the condition in @@ -181,6 +199,35 @@ possible error, and the 2nd argument is the found feed (which can be either the root meta feed `{ seed, keys }` or a sub feed `{ feedpurpose, subfeed, keys, metadata }`). +### 🌻 `sbot.metafeeds.findOrCreate(metafeed, visit, details, cb)` + +_Looks for the first subfeed of `metafeed` that satisfies the condition in +`visit`, or creates it matching the properties in `details`._ + +`metafeed` can be either `null` or a meta feed object `{ seed, keys }` (as +returned by `create()`). If it's null, then the root meta feed will be created, +if and only if it does not already exist. If it's null and the root meta feed +exists, the root meta feed will be returned via the `cb`. Alternatively, you can +call this API with just the callback: `sbot.metafeeds.findOrCreate(cb)`. + +`visit` can be either `null` or a function of the shape +`({feedpurpose,subfeed,metadata,keys}) => boolean`. If it's null, then one +arbitrary subfeed under `metafeed` is returned. + +`details` can be `null` only if if `metafeed` is null, but usually it's an +object with the properties: + +- `feedpurpose`: any string to characterize the purpose of this new subfeed +- `feedformat`: the string `'classic'` or the string `'bendy butt'` +- `metadata`: an optional object containing other fields + +The response is delivered to the callback `cb`, where the 1st argument is the +possible error, and the 2nd argument is the created feed (which can be either +the root meta feed `{ seed, keys }` or a sub feed +`{ feedpurpose, subfeed, keys, metadata }`). + + + ## Validation Exposed via the internal API. @@ -327,3 +350,4 @@ LGPL-3.0 [ssb-keys]: https://github.com/ssb-js/ssb-keys [ssb meta feed spec]: https://github.com/ssb-ngi-pointer/ssb-meta-feed-spec +[pull-stream]: https://github.com/pull-stream/pull-stream/ From e11aa9f2ee2d18eaea781752dfc3d9df9a724243 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 14:42:11 +0300 Subject: [PATCH 4/4] improve README for branchStream --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bc3ced6..19ba8f8 100644 --- a/README.md +++ b/README.md @@ -128,23 +128,22 @@ basis to make sure that `findByIdSync` will operate at the correct time when the Returns a **[pull-stream] source** of all "branches" in the meta feed trees. A "branch" is an array where the first item is the root meta feed and the -subsequent items are the children and grandchildren of the root. A branch looks -like this: +subsequent items are the children and grandchildren (and etc) of the root. A +branch looks like this: ```js [ [rootMetafeedId, null], [childMetafeedId, childDetails], [grandchildMetafeedId, grandchildDetails], - [grandgrandchildSubfeedId, grandgrandChildDetails], ] ``` -Or in general, an `Array<[FeedId, Details | null]>`. **details** if the object -with `{feedpurpose, feedformat, metafeed, metadata}` like in `findById`. +Or in general, an `Array<[FeedId, Details | null]>`. The **details** object has +the shape `{ feedpurpose, feedformat, metafeed, metadata }` like in `findById`. `branchStream` will emit all possible branches, which means sub-branches are -included. For instance, in the example above, `branchStream` would also emit: +included. For instance, in the example above, `branchStream` would emit: ```js [