From f54698d843bc37ac03726c9fcf97803066ce2589 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 17 Aug 2021 14:05:39 +0300 Subject: [PATCH 01/94] work in progress request manager --- index.js | 10 ++++--- req-manager.js | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 req-manager.js diff --git a/index.js b/index.js index 10efaf7..a7aa0ac 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ // SPDX-License-Identifier: LGPL-3.0-only const pull = require('pull-stream') +const RequestManager = require('./request-manager') exports.name = 'replicationScheduler' exports.version = '1.0.0' @@ -23,7 +24,9 @@ exports.init = function (ssb, config) { // true in most cases. These three blocks below may sometimes overlap, but // that's okay, as long as we cover *all* cases. - // Replicate myself + const requestManager = new RequestManager(ssb) + + // Replicate myself ASAP, without request manager ssb.ebt.request(ssb.id, true) // For each edge in the social graph, call either `request` or `block` @@ -35,7 +38,8 @@ exports.init = function (ssb, config) { const value = graph[source][dest] // Only if I am the `source` and `value >= 0`, request replication if (source === ssb.id) { - ssb.ebt.request(dest, value >= 0) + if (value >= 0) requestManager.add(dest) + else ssb.ebt.request(dest, false) } // Compute every block edge, unless I am the edge destination if (dest !== ssb.id) { @@ -54,7 +58,7 @@ exports.init = function (ssb, config) { const value = hops[dest] // myself or friendly peers if (value >= 0) { - ssb.ebt.request(dest, true) + requestManager.add(dest) ssb.ebt.block(ssb.id, dest, false) } // blocked peers diff --git a/req-manager.js b/req-manager.js new file mode 100644 index 0000000..80ee9ba --- /dev/null +++ b/req-manager.js @@ -0,0 +1,72 @@ +const pull = require('pull-stream') + +module.exports = class RequestManager { + constructor(ssb) { + this._ssb = ssb + this._requestables = new Set() + this._requestedFully = new Set() + this._requestedPartially = new Set() + this._flushing = false + this._wantsMoreFlushing = false + } + + add(feedId) { + this._requestables.add(feedId) + this._flush() + } + + _requestFully(feedId) { + this._requestables.delete(feedId) + this._requestedFully.add(feedId) + this._ssb.ebt.request(feedId, true) + } + + _requestPartially(feedId) { + this._requestables.delete(feedId) + this._requestedPartially.add(feedId) + // FIXME: implement + } + + _supportsPartialReplication(feedId, cb) { + // FIXME: implement + cb(null, false) + } + + _flush() { + if (this._flushing) { + this._wantsMoreFlushing = true + return + } + this._wantsMoreFlushing = false + + pull( + pull.values([...this._requestables]), + pull.asyncMap((feedId, cb) => { + if (feedId === this._ssb.id) return cb(null, [feedId, false]) + + this._supportsPartialReplication(feedId, (err, partially) => { + if (err) cb(err) + else cb(null, [feedId, partially]) + }) + }), + pull.drain( + ([feedId, partially]) => { + if (partially) { + this._requestPartially(feedId) + } else { + this._requestFully(feedId) + } + }, + (err) => { + if (err) console.error(err) + this._flushing = false + if (this._wantsMoreFlushing) { + setTimeout(() => { + this._flush() + }) + } + } + ) + ) + } +} From 8ddd5e2d7dca13f30395591b3086927d4b9f48dd Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 17 Aug 2021 14:08:20 +0300 Subject: [PATCH 02/94] remove self feed ID logic from req-manager --- req-manager.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index 80ee9ba..7cfa63c 100644 --- a/req-manager.js +++ b/req-manager.js @@ -42,8 +42,6 @@ module.exports = class RequestManager { pull( pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { - if (feedId === this._ssb.id) return cb(null, [feedId, false]) - this._supportsPartialReplication(feedId, (err, partially) => { if (err) cb(err) else cb(null, [feedId, partially]) From cf5b14815e6df5184b6c71bd1acd28ae245187e6 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 17 Aug 2021 14:09:41 +0300 Subject: [PATCH 03/94] fix typo in import --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index a7aa0ac..14be860 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,7 @@ // SPDX-License-Identifier: LGPL-3.0-only const pull = require('pull-stream') -const RequestManager = require('./request-manager') +const RequestManager = require('./req-manager') exports.name = 'replicationScheduler' exports.version = '1.0.0' From f45bac752120b2abe27bf2867623f43b4c4c43c8 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 17 Aug 2021 14:35:04 +0300 Subject: [PATCH 04/94] add configuration rules --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++----- index.js | 12 ++++++++--- req-manager.js | 15 +++++++++++-- 3 files changed, 74 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index dea25ef..c7415bc 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: CC0-1.0 # ssb-replication-scheduler -*Triggers replication of feeds identified as friendly in the social graph.* +_Triggers replication of feeds identified as friendly in the social graph._ Depends on ssb-friends APIs, and calls ssb-ebt APIs. @@ -41,10 +41,10 @@ Add this secret-stack plugin like this: ## Usage -There are no APIs, and nothing else you need to do. As soon as the SSB peer is -initialized, `ssb-replication-scheduler` will automatically query the social -graph, and either request replication or stop replication, depending whether the -feed is friendly or blocked. +Typically there is nothing you need to do after installing this plugin. As soon +as the SSB peer is initialized, `ssb-replication-scheduler` will automatically +query the social graph, and either request replication or stop replication, +depending whether the feed is friendly or blocked. **Opinions embedded in the scheduler:** @@ -58,6 +58,53 @@ feed is friendly or blocked. - Replication is strictly disabled for: - Any feed you explicitly block +### Configuration + +Some parameters and opinions can be configured by the user or by application +code through the conventional [ssb-config](https://github.com/ssbc/ssb-config) +object. The possible options are listed below: + +````js +{ + replicationScheduler: { + /** + * If `partially` is an array, it tells the replication scheduler to perform + * partial replication, whenever remote feeds support it. The array + * describes the tree of meta feeds that we are interested in replicating. + * If `partially` is `false (which it is, by default), then all friendly + * feeds will be requested in full. + * + * Example: + * + * ```js + * partially: [ + * { + * feedpurpose: 'indexes', + * children: [ + * { metadata: { querylang: 'ssb-ql-0', query: { type: 'post' } } }, + * { metadata: { querylang: 'ssb-ql-0', query: { type: 'vote' } } }, + * { metadata: { querylang: 'ssb-ql-0', query: { type: 'about' } } }, + * { metadata: { querylang: 'ssb-ql-0', query: { type: 'contact' } } } + * ] + * }, + * { feedpurpose: 'coolgame' }, + * { feedpurpose: 'git-ssb' }, + * ] + * ``` + */ + partially: false, + } +} +```` + +### muxrpc APIs + +#### `ssb.replicationScheduler.reconfigure(config) => void` + +At any point during the execution of your program, you can reconfigure the +replication rules using this API. The configuration object passed to this API +has the same shape as `config.replicationScheduler` (see above) has. + ## License LGPL-3.0 diff --git a/index.js b/index.js index 14be860..91ad39d 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,9 @@ const RequestManager = require('./req-manager') exports.name = 'replicationScheduler' exports.version = '1.0.0' -exports.manifest = {} +exports.manifest = { + reconfigure: 'sync', +} exports.init = function (ssb, config) { if (!ssb.ebt) { @@ -24,7 +26,7 @@ exports.init = function (ssb, config) { // true in most cases. These three blocks below may sometimes overlap, but // that's okay, as long as we cover *all* cases. - const requestManager = new RequestManager(ssb) + const requestManager = new RequestManager(ssb, config) // Replicate myself ASAP, without request manager ssb.ebt.request(ssb.id, true) @@ -74,5 +76,9 @@ exports.init = function (ssb, config) { }) ) - return {} + return { + reconfigure(opts) { + requestManager.reconfigure(opts) + }, + } } diff --git a/req-manager.js b/req-manager.js index 7cfa63c..8f5559a 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,8 +1,13 @@ const pull = require('pull-stream') +const DEFAULT_OPTS = { + partially: false, +} + module.exports = class RequestManager { - constructor(ssb) { + constructor(ssb, config) { this._ssb = ssb + this._opts = config.replicationScheduler || DEFAULT_OPTS this._requestables = new Set() this._requestedFully = new Set() this._requestedPartially = new Set() @@ -15,6 +20,10 @@ module.exports = class RequestManager { this._flush() } + reconfigure(opts) { + this._opts = { ...this._opts, opts } + } + _requestFully(feedId) { this._requestables.delete(feedId) this._requestedFully.add(feedId) @@ -24,7 +33,7 @@ module.exports = class RequestManager { _requestPartially(feedId) { this._requestables.delete(feedId) this._requestedPartially.add(feedId) - // FIXME: implement + // FIXME: implement. Go through this._opts.partially to detect subfeeds } _supportsPartialReplication(feedId, cb) { @@ -42,6 +51,8 @@ module.exports = class RequestManager { pull( pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { + if (!this._opts.partially) return cb(null, false) + this._supportsPartialReplication(feedId, (err, partially) => { if (err) cb(err) else cb(null, [feedId, partially]) From 10fe0e4583fd5fbfb30be731099ceb28af736b84 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 17 Aug 2021 14:41:26 +0300 Subject: [PATCH 05/94] fix small mistake --- req-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/req-manager.js b/req-manager.js index 8f5559a..fb79e29 100644 --- a/req-manager.js +++ b/req-manager.js @@ -51,7 +51,7 @@ module.exports = class RequestManager { pull( pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { - if (!this._opts.partially) return cb(null, false) + if (!this._opts.partially) return cb(null, [feedId, false]) this._supportsPartialReplication(feedId, (err, partially) => { if (err) cb(err) From c0be934a012fd47cd26ac37824ef1fe626995cd1 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 20 Aug 2021 13:26:11 +0300 Subject: [PATCH 06/94] renamed "partially" to "partialReplication" --- README.md | 43 +++++++++++++++++++++++++++---------------- req-manager.js | 4 ++-- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c7415bc..cfe2cfb 100644 --- a/README.md +++ b/README.md @@ -68,31 +68,42 @@ object. The possible options are listed below: { replicationScheduler: { /** - * If `partially` is an array, it tells the replication scheduler to perform - * partial replication, whenever remote feeds support it. The array - * describes the tree of meta feeds that we are interested in replicating. - * If `partially` is `false (which it is, by default), then all friendly - * feeds will be requested in full. + * If `partialReplication` is an object, it tells the replication scheduler + * to perform partial replication, whenever remote feeds support it. The + * object describes the tree of meta feeds that we are interested in + * replicating. If `partialReplication` is `false` (which it is, by + * default), then all friendly feeds will be requested in full. + * + * The tree of objects and arrays describes which keys in the metafeeds and + * subfeeds must match exactly the values given. So that if we write + * `feedpurposes: 'indexes'`, then we are interested in matching the + * metafeed that has the field `feedpurposes` exactly matching the value + * "indexes". All specified fields must match, but fields omitted are + * allowed to be any value. If the value is the special string "$main" or + * "$root", then they refer to (respectively) the IDs of the main feed + * and of the root meta feed. * * Example: * * ```js - * partially: [ - * { + * partialReplication: { + * children: [ + * { * feedpurpose: 'indexes', * children: [ - * { metadata: { querylang: 'ssb-ql-0', query: { type: 'post' } } }, - * { metadata: { querylang: 'ssb-ql-0', query: { type: 'vote' } } }, - * { metadata: { querylang: 'ssb-ql-0', query: { type: 'about' } } }, - * { metadata: { querylang: 'ssb-ql-0', query: { type: 'contact' } } } + * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'post' } } }, + * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'vote' } } }, + * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'about' } } }, + * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'contact' } } } * ] - * }, - * { feedpurpose: 'coolgame' }, - * { feedpurpose: 'git-ssb' }, - * ] + * }, + * { feedpurpose: 'coolgame' }, + * { feedpurpose: 'git-ssb' }, + * ] + * } * ``` */ - partially: false, + partialReplication: false, } } ```` diff --git a/req-manager.js b/req-manager.js index fb79e29..42e6c11 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,7 +1,7 @@ const pull = require('pull-stream') const DEFAULT_OPTS = { - partially: false, + partialReplication: false, } module.exports = class RequestManager { @@ -51,7 +51,7 @@ module.exports = class RequestManager { pull( pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { - if (!this._opts.partially) return cb(null, [feedId, false]) + if (!this._opts.partialReplication) return cb(null, [feedId, false]) this._supportsPartialReplication(feedId, (err, partially) => { if (err) cb(err) From 42d54ccf47e7b56b43ea40cb77c5ab2e0f3bcc4b Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 20 Aug 2021 13:30:34 +0300 Subject: [PATCH 07/94] move DEFAULT_OPTS to index.js --- index.js | 10 ++++++++-- req-manager.js | 8 ++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/index.js b/index.js index 91ad39d..7e123f5 100644 --- a/index.js +++ b/index.js @@ -5,6 +5,10 @@ const pull = require('pull-stream') const RequestManager = require('./req-manager') +const DEFAULT_OPTS = { + partialReplication: false, +} + exports.name = 'replicationScheduler' exports.version = '1.0.0' exports.manifest = { @@ -21,13 +25,15 @@ exports.init = function (ssb, config) { ) } + const opts = config.replicationScheduler || DEFAULT_OPTS + + const requestManager = new RequestManager(ssb, opts) + // Note: ssb.ebt.request and ssb.ebt.block are idempotent operations, // so it's safe to call these methods redundantly, which is most likely // true in most cases. These three blocks below may sometimes overlap, but // that's okay, as long as we cover *all* cases. - const requestManager = new RequestManager(ssb, config) - // Replicate myself ASAP, without request manager ssb.ebt.request(ssb.id, true) diff --git a/req-manager.js b/req-manager.js index 42e6c11..633d645 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,13 +1,9 @@ const pull = require('pull-stream') -const DEFAULT_OPTS = { - partialReplication: false, -} - module.exports = class RequestManager { - constructor(ssb, config) { + constructor(ssb, opts) { this._ssb = ssb - this._opts = config.replicationScheduler || DEFAULT_OPTS + this._opts = opts this._requestables = new Set() this._requestedFully = new Set() this._requestedPartially = new Set() From 4d36c287a5e61557656a49ecea334f456a0e085a Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 20 Aug 2021 14:41:39 +0300 Subject: [PATCH 08/94] update README --- README.md | 98 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index cfe2cfb..71683a1 100644 --- a/README.md +++ b/README.md @@ -64,49 +64,79 @@ Some parameters and opinions can be configured by the user or by application code through the conventional [ssb-config](https://github.com/ssbc/ssb-config) object. The possible options are listed below: -````js +```js { replicationScheduler: { /** * If `partialReplication` is an object, it tells the replication scheduler - * to perform partial replication, whenever remote feeds support it. The - * object describes the tree of meta feeds that we are interested in - * replicating. If `partialReplication` is `false` (which it is, by - * default), then all friendly feeds will be requested in full. + * to perform partial replication, whenever remote feeds support it. If + * `partialReplication` is `false` (which it is, by default), then all + * friendly feeds will be requested in full. * - * The tree of objects and arrays describes which keys in the metafeeds and - * subfeeds must match exactly the values given. So that if we write - * `feedpurposes: 'indexes'`, then we are interested in matching the - * metafeed that has the field `feedpurposes` exactly matching the value - * "indexes". All specified fields must match, but fields omitted are - * allowed to be any value. If the value is the special string "$main" or - * "$root", then they refer to (respectively) the IDs of the main feed - * and of the root meta feed. - * - * Example: - * - * ```js - * partialReplication: { - * children: [ - * { - * feedpurpose: 'indexes', - * children: [ - * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'post' } } }, - * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'vote' } } }, - * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'about' } } }, - * { metadata: { querylang: 'ssb-ql-0', query: { author: '$main', type: 'contact' } } } - * ] - * }, - * { feedpurpose: 'coolgame' }, - * { feedpurpose: 'git-ssb' }, - * ] - * } - * ``` + * Read below more about this configuration. */ partialReplication: false, } } -```` +``` + +#### Configuring partial replication + +The `config.replicationScheduler.partialReplication` object describes the tree +of meta feeds that we are interested in replicating. + +It is recursively made up of objects and arrays describes which **keys** in the +metafeeds and subfeeds must match exactly the **values** given. So that if we +write `feedpurposes: 'indexes'`, it means we are interested in matching the +metafeed that has the field `feedpurposes` exactly matching the value "indexes". +All specified fields must match, but omitted fields are allowed to be any value. + +The field `subfeeds` is not matching an actual field, instead, it is assumes we +are dealing with a meta feed and this is describing its subfeeds that we would +like to replicate. + +If the value is the special string `"$main"` or `"$root"`, then they refer to +(respectively) the IDs of the *main feed* and of the *root meta feed*. + +Example: + +```js +partialReplication: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post' }, + }, + }, + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'vote' }, + }, + }, + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'about' }, + }, + }, + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'contact' }, + }, + }, + ], + }, + { feedpurpose: 'coolgame' }, + { feedpurpose: 'git-ssb' }, + ] +} +``` ### muxrpc APIs From 03782bc6d3152510b57a1ff63563c52a8d2b30b7 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 23 Aug 2021 13:56:10 +0300 Subject: [PATCH 09/94] protect RequestManager from adding duplicates --- req-manager.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/req-manager.js b/req-manager.js index 633d645..51e7bdc 100644 --- a/req-manager.js +++ b/req-manager.js @@ -12,6 +12,9 @@ module.exports = class RequestManager { } add(feedId) { + if (this._requestedFully.has(feedId)) return + if (this._requestedPartially.has(feedId)) return + this._requestables.add(feedId) this._flush() } From 02b632c1f9c31b5d08ed524f7d0f8844639cff3a Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 23 Aug 2021 13:56:39 +0300 Subject: [PATCH 10/94] update some FIXME comments --- req-manager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index 51e7bdc..6b1aad0 100644 --- a/req-manager.js +++ b/req-manager.js @@ -16,7 +16,7 @@ module.exports = class RequestManager { if (this._requestedPartially.has(feedId)) return this._requestables.add(feedId) - this._flush() + this._flush() // FIXME: this may need some debouncing } reconfigure(opts) { @@ -32,7 +32,7 @@ module.exports = class RequestManager { _requestPartially(feedId) { this._requestables.delete(feedId) this._requestedPartially.add(feedId) - // FIXME: implement. Go through this._opts.partially to detect subfeeds + // FIXME: go through this._opts.partialReplication to detect subfeeds } _supportsPartialReplication(feedId, cb) { From 15845c1c8e1f199f2fa40699b4c2c0ecdfd3416d Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 23 Aug 2021 13:57:58 +0300 Subject: [PATCH 11/94] first draft of MetafeedFinder --- index.js | 4 +- metafeed-finder.js | 170 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 9 ++- req-manager.js | 9 ++- 4 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 metafeed-finder.js diff --git a/index.js b/index.js index 7e123f5..aecc5f5 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ // SPDX-License-Identifier: LGPL-3.0-only const pull = require('pull-stream') +const MetafeedFinder = require('./metafeed-finder') const RequestManager = require('./req-manager') const DEFAULT_OPTS = { @@ -27,7 +28,8 @@ exports.init = function (ssb, config) { const opts = config.replicationScheduler || DEFAULT_OPTS - const requestManager = new RequestManager(ssb, opts) + const metafeedFinder = new MetafeedFinder(ssb, opts) + const requestManager = new RequestManager(ssb, opts, metafeedFinder) // Note: ssb.ebt.request and ssb.ebt.block are idempotent operations, // so it's safe to call these methods redundantly, which is most likely diff --git a/metafeed-finder.js b/metafeed-finder.js new file mode 100644 index 0000000..125e940 --- /dev/null +++ b/metafeed-finder.js @@ -0,0 +1,170 @@ +const pull = require('pull-stream') +const Ref = require('ssb-ref') +const debug = require('debug')('ssb:replication-scheduler') +const SSBURI = require('ssb-uri2') +const detectSsbNetworkErrorSeverity = require('ssb-network-errors') +const { where, type, toPullStream } = require('ssb-db2/operators') + +const DEFAULT_PERIOD = 500 + +module.exports = class MetafeedFinder { + constructor(ssb, opts, period) { + this._ssb = ssb + this._opts = opts + this._period = period || DEFAULT_PERIOD + this._map = new Map() + this._requestsByMainfeedId = new Map() + this._latestRequestTime = 0 + this._timer = null + + if (this._opts.partialReplication) { + this._loadAllFromLog() + } + } + + get(mainFeedId, cb) { + if (this._map.has(mainFeedId)) { + const metaFeedId = this._map.get(mainFeedId) + cb(null, metaFeedId) + return + } else { + this._request(mainFeedId, cb) + } + } + + _loadAllFromLog() { + if (!ssb.db || !ssb.db.query) { + throw new Error( + 'ssb-replication-scheduler expects ssb-db2 to be installed, to use partial replication' + ) + } + + pull( + this._ssb.db.query(where(type('metafeed/announce')), toPullStream()), + pull.drain((msg) => { + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) + if (!mainFeedId || !metaFeedId) return + this._map.set(mainFeedId, metaFeedId) + }) + ) + } + + _pluckFromAnnounceMsg(msg) { + const { author, content } = msg.value + if (!Ref.isFeedId(author)) return [] + const mainFeedId = author + if (!SSBURI.isBendyButtV1FeedSSBURI(content.metafeed)) return [] + const metaFeedId = content.metafeed + return [mainFeedId, metaFeedId] + } + + _request(mainFeedId, cb) { + const callbacks = this._requestsByMainfeedId.get(mainFeedId) || [] + callbacks.push(cb) + this._requestsByMainfeedId.set(mainFeedId, callbacks) + this._latestRequestTime = Date.now() + this._scheduleDebouncedFlush() + } + + _persist(msg) { + this._ssb.db.addOOO(msg.value, (err) => { + if (err) { + debug( + 'failed to addOOO for a metafeed/announce: %s', + err.message || err + ) + } + }) + } + + async _forEachNeighborPeer(run) { + for (const peerId of Object.keys(this._ssb.peers)) { + for (const rpc of this._ssb.peers[peerId]) { + const goToNext = await new Promise((resolve) => run(rpc, resolve)) + if (!goToNext) return + } + } + } + + _scheduleDebouncedFlush() { + if (this._timer) return // Timer is already enabled + this._timer = setInterval(() => { + // Turn off the timer if there is nothing to flush + if (this._requestsByMainfeedId.size === 0) { + clearInterval(this._timer) + this._timer = null + } + // Flush if enough time has passed + else if (Date.now() - this._latestRequestTime > this._period) + this._flush() + }, this._period * 0.5) + } + + _makeQL1(map) { + const query = { + op: 'or', + args: [], + } + for (const mainFeedId of map.keys()) { + query.args.push({ + op: 'and', + args: [ + { op: 'type', string: 'metafeed/announce' }, + { op: 'author', feed: mainFeedId }, + ], + }) + } + return query + } + + async _flush() { + let drainer + const requests = new Map(this._requestsByMainfeedId) + this._requestsByMainfeedId.clear() + + await this._forEachNeighborPeer((rpc, goToNextNeighbor) => { + pull( + rpc.getSubset(this._makeQL1(requests), { querylang: 'ssb-ql-1' }), + (drainer = pull.drain( + (msg) => { + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) + if (!mainFeedId || !metaFeedId) return + if (requests.has(mainFeedId)) { + this._map.set(mainFeedId, metaFeedId) + this._persist(msg) + const callbacks = requests.get(mainFeedId) + requests.delete(mainFeedId) + for (const cb of callbacks) cb(null, metaFeedId) + if (requests.size === 0) { + drainer.abort() + goToNextNeighbor(false) + } else { + goToNextNeighbor(true) + } + } + }, + (err) => { + if (err && detectSsbNetworkErrorSeverity(err) >= 2) { + debug( + 'failed "getSubset" muxrpc at peer %s because: %s', + rpc.id, + err.message || err + ) + } + goToNextNeighbor(true) + } + )) + ) + }) + + if (requests.size > 0) { + // We couldn't find metaFeedIds for some mainFeedIds, so we assume there + // none. Note, this may give false negatives depending on who you're + // connected to! + for (const callbacks of requests.values()) { + for (const cb of callbacks) cb(null, null) + } + requests.clear() + } + } +} diff --git a/package.json b/package.json index 1c5453a..f59e5ab 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,12 @@ "node": ">=10" }, "dependencies": { - "pull-stream": "^3.6.0" + "debug": "^4.3.2", + "pull-stream": "^3.6.0", + "ssb-network-errors": "^1.0.0", + "ssb-ref": "^2.13.9", + "ssb-subset-ql": "^0.2.0", + "ssb-uri2": "^1.2.0" }, "devDependencies": { "cat-names": "^3.0.0", @@ -36,7 +41,7 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-db2": "^2.2.0", + "ssb-db2": "github:ssb-ngi-pointer/ssb-db2#support-bendy-butt-uri", "ssb-ebt": "^7.0.0", "ssb-fixtures": "^2.4.1", "ssb-friends": "^5.0.0", diff --git a/req-manager.js b/req-manager.js index 6b1aad0..80bb6db 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,9 +1,10 @@ const pull = require('pull-stream') module.exports = class RequestManager { - constructor(ssb, opts) { + constructor(ssb, opts, metafeedFinder) { this._ssb = ssb this._opts = opts + this._metafeedFinder = metafeedFinder this._requestables = new Set() this._requestedFully = new Set() this._requestedPartially = new Set() @@ -36,8 +37,10 @@ module.exports = class RequestManager { } _supportsPartialReplication(feedId, cb) { - // FIXME: implement - cb(null, false) + this._metafeedFinder.get(feedId, (err, metafeedId) => { + if (err) cb(err) + else cb(null, !!metafeedId) + }) } _flush() { From bd9fa09f83bf7c234a416cfd3b0ec8c738991035 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 23 Aug 2021 14:24:49 +0300 Subject: [PATCH 12/94] add debouncing flush to RequestManager --- req-manager.js | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/req-manager.js b/req-manager.js index 80bb6db..13bb1d5 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,15 +1,20 @@ const pull = require('pull-stream') +const DEFAULT_PERIOD = 500 + module.exports = class RequestManager { - constructor(ssb, opts, metafeedFinder) { + constructor(ssb, opts, metafeedFinder, period) { this._ssb = ssb this._opts = opts this._metafeedFinder = metafeedFinder + this._period = period || DEFAULT_PERIOD this._requestables = new Set() this._requestedFully = new Set() this._requestedPartially = new Set() this._flushing = false this._wantsMoreFlushing = false + this._latestAdd = 0 + this._timer = null } add(feedId) { @@ -17,7 +22,8 @@ module.exports = class RequestManager { if (this._requestedPartially.has(feedId)) return this._requestables.add(feedId) - this._flush() // FIXME: this may need some debouncing + this._latestAdd = Date.now() + this._scheduleDebouncedFlush() } reconfigure(opts) { @@ -43,13 +49,26 @@ module.exports = class RequestManager { }) } - _flush() { + _scheduleDebouncedFlush() { if (this._flushing) { this._wantsMoreFlushing = true return } this._wantsMoreFlushing = false + if (this._timer) return // Timer is already enabled + this._timer = setInterval(() => { + // Turn off the timer if there is nothing to flush + if (this._requestables.size === 0) { + clearInterval(this._timer) + this._timer = null + } + // Flush if enough time has passed + else if (Date.now() - this._latestAdd > this._period) this._flush() + }, this._period * 0.5) + } + + _flush() { pull( pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { @@ -72,9 +91,7 @@ module.exports = class RequestManager { if (err) console.error(err) this._flushing = false if (this._wantsMoreFlushing) { - setTimeout(() => { - this._flush() - }) + this._scheduleDebouncedFlush() } } ) From b2a18e5503de4d08f9efbd11a15402d7d28ee70d Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 7 Sep 2021 17:03:24 +0300 Subject: [PATCH 13/94] fix tests and add debouncePeriod opts --- package.json | 4 +- req-manager.js | 16 ++++-- test/integration/block.js | 9 ++++ test/integration/block2.js | 6 +++ test/integration/block3.js | 92 ++++++++++++++++++--------------- test/integration/db2-support.js | 2 +- test/integration/generate.js | 6 +++ test/integration/hops.js | 36 +++++++++++++ test/integration/random.js | 6 +++ test/unit/blocks.js | 12 ++++- test/unit/follows.js | 12 ++++- test/unit/self.js | 6 ++- 12 files changed, 153 insertions(+), 54 deletions(-) diff --git a/package.json b/package.json index f59e5ab..eec3688 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "ssb-network-errors": "^1.0.0", "ssb-ref": "^2.13.9", "ssb-subset-ql": "^0.2.0", - "ssb-uri2": "^1.2.0" + "ssb-uri2": "^1.5.2" }, "devDependencies": { "cat-names": "^3.0.0", @@ -41,7 +41,7 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-db2": "github:ssb-ngi-pointer/ssb-db2#support-bendy-butt-uri", + "ssb-db2": "github:ssb-ngi-pointer/ssb-db2#support-bendy-butt", "ssb-ebt": "^7.0.0", "ssb-fixtures": "^2.4.1", "ssb-friends": "^5.0.0", diff --git a/req-manager.js b/req-manager.js index 13bb1d5..49c0f92 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,13 +1,16 @@ const pull = require('pull-stream') -const DEFAULT_PERIOD = 500 +const DEFAULT_PERIOD = 150 module.exports = class RequestManager { - constructor(ssb, opts, metafeedFinder, period) { + constructor(ssb, opts, metafeedFinder) { this._ssb = ssb this._opts = opts this._metafeedFinder = metafeedFinder - this._period = period || DEFAULT_PERIOD + this._period = + typeof opts.debouncePeriod === 'number' + ? opts.debouncePeriod + : DEFAULT_PERIOD this._requestables = new Set() this._requestedFully = new Set() this._requestedPartially = new Set() @@ -56,6 +59,11 @@ module.exports = class RequestManager { } this._wantsMoreFlushing = false + if (this._period === 0) { + this._flush() + return + } + if (this._timer) return // Timer is already enabled this._timer = setInterval(() => { // Turn off the timer if there is nothing to flush @@ -66,9 +74,11 @@ module.exports = class RequestManager { // Flush if enough time has passed else if (Date.now() - this._latestAdd > this._period) this._flush() }, this._period * 0.5) + if (this._timer.unref) this._timer.unref() } _flush() { + this._flushing = true pull( pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { diff --git a/test/integration/block.js b/test/integration/block.js index 1e06067..7c8d916 100644 --- a/test/integration/block.js +++ b/test/integration/block.js @@ -36,6 +36,9 @@ const alice = createSsbServer({ friends: { hops: 1, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSsbServer({ @@ -45,6 +48,9 @@ const bob = createSsbServer({ friends: { hops: 1, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const carol = createSsbServer({ @@ -54,6 +60,9 @@ const carol = createSsbServer({ friends: { hops: 1, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) tape('alice blocks bob, and bob cannot connect to alice', async (t) => { diff --git a/test/integration/block2.js b/test/integration/block2.js index e63f529..fa59515 100644 --- a/test/integration/block2.js +++ b/test/integration/block2.js @@ -25,12 +25,18 @@ const alice = createSsbServer({ temp: 'test-block2-alice', timeout: CONNECTION_TIMEOUT, keys: ssbKeys.generate(), + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSsbServer({ temp: 'test-block2-bob', timeout: CONNECTION_TIMEOUT, keys: ssbKeys.generate(), + replicationScheduler: { + debouncePeriod: 0, + }, }) tape( diff --git a/test/integration/block3.js b/test/integration/block3.js index ceb0f8f..78cd16b 100644 --- a/test/integration/block3.js +++ b/test/integration/block3.js @@ -33,60 +33,66 @@ const alice = createSsbServer({ temp: 'test-block3-alice', timeout: CONNECTION_TIMEOUT, keys: ssbKeys.generate(), + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSsbServer({ temp: 'test-block3-bob', timeout: CONNECTION_TIMEOUT, keys: ssbKeys.generate(), + replicationScheduler: { + debouncePeriod: 0, + }, }) const carol = createSsbServer({ temp: 'test-block3-carol', timeout: CONNECTION_TIMEOUT, keys: ssbKeys.generate(), + replicationScheduler: { + debouncePeriod: 0, + }, }) -tape( - 'alice blocks bob while he is connected, she should disconnect him', - async (t) => { - t.plan(3) - - // in the beginning alice and bob follow each other - await Promise.all([ - pify(alice.publish)(u.follow(bob.id)), - pify(bob.publish)(u.follow(alice.id)), - pify(carol.publish)(u.follow(alice.id)), - ]) - - await Promise.all([ - pify(bob.connect)(carol.getAddress()), - pify(carol.connect)(alice.getAddress()), - ]) - - const msgAtBob = await u.readOnceFromDB(bob) - - // should be the alice's follow(bob) message. - t.equal(msgAtBob.value.author, alice.id) - t.equal(msgAtBob.value.content.contact, bob.id) - - await pify(alice.publish)(u.block(bob.id)) - - await sleep(REPLICATION_TIMEOUT) - - const clockBob = await pify(bob.getVectorClock)() - t.equals( - clockBob[alice.id], - 1, - 'bob does not receive the message where alice blocked him' - ) - - await Promise.all([ - pify(alice.close)(true), - pify(bob.close)(true), - pify(carol.close)(true), - ]) - - t.end() - } -) +tape('alice blocks bob and both are connected to carla', async (t) => { + t.plan(3) + + // in the beginning alice and bob follow each other + await Promise.all([ + pify(alice.publish)(u.follow(bob.id)), + pify(bob.publish)(u.follow(alice.id)), + pify(carol.publish)(u.follow(alice.id)), + ]) + + await Promise.all([ + pify(bob.connect)(carol.getAddress()), + pify(carol.connect)(alice.getAddress()), + ]) + + const msgAtBob = await u.readOnceFromDB(bob) + + // should be the alice's follow(bob) message. + t.equal(msgAtBob.value.author, alice.id) + t.equal(msgAtBob.value.content.contact, bob.id) + + await pify(alice.publish)(u.block(bob.id)) + + await sleep(REPLICATION_TIMEOUT) + + const clockBob = await pify(bob.getVectorClock)() + t.equals( + clockBob[alice.id], + 1, + 'bob does not receive the message where alice blocked him' + ) + + await Promise.all([ + pify(alice.close)(true), + pify(bob.close)(true), + pify(carol.close)(true), + ]) + + t.end() +}) diff --git a/test/integration/db2-support.js b/test/integration/db2-support.js index 267c5eb..754d2fd 100644 --- a/test/integration/db2-support.js +++ b/test/integration/db2-support.js @@ -20,7 +20,7 @@ const createSsbServer = SecretStack({ caps }) .use(require('../..')) const CONNECTION_TIMEOUT = 500 // ms -const REPLICATION_TIMEOUT = 4 * CONNECTION_TIMEOUT +const REPLICATION_TIMEOUT = 6 * CONNECTION_TIMEOUT tape('replicate between 3 peers, using ssb-db2', async (t) => { rimraf.sync(path.join(os.tmpdir(), 'server-alice')) diff --git a/test/integration/generate.js b/test/integration/generate.js index 3bcbc83..c8979ff 100644 --- a/test/integration/generate.js +++ b/test/integration/generate.js @@ -63,6 +63,9 @@ test('tests large-scale EBT replication', async (t) => { timeout: CONNECTION_TIMEOUT, keys: aliceKeys, friends: { hops: 2 }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSbot({ @@ -70,6 +73,9 @@ test('tests large-scale EBT replication', async (t) => { timeout: CONNECTION_TIMEOUT, keys: bobKeys, friends: { hops: 2 }, + replicationScheduler: { + debouncePeriod: 0, + }, }) t.ok(alice.getAddress(), 'alice has an address') diff --git a/test/integration/hops.js b/test/integration/hops.js index 69b18a7..f6f27b4 100644 --- a/test/integration/hops.js +++ b/test/integration/hops.js @@ -42,6 +42,9 @@ tape('hops 1', async (t) => { friends: { hops: 1, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSsbServer({ @@ -51,6 +54,9 @@ tape('hops 1', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const carol = createSsbServer({ @@ -60,6 +66,9 @@ tape('hops 1', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const david = createSsbServer({ @@ -69,6 +78,9 @@ tape('hops 1', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) // alice follows bob, bob follows carol, carol follows david @@ -116,6 +128,9 @@ tape('hops 2', async (t) => { friends: { hops: 2, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSsbServer({ @@ -125,6 +140,9 @@ tape('hops 2', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const carol = createSsbServer({ @@ -134,6 +152,9 @@ tape('hops 2', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const david = createSsbServer({ @@ -143,6 +164,9 @@ tape('hops 2', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const [rpcAB, rpcBC, rpcCD] = await Promise.all([ @@ -182,6 +206,9 @@ tape('hops 2', async (t) => { friends: { hops: 3, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const bob = createSsbServer({ @@ -191,6 +218,9 @@ tape('hops 2', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const carol = createSsbServer({ @@ -200,6 +230,9 @@ tape('hops 2', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const david = createSsbServer({ @@ -209,6 +242,9 @@ tape('hops 2', async (t) => { friends: { hops: UNLIMITED, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) const [rpcAB, rpcBC, rpcCD] = await Promise.all([ diff --git a/test/integration/random.js b/test/integration/random.js index 7ca91fa..9cd4bdf 100644 --- a/test/integration/random.js +++ b/test/integration/random.js @@ -95,6 +95,9 @@ const alice = createSsbServer({ friends: { hops: 1, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) let liveMsgCount = 0 @@ -184,6 +187,9 @@ tape('replicate social network for animals', async (t) => { friends: { hops: 1, }, + replicationScheduler: { + debouncePeriod: 0, + }, }) if (!bob.friends) { diff --git a/test/unit/blocks.js b/test/unit/blocks.js index 6ae79a8..93b4b77 100644 --- a/test/unit/blocks.js +++ b/test/unit/blocks.js @@ -54,7 +54,11 @@ tape('listen to friends stream and ebt.blocks initial blocked peers', (t) => { }, }) .use(require('../..')) - .call(null, {}) + .call(null, { + replicationScheduler: { + debouncePeriod: 0, + }, + }) }) tape('listen to friends stream ebt.blocks subsequent blocks', (t) => { @@ -124,5 +128,9 @@ tape('listen to friends stream ebt.blocks subsequent blocks', (t) => { }, }) .use(require('../..')) - .call(null, {}) + .call(null, { + replicationScheduler: { + debouncePeriod: 0, + }, + }) }) diff --git a/test/unit/follows.js b/test/unit/follows.js index 1690763..0b37f1e 100644 --- a/test/unit/follows.js +++ b/test/unit/follows.js @@ -54,7 +54,11 @@ tape('listen to friends stream and replicates initial follows', (t) => { }, }) .use(require('../..')) - .call(null, {}) + .call(null, { + replicationScheduler: { + debouncePeriod: 0, + }, + }) }) tape('listen to friends stream and replicates subsequent follows', (t) => { @@ -102,5 +106,9 @@ tape('listen to friends stream and replicates subsequent follows', (t) => { }, }) .use(require('../..')) - .call(null, {}) + .call(null, { + replicationScheduler: { + debouncePeriod: 0, + }, + }) }) diff --git a/test/unit/self.js b/test/unit/self.js index ecf2300..4fb9588 100644 --- a/test/unit/self.js +++ b/test/unit/self.js @@ -41,5 +41,9 @@ tape('replicates myself', (t) => { }, }) .use(require('../..')) - .call(null, {}) + .call(null, { + replicationScheduler: { + debouncePeriod: 0, + }, + }) }) From f3ffd1e0e5160d2bba75192b488253a96a05aab7 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 8 Sep 2021 12:53:28 +0300 Subject: [PATCH 14/94] update dependencies --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index eec3688..fb51aad 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "pull-stream": "^3.6.0", "ssb-network-errors": "^1.0.0", "ssb-ref": "^2.13.9", - "ssb-subset-ql": "^0.2.0", + "ssb-subset-ql": "^0.2.1", "ssb-uri2": "^1.5.2" }, "devDependencies": { @@ -41,11 +41,11 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-db2": "github:ssb-ngi-pointer/ssb-db2#support-bendy-butt", + "ssb-db2": "^2.4.0", "ssb-ebt": "^7.0.0", "ssb-fixtures": "^2.4.1", - "ssb-friends": "^5.0.0", - "ssb-keys": "^8.1.0", + "ssb-friends": "^5.1.0", + "ssb-keys": "^8.2.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" }, From 266ce93d372554d57ee236135065d8e88bae4742 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 9 Sep 2021 12:41:04 +0300 Subject: [PATCH 15/94] update ssb-subset-ql --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fb51aad..7ae6b68 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "pull-stream": "^3.6.0", "ssb-network-errors": "^1.0.0", "ssb-ref": "^2.13.9", - "ssb-subset-ql": "^0.2.1", + "ssb-subset-ql": "~0.5.0", "ssb-uri2": "^1.5.2" }, "devDependencies": { From 32f688ee07025bc185134d03dd6aa5ff9c123f34 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 9 Sep 2021 12:44:40 +0300 Subject: [PATCH 16/94] improve RequestManager with partialReplication traversal --- req-manager.js | 145 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 4 deletions(-) diff --git a/req-manager.js b/req-manager.js index 49c0f92..313f823 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,4 +1,8 @@ const pull = require('pull-stream') +const Ref = require('ssb-ref') +const { QL0 } = require('ssb-subset-ql') +const { isBendyButtV1FeedSSBURI } = require('ssb-uri2') +const { where, author, live, toPullStream } = require('ssb-db2/operators') const DEFAULT_PERIOD = 150 @@ -33,16 +37,149 @@ module.exports = class RequestManager { this._opts = { ...this._opts, opts } } + /** + * @param {string} feedId classic feed ref or bendybutt feed URI + */ _requestFully(feedId) { this._requestables.delete(feedId) this._requestedFully.add(feedId) this._ssb.ebt.request(feedId, true) + // FIXME: needs to support bendybutt and classic } - _requestPartially(feedId) { - this._requestables.delete(feedId) - this._requestedPartially.add(feedId) - // FIXME: go through this._opts.partialReplication to detect subfeeds + /** + * @param {string} mainFeedId classic feed ref which has announced a root MF + */ + _requestPartially(mainFeedId) { + this._requestables.delete(mainFeedId) + this._requestedPartially.add(mainFeedId) + + // Get metafeedId for this feedId + this._metafeedFinder.get(mainFeedId, (err, metafeedId) => { + if (err) { + console.error(err) + } else if (!metafeedId) { + console.error('cannot partially replicate ' + mainFeedId) + } else { + this._traverse(metafeedId, this._opts.partialReplication, mainFeedId) + } + }) + } + + /** + * @param {string | object} input a metafeedId or a msg concerning a metafeed + * @param {object} template one of the nodes in opts.partialReplication + * @param {string} mainFeedId + */ + _traverse(input, template, mainFeedId) { + if (!this._matchesTemplate(input, template)) return + + const metafeedId = + typeof input === 'string' ? input : input.value.content.subfeed + + this._requestFully(metafeedId) + + // ssb-db2 live query for metafeedId + // * for every *tombstoned* subfeed, call ssb.ebt.request(subfeedId, false) + // * for every *added* subfeed, match against template.subfeeds + // * if not matched, ignore + // * if matched and is classic feed, _requestFully(subfeedId) + // * if matched with subfeeds and is bendybutt feed, then + // * traverse(mainfeedId, subfeedId, matchedTemplate) + pull( + this._ssb.db.query( + where(author(metafeedId)), + live({ old: true }), + toPullStream() + ), + pull.filter((msg) => typeof msg.value.content !== 'string'), + // FIXME: this drain multiplied by N peers with support for + // partialReplication multiplied by M sub-meta-feeds for each means N*M + // live drains. Sounds like a performance nightmare, if N > 1000. + // + // Should we do instead one drain for all bendybutt messages and based + // on the bbmsg look up who it belongs to and then traverse the template? + pull.drain((msg) => { + const { type, subfeed } = msg.value.content + if (type.startsWith('metafeed/add/')) { + for (const childTemplate of template.subfeeds) { + if (this._matchesTemplate(msg, childTemplate, mainFeedId)) { + if (isBendyButtV1FeedSSBURI(subfeed)) { + this._traverse(msg, childTemplate, mainFeedId) + } else if (Ref.isFeedId(subfeed)) { + // FIXME: what if `subfeed` is an index feed? + this._requestedFully(subfeed) + } else { + console.error('cannot replicate unknown feed type: ' + subfeed) + } + break + } + } + } else if (type === 'metafeed/tombstone') { + this._ssb.ebt.request(subfeed, false) + } + }) + ) + } + + _matchesTemplate(input, template, mainFeedId) { + // Input is `metafeedId` + if (typeof input === 'string' && isBendyButtV1FeedSSBURI(input)) { + const keys = Object.keys(template) + return ( + keys.length === 1 && + keys[0] === 'subfeeds' && + Array.isArray(template.subfeeds) + ) + } + + // Input is a bendybutt message + if (typeof input === 'object' && input.value && input.value.content) { + const msg = input + const content = msg.value.content + + // If present, feedpurpose must match + if ( + template.feedpurpose && + content.feedpurpose !== template.feedpurpose + ) { + return false + } + + // If present, metadata must match + if (template.metadata && content.metadata) { + // If querylang is present, match ssb-ql-0 queries + if (template.metadata.querylang !== content.metadata.querylang) { + return false + } + if (template.metadata.querylang === 'ssb-ql-0') { + if (!QL0.parse(content.metadata.query)) return false + if (template.metadata.query) { + if (template.metadata.query.author === '$main') { + template.metadata.query.author = mainFeedId + } + if ( + !QL0.isEquals(content.metadata.query, template.metadata.query) + ) { + return false + } + } + } + + // Any other metadata field must match exactly + for (const field of Object.keys(template.metadata)) { + // Ignore these because we already handled them: + if (field === 'query') continue + if (field === 'querylang') continue + + if (content.metadata[field] !== template.metadata[field]) return false + } + } + + return true + } + + return false } _supportsPartialReplication(feedId, cb) { From 073d43ad9517553052bb1a03a48e0f0b607a108e Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 10 Sep 2021 16:22:18 +0300 Subject: [PATCH 17/94] add one code comment --- req-manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/req-manager.js b/req-manager.js index 313f823..cbbf015 100644 --- a/req-manager.js +++ b/req-manager.js @@ -116,6 +116,7 @@ module.exports = class RequestManager { } } } else if (type === 'metafeed/tombstone') { + // FIXME: what if this happens very quickly after add? this._ssb.ebt.request(subfeed, false) } }) From 7732163c3cfc90b519b31d5a586843af51d16027 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 13 Sep 2021 15:12:40 +0300 Subject: [PATCH 18/94] update deps --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 7ae6b68..a912753 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "pull-stream": "^3.6.0", "ssb-network-errors": "^1.0.0", "ssb-ref": "^2.13.9", - "ssb-subset-ql": "~0.5.0", + "ssb-subset-ql": "~0.6.1", "ssb-uri2": "^1.5.2" }, "devDependencies": { @@ -32,7 +32,7 @@ "husky": "^4.3.0", "mkdirp": "^1.0.4", "nyc": "^15.1.0", - "prettier": "^2.1.2", + "prettier": "^2.4.0", "pretty-quick": "^3.1.0", "promisify-4loc": "^1.0.0", "pull-paramap": "^1.2.2", @@ -41,9 +41,9 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-db2": "^2.4.0", + "ssb-db2": "^2.5.0", "ssb-ebt": "^7.0.0", - "ssb-fixtures": "^2.4.1", + "ssb-fixtures": "^2.5.1", "ssb-friends": "^5.1.0", "ssb-keys": "^8.2.0", "tap-spec": "^5.0.0", From 7c4ef2d5278977e77c8ec04d565e1a7277adef1c Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 13 Sep 2021 18:37:28 +0300 Subject: [PATCH 19/94] fix typos in metafeed-finder --- metafeed-finder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 125e940..82aa831 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -33,7 +33,7 @@ module.exports = class MetafeedFinder { } _loadAllFromLog() { - if (!ssb.db || !ssb.db.query) { + if (!this._ssb.db || !this._ssb.db.query) { throw new Error( 'ssb-replication-scheduler expects ssb-db2 to be installed, to use partial replication' ) From e4737f0f9481dc2316d8efeeb7ced270b1621242 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 14 Sep 2021 15:13:52 +0300 Subject: [PATCH 20/94] validate metafeed/announce in metafeed-finder --- metafeed-finder.js | 22 +++++++++++++++------- package.json | 2 ++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 82aa831..ce408a1 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -1,9 +1,8 @@ const pull = require('pull-stream') -const Ref = require('ssb-ref') const debug = require('debug')('ssb:replication-scheduler') -const SSBURI = require('ssb-uri2') const detectSsbNetworkErrorSeverity = require('ssb-network-errors') const { where, type, toPullStream } = require('ssb-db2/operators') +const { validateMetafeedAnnounce } = require('ssb-meta-feeds/validate') const DEFAULT_PERIOD = 500 @@ -41,6 +40,7 @@ module.exports = class MetafeedFinder { pull( this._ssb.db.query(where(type('metafeed/announce')), toPullStream()), + pull.filter(this._validateMetafeedAnnounce), pull.drain((msg) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) if (!mainFeedId || !metaFeedId) return @@ -49,12 +49,19 @@ module.exports = class MetafeedFinder { ) } + _validateMetafeedAnnounce(msg) { + const err = validateMetafeedAnnounce(msg) + if (err) { + console.warn(err) + return false + } else { + return true + } + } + _pluckFromAnnounceMsg(msg) { - const { author, content } = msg.value - if (!Ref.isFeedId(author)) return [] - const mainFeedId = author - if (!SSBURI.isBendyButtV1FeedSSBURI(content.metafeed)) return [] - const metaFeedId = content.metafeed + const mainFeedId = msg.value.author + const metaFeedId = msg.value.content.metafeed return [mainFeedId, metaFeedId] } @@ -125,6 +132,7 @@ module.exports = class MetafeedFinder { await this._forEachNeighborPeer((rpc, goToNextNeighbor) => { pull( rpc.getSubset(this._makeQL1(requests), { querylang: 'ssb-ql-1' }), + pull.filter(this._validateMetafeedAnnounce), (drainer = pull.drain( (msg) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) diff --git a/package.json b/package.json index a912753..986b64f 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,9 @@ "ssb-ebt": "^7.0.0", "ssb-fixtures": "^2.5.1", "ssb-friends": "^5.1.0", + "ssb-index-feed-writer": "~0.5.1", "ssb-keys": "^8.2.0", + "ssb-meta-feeds": "~0.20.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" }, From 5f25cf03a316a1d14ae024d6c9ca2c8505fe489d Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 14 Sep 2021 17:18:01 +0300 Subject: [PATCH 21/94] update dev deps --- package.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 986b64f..fbc0fe3 100644 --- a/package.json +++ b/package.json @@ -41,13 +41,14 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-db2": "^2.5.0", + "ssb-db2": "^2.5.2", "ssb-ebt": "^7.0.0", - "ssb-fixtures": "^2.5.1", + "ssb-fixtures": "^2.5.3", "ssb-friends": "^5.1.0", - "ssb-index-feed-writer": "~0.5.1", + "ssb-index-feed-writer": "~0.6.0", "ssb-keys": "^8.2.0", - "ssb-meta-feeds": "~0.20.0", + "ssb-meta-feeds": "~0.21.0", + "ssb-meta-feeds-rpc": "~0.2.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" }, From f5ff2eec8b122d822dc92a52d6382c6a4a062999 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 14 Sep 2021 17:19:20 +0300 Subject: [PATCH 22/94] update some code comments --- index.js | 1 + req-manager.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index aecc5f5..8a63950 100644 --- a/index.js +++ b/index.js @@ -53,6 +53,7 @@ exports.init = function (ssb, config) { } // Compute every block edge, unless I am the edge destination if (dest !== ssb.id) { + // FIXME: consider meta feed and subfeed blocking ssb.ebt.block(source, dest, value === -1) } } diff --git a/req-manager.js b/req-manager.js index cbbf015..07b285e 100644 --- a/req-manager.js +++ b/req-manager.js @@ -44,7 +44,7 @@ module.exports = class RequestManager { this._requestables.delete(feedId) this._requestedFully.add(feedId) this._ssb.ebt.request(feedId, true) - // FIXME: needs to support bendybutt and classic + // FIXME: needs to support bendybutt and classic, or not?? } /** From 8d34e518efdbb2c28589aafc239ee794222e1284 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 14 Sep 2021 17:19:42 +0300 Subject: [PATCH 23/94] improve RequestManager's filtering of public msgs --- req-manager.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/req-manager.js b/req-manager.js index 07b285e..d6f66f4 100644 --- a/req-manager.js +++ b/req-manager.js @@ -2,7 +2,14 @@ const pull = require('pull-stream') const Ref = require('ssb-ref') const { QL0 } = require('ssb-subset-ql') const { isBendyButtV1FeedSSBURI } = require('ssb-uri2') -const { where, author, live, toPullStream } = require('ssb-db2/operators') +const { + where, + and, + author, + isPublic, + live, + toPullStream, +} = require('ssb-db2/operators') const DEFAULT_PERIOD = 150 @@ -88,11 +95,10 @@ module.exports = class RequestManager { // * traverse(mainfeedId, subfeedId, matchedTemplate) pull( this._ssb.db.query( - where(author(metafeedId)), + where(and(author(metafeedId), isPublic())), live({ old: true }), toPullStream() ), - pull.filter((msg) => typeof msg.value.content !== 'string'), // FIXME: this drain multiplied by N peers with support for // partialReplication multiplied by M sub-meta-feeds for each means N*M // live drains. Sounds like a performance nightmare, if N > 1000. From 1f445cac7307d0cb3e3746f3fc91a96b4f6d415d Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 14 Sep 2021 17:22:28 +0300 Subject: [PATCH 24/94] refactor MetafeedFinder: pluckFromAnnounceMsg can be trusted --- metafeed-finder.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index ce408a1..0aae1f3 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -43,7 +43,6 @@ module.exports = class MetafeedFinder { pull.filter(this._validateMetafeedAnnounce), pull.drain((msg) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) - if (!mainFeedId || !metaFeedId) return this._map.set(mainFeedId, metaFeedId) }) ) @@ -136,7 +135,6 @@ module.exports = class MetafeedFinder { (drainer = pull.drain( (msg) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) - if (!mainFeedId || !metaFeedId) return if (requests.has(mainFeedId)) { this._map.set(mainFeedId, metaFeedId) this._persist(msg) From 30dcb0a5d7350cbb07b4f2f5bf0c5b3acf3d4fd8 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 14 Sep 2021 17:42:57 +0300 Subject: [PATCH 25/94] for now, self feed ID always fully replicates itself --- req-manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/req-manager.js b/req-manager.js index d6f66f4..6896a11 100644 --- a/req-manager.js +++ b/req-manager.js @@ -227,6 +227,7 @@ module.exports = class RequestManager { pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { if (!this._opts.partialReplication) return cb(null, [feedId, false]) + if (feedId === this._ssb.id) return cb(null, [feedId, false]) this._supportsPartialReplication(feedId, (err, partially) => { if (err) cb(err) From 89a83c2c70031553f9d9a6d7ce6b976dd6dda0a1 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 14:29:47 +0300 Subject: [PATCH 26/94] add more debug() to MetafeedFinder --- metafeed-finder.js | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 0aae1f3..c76a432 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -41,10 +41,18 @@ module.exports = class MetafeedFinder { pull( this._ssb.db.query(where(type('metafeed/announce')), toPullStream()), pull.filter(this._validateMetafeedAnnounce), - pull.drain((msg) => { - const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) - this._map.set(mainFeedId, metaFeedId) - }) + pull.drain( + (msg) => { + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) + this._map.set(mainFeedId, metaFeedId) + }, + () => { + debug( + 'loaded Map of all known main=>rootMF from disk, total %d', + this._map.size + ) + } + ) ) } @@ -129,6 +137,7 @@ module.exports = class MetafeedFinder { this._requestsByMainfeedId.clear() await this._forEachNeighborPeer((rpc, goToNextNeighbor) => { + debug('"getSubset" on peer %s for metafeed/announce messages', rpc.id) pull( rpc.getSubset(this._makeQL1(requests), { querylang: 'ssb-ql-1' }), pull.filter(this._validateMetafeedAnnounce), @@ -136,6 +145,11 @@ module.exports = class MetafeedFinder { (msg) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) if (requests.has(mainFeedId)) { + debug( + 'learned that main %s maps to rootMF %s', + mainFeedId, + metaFeedId + ) this._map.set(mainFeedId, metaFeedId) this._persist(msg) const callbacks = requests.get(mainFeedId) From 41ff7838315f92d42688dd488accf9b7a786926c Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 14:31:10 +0300 Subject: [PATCH 27/94] fix usage of getSubset rpc --- metafeed-finder.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index c76a432..796dee7 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -43,7 +43,7 @@ module.exports = class MetafeedFinder { pull.filter(this._validateMetafeedAnnounce), pull.drain( (msg) => { - const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg.value) this._map.set(mainFeedId, metaFeedId) }, () => { @@ -66,9 +66,9 @@ module.exports = class MetafeedFinder { } } - _pluckFromAnnounceMsg(msg) { - const mainFeedId = msg.value.author - const metaFeedId = msg.value.content.metafeed + _pluckFromAnnounceMsg(msgVal) { + const mainFeedId = msgVal.author + const metaFeedId = msgVal.content.metafeed return [mainFeedId, metaFeedId] } @@ -80,8 +80,8 @@ module.exports = class MetafeedFinder { this._scheduleDebouncedFlush() } - _persist(msg) { - this._ssb.db.addOOO(msg.value, (err) => { + _persist(msgVal) { + this._ssb.db.addOOO(msgVal, (err) => { if (err) { debug( 'failed to addOOO for a metafeed/announce: %s', @@ -140,10 +140,10 @@ module.exports = class MetafeedFinder { debug('"getSubset" on peer %s for metafeed/announce messages', rpc.id) pull( rpc.getSubset(this._makeQL1(requests), { querylang: 'ssb-ql-1' }), - pull.filter(this._validateMetafeedAnnounce), + pull.filter((value) => this._validateMetafeedAnnounce({ value })), (drainer = pull.drain( - (msg) => { - const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg) + (msgVal) => { + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) if (requests.has(mainFeedId)) { debug( 'learned that main %s maps to rootMF %s', @@ -151,7 +151,7 @@ module.exports = class MetafeedFinder { metaFeedId ) this._map.set(mainFeedId, metaFeedId) - this._persist(msg) + this._persist(msgVal) const callbacks = requests.get(mainFeedId) requests.delete(mainFeedId) for (const cb of callbacks) cb(null, metaFeedId) From d734877c87a36c6e7d57de31786d544e254f3abb Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:00:53 +0300 Subject: [PATCH 28/94] add a FIXME comment --- req-manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/req-manager.js b/req-manager.js index 6896a11..f1a28b5 100644 --- a/req-manager.js +++ b/req-manager.js @@ -42,6 +42,7 @@ module.exports = class RequestManager { reconfigure(opts) { this._opts = { ...this._opts, opts } + // FIXME: trigger a recalculation somehow } /** From 59aca75983456b8f6bd48c87140a6ce18fec6a42 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:01:32 +0300 Subject: [PATCH 29/94] fix mistakes in RequestManager --- req-manager.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index f1a28b5..79c781c 100644 --- a/req-manager.js +++ b/req-manager.js @@ -32,6 +32,7 @@ module.exports = class RequestManager { } add(feedId) { + if (this._requestables.has(feedId)) return if (this._requestedFully.has(feedId)) return if (this._requestedPartially.has(feedId)) return @@ -115,7 +116,7 @@ module.exports = class RequestManager { this._traverse(msg, childTemplate, mainFeedId) } else if (Ref.isFeedId(subfeed)) { // FIXME: what if `subfeed` is an index feed? - this._requestedFully(subfeed) + this._requestFully(subfeed) } else { console.error('cannot replicate unknown feed type: ' + subfeed) } @@ -217,7 +218,11 @@ module.exports = class RequestManager { this._timer = null } // Flush if enough time has passed - else if (Date.now() - this._latestAdd > this._period) this._flush() + else if (Date.now() - this._latestAdd > this._period) { + clearInterval(this._timer) + this._timer = null + this._flush() + } }, this._period * 0.5) if (this._timer.unref) this._timer.unref() } From c7ec4c60eb334bc657dfb90ef3ad24791573df12 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:08:33 +0300 Subject: [PATCH 30/94] MetafeedFinder uses metafeeds.find in case of self ID --- metafeed-finder.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/metafeed-finder.js b/metafeed-finder.js index 796dee7..4a24662 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -26,6 +26,18 @@ module.exports = class MetafeedFinder { const metaFeedId = this._map.get(mainFeedId) cb(null, metaFeedId) return + } else if (mainFeedId === this._ssb.id) { + this._ssb.metafeeds.find((err, rootMF) => { + if (err) { + return cb(err) + } else if (!rootMF) { + cb(null, null) + } else { + const metaFeedId = rootMF.keys.id + this._map.set(mainFeedId, metaFeedId) + cb(null, metaFeedId) + } + }) } else { this._request(mainFeedId, cb) } From f571fd9f0fd2f3bb9458772992e39d0183465ba6 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:21:06 +0300 Subject: [PATCH 31/94] tweak a debug in MetafeedFinder --- metafeed-finder.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 4a24662..6f64a61 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -158,7 +158,7 @@ module.exports = class MetafeedFinder { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) if (requests.has(mainFeedId)) { debug( - 'learned that main %s maps to rootMF %s', + 'learned that main %s has rootMF %s', mainFeedId, metaFeedId ) From 5219fff7f1e40863acb64a251facad4ca4c5d43b Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:26:28 +0300 Subject: [PATCH 32/94] remove one debug in MetafeedFinder --- metafeed-finder.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 6f64a61..5a25671 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -157,11 +157,6 @@ module.exports = class MetafeedFinder { (msgVal) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) if (requests.has(mainFeedId)) { - debug( - 'learned that main %s has rootMF %s', - mainFeedId, - metaFeedId - ) this._map.set(mainFeedId, metaFeedId) this._persist(msgVal) const callbacks = requests.get(mainFeedId) From 6a926ca2f5129a9613b5e1128f54185ae8ecd630 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:28:45 +0300 Subject: [PATCH 33/94] remove some obsolete FIXME comments --- req-manager.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index 79c781c..1e7f975 100644 --- a/req-manager.js +++ b/req-manager.js @@ -53,7 +53,6 @@ module.exports = class RequestManager { this._requestables.delete(feedId) this._requestedFully.add(feedId) this._ssb.ebt.request(feedId, true) - // FIXME: needs to support bendybutt and classic, or not?? } /** @@ -115,7 +114,6 @@ module.exports = class RequestManager { if (isBendyButtV1FeedSSBURI(subfeed)) { this._traverse(msg, childTemplate, mainFeedId) } else if (Ref.isFeedId(subfeed)) { - // FIXME: what if `subfeed` is an index feed? this._requestFully(subfeed) } else { console.error('cannot replicate unknown feed type: ' + subfeed) From 047f97f7cf5fb306f5ae0878bf3fd4b0473a6731 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:31:53 +0300 Subject: [PATCH 34/94] integration test for partial replication --- package.json | 2 +- req-manager.js | 2 +- test/integration/index-feeds.js | 204 ++++++++++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 test/integration/index-feeds.js diff --git a/package.json b/package.json index fbc0fe3..3d6f937 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", "ssb-db2": "^2.5.2", - "ssb-ebt": "^7.0.0", + "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", "ssb-fixtures": "^2.5.3", "ssb-friends": "^5.1.0", "ssb-index-feed-writer": "~0.6.0", diff --git a/req-manager.js b/req-manager.js index 1e7f975..b714c35 100644 --- a/req-manager.js +++ b/req-manager.js @@ -231,7 +231,7 @@ module.exports = class RequestManager { pull.values([...this._requestables]), pull.asyncMap((feedId, cb) => { if (!this._opts.partialReplication) return cb(null, [feedId, false]) - if (feedId === this._ssb.id) return cb(null, [feedId, false]) + if (feedId === this._ssb.id) return cb(null, [feedId, true]) this._supportsPartialReplication(feedId, (err, partially) => { if (err) cb(err) diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js new file mode 100644 index 0000000..82f42a9 --- /dev/null +++ b/test/integration/index-feeds.js @@ -0,0 +1,204 @@ +const tape = require('tape') +const path = require('path') +const os = require('os') +const rimraf = require('rimraf') +const caps = require('ssb-caps') +const SecretStack = require('secret-stack') +const pify = require('promisify-4loc') +const { + where, + and, + type, + author, + count, + toPromise, +} = require('ssb-db2/operators') +const sleep = require('util').promisify(setTimeout) +const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') +const indexedEBTFormat = require('ssb-ebt/formats/indexed') +const { keysFor } = require('../misc/util') + +const createSsbServer = SecretStack({ caps }) + .use(require('ssb-db2')) + .use(require('ssb-db2/compat/ebt')) + .use(require('ssb-ebt')) + .use(require('ssb-friends')) + .use(require('ssb-meta-feeds')) + .use(require('ssb-meta-feeds-rpc')) + .use(require('ssb-index-feed-writer')) + .use(require('../..')) + +const CONNECTION_TIMEOUT = 500 // ms +const REPLICATION_TIMEOUT = 10000 // ms +const INDEX_WRITING_TIMEOUT = 2000 // ms + +const aliceKeys = keysFor('alice') +const bobKeys = keysFor('bob') + +tape('setup', async (t) => { + rimraf.sync(path.join(os.tmpdir(), 'server-alice')) + rimraf.sync(path.join(os.tmpdir(), 'server-bob')) + + const alice = createSsbServer({ + path: path.join(os.tmpdir(), 'server-alice'), + keys: aliceKeys, + timeout: CONNECTION_TIMEOUT, + indexFeedWriter: { + autostart: [{ type: 'post', private: false }], + }, + }) + + const bob = createSsbServer({ + path: path.join(os.tmpdir(), 'server-bob'), + keys: bobKeys, + timeout: CONNECTION_TIMEOUT, + }) + + t.pass('started both peers') + t.pass('alice is ' + alice.id) + t.pass('bob is ' + bob.id) + + // Wait for all bots to be ready + await sleep(500) + + const following = true + await Promise.all([ + // All peers publish a post + pify(alice.db.publish)({ type: 'post', text: 'My name is Alice' }), + pify(bob.db.publish)({ type: 'post', text: 'My name is Bob' }), + + // alice and bob follow each other + pify(alice.db.publish)({ type: 'contact', contact: bob.id, following }), + pify(bob.db.publish)({ type: 'contact', contact: alice.id, following }), + ]) + t.pass('published all the messages') + + await sleep(INDEX_WRITING_TIMEOUT) + t.pass('waited for Alice to publish meta feed msgs') + + await Promise.all([pify(alice.close)(true), pify(bob.close)(true)]) + + t.end() +}) + +tape('alice writes index feeds and bob replicates them', async (t) => { + const alice = createSsbServer({ + path: path.join(os.tmpdir(), 'server-alice'), + keys: aliceKeys, + timeout: CONNECTION_TIMEOUT, + indexFeedWriter: { + autostart: [{ type: 'post', private: false }], + }, + replicationScheduler: { + partialReplication: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post' }, + }, + }, + ], + }, + ], + }, + }, + }) + + const bob = createSsbServer({ + path: path.join(os.tmpdir(), 'server-bob'), + keys: bobKeys, + timeout: CONNECTION_TIMEOUT, + replicationScheduler: { + partialReplication: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post' }, + }, + }, + ], + }, + ], + }, + }, + }) + + // FIXME: these should be inside the implementation of replication-scheduler + alice.ebt.registerFormat('indexedfeed', indexedEBTFormat()) + alice.ebt.registerFormat('bendybutt-v1', bendyButtEBTFormat) + bob.ebt.registerFormat('indexedfeed', indexedEBTFormat()) + bob.ebt.registerFormat('bendybutt-v1', bendyButtEBTFormat) + t.pass('registered ebt formats') + + const connectionBA = await pify(bob.connect)(alice.getAddress()) + t.pass('peers are connected to each other') + + await sleep(REPLICATION_TIMEOUT) + t.pass('replication period is over') + + // Alice fully replicates Bob + t.equals( + await alice.db.query( + where(and(type('post'), author(bob.id))), + count(), + toPromise() + ), + 1, + 'alice has 1 post from bob' + ) + + t.equals( + await alice.db.query( + where(and(type('contact'), author(bob.id))), + count(), + toPromise() + ), + 1, + 'alice has 1 contact from bob' + ) + + // Bob partially replicates Alice + t.equals( + await bob.db.query( + where(and(type('post'), author(alice.id))), + count(), + toPromise() + ), + 1, + 'bob has 1 post from alice' + ) + + t.equals( + await bob.db.query( + where(and(type('contact'), author(alice.id))), + count(), + toPromise() + ), + 0, + 'bob does NOT have contact msgs from alice' + ) + + t.equals( + await bob.db.query( + where(and(type('metafeed/announce'), author(alice.id))), + count(), + toPromise() + ), + 1, + 'bob has 1 metafeed/announce from alice' + ) + + await pify(connectionBA.close)(true) + + await Promise.all([pify(alice.close)(true), pify(bob.close)(true)]) + + t.end() +}) From 54da68eeb4aa6ad1d839c2b229456ba60a2a1ad1 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 15 Sep 2021 15:39:56 +0300 Subject: [PATCH 35/94] remove all FIXME from code --- index.js | 1 - req-manager.js | 15 --------------- test/integration/index-feeds.js | 1 - 3 files changed, 17 deletions(-) diff --git a/index.js b/index.js index 8a63950..aecc5f5 100644 --- a/index.js +++ b/index.js @@ -53,7 +53,6 @@ exports.init = function (ssb, config) { } // Compute every block edge, unless I am the edge destination if (dest !== ssb.id) { - // FIXME: consider meta feed and subfeed blocking ssb.ebt.block(source, dest, value === -1) } } diff --git a/req-manager.js b/req-manager.js index b714c35..236d12f 100644 --- a/req-manager.js +++ b/req-manager.js @@ -43,7 +43,6 @@ module.exports = class RequestManager { reconfigure(opts) { this._opts = { ...this._opts, opts } - // FIXME: trigger a recalculation somehow } /** @@ -87,25 +86,12 @@ module.exports = class RequestManager { this._requestFully(metafeedId) - // ssb-db2 live query for metafeedId - // * for every *tombstoned* subfeed, call ssb.ebt.request(subfeedId, false) - // * for every *added* subfeed, match against template.subfeeds - // * if not matched, ignore - // * if matched and is classic feed, _requestFully(subfeedId) - // * if matched with subfeeds and is bendybutt feed, then - // * traverse(mainfeedId, subfeedId, matchedTemplate) pull( this._ssb.db.query( where(and(author(metafeedId), isPublic())), live({ old: true }), toPullStream() ), - // FIXME: this drain multiplied by N peers with support for - // partialReplication multiplied by M sub-meta-feeds for each means N*M - // live drains. Sounds like a performance nightmare, if N > 1000. - // - // Should we do instead one drain for all bendybutt messages and based - // on the bbmsg look up who it belongs to and then traverse the template? pull.drain((msg) => { const { type, subfeed } = msg.value.content if (type.startsWith('metafeed/add/')) { @@ -122,7 +108,6 @@ module.exports = class RequestManager { } } } else if (type === 'metafeed/tombstone') { - // FIXME: what if this happens very quickly after add? this._ssb.ebt.request(subfeed, false) } }) diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 82f42a9..79cbbae 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -131,7 +131,6 @@ tape('alice writes index feeds and bob replicates them', async (t) => { }, }) - // FIXME: these should be inside the implementation of replication-scheduler alice.ebt.registerFormat('indexedfeed', indexedEBTFormat()) alice.ebt.registerFormat('bendybutt-v1', bendyButtEBTFormat) bob.ebt.registerFormat('indexedfeed', indexedEBTFormat()) From 382b727b7562203bf439a5f47393b58168f2dc98 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 16 Sep 2021 18:14:04 +0300 Subject: [PATCH 36/94] introduce different templates for each hops level --- README.md | 271 ++++++++++++++++++++++++++------ index.js | 4 +- metafeed-finder.js | 6 +- package.json | 2 +- req-manager.js | 73 +++++---- test/integration/index-feeds.js | 57 ++++--- 6 files changed, 314 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 71683a1..26ea84d 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ Depends on ssb-friends APIs, and calls ssb-ebt APIs. - Requires **Node.js 10** or higher - Requires **ssb-db** or **ssb-db2** -- Requires **ssb-friends** version **5.0** or higher -- Requires **ssb-ebt** version **7.0** or higher +- Requires [**ssb-friends**](https://github.com/ssbc/ssb-friends) version **5.0** or higher +- Requires [**ssb-ebt**](https://github.com/ssbc/ssb-ebt) version **7.0** or higher ``` npm install --save ssb-replication-scheduler @@ -58,7 +58,7 @@ depending whether the feed is friendly or blocked. - Replication is strictly disabled for: - Any feed you explicitly block -### Configuration +## Configuration Some parameters and opinions can be configured by the user or by application code through the conventional [ssb-config](https://github.com/ssbc/ssb-config) @@ -70,24 +70,89 @@ object. The possible options are listed below: /** * If `partialReplication` is an object, it tells the replication scheduler * to perform partial replication, whenever remote feeds support it. If - * `partialReplication` is `false` (which it is, by default), then all + * `partialReplication` is `null` (which it is, by default), then all * friendly feeds will be requested in full. * * Read below more about this configuration. */ - partialReplication: false, + partialReplication: null, } } ``` -#### Configuring partial replication +### Configuring partial replication The `config.replicationScheduler.partialReplication` object describes the tree -of meta feeds that we are interested in replicating. +of meta feeds that we are interested in replicating, for each hops level. For +each hops level we have a certain *template* to describe how replication should +work at that level. Notice that this configuration cannot specify **who** we +replicate (that's the job of ssb-friends and your chosen `hops`, see the *Usage* +section above), this configuration just specifies **how** should we replicate a +friendly peer, in other words, the level of granularity for those peers. -It is recursively made up of objects and arrays describes which **keys** in the -metafeeds and subfeeds must match exactly the **values** given. So that if we -write `feedpurposes: 'indexes'`, it means we are interested in matching the +#### Template per hops + +The high-level overview of the `partialReplication` configuration is: + +```js +replicationScheduler: { + partialReplication: { + 0: TEMPLATE_FOR_HOPS_0, + 1: TEMPLATE_FOR_HOPS_1, + 2: TEMPLATE_FOR_HOPS_2_AND_ABOVE, + } +} +``` + +Soon we'll show how those `TEMPLATE_FOR_HOPS` work, but for now notice that the +highest number will handle all the hops beyond that number, e.g. notice how `2` +is the highest number and it means that `TEMPLATE_FOR_HOPS_2_AND_ABOVE` +configures how to replicate peers at hops 2 or 3 or 4 or higher. There's nothing +special about the number 2, it could also have been this: + +```js +replicationScheduler: { + partialReplication: { + 0: TEMPLATE_FOR_HOPS_0, + 1: TEMPLATE_FOR_HOPS_1_AND_ABOVE, + } +} +``` + +Or even this (which means we use the same template for all peers, regardless of +their hops distance): + +```js +replicationScheduler: { + partialReplication: { + 0: TEMPLATE_FOR_HOPS_0_AND_ABOVE, + } +} +``` + +Or even fractional numbers: + +```js +replicationScheduler: { + partialReplication: { + 0: TEMPLATE_FOR_HOPS_0, + 0.5: TEMPLATE_FOR_HOPS_HALF, + 1: TEMPLATE_FOR_HOPS_1_AND_ABOVE, + } +} +``` + +#### Template structure + +A **Template** is JSON which describes how should we do partial replication. If +the template is `null` or a falsy value, then it means that for that hops level +we don't do partial replication and we **will** do **full** replication (which +means pre-2022 SSB replication of the peer's `main` feed). + +When the template is a JSON tree of objects and arrays, where the root of the +tree is always the _root meta feed_. The template describes which **keys** in +the metafeeds and subfeeds must match exactly the **values** given. So that if +we write `feedpurposes: 'indexes'`, it means we are interested in matching the metafeed that has the field `feedpurposes` exactly matching the value "indexes". All specified fields must match, but omitted fields are allowed to be any value. @@ -95,52 +160,170 @@ The field `subfeeds` is not matching an actual field, instead, it is assumes we are dealing with a meta feed and this is describing its subfeeds that we would like to replicate. -If the value is the special string `"$main"` or `"$root"`, then they refer to -(respectively) the IDs of the *main feed* and of the *root meta feed*. +#### Special variables + +Some keys and some values are special, in the sense that they are not taken +literally, but are going to be substituted by other context-relative values. +These special variables are always prefixed with **`$`**. + +- Special keys + - `$format` +- Special values + - `$main` + - `$root` + +The field *key* `$format` refers to [ssb-ebt](https://github.com/ssbc/ssb-ebt) +"replication formats" and can be included in a template to specify which +replication format to use in ssb-ebt. The value of this field should be the +format's name as a string. + +If the value of a field, e.g. in ssb-ql-0 queries, are the special strings +`"$main"` or `"$root"`, then they respectively refer to the IDs of the _main +feed_ and of the _root meta feed_. + +#### Example + +In the example below, we set up partial replication with the meaning: -Example: +- For hops 0 (that is, "yourself"), replicate some app feeds and all index feeds +- For hops 1 (direct friends), replicate only some index feeds +- For hops 2 and beyond, replicate only about index feed ```js partialReplication: { - subfeeds: [ - { - feedpurpose: 'indexes', - subfeeds: [ - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'post' }, + 0: { + subfeeds: [ + { feedpurpose: 'coolgame' }, + { feedpurpose: 'git-ssb' }, + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: null, private: true }, + }, + $format: 'indexed', }, - }, - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'vote' }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post', private: false }, + }, + $format: 'indexed', }, - }, - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'about' }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'vote', private: false }, + }, + $format: 'indexed', + }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'about', private: false }, + }, + $format: 'indexed', + }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'contact', private: false }, + }, + $format: 'indexed', + }, + ], + }, + ], + }, + + 1: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: null, private: true }, + }, + $format: 'indexed', + }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post', private: false }, + }, + $format: 'indexed', + }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'vote', private: false }, + }, + $format: 'indexed', + }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'about', private: false }, + }, + $format: 'indexed', + }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'contact', private: false }, + }, + $format: 'indexed', + }, + ], + }, + ], + }, + + 2: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'about', private: false }, + }, + $format: 'indexed', }, - }, - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'contact' }, + { + feedpurpose: 'index', + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'contact', private: false }, + }, + $format: 'indexed', }, - }, - ], - }, - { feedpurpose: 'coolgame' }, - { feedpurpose: 'git-ssb' }, - ] + ], + }, + ], + }, } ``` -### muxrpc APIs +## muxrpc APIs -#### `ssb.replicationScheduler.reconfigure(config) => void` +### `ssb.replicationScheduler.reconfigure(config) => void` At any point during the execution of your program, you can reconfigure the replication rules using this API. The configuration object passed to this API diff --git a/index.js b/index.js index aecc5f5..1306e93 100644 --- a/index.js +++ b/index.js @@ -48,7 +48,7 @@ exports.init = function (ssb, config) { const value = graph[source][dest] // Only if I am the `source` and `value >= 0`, request replication if (source === ssb.id) { - if (value >= 0) requestManager.add(dest) + if (value >= 0) requestManager.add(dest, value) else ssb.ebt.request(dest, false) } // Compute every block edge, unless I am the edge destination @@ -68,7 +68,7 @@ exports.init = function (ssb, config) { const value = hops[dest] // myself or friendly peers if (value >= 0) { - requestManager.add(dest) + requestManager.add(dest, value) ssb.ebt.block(ssb.id, dest, false) } // blocked peers diff --git a/metafeed-finder.js b/metafeed-finder.js index 5a25671..4ad490d 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -16,7 +16,11 @@ module.exports = class MetafeedFinder { this._latestRequestTime = 0 this._timer = null - if (this._opts.partialReplication) { + // If at least one hops template is configured, then load + if ( + this._opts.partialReplication && + Object.values(this._opts.partialReplication).some((templ) => !!templ) + ) { this._loadAllFromLog() } } diff --git a/package.json b/package.json index 3d6f937..b7d2c86 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "ssb-friends": "^5.1.0", "ssb-index-feed-writer": "~0.6.0", "ssb-keys": "^8.2.0", - "ssb-meta-feeds": "~0.21.0", + "ssb-meta-feeds": "~0.22.0", "ssb-meta-feeds-rpc": "~0.2.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" diff --git a/req-manager.js b/req-manager.js index 236d12f..94528a3 100644 --- a/req-manager.js +++ b/req-manager.js @@ -22,21 +22,24 @@ module.exports = class RequestManager { typeof opts.debouncePeriod === 'number' ? opts.debouncePeriod : DEFAULT_PERIOD - this._requestables = new Set() - this._requestedFully = new Set() - this._requestedPartially = new Set() + this._requestables = new Map() + this._requestedDirectly = new Map() + this._requestedIndirectly = new Map() this._flushing = false this._wantsMoreFlushing = false this._latestAdd = 0 this._timer = null + this._hopsLevels = !this._opts.partialReplication + ? [] + : Object.keys(this._opts.partialReplication).map(Number).sort() } - add(feedId) { + add(feedId, hops) { if (this._requestables.has(feedId)) return - if (this._requestedFully.has(feedId)) return - if (this._requestedPartially.has(feedId)) return + if (this._requestedDirectly.has(feedId)) return + if (this._requestedIndirectly.has(feedId)) return - this._requestables.add(feedId) + this._requestables.set(feedId, hops) this._latestAdd = Date.now() this._scheduleDebouncedFlush() } @@ -48,18 +51,21 @@ module.exports = class RequestManager { /** * @param {string} feedId classic feed ref or bendybutt feed URI */ - _requestFully(feedId) { + _requestDirectly(feedId, ebtFormat = undefined) { + const hops = this._requestables.get(feedId) + this._requestedDirectly.set(feedId, hops) this._requestables.delete(feedId) - this._requestedFully.add(feedId) - this._ssb.ebt.request(feedId, true) + this._ssb.ebt.request(feedId, true, ebtFormat) } /** * @param {string} mainFeedId classic feed ref which has announced a root MF + * @param {object} template one of the nodes in opts.partialReplication */ - _requestPartially(mainFeedId) { + _requestIndirectly(mainFeedId, template) { + const hops = this._requestables.get(mainFeedId) + this._requestedIndirectly.set(mainFeedId, hops) this._requestables.delete(mainFeedId) - this._requestedPartially.add(mainFeedId) // Get metafeedId for this feedId this._metafeedFinder.get(mainFeedId, (err, metafeedId) => { @@ -68,7 +74,7 @@ module.exports = class RequestManager { } else if (!metafeedId) { console.error('cannot partially replicate ' + mainFeedId) } else { - this._traverse(metafeedId, this._opts.partialReplication, mainFeedId) + this._traverse(metafeedId, template, mainFeedId) } }) } @@ -84,7 +90,7 @@ module.exports = class RequestManager { const metafeedId = typeof input === 'string' ? input : input.value.content.subfeed - this._requestFully(metafeedId) + this._requestDirectly(metafeedId) pull( this._ssb.db.query( @@ -100,7 +106,7 @@ module.exports = class RequestManager { if (isBendyButtV1FeedSSBURI(subfeed)) { this._traverse(msg, childTemplate, mainFeedId) } else if (Ref.isFeedId(subfeed)) { - this._requestFully(subfeed) + this._requestDirectly(subfeed, childTemplate['$format']) } else { console.error('cannot replicate unknown feed type: ' + subfeed) } @@ -181,6 +187,14 @@ module.exports = class RequestManager { }) } + _findTemplateForHops(hops) { + if (!this._opts.partialReplication) return null + const eligible = this._hopsLevels.filter((h) => h >= hops) + const picked = Math.min(...eligible) + const x = this._opts.partialReplication[picked] + return x + } + _scheduleDebouncedFlush() { if (this._flushing) { this._wantsMoreFlushing = true @@ -213,22 +227,27 @@ module.exports = class RequestManager { _flush() { this._flushing = true pull( - pull.values([...this._requestables]), - pull.asyncMap((feedId, cb) => { - if (!this._opts.partialReplication) return cb(null, [feedId, false]) - if (feedId === this._ssb.id) return cb(null, [feedId, true]) - - this._supportsPartialReplication(feedId, (err, partially) => { - if (err) cb(err) - else cb(null, [feedId, partially]) + pull.values([...this._requestables.entries()]), + pull.asyncMap(([feedId, hops], cb) => { + const template = this._findTemplateForHops(hops) + if (!template) return cb(null, [feedId, null]) + + this._supportsPartialReplication(feedId, (err, supports) => { + if (err) { + cb(err) + } else if (supports) { + cb(null, [feedId, template]) + } else { + cb(null, [feedId, null]) + } }) }), pull.drain( - ([feedId, partially]) => { - if (partially) { - this._requestPartially(feedId) + ([feedId, template]) => { + if (template) { + this._requestIndirectly(feedId, template) } else { - this._requestFully(feedId) + this._requestDirectly(feedId) } }, (err) => { diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 79cbbae..763edcc 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -91,19 +91,24 @@ tape('alice writes index feeds and bob replicates them', async (t) => { }, replicationScheduler: { partialReplication: { - subfeeds: [ - { - feedpurpose: 'indexes', - subfeeds: [ - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'post' }, + 0: { + subfeeds: [ + { feedpurpose: 'main' }, + { + feedpurpose: 'indexes', + subfeeds: [ + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post', private: false }, + }, + $format: 'indexed' }, - }, - ], - }, - ], + ], + }, + ], + }, + 1: null, }, }, }) @@ -114,19 +119,23 @@ tape('alice writes index feeds and bob replicates them', async (t) => { timeout: CONNECTION_TIMEOUT, replicationScheduler: { partialReplication: { - subfeeds: [ - { - feedpurpose: 'indexes', - subfeeds: [ - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'post' }, + 0: null, + 1: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post', private: false }, + }, + $format: 'indexed' }, - }, - ], - }, - ], + ], + }, + ], + }, }, }, }) From 1a8b49d723c89640f37b079598e98c05f40878f6 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 16 Sep 2021 18:14:33 +0300 Subject: [PATCH 37/94] register EBT format in RequestManager, if needed --- req-manager.js | 11 +++++++++++ test/integration/index-feeds.js | 6 ------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/req-manager.js b/req-manager.js index 94528a3..b455531 100644 --- a/req-manager.js +++ b/req-manager.js @@ -10,6 +10,8 @@ const { live, toPullStream, } = require('ssb-db2/operators') +const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') +const indexedEBTFormat = require('ssb-ebt/formats/indexed') const DEFAULT_PERIOD = 150 @@ -32,6 +34,15 @@ module.exports = class RequestManager { this._hopsLevels = !this._opts.partialReplication ? [] : Object.keys(this._opts.partialReplication).map(Number).sort() + + // If at least one hops template is configured, then setup ssb-ebt + if ( + this._opts.partialReplication && + Object.values(this._opts.partialReplication).some((templ) => !!templ) + ) { + this._ssb.ebt.registerFormat(indexedEBTFormat) + this._ssb.ebt.registerFormat(bendyButtEBTFormat) + } } add(feedId, hops) { diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 763edcc..326e637 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -140,12 +140,6 @@ tape('alice writes index feeds and bob replicates them', async (t) => { }, }) - alice.ebt.registerFormat('indexedfeed', indexedEBTFormat()) - alice.ebt.registerFormat('bendybutt-v1', bendyButtEBTFormat) - bob.ebt.registerFormat('indexedfeed', indexedEBTFormat()) - bob.ebt.registerFormat('bendybutt-v1', bendyButtEBTFormat) - t.pass('registered ebt formats') - const connectionBA = await pify(bob.connect)(alice.getAddress()) t.pass('peers are connected to each other') From 2987ed0c78e2083f8ff5ff55f352a87fd5d21071 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 17 Sep 2021 17:11:41 +0300 Subject: [PATCH 38/94] do only one database pull-drain in RequestManager --- index.js | 2 +- metafeed-finder.js | 8 ++ package.json | 2 +- req-manager.js | 247 +++++++++++++++++++++------------------------ template.js | 113 +++++++++++++++++++++ 5 files changed, 237 insertions(+), 135 deletions(-) create mode 100644 template.js diff --git a/index.js b/index.js index 1306e93..57d7e3c 100644 --- a/index.js +++ b/index.js @@ -7,7 +7,7 @@ const MetafeedFinder = require('./metafeed-finder') const RequestManager = require('./req-manager') const DEFAULT_OPTS = { - partialReplication: false, + partialReplication: null, } exports.name = 'replicationScheduler' diff --git a/metafeed-finder.js b/metafeed-finder.js index 4ad490d..5849289 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -12,6 +12,7 @@ module.exports = class MetafeedFinder { this._opts = opts this._period = period || DEFAULT_PERIOD this._map = new Map() + this._inverseMap = new Map() this._requestsByMainfeedId = new Map() this._latestRequestTime = 0 this._timer = null @@ -39,6 +40,7 @@ module.exports = class MetafeedFinder { } else { const metaFeedId = rootMF.keys.id this._map.set(mainFeedId, metaFeedId) + this._inverseMap.set(metaFeedId, mainFeedId) cb(null, metaFeedId) } }) @@ -47,6 +49,10 @@ module.exports = class MetafeedFinder { } } + getInverse(metaFeedId) { + return this._inverseMap.get(metaFeedId) + } + _loadAllFromLog() { if (!this._ssb.db || !this._ssb.db.query) { throw new Error( @@ -61,6 +67,7 @@ module.exports = class MetafeedFinder { (msg) => { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg.value) this._map.set(mainFeedId, metaFeedId) + this._inverseMap.set(metaFeedId, mainFeedId) }, () => { debug( @@ -162,6 +169,7 @@ module.exports = class MetafeedFinder { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) if (requests.has(mainFeedId)) { this._map.set(mainFeedId, metaFeedId) + this._inverseMap.set(metaFeedId, mainFeedId) this._persist(msgVal) const callbacks = requests.get(mainFeedId) requests.delete(mainFeedId) diff --git a/package.json b/package.json index b7d2c86..b95be88 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "ssb-friends": "^5.1.0", "ssb-index-feed-writer": "~0.6.0", "ssb-keys": "^8.2.0", - "ssb-meta-feeds": "~0.22.0", + "ssb-meta-feeds": "~0.22.1", "ssb-meta-feeds-rpc": "~0.2.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" diff --git a/req-manager.js b/req-manager.js index b455531..f1df6bd 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,17 +1,15 @@ const pull = require('pull-stream') -const Ref = require('ssb-ref') -const { QL0 } = require('ssb-subset-ql') -const { isBendyButtV1FeedSSBURI } = require('ssb-uri2') const { where, and, - author, + authorIsBendyButtV1, isPublic, live, toPullStream, } = require('ssb-db2/operators') const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') const indexedEBTFormat = require('ssb-ebt/formats/indexed') +const Template = require('./template') const DEFAULT_PERIOD = 150 @@ -27,21 +25,20 @@ module.exports = class RequestManager { this._requestables = new Map() this._requestedDirectly = new Map() this._requestedIndirectly = new Map() + this._tombstoned = new Set() this._flushing = false this._wantsMoreFlushing = false this._latestAdd = 0 this._timer = null - this._hopsLevels = !this._opts.partialReplication - ? [] - : Object.keys(this._opts.partialReplication).map(Number).sort() + this._hasCloseHook = false + this._templates = this._setupTemplates(this._opts.partialReplication) // If at least one hops template is configured, then setup ssb-ebt - if ( - this._opts.partialReplication && - Object.values(this._opts.partialReplication).some((templ) => !!templ) - ) { - this._ssb.ebt.registerFormat(indexedEBTFormat) + if (this._templates) { this._ssb.ebt.registerFormat(bendyButtEBTFormat) + if (this._someTemplate((t) => t.hasIndexLeaf())) { + this._ssb.ebt.registerFormat(indexedEBTFormat) + } } } @@ -59,6 +56,103 @@ module.exports = class RequestManager { this._opts = { ...this._opts, opts } } + _setupTemplates(optsPartialReplication) { + if (!optsPartialReplication) return null + if (Object.values(optsPartialReplication).every((t) => !t)) return null + const hopsArr = Object.keys(optsPartialReplication).map(Number) + const templates = new Map() + for (const hops of hopsArr) { + if (optsPartialReplication[hops]) { + templates.set(hops, new Template(optsPartialReplication[hops])) + } else { + templates.set(hops, null) + } + } + return templates + } + + _someTemplate(fn) { + if (!this._templates) return false + return [...this._templates.values()].filter((t) => !!t).some(fn) + } + + _findTemplateForHops(hops) { + if (!this._templates) return null + const eligibleHopsArr = [...this._templates.keys()].filter((h) => h >= hops) + const pickedHops = Math.min(...eligibleHopsArr) + return this._templates.get(pickedHops) + } + + _setupCloseHook() { + this._hasCloseHook = true + const that = this + this._ssb.close.hook(function (fn, args) { + if (that._liveDrainer) that._liveDrainer.abort() + fn.apply(this, args) + }) + } + + _scanBendyButtFeeds() { + if (this._liveDrainer) this._liveDrainer.abort() + if (!this._hasCloseHook) this._setupCloseHook() + + pull( + this._ssb.db.query( + where(and(authorIsBendyButtV1(), isPublic())), + live({ old: true }), + toPullStream() + ), + pull.filter((msg) => this._ssb.metafeeds.validate.isValid(msg)), + (this._liveDrainer = pull.drain( + (msg) => { + const { type, subfeed } = msg.value.content + if (this._tombstoned.has(subfeed)) return + + if (type.startsWith('metafeed/add/')) { + const path = this._getMetafeedTreePath(msg) + const metaFeedId = path[0] + const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) + if (!this._requestedIndirectly.has(mainFeedId)) return + const hops = this._requestedIndirectly.get(mainFeedId) + const template = this._findTemplateForHops(hops) + if (!template) return + const matchedNode = template.matchPath(path, mainFeedId) + if (!matchedNode) return + this._requestDirectly(subfeed, matchedNode['$format']) + } else if (type === 'metafeed/tombstone') { + this._tombstoned.add(subfeed) + this._ssb.ebt.request(subfeed, false) + } + }, + (err) => { + if (err) return cb(err) + } + )) + ) + } + + /** + * Returns an array that represents the path from root meta feed to the given + * leaf `msg`. + * + * @param {*} msg a metafeed message + * @returns {Array} + */ + _getMetafeedTreePath(msg) { + const details = this._ssb.metafeeds.findByIdSync(msg.value.content.subfeed) + const path = [details] + while (true) { + const head = path[0] + const details = this._ssb.metafeeds.findByIdSync(head.metafeed) + if (details) { + path.unshift(details) + } else { + path.unshift(head.metafeed) + return path + } + } + } + /** * @param {string} feedId classic feed ref or bendybutt feed URI */ @@ -85,112 +179,11 @@ module.exports = class RequestManager { } else if (!metafeedId) { console.error('cannot partially replicate ' + mainFeedId) } else { - this._traverse(metafeedId, template, mainFeedId) + this._requestDirectly(metafeedId) } }) } - /** - * @param {string | object} input a metafeedId or a msg concerning a metafeed - * @param {object} template one of the nodes in opts.partialReplication - * @param {string} mainFeedId - */ - _traverse(input, template, mainFeedId) { - if (!this._matchesTemplate(input, template)) return - - const metafeedId = - typeof input === 'string' ? input : input.value.content.subfeed - - this._requestDirectly(metafeedId) - - pull( - this._ssb.db.query( - where(and(author(metafeedId), isPublic())), - live({ old: true }), - toPullStream() - ), - pull.drain((msg) => { - const { type, subfeed } = msg.value.content - if (type.startsWith('metafeed/add/')) { - for (const childTemplate of template.subfeeds) { - if (this._matchesTemplate(msg, childTemplate, mainFeedId)) { - if (isBendyButtV1FeedSSBURI(subfeed)) { - this._traverse(msg, childTemplate, mainFeedId) - } else if (Ref.isFeedId(subfeed)) { - this._requestDirectly(subfeed, childTemplate['$format']) - } else { - console.error('cannot replicate unknown feed type: ' + subfeed) - } - break - } - } - } else if (type === 'metafeed/tombstone') { - this._ssb.ebt.request(subfeed, false) - } - }) - ) - } - - _matchesTemplate(input, template, mainFeedId) { - // Input is `metafeedId` - if (typeof input === 'string' && isBendyButtV1FeedSSBURI(input)) { - const keys = Object.keys(template) - return ( - keys.length === 1 && - keys[0] === 'subfeeds' && - Array.isArray(template.subfeeds) - ) - } - - // Input is a bendybutt message - if (typeof input === 'object' && input.value && input.value.content) { - const msg = input - const content = msg.value.content - - // If present, feedpurpose must match - if ( - template.feedpurpose && - content.feedpurpose !== template.feedpurpose - ) { - return false - } - - // If present, metadata must match - if (template.metadata && content.metadata) { - // If querylang is present, match ssb-ql-0 queries - if (template.metadata.querylang !== content.metadata.querylang) { - return false - } - if (template.metadata.querylang === 'ssb-ql-0') { - if (!QL0.parse(content.metadata.query)) return false - if (template.metadata.query) { - if (template.metadata.query.author === '$main') { - template.metadata.query.author = mainFeedId - } - if ( - !QL0.isEquals(content.metadata.query, template.metadata.query) - ) { - return false - } - } - } - - // Any other metadata field must match exactly - for (const field of Object.keys(template.metadata)) { - // Ignore these because we already handled them: - if (field === 'query') continue - if (field === 'querylang') continue - - if (content.metadata[field] !== template.metadata[field]) return false - } - } - - return true - } - - return false - } - _supportsPartialReplication(feedId, cb) { this._metafeedFinder.get(feedId, (err, metafeedId) => { if (err) cb(err) @@ -198,14 +191,6 @@ module.exports = class RequestManager { }) } - _findTemplateForHops(hops) { - if (!this._opts.partialReplication) return null - const eligible = this._hopsLevels.filter((h) => h >= hops) - const picked = Math.min(...eligible) - const x = this._opts.partialReplication[picked] - return x - } - _scheduleDebouncedFlush() { if (this._flushing) { this._wantsMoreFlushing = true @@ -241,28 +226,24 @@ module.exports = class RequestManager { pull.values([...this._requestables.entries()]), pull.asyncMap(([feedId, hops], cb) => { const template = this._findTemplateForHops(hops) - if (!template) return cb(null, [feedId, null]) + if (!template) return cb(null, [feedId, false]) this._supportsPartialReplication(feedId, (err, supports) => { - if (err) { - cb(err) - } else if (supports) { - cb(null, [feedId, template]) - } else { - cb(null, [feedId, null]) - } + if (err) cb(err) + else cb(null, [feedId, supports]) }) }), pull.drain( - ([feedId, template]) => { - if (template) { - this._requestIndirectly(feedId, template) + ([feedId, supportsPartialReplication]) => { + if (supportsPartialReplication) { + this._requestIndirectly(feedId) } else { this._requestDirectly(feedId) } }, (err) => { if (err) console.error(err) + if (this._templates) this._scanBendyButtFeeds() this._flushing = false if (this._wantsMoreFlushing) { this._scheduleDebouncedFlush() diff --git a/template.js b/template.js new file mode 100644 index 0000000..ebb9d55 --- /dev/null +++ b/template.js @@ -0,0 +1,113 @@ +const { isBendyButtV1FeedSSBURI } = require('ssb-uri2') +const { QL0 } = require('ssb-subset-ql') + +/** + * Algorithms for the "replication template" object from the SSB config. + */ +module.exports = class Template { + constructor(node) { + this._rootNode = node + } + + hasIndexLeaf() { + const indexes = this._find( + this._rootNode, + (node) => node.feedpurpose === 'indexes' + ) + if (!indexes) return false + const leaf = this._find(indexes, (node) => node['$format'] === 'indexed') + return !!leaf + } + + _find(node, fn) { + if (!node) return null + if (fn(node)) return node + + if (node.subfeeds && Array.isArray(node.subfeeds)) { + for (const childNode of node.subfeeds) { + if (this._find(childNode, fn)) { + return childNode + } + } + } + } + + matchPath(path, mainFeedId) { + return this._matchPath(path, this._rootNode, mainFeedId) + } + + _matchPath(path, node, mainFeedId) { + if (path.length === 0) return null + const head = path[0] + + // Head is `metafeedId` + if (typeof head === 'string' && isBendyButtV1FeedSSBURI(head)) { + const keys = Object.keys(node) + const rootMatches = + keys.length === 1 && + keys[0] === 'subfeeds' && + Array.isArray(node.subfeeds) + + if (!rootMatches) { + return null + } else { + const childPath = path.slice(1) + for (const childNode of node.subfeeds) { + const matched = this._matchPath(childPath, childNode, mainFeedId) + if (matched) return matched + } + return null + } + } + + // Head is a subfeed details + if (typeof head === 'object') { + // If present, feedpurpose must match + if (node.feedpurpose && head.feedpurpose !== node.feedpurpose) { + return null + } + + // If present, metadata must match + if (node.metadata && head.metadata) { + // If querylang is present, match ssb-ql-0 queries + if (node.metadata.querylang !== head.metadata.querylang) { + return null + } + if (node.metadata.querylang === 'ssb-ql-0') { + if (!QL0.parse(head.metadata.query)) return null + if (node.metadata.query) { + const nodeQuery = { ...node.metadata.query } + if (nodeQuery.author === '$main') nodeQuery.author = mainFeedId + if (!QL0.isEquals(head.metadata.query, nodeQuery)) { + return null + } + } + } + + // Any other metadata field must match exactly + for (const field of Object.keys(node.metadata)) { + // Ignore these because we already handled them: + if (field === 'query') continue + if (field === 'querylang') continue + + if (head.metadata[field] !== node.metadata[field]) { + return null + } + } + } + + if (Array.isArray(node.subfeeds)) { + const childPath = path.slice(1) + for (const childNode of node.subfeeds) { + const matched = this._matchPath(childPath, childNode, mainFeedId) + if (matched) return matched + } + return node + } else { + return node + } + } + + return null + } +} From 13b024f610454217b0657144721668958408129b Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 17 Sep 2021 17:21:29 +0300 Subject: [PATCH 39/94] protect RequestManager against quick tombstoning --- req-manager.js | 23 +++++++++++++---------- test/integration/index-feeds.js | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/req-manager.js b/req-manager.js index f1df6bd..b411e17 100644 --- a/req-manager.js +++ b/req-manager.js @@ -109,16 +109,19 @@ module.exports = class RequestManager { if (this._tombstoned.has(subfeed)) return if (type.startsWith('metafeed/add/')) { - const path = this._getMetafeedTreePath(msg) - const metaFeedId = path[0] - const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - if (!this._requestedIndirectly.has(mainFeedId)) return - const hops = this._requestedIndirectly.get(mainFeedId) - const template = this._findTemplateForHops(hops) - if (!template) return - const matchedNode = template.matchPath(path, mainFeedId) - if (!matchedNode) return - this._requestDirectly(subfeed, matchedNode['$format']) + setTimeout(() => { + if (this._tombstoned.has(subfeed)) return + const path = this._getMetafeedTreePath(msg) + const metaFeedId = path[0] + const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) + if (!this._requestedIndirectly.has(mainFeedId)) return + const hops = this._requestedIndirectly.get(mainFeedId) + const template = this._findTemplateForHops(hops) + if (!template) return + const matchedNode = template.matchPath(path, mainFeedId) + if (!matchedNode) return + this._requestDirectly(subfeed, matchedNode['$format']) + }, this._period) } else if (type === 'metafeed/tombstone') { this._tombstoned.add(subfeed) this._ssb.ebt.request(subfeed, false) diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 326e637..24f9b2e 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -146,6 +146,23 @@ tape('alice writes index feeds and bob replicates them', async (t) => { await sleep(REPLICATION_TIMEOUT) t.pass('replication period is over') + console.log('ALICE HAS----------------') + console.log( + (await alice.db.query(toPromise())).map((msg) => [ + msg.value.author, + msg.value.content, + ]) + ) + console.log('-------------------------\n') + console.log('BOB HAS------------------') + console.log( + (await bob.db.query(toPromise())).map((msg) => [ + msg.value.author, + msg.value.content, + ]) + ) + console.log('-------------------------') + // Alice fully replicates Bob t.equals( await alice.db.query( From c590b118f6b794e2700952b031bd3cfb66f1f993 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 17 Sep 2021 17:22:19 +0300 Subject: [PATCH 40/94] remove console.logs from tests --- test/integration/index-feeds.js | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 24f9b2e..326e637 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -146,23 +146,6 @@ tape('alice writes index feeds and bob replicates them', async (t) => { await sleep(REPLICATION_TIMEOUT) t.pass('replication period is over') - console.log('ALICE HAS----------------') - console.log( - (await alice.db.query(toPromise())).map((msg) => [ - msg.value.author, - msg.value.content, - ]) - ) - console.log('-------------------------\n') - console.log('BOB HAS------------------') - console.log( - (await bob.db.query(toPromise())).map((msg) => [ - msg.value.author, - msg.value.content, - ]) - ) - console.log('-------------------------') - // Alice fully replicates Bob t.equals( await alice.db.query( From 191953a3953173be8ab11387e0c59f948290a6f6 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 17 Sep 2021 18:36:38 +0300 Subject: [PATCH 41/94] remove leftovers --- req-manager.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index b411e17..57151d7 100644 --- a/req-manager.js +++ b/req-manager.js @@ -168,9 +168,8 @@ module.exports = class RequestManager { /** * @param {string} mainFeedId classic feed ref which has announced a root MF - * @param {object} template one of the nodes in opts.partialReplication */ - _requestIndirectly(mainFeedId, template) { + _requestIndirectly(mainFeedId) { const hops = this._requestables.get(mainFeedId) this._requestedIndirectly.set(mainFeedId, hops) this._requestables.delete(mainFeedId) From 8354bd8c35e8c9cec8a6901b03474cac9e77149f Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 17 Sep 2021 18:40:15 +0300 Subject: [PATCH 42/94] test and implement reconfigure() --- README.md | 42 ++------------------ req-manager.js | 12 +++++- template.js | 4 +- test/integration/index-feeds.js | 68 +++++++++++++++++++++++++++------ 4 files changed, 72 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 26ea84d..76a508e 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,9 @@ object. The possible options are listed below: The `config.replicationScheduler.partialReplication` object describes the tree of meta feeds that we are interested in replicating, for each hops level. For -each hops level we have a certain *template* to describe how replication should +each hops level we have a certain _template_ to describe how replication should work at that level. Notice that this configuration cannot specify **who** we -replicate (that's the job of ssb-friends and your chosen `hops`, see the *Usage* +replicate (that's the job of ssb-friends and your chosen `hops`, see the _Usage_ section above), this configuration just specifies **how** should we replicate a friendly peer, in other words, the level of granularity for those peers. @@ -172,7 +172,7 @@ These special variables are always prefixed with **`$`**. - `$main` - `$root` -The field *key* `$format` refers to [ssb-ebt](https://github.com/ssbc/ssb-ebt) +The field _key_ `$format` refers to [ssb-ebt](https://github.com/ssbc/ssb-ebt) "replication formats" and can be included in a template to specify which replication format to use in ssb-ebt. The value of this field should be the format's name as a string. @@ -200,42 +200,6 @@ partialReplication: { subfeeds: [ { feedpurpose: 'index', - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: null, private: true }, - }, - $format: 'indexed', - }, - { - feedpurpose: 'index', - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'post', private: false }, - }, - $format: 'indexed', - }, - { - feedpurpose: 'index', - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'vote', private: false }, - }, - $format: 'indexed', - }, - { - feedpurpose: 'index', - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'about', private: false }, - }, - $format: 'indexed', - }, - { - feedpurpose: 'index', - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'contact', private: false }, - }, $format: 'indexed', }, ], diff --git a/req-manager.js b/req-manager.js index 57151d7..f449b0b 100644 --- a/req-manager.js +++ b/req-manager.js @@ -53,7 +53,17 @@ module.exports = class RequestManager { } reconfigure(opts) { - this._opts = { ...this._opts, opts } + this._opts = { ...this._opts, ...opts } + this._templates = this._setupTemplates(this._opts.partialReplication) + for (const [feedId, hops] of this._requestedDirectly) { + this._requestables.set(feedId, hops) + this._requestedDirectly.delete(feedId) + } + for (const [feedId, hops] of this._requestedIndirectly) { + this._requestables.set(feedId, hops) + this._requestedIndirectly.delete(feedId) + } + this._flush() } _setupTemplates(optsPartialReplication) { diff --git a/template.js b/template.js index ebb9d55..5744f8f 100644 --- a/template.js +++ b/template.js @@ -96,13 +96,13 @@ module.exports = class Template { } } - if (Array.isArray(node.subfeeds)) { + if (Array.isArray(node.subfeeds) && path.length >= 2) { const childPath = path.slice(1) for (const childNode of node.subfeeds) { const matched = this._matchPath(childPath, childNode, mainFeedId) if (matched) return matched } - return node + return null } else { return node } diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 326e637..6ca0bf8 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -14,8 +14,6 @@ const { toPromise, } = require('ssb-db2/operators') const sleep = require('util').promisify(setTimeout) -const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') -const indexedEBTFormat = require('ssb-ebt/formats/indexed') const { keysFor } = require('../misc/util') const createSsbServer = SecretStack({ caps }) @@ -29,8 +27,8 @@ const createSsbServer = SecretStack({ caps }) .use(require('../..')) const CONNECTION_TIMEOUT = 500 // ms -const REPLICATION_TIMEOUT = 10000 // ms -const INDEX_WRITING_TIMEOUT = 2000 // ms +const REPLICATION_TIMEOUT = 6000 // ms +const INDEX_WRITING_TIMEOUT = 3000 // ms const aliceKeys = keysFor('alice') const bobKeys = keysFor('bob') @@ -44,7 +42,10 @@ tape('setup', async (t) => { keys: aliceKeys, timeout: CONNECTION_TIMEOUT, indexFeedWriter: { - autostart: [{ type: 'post', private: false }], + autostart: [ + { type: 'post', private: false }, + { type: 'contact', private: false }, + ], }, }) @@ -87,7 +88,10 @@ tape('alice writes index feeds and bob replicates them', async (t) => { keys: aliceKeys, timeout: CONNECTION_TIMEOUT, indexFeedWriter: { - autostart: [{ type: 'post', private: false }], + autostart: [ + { type: 'post', private: false }, + { type: 'contact', private: false }, + ], }, replicationScheduler: { partialReplication: { @@ -98,11 +102,8 @@ tape('alice writes index feeds and bob replicates them', async (t) => { feedpurpose: 'indexes', subfeeds: [ { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'post', private: false }, - }, - $format: 'indexed' + feedpurpose: 'index', + $format: 'indexed', }, ], }, @@ -130,7 +131,7 @@ tape('alice writes index feeds and bob replicates them', async (t) => { querylang: 'ssb-ql-0', query: { author: '$main', type: 'post', private: false }, }, - $format: 'indexed' + $format: 'indexed', }, ], }, @@ -198,6 +199,49 @@ tape('alice writes index feeds and bob replicates them', async (t) => { 'bob has 1 metafeed/announce from alice' ) + bob.replicationScheduler.reconfigure({ + partialReplication: { + 0: null, + 1: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'post', private: false }, + }, + $format: 'indexed', + }, + { + metadata: { + querylang: 'ssb-ql-0', + query: { author: '$main', type: 'contact', private: false }, + }, + $format: 'indexed', + }, + ], + }, + ], + }, + }, + }) + t.pass('reconfigure bob') + + await sleep(REPLICATION_TIMEOUT) + t.pass('replication period is over') + + t.equals( + await bob.db.query( + where(and(type('contact'), author(alice.id))), + count(), + toPromise() + ), + 1, + 'bob has 1 contact msgs from alice' + ) + await pify(connectionBA.close)(true) await Promise.all([pify(alice.close)(true), pify(bob.close)(true)]) From 4f9e91f237d4e87e9eacd224019ab0019bb2b02b Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Sun, 19 Sep 2021 20:40:09 +0300 Subject: [PATCH 43/94] refactor some names in RequestManager --- req-manager.js | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/req-manager.js b/req-manager.js index f449b0b..c822f90 100644 --- a/req-manager.js +++ b/req-manager.js @@ -23,8 +23,8 @@ module.exports = class RequestManager { ? opts.debouncePeriod : DEFAULT_PERIOD this._requestables = new Map() - this._requestedDirectly = new Map() - this._requestedIndirectly = new Map() + this._requested = new Map() + this._requestedPartially = new Map() this._tombstoned = new Set() this._flushing = false this._wantsMoreFlushing = false @@ -44,8 +44,8 @@ module.exports = class RequestManager { add(feedId, hops) { if (this._requestables.has(feedId)) return - if (this._requestedDirectly.has(feedId)) return - if (this._requestedIndirectly.has(feedId)) return + if (this._requested.has(feedId)) return + if (this._requestedPartially.has(feedId)) return this._requestables.set(feedId, hops) this._latestAdd = Date.now() @@ -55,13 +55,13 @@ module.exports = class RequestManager { reconfigure(opts) { this._opts = { ...this._opts, ...opts } this._templates = this._setupTemplates(this._opts.partialReplication) - for (const [feedId, hops] of this._requestedDirectly) { + for (const [feedId, hops] of this._requested) { this._requestables.set(feedId, hops) - this._requestedDirectly.delete(feedId) + this._requested.delete(feedId) } - for (const [feedId, hops] of this._requestedIndirectly) { + for (const [feedId, hops] of this._requestedPartially) { this._requestables.set(feedId, hops) - this._requestedIndirectly.delete(feedId) + this._requestedPartially.delete(feedId) } this._flush() } @@ -124,13 +124,13 @@ module.exports = class RequestManager { const path = this._getMetafeedTreePath(msg) const metaFeedId = path[0] const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - if (!this._requestedIndirectly.has(mainFeedId)) return - const hops = this._requestedIndirectly.get(mainFeedId) + if (!this._requestedPartially.has(mainFeedId)) return + const hops = this._requestedPartially.get(mainFeedId) const template = this._findTemplateForHops(hops) if (!template) return const matchedNode = template.matchPath(path, mainFeedId) if (!matchedNode) return - this._requestDirectly(subfeed, matchedNode['$format']) + this._request(subfeed, matchedNode['$format']) }, this._period) } else if (type === 'metafeed/tombstone') { this._tombstoned.add(subfeed) @@ -169,9 +169,9 @@ module.exports = class RequestManager { /** * @param {string} feedId classic feed ref or bendybutt feed URI */ - _requestDirectly(feedId, ebtFormat = undefined) { + _request(feedId, ebtFormat = undefined) { const hops = this._requestables.get(feedId) - this._requestedDirectly.set(feedId, hops) + this._requested.set(feedId, hops) this._requestables.delete(feedId) this._ssb.ebt.request(feedId, true, ebtFormat) } @@ -179,9 +179,9 @@ module.exports = class RequestManager { /** * @param {string} mainFeedId classic feed ref which has announced a root MF */ - _requestIndirectly(mainFeedId) { + _requestPartially(mainFeedId) { const hops = this._requestables.get(mainFeedId) - this._requestedIndirectly.set(mainFeedId, hops) + this._requestedPartially.set(mainFeedId, hops) this._requestables.delete(mainFeedId) // Get metafeedId for this feedId @@ -191,7 +191,7 @@ module.exports = class RequestManager { } else if (!metafeedId) { console.error('cannot partially replicate ' + mainFeedId) } else { - this._requestDirectly(metafeedId) + this._request(metafeedId) } }) } @@ -248,9 +248,9 @@ module.exports = class RequestManager { pull.drain( ([feedId, supportsPartialReplication]) => { if (supportsPartialReplication) { - this._requestIndirectly(feedId) + this._requestPartially(feedId) } else { - this._requestDirectly(feedId) + this._request(feedId) } }, (err) => { From 77a281820e75d464014a63be1f411361c5069d75 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 20 Sep 2021 11:04:59 +0300 Subject: [PATCH 44/94] refactor RequestManager internals a bit --- req-manager.js | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/req-manager.js b/req-manager.js index c822f90..08e124d 100644 --- a/req-manager.js +++ b/req-manager.js @@ -166,9 +166,6 @@ module.exports = class RequestManager { } } - /** - * @param {string} feedId classic feed ref or bendybutt feed URI - */ _request(feedId, ebtFormat = undefined) { const hops = this._requestables.get(feedId) this._requested.set(feedId, hops) @@ -176,20 +173,17 @@ module.exports = class RequestManager { this._ssb.ebt.request(feedId, true, ebtFormat) } - /** - * @param {string} mainFeedId classic feed ref which has announced a root MF - */ - _requestPartially(mainFeedId) { - const hops = this._requestables.get(mainFeedId) - this._requestedPartially.set(mainFeedId, hops) - this._requestables.delete(mainFeedId) + _requestPartially(feedId) { + const hops = this._requestables.get(feedId) + this._requestedPartially.set(feedId, hops) + this._requestables.delete(feedId) - // Get metafeedId for this feedId - this._metafeedFinder.get(mainFeedId, (err, metafeedId) => { + // Get metafeedId for this (main) feedId + this._metafeedFinder.get(feedId, (err, metafeedId) => { if (err) { console.error(err) } else if (!metafeedId) { - console.error('cannot partially replicate ' + mainFeedId) + console.error('cannot partially replicate ' + feedId) } else { this._request(metafeedId) } From 05ace94443b7ca4bc6201f1625ede8b054c6a67d Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 20 Sep 2021 11:05:37 +0300 Subject: [PATCH 45/94] avoid unnecessary database scans in RequestManager --- req-manager.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/req-manager.js b/req-manager.js index 08e124d..9aef1a4 100644 --- a/req-manager.js +++ b/req-manager.js @@ -63,6 +63,8 @@ module.exports = class RequestManager { this._requestables.set(feedId, hops) this._requestedPartially.delete(feedId) } + this._liveDrainer.abort() + this._liveDrainer = null this._flush() } @@ -103,7 +105,7 @@ module.exports = class RequestManager { } _scanBendyButtFeeds() { - if (this._liveDrainer) this._liveDrainer.abort() + if (this._liveDrainer) return if (!this._hasCloseHook) this._setupCloseHook() pull( From 1328eb4dcac49521c1837bd109f233e85a3d1a6a Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 20 Sep 2021 12:24:34 +0300 Subject: [PATCH 46/94] fix README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76a508e..708a50e 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,8 @@ feed_ and of the _root meta feed_. In the example below, we set up partial replication with the meaning: - For hops 0 (that is, "yourself"), replicate some app feeds and all index feeds -- For hops 1 (direct friends), replicate only some index feeds -- For hops 2 and beyond, replicate only about index feed +- For hops 1 (direct friends), replicate only 5 specific index feeds +- For hops 2 and beyond, replicate only 2 specific index feeds ```js partialReplication: { From 488e5265340aba33e6c2dbeb41b5794c45006287 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 20 Sep 2021 12:24:55 +0300 Subject: [PATCH 47/94] improve integration/block3 test --- test/integration/block3.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/block3.js b/test/integration/block3.js index 78cd16b..0ef7c8b 100644 --- a/test/integration/block3.js +++ b/test/integration/block3.js @@ -77,14 +77,16 @@ tape('alice blocks bob and both are connected to carla', async (t) => { t.equal(msgAtBob.value.author, alice.id) t.equal(msgAtBob.value.content.contact, bob.id) + const clockBob = await pify(bob.getVectorClock)() + await pify(alice.publish)(u.block(bob.id)) await sleep(REPLICATION_TIMEOUT) - const clockBob = await pify(bob.getVectorClock)() + const clockBob2 = await pify(bob.getVectorClock)() t.equals( + clockBob2[alice.id], clockBob[alice.id], - 1, 'bob does not receive the message where alice blocked him' ) From 6df9f5ad8722eecd717a4ab48a61490703f63335 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 20 Sep 2021 13:35:44 +0300 Subject: [PATCH 48/94] improve coverage script --- .gitignore | 3 ++- package.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 67348d1..aab0419 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ /node_modules /pnpm-lock -/.nyc_output \ No newline at end of file +/.nyc_output +/coverage \ No newline at end of file diff --git a/package.json b/package.json index b95be88..9e7618f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ }, "scripts": { "test": "tape 'test/@(unit|integration)/*.js' | tap-spec", - "coverage": "nyc npm run test", + "coverage": "nyc --reporter=lcov npm test", "format-code": "prettier --write \"*.js\" \"test/(integration|misc|unit)/*.js\"", "format-code-staged": "pretty-quick --staged --pattern \"*.js\" --pattern \"test/(integration|misc|unit)/*.js\"" }, From 3a48d2f461ef2acae91e470fafec6ffd19f9d003 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 22 Sep 2021 15:41:24 +0300 Subject: [PATCH 49/94] add one test utility --- test/misc/util.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/misc/util.js b/test/misc/util.js index 7ce70d8..af7dba8 100644 --- a/test/misc/util.js +++ b/test/misc/util.js @@ -34,6 +34,14 @@ exports.block = function unfollow(id) { } } +exports.unblock = function (id) { + return { + type: 'contact', + contact: id, + blocking: false, + } +} + exports.readOnceFromDB = function (sbot) { return new Promise((resolve) => { var cancel = sbot.post((msg) => { From 5ec114e77858a083e527200f195cd16ea4e44854 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 22 Sep 2021 15:59:53 +0300 Subject: [PATCH 50/94] RequestManager can handle blocks and unfollows --- index.js | 23 +-- metafeed-finder.js | 6 +- package.json | 10 +- req-manager.js | 265 +++++++++++++++++++++----------- template.js | 46 +++--- test/integration/index-feeds.js | 231 ++++++++++++++++++++++++---- 6 files changed, 420 insertions(+), 161 deletions(-) diff --git a/index.js b/index.js index 57d7e3c..e5aceea 100644 --- a/index.js +++ b/index.js @@ -48,11 +48,10 @@ exports.init = function (ssb, config) { const value = graph[source][dest] // Only if I am the `source` and `value >= 0`, request replication if (source === ssb.id) { - if (value >= 0) requestManager.add(dest, value) - else ssb.ebt.request(dest, false) + requestManager.add(dest, value) } - // Compute every block edge, unless I am the edge destination - if (dest !== ssb.id) { + // Compute every block edge unrelated to me + if (source !== ssb.id && dest !== ssb.id) { ssb.ebt.block(source, dest, value === -1) } } @@ -65,21 +64,7 @@ exports.init = function (ssb, config) { ssb.friends.hopStream({ old: true, live: true }), pull.drain((hops) => { for (const dest of Object.keys(hops)) { - const value = hops[dest] - // myself or friendly peers - if (value >= 0) { - requestManager.add(dest, value) - ssb.ebt.block(ssb.id, dest, false) - } - // blocked peers - else if (value === -1) { - ssb.ebt.request(dest, false) - ssb.ebt.block(ssb.id, dest, true) - } - // unfollowed/unblocked peers - else if (value < -1) { - ssb.ebt.request(dest, false) - } + requestManager.add(dest, hops[dest]) } }) ) diff --git a/metafeed-finder.js b/metafeed-finder.js index 5849289..9e7f3b0 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -26,7 +26,7 @@ module.exports = class MetafeedFinder { } } - get(mainFeedId, cb) { + fetch(mainFeedId, cb) { if (this._map.has(mainFeedId)) { const metaFeedId = this._map.get(mainFeedId) cb(null, metaFeedId) @@ -49,6 +49,10 @@ module.exports = class MetafeedFinder { } } + get(mainFeedId) { + return this._map.get(mainFeedId) + } + getInverse(metaFeedId) { return this._inverseMap.get(metaFeedId) } diff --git a/package.json b/package.json index 9e7618f..95d35e8 100644 --- a/package.json +++ b/package.json @@ -21,10 +21,11 @@ "dependencies": { "debug": "^4.3.2", "pull-stream": "^3.6.0", + "ssb-db2": "^2.5.2", + "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", + "ssb-meta-feeds": "ssb-ngi-pointer/ssb-meta-feeds#tree-api", "ssb-network-errors": "^1.0.0", - "ssb-ref": "^2.13.9", - "ssb-subset-ql": "~0.6.1", - "ssb-uri2": "^1.5.2" + "ssb-subset-ql": "~0.6.1" }, "devDependencies": { "cat-names": "^3.0.0", @@ -41,13 +42,10 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-db2": "^2.5.2", - "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", "ssb-fixtures": "^2.5.3", "ssb-friends": "^5.1.0", "ssb-index-feed-writer": "~0.6.0", "ssb-keys": "^8.2.0", - "ssb-meta-feeds": "~0.22.1", "ssb-meta-feeds-rpc": "~0.2.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" diff --git a/req-manager.js b/req-manager.js index 9aef1a4..5414a2a 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,12 +1,4 @@ const pull = require('pull-stream') -const { - where, - and, - authorIsBendyButtV1, - isPublic, - live, - toPullStream, -} = require('ssb-db2/operators') const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') const indexedEBTFormat = require('ssb-ebt/formats/indexed') const Template = require('./template') @@ -22,10 +14,12 @@ module.exports = class RequestManager { typeof opts.debouncePeriod === 'number' ? opts.debouncePeriod : DEFAULT_PERIOD - this._requestables = new Map() - this._requested = new Map() - this._requestedPartially = new Map() - this._tombstoned = new Set() + this._requestables = new Map() // feedId => hops + this._requested = new Map() // feedId => hops + this._requestedPartially = new Map() // feedId => hops + this._unrequested = new Map() // feedId => hops when it used to be requested + this._blocked = new Map() // feedId => hops before it was blocked + // FIXME: how should we handle tombstoning? this._flushing = false this._wantsMoreFlushing = false this._latestAdd = 0 @@ -43,13 +37,25 @@ module.exports = class RequestManager { } add(feedId, hops) { - if (this._requestables.has(feedId)) return - if (this._requested.has(feedId)) return - if (this._requestedPartially.has(feedId)) return + if (hops === -1) return this._block(feedId) + if (hops < -1) return this._unrequest(feedId) + + const sameHops = hops === this._getCurrentHops(feedId) + if (sameHops && this._requestables.has(feedId)) return + if (sameHops && this._requested.has(feedId)) return + if (sameHops && this._requestedPartially.has(feedId)) return - this._requestables.set(feedId, hops) - this._latestAdd = Date.now() - this._scheduleDebouncedFlush() + if (!sameHops) { + this._requestables.delete(feedId) + this._requested.delete(feedId) + this._requestedPartially.delete(feedId) + this._unrequested.delete(feedId) + this._blocked.delete(feedId) + + this._requestables.set(feedId, hops) + this._latestAdd = Date.now() + this._scheduleDebouncedFlush() + } } reconfigure(opts) { @@ -63,7 +69,7 @@ module.exports = class RequestManager { this._requestables.set(feedId, hops) this._requestedPartially.delete(feedId) } - this._liveDrainer.abort() + if (this._liveDrainer) this._liveDrainer.abort() this._liveDrainer = null this._flush() } @@ -71,7 +77,9 @@ module.exports = class RequestManager { _setupTemplates(optsPartialReplication) { if (!optsPartialReplication) return null if (Object.values(optsPartialReplication).every((t) => !t)) return null - const hopsArr = Object.keys(optsPartialReplication).map(Number) + const hopsArr = Object.keys(optsPartialReplication) + .map(Number) + .filter((x) => x >= 0) const templates = new Map() for (const hops of hopsArr) { if (optsPartialReplication[hops]) { @@ -104,84 +112,78 @@ module.exports = class RequestManager { }) } + _getCurrentHops(feedId) { + return ( + this._requestables.get(feedId) || + this._requested.get(feedId) || + this._requestedPartially.get(feedId) || + null + ) + } + _scanBendyButtFeeds() { if (this._liveDrainer) return if (!this._hasCloseHook) this._setupCloseHook() pull( - this._ssb.db.query( - where(and(authorIsBendyButtV1(), isPublic())), - live({ old: true }), - toPullStream() - ), - pull.filter((msg) => this._ssb.metafeeds.validate.isValid(msg)), - (this._liveDrainer = pull.drain( - (msg) => { - const { type, subfeed } = msg.value.content - if (this._tombstoned.has(subfeed)) return - - if (type.startsWith('metafeed/add/')) { - setTimeout(() => { - if (this._tombstoned.has(subfeed)) return - const path = this._getMetafeedTreePath(msg) - const metaFeedId = path[0] - const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - if (!this._requestedPartially.has(mainFeedId)) return - const hops = this._requestedPartially.get(mainFeedId) - const template = this._findTemplateForHops(hops) - if (!template) return - const matchedNode = template.matchPath(path, mainFeedId) - if (!matchedNode) return - this._request(subfeed, matchedNode['$format']) - }, this._period) - } else if (type === 'metafeed/tombstone') { - this._tombstoned.add(subfeed) - this._ssb.ebt.request(subfeed, false) - } - }, - (err) => { - if (err) return cb(err) - } - )) + this._ssb.metafeeds.branchStream({ old: true, live: true }), + (this._liveDrainer = pull.drain((branch) => { + const metaFeedId = branch[0][0] + const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) + this._handleBranch(branch, mainFeedId) + })) ) + + // FIXME: pull tombstoneBranchStream and do ssb.ebt.request(tombId, false) } - /** - * Returns an array that represents the path from root meta feed to the given - * leaf `msg`. - * - * @param {*} msg a metafeed message - * @returns {Array} - */ - _getMetafeedTreePath(msg) { - const details = this._ssb.metafeeds.findByIdSync(msg.value.content.subfeed) - const path = [details] - while (true) { - const head = path[0] - const details = this._ssb.metafeeds.findByIdSync(head.metafeed) - if (details) { - path.unshift(details) - } else { - path.unshift(head.metafeed) - return path + _matchBranchWith(hops, branch, mainFeedId) { + const template = this._findTemplateForHops(hops) + if (!template) return + return template.matchBranch(branch, mainFeedId) + } + + _handleBranch(branch, mainFeedId) { + const last = branch[branch.length - 1] + const subfeed = last[0] + + if (this._requestedPartially.has(mainFeedId)) { + const hops = this._requestedPartially.get(mainFeedId) + const matchedNode = this._matchBranchWith(hops, branch, mainFeedId) + if (!matchedNode) return + this._request(subfeed, matchedNode['$format']) + return + } + + // Unrequest subfeed if main feed was unrequested + if (this._unrequested.has(mainFeedId)) { + const prevHops = this._unrequested.get(mainFeedId) + if (prevHops === null) { + this._unrequest(subfeed) + return } + const matchedNode = this._matchBranchWith(prevHops, branch, mainFeedId) + if (!matchedNode) return + this._unrequest(subfeed, matchedNode['$format']) + return } - } - _request(feedId, ebtFormat = undefined) { - const hops = this._requestables.get(feedId) - this._requested.set(feedId, hops) - this._requestables.delete(feedId) - this._ssb.ebt.request(feedId, true, ebtFormat) + // Block subfeed if main feed was blocked + if (this._blocked.has(mainFeedId)) { + const prevHops = this._blocked.get(mainFeedId) + if (prevHops === null) { + this._block(subfeed) + return + } + const matchedNode = this._matchBranchWith(prevHops, branch, mainFeedId) + if (!matchedNode) return + this._block(subfeed, matchedNode['$format']) + return + } } - _requestPartially(feedId) { - const hops = this._requestables.get(feedId) - this._requestedPartially.set(feedId, hops) - this._requestables.delete(feedId) - - // Get metafeedId for this (main) feedId - this._metafeedFinder.get(feedId, (err, metafeedId) => { + _fetchAndRequestMetafeed(feedId) { + this._metafeedFinder.fetch(feedId, (err, metafeedId) => { if (err) { console.error(err) } else if (!metafeedId) { @@ -192,8 +194,99 @@ module.exports = class RequestManager { }) } + _requestPartially(feedId) { + if (this._requestedPartially.has(feedId)) return + if (!this._requestables.has(feedId)) return + + const hops = this._requestables.get(feedId) + this._requestedPartially.set(feedId, hops) + this._requestables.delete(feedId) + + // We may already have the meta feed, so continue replicating + const root = this._metafeedFinder.get(feedId) + if (root) { + let branchesFound = false + pull( + this._ssb.metafeeds.branchStream({ root, old: true, live: false }), + pull.drain( + (branch) => { + branchesFound = true + this._handleBranch(branch, feedId) + }, + () => { + if (branchesFound === false) { + this._fetchAndRequestMetafeed(feedId) + } + } + ) + ) + } + // Fetch metafeedId for this (main) feedId for the first time + else { + this._fetchAndRequestMetafeed(feedId) + } + } + + _request(feedId, ebtFormat = undefined) { + if (this._requested.has(feedId)) return + + if (this._requestables.has(feedId)) { + const hops = this._requestables.get(feedId) + this._requested.set(feedId, hops) + this._requestables.delete(feedId) + } else { + this._requested.set(feedId, null) + } + this._ssb.ebt.block(this._ssb.id, feedId, false, ebtFormat) + this._ssb.ebt.request(feedId, true, ebtFormat) + } + + _unrequest(feedId, ebtFormat = undefined) { + if (this._unrequested.has(feedId)) return + const hops = this._getCurrentHops(feedId) + this._requestables.delete(feedId) + this._requested.delete(feedId) + this._requestedPartially.delete(feedId) + this._unrequested.set(feedId, hops) + this._ssb.ebt.request(feedId, false, ebtFormat) + this._ssb.ebt.block(this._ssb.id, feedId, false, ebtFormat) + + // Weave through all of the subfeeds and unrequest them too + const root = this._metafeedFinder.get(feedId) + if (root) { + pull( + this._ssb.metafeeds.branchStream({ root, old: true, live: false }), + pull.drain((branch) => { + this._handleBranch(branch, feedId) + }) + ) + } + } + + _block(feedId, ebtFormat = undefined) { + if (this._blocked.has(feedId)) return + const hops = this._getCurrentHops(feedId) + this._requestables.delete(feedId) + this._requested.delete(feedId) + this._requestedPartially.delete(feedId) + this._blocked.set(feedId, hops) + this._ssb.ebt.request(feedId, false, ebtFormat) + this._ssb.ebt.block(this._ssb.id, feedId, true, ebtFormat) + + // Weave through all of the subfeeds and block them too + const root = this._metafeedFinder.get(feedId) + if (root) { + pull( + this._ssb.metafeeds.branchStream({ root, old: true, live: false }), + pull.drain((branch) => { + this._handleBranch(branch, feedId) + }) + ) + } + } + _supportsPartialReplication(feedId, cb) { - this._metafeedFinder.get(feedId, (err, metafeedId) => { + this._metafeedFinder.fetch(feedId, (err, metafeedId) => { if (err) cb(err) else cb(null, !!metafeedId) }) diff --git a/template.js b/template.js index 5744f8f..e34d019 100644 --- a/template.js +++ b/template.js @@ -1,4 +1,3 @@ -const { isBendyButtV1FeedSSBURI } = require('ssb-uri2') const { QL0 } = require('ssb-subset-ql') /** @@ -32,16 +31,17 @@ module.exports = class Template { } } - matchPath(path, mainFeedId) { - return this._matchPath(path, this._rootNode, mainFeedId) + matchBranch(branch, mainFeedId) { + return this._matchBranch(branch, this._rootNode, mainFeedId) } - _matchPath(path, node, mainFeedId) { - if (path.length === 0) return null - const head = path[0] + _matchBranch(branch, node, mainFeedId) { + if (branch.length === 0) return null + const head = branch[0] + const [feedId, details] = head - // Head is `metafeedId` - if (typeof head === 'string' && isBendyButtV1FeedSSBURI(head)) { + // Head is a root meta feed: + if (!details) { const keys = Object.keys(node) const rootMatches = keys.length === 1 && @@ -50,35 +50,37 @@ module.exports = class Template { if (!rootMatches) { return null - } else { - const childPath = path.slice(1) + } else if (branch.length >= 2) { + const childBranch = branch.slice(1) for (const childNode of node.subfeeds) { - const matched = this._matchPath(childPath, childNode, mainFeedId) + const matched = this._matchBranch(childBranch, childNode, mainFeedId) if (matched) return matched } return null + } else { + return node } } - // Head is a subfeed details - if (typeof head === 'object') { + // Head is a subfeed: + if (details) { // If present, feedpurpose must match - if (node.feedpurpose && head.feedpurpose !== node.feedpurpose) { + if (node.feedpurpose && details.feedpurpose !== node.feedpurpose) { return null } // If present, metadata must match - if (node.metadata && head.metadata) { + if (node.metadata && details.metadata) { // If querylang is present, match ssb-ql-0 queries - if (node.metadata.querylang !== head.metadata.querylang) { + if (node.metadata.querylang !== details.metadata.querylang) { return null } if (node.metadata.querylang === 'ssb-ql-0') { - if (!QL0.parse(head.metadata.query)) return null + if (!QL0.parse(details.metadata.query)) return null if (node.metadata.query) { const nodeQuery = { ...node.metadata.query } if (nodeQuery.author === '$main') nodeQuery.author = mainFeedId - if (!QL0.isEquals(head.metadata.query, nodeQuery)) { + if (!QL0.isEquals(details.metadata.query, nodeQuery)) { return null } } @@ -90,16 +92,16 @@ module.exports = class Template { if (field === 'query') continue if (field === 'querylang') continue - if (head.metadata[field] !== node.metadata[field]) { + if (details.metadata[field] !== node.metadata[field]) { return null } } } - if (Array.isArray(node.subfeeds) && path.length >= 2) { - const childPath = path.slice(1) + if (Array.isArray(node.subfeeds) && branch.length >= 2) { + const childBranch = branch.slice(1) for (const childNode of node.subfeeds) { - const matched = this._matchPath(childPath, childNode, mainFeedId) + const matched = this._matchBranch(childBranch, childNode, mainFeedId) if (matched) return matched } return null diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 6ca0bf8..b3b77c1 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -10,11 +10,12 @@ const { and, type, author, + authorIsBendyButtV1, count, toPromise, } = require('ssb-db2/operators') const sleep = require('util').promisify(setTimeout) -const { keysFor } = require('../misc/util') +const u = require('../misc/util') const createSsbServer = SecretStack({ caps }) .use(require('ssb-db2')) @@ -26,21 +27,31 @@ const createSsbServer = SecretStack({ caps }) .use(require('ssb-index-feed-writer')) .use(require('../..')) -const CONNECTION_TIMEOUT = 500 // ms -const REPLICATION_TIMEOUT = 6000 // ms -const INDEX_WRITING_TIMEOUT = 3000 // ms +const CONNECTION_TIMEOUT = 1e3 +const INACTIVITY_TIMEOUT = 60e3 +const REPLICATION_TIMEOUT = 8e3 +const INDEX_WRITING_TIMEOUT = 3e3 -const aliceKeys = keysFor('alice') -const bobKeys = keysFor('bob') +const aliceKeys = u.keysFor('alice') +const bobKeys = u.keysFor('bob') +const carolKeys = u.keysFor('carol') +let alice +let bob +let carol +let connectionBA +let connectionCA +let connectionCB tape('setup', async (t) => { rimraf.sync(path.join(os.tmpdir(), 'server-alice')) rimraf.sync(path.join(os.tmpdir(), 'server-bob')) + rimraf.sync(path.join(os.tmpdir(), 'server-carol')) - const alice = createSsbServer({ + alice = createSsbServer({ path: path.join(os.tmpdir(), 'server-alice'), keys: aliceKeys, timeout: CONNECTION_TIMEOUT, + timers: { inactivity: INACTIVITY_TIMEOUT }, indexFeedWriter: { autostart: [ { type: 'post', private: false }, @@ -49,9 +60,10 @@ tape('setup', async (t) => { }, }) - const bob = createSsbServer({ + bob = createSsbServer({ path: path.join(os.tmpdir(), 'server-bob'), keys: bobKeys, + timers: { inactivity: INACTIVITY_TIMEOUT }, timeout: CONNECTION_TIMEOUT, }) @@ -62,15 +74,14 @@ tape('setup', async (t) => { // Wait for all bots to be ready await sleep(500) - const following = true await Promise.all([ // All peers publish a post pify(alice.db.publish)({ type: 'post', text: 'My name is Alice' }), pify(bob.db.publish)({ type: 'post', text: 'My name is Bob' }), // alice and bob follow each other - pify(alice.db.publish)({ type: 'contact', contact: bob.id, following }), - pify(bob.db.publish)({ type: 'contact', contact: alice.id, following }), + pify(alice.db.publish)(u.follow(bob.id)), + pify(bob.db.publish)(u.follow(alice.id)), ]) t.pass('published all the messages') @@ -83,10 +94,11 @@ tape('setup', async (t) => { }) tape('alice writes index feeds and bob replicates them', async (t) => { - const alice = createSsbServer({ + alice = createSsbServer({ path: path.join(os.tmpdir(), 'server-alice'), keys: aliceKeys, timeout: CONNECTION_TIMEOUT, + timers: { inactivity: INACTIVITY_TIMEOUT }, indexFeedWriter: { autostart: [ { type: 'post', private: false }, @@ -114,10 +126,11 @@ tape('alice writes index feeds and bob replicates them', async (t) => { }, }) - const bob = createSsbServer({ + bob = createSsbServer({ path: path.join(os.tmpdir(), 'server-bob'), keys: bobKeys, timeout: CONNECTION_TIMEOUT, + timers: { inactivity: INACTIVITY_TIMEOUT }, replicationScheduler: { partialReplication: { 0: null, @@ -141,7 +154,13 @@ tape('alice writes index feeds and bob replicates them', async (t) => { }, }) - const connectionBA = await pify(bob.connect)(alice.getAddress()) + t.equals( + await alice.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 4, // add main + add indexes + add post index + add contact index + 'alice has 4 bendybutt msgs' + ) + + connectionBA = await pify(bob.connect)(alice.getAddress()) t.pass('peers are connected to each other') await sleep(REPLICATION_TIMEOUT) @@ -199,6 +218,75 @@ tape('alice writes index feeds and bob replicates them', async (t) => { 'bob has 1 metafeed/announce from alice' ) + t.equals( + await bob.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 4, // add main + add indexes + add post index + add contact index + 'bob replicated 4 bendybutt msgs' + ) + + await pify(connectionBA.close)(true) + + t.end() +}) + +tape('carol acts as an intermediate between alice and bob', async (t) => { + carol = createSsbServer({ + path: path.join(os.tmpdir(), 'server-carol'), + keys: carolKeys, + timeout: CONNECTION_TIMEOUT, + timers: { inactivity: INACTIVITY_TIMEOUT }, + friends: { hops: 2 }, + replicationScheduler: { + partialReplication: { + 0: null, + 1: { + subfeeds: [ + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + $format: 'indexed', + }, + ], + }, + ], + }, + }, + }, + }) + t.pass('carol initialized') + + // This needs to happen before publishing follows, otherwise carol + // defaults to normal replication (which means she won't replicate meta feeds) + connectionCA = await pify(carol.connect)(alice.getAddress()) + t.pass('carol is connected to alice') + + await pify(carol.db.publish)(u.follow(alice.id)) + t.pass('carol follows alice') + + await sleep(REPLICATION_TIMEOUT) + t.pass('replication period is over') + + t.equals( + await carol.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 4, // add main + add indexes + add post index + add contact index + 'carol replicated 4 bendybutt msgs' + ) + + t.equals( + await carol.db.query(where(author(alice.id)), count(), toPromise()), + 3, // post + contact + metafeed/announce + 'carol replicated all of alices msgs' + ) + + connectionCB = await pify(carol.connect)(bob.getAddress()) + t.pass('carol is connected to bob') + + t.end() +}) + +tape('bob reconfigures to replicate everything from alice', async (t) => { bob.replicationScheduler.reconfigure({ partialReplication: { 0: null, @@ -208,17 +296,7 @@ tape('alice writes index feeds and bob replicates them', async (t) => { feedpurpose: 'indexes', subfeeds: [ { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'post', private: false }, - }, - $format: 'indexed', - }, - { - metadata: { - querylang: 'ssb-ql-0', - query: { author: '$main', type: 'contact', private: false }, - }, + feedpurpose: 'index', $format: 'indexed', }, ], @@ -242,9 +320,108 @@ tape('alice writes index feeds and bob replicates them', async (t) => { 'bob has 1 contact msgs from alice' ) - await pify(connectionBA.close)(true) + t.end() +}) - await Promise.all([pify(alice.close)(true), pify(bob.close)(true)]) +tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { + // FIXME: this disconnection and subsequent reconnection is only required + // because index writing is buggy because we still don't have transactions + // in ssb-db2. Once ssb-db2 is patched (and ssb-index-feed-writer), we can + // remove this disconnection+reconnection. + await pify(connectionCA.close)(true) + await pify(connectionCB.close)(true) + t.pass('close carols connections') + + await pify(bob.db.publish)(u.block(alice.id)) + t.pass('bob blocked alice') + + await pify(alice.db.publish)({ type: 'post', text: 'Whatever' }) + t.pass('alice published a new post') + + const aliceRootMF = await pify(alice.metafeeds.find)() + await pify(alice.metafeeds.create)(aliceRootMF, { + feedpurpose: 'mygame', + feedformat: 'classic', + metadata: { + score: 0, + whateverElse: true, + }, + }) + t.pass('alice created a game subfeed') + + connectionCA = await pify(carol.connect)(alice.getAddress()) + connectionCB = await pify(carol.connect)(bob.getAddress()) + t.pass('reset carols connections') + + await sleep(REPLICATION_TIMEOUT) + t.pass('replication period is over') + + t.equals( + await alice.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 5, // add main + add indexes + add post index + add contact index + add game + 'alice has 5 bendybutt msgs' + ) + + t.equals( + await bob.db.query( + where(and(type('post'), author(alice.id))), + count(), + toPromise() + ), + 1, + 'bob has 1 post from alice' + ) + + t.equals( + await bob.db.query( + where(and(type('contact'), author(alice.id))), + count(), + toPromise() + ), + 1, + 'bob has 1 contact msg from alice' + ) + + t.equals( + await bob.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 4, // add main + add indexes + add post index + add contact index + 'bob replicated 4 bendybutt msgs' + ) + + t.end() +}) + +tape('once bob unblocks alice, he replicates her subfeeds', async (t) => { + await pify(bob.db.publish)(u.follow(alice.id)) + + await sleep(REPLICATION_TIMEOUT * 2) + t.pass('replication period is over') + + t.equals( + await bob.db.query( + where(and(type('post'), author(alice.id))), + count(), + toPromise() + ), + 2, + 'bob has 2 posts from alice' + ) + + t.equals( + await bob.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 5, // add main + add indexes + add post index + add contact index + add game + 'bob replicated 5 bendybutt msgs' + ) + + t.end() +}) + +tape('teardown', async (t) => { + await Promise.all([ + pify(alice.close)(true), + pify(bob.close)(true), + pify(carol.close)(true), + ]) t.end() }) From 2c818cc14285f51fafa894d1eb3fbae48954df87 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 11:31:41 +0300 Subject: [PATCH 51/94] remove a hack from tests now that ssb-ebt was fixed --- test/integration/index-feeds.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index b3b77c1..103f20a 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -324,14 +324,6 @@ tape('bob reconfigures to replicate everything from alice', async (t) => { }) tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { - // FIXME: this disconnection and subsequent reconnection is only required - // because index writing is buggy because we still don't have transactions - // in ssb-db2. Once ssb-db2 is patched (and ssb-index-feed-writer), we can - // remove this disconnection+reconnection. - await pify(connectionCA.close)(true) - await pify(connectionCB.close)(true) - t.pass('close carols connections') - await pify(bob.db.publish)(u.block(alice.id)) t.pass('bob blocked alice') @@ -349,10 +341,6 @@ tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { }) t.pass('alice created a game subfeed') - connectionCA = await pify(carol.connect)(alice.getAddress()) - connectionCB = await pify(carol.connect)(bob.getAddress()) - t.pass('reset carols connections') - await sleep(REPLICATION_TIMEOUT) t.pass('replication period is over') From 1e3fbdcef5d73a87a5e2d17e4324f1dfdf8cbbda Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 13:25:20 +0300 Subject: [PATCH 52/94] rename ssb-meta-feeds-rpc to ssb-subset-rpc --- package.json | 2 +- test/integration/index-feeds.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 95d35e8..4fa03b7 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "ssb-friends": "^5.1.0", "ssb-index-feed-writer": "~0.6.0", "ssb-keys": "^8.2.0", - "ssb-meta-feeds-rpc": "~0.2.0", + "ssb-subset-rpc": "~0.3.0", "tap-spec": "^5.0.0", "tape": "^5.2.2" }, diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 103f20a..32064fb 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -23,7 +23,7 @@ const createSsbServer = SecretStack({ caps }) .use(require('ssb-ebt')) .use(require('ssb-friends')) .use(require('ssb-meta-feeds')) - .use(require('ssb-meta-feeds-rpc')) + .use(require('ssb-subset-rpc')) .use(require('ssb-index-feed-writer')) .use(require('../..')) From e986c85a5622c541643ba3ed1f26e666b55a2352 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 13:30:00 +0300 Subject: [PATCH 53/94] refactor the test a bit --- test/integration/index-feeds.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/integration/index-feeds.js b/test/integration/index-feeds.js index 32064fb..80eede6 100644 --- a/test/integration/index-feeds.js +++ b/test/integration/index-feeds.js @@ -38,9 +38,6 @@ const carolKeys = u.keysFor('carol') let alice let bob let carol -let connectionBA -let connectionCA -let connectionCB tape('setup', async (t) => { rimraf.sync(path.join(os.tmpdir(), 'server-alice')) @@ -160,7 +157,7 @@ tape('alice writes index feeds and bob replicates them', async (t) => { 'alice has 4 bendybutt msgs' ) - connectionBA = await pify(bob.connect)(alice.getAddress()) + const connectionBA = await pify(bob.connect)(alice.getAddress()) t.pass('peers are connected to each other') await sleep(REPLICATION_TIMEOUT) @@ -259,7 +256,7 @@ tape('carol acts as an intermediate between alice and bob', async (t) => { // This needs to happen before publishing follows, otherwise carol // defaults to normal replication (which means she won't replicate meta feeds) - connectionCA = await pify(carol.connect)(alice.getAddress()) + await pify(carol.connect)(alice.getAddress()) t.pass('carol is connected to alice') await pify(carol.db.publish)(u.follow(alice.id)) @@ -280,7 +277,7 @@ tape('carol acts as an intermediate between alice and bob', async (t) => { 'carol replicated all of alices msgs' ) - connectionCB = await pify(carol.connect)(bob.getAddress()) + await pify(carol.connect)(bob.getAddress()) t.pass('carol is connected to bob') t.end() From 0ae3fe9fd225aec5c1bb750df7f0600fdbb2ddaf Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 14:05:42 +0300 Subject: [PATCH 54/94] rename the test file --- test/integration/{index-feeds.js => partial.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/integration/{index-feeds.js => partial.js} (100%) diff --git a/test/integration/index-feeds.js b/test/integration/partial.js similarity index 100% rename from test/integration/index-feeds.js rename to test/integration/partial.js From 59ab6e3e9a5aa7e4fb3301f227035bcdc0687d4a Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 15:04:48 +0300 Subject: [PATCH 55/94] update ssb-meta-feeds to 0.23.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4fa03b7..d3d4d13 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "pull-stream": "^3.6.0", "ssb-db2": "^2.5.2", "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", - "ssb-meta-feeds": "ssb-ngi-pointer/ssb-meta-feeds#tree-api", + "ssb-meta-feeds": "~0.23.0", "ssb-network-errors": "^1.0.0", "ssb-subset-ql": "~0.6.1" }, From cabf98fdd93bf9e0277dc19e4e0612ea0db87c13 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 17:03:29 +0300 Subject: [PATCH 56/94] add some code comments --- metafeed-finder.js | 6 +++--- req-manager.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 9e7f3b0..f075bca 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -11,9 +11,9 @@ module.exports = class MetafeedFinder { this._ssb = ssb this._opts = opts this._period = period || DEFAULT_PERIOD - this._map = new Map() - this._inverseMap = new Map() - this._requestsByMainfeedId = new Map() + this._map = new Map() // mainFeedId => rootMetaFeedId + this._inverseMap = new Map() // rootMetaFeedId => mainFeedId + this._requestsByMainfeedId = new Map() // mainFeedId => Array this._latestRequestTime = 0 this._timer = null diff --git a/req-manager.js b/req-manager.js index 5414a2a..4151c61 100644 --- a/req-manager.js +++ b/req-manager.js @@ -3,7 +3,7 @@ const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') const indexedEBTFormat = require('ssb-ebt/formats/indexed') const Template = require('./template') -const DEFAULT_PERIOD = 150 +const DEFAULT_PERIOD = 150 // ms module.exports = class RequestManager { constructor(ssb, opts, metafeedFinder) { From 47d8be93c8887e41299e1cb8a291bc9c0dbf2a53 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 17:09:10 +0300 Subject: [PATCH 57/94] speed up test/intergration/partial --- test/integration/partial.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/partial.js b/test/integration/partial.js index 80eede6..dae2cd9 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -29,7 +29,7 @@ const createSsbServer = SecretStack({ caps }) const CONNECTION_TIMEOUT = 1e3 const INACTIVITY_TIMEOUT = 60e3 -const REPLICATION_TIMEOUT = 8e3 +const REPLICATION_TIMEOUT = 2e3 const INDEX_WRITING_TIMEOUT = 3e3 const aliceKeys = u.keysFor('alice') @@ -103,6 +103,7 @@ tape('alice writes index feeds and bob replicates them', async (t) => { ], }, replicationScheduler: { + debouncePeriod: 1, partialReplication: { 0: { subfeeds: [ @@ -129,6 +130,7 @@ tape('alice writes index feeds and bob replicates them', async (t) => { timeout: CONNECTION_TIMEOUT, timers: { inactivity: INACTIVITY_TIMEOUT }, replicationScheduler: { + debouncePeriod: 1, partialReplication: { 0: null, 1: { @@ -234,6 +236,7 @@ tape('carol acts as an intermediate between alice and bob', async (t) => { timers: { inactivity: INACTIVITY_TIMEOUT }, friends: { hops: 2 }, replicationScheduler: { + debouncePeriod: 1, partialReplication: { 0: null, 1: { From e528b5bc96759d13ee17f706e0b8f4b59c6e2d24 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 17:09:29 +0300 Subject: [PATCH 58/94] reconfigure() supports opts.debouncePeriod --- req-manager.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/req-manager.js b/req-manager.js index 4151c61..9b704ed 100644 --- a/req-manager.js +++ b/req-manager.js @@ -60,6 +60,10 @@ module.exports = class RequestManager { reconfigure(opts) { this._opts = { ...this._opts, ...opts } + this._period = + typeof opts.debouncePeriod === 'number' + ? opts.debouncePeriod + : this._period this._templates = this._setupTemplates(this._opts.partialReplication) for (const [feedId, hops] of this._requested) { this._requestables.set(feedId, hops) From e9f1c9fa860475444c069faa230d7e23071571b9 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 17:54:43 +0300 Subject: [PATCH 59/94] use graphStream only for computing third-party blocks --- index.js | 11 +---------- test/unit/blocks.js | 24 +++++++++--------------- test/unit/follows.js | 20 ++++++++------------ 3 files changed, 18 insertions(+), 37 deletions(-) diff --git a/index.js b/index.js index e5aceea..5441f49 100644 --- a/index.js +++ b/index.js @@ -31,11 +31,6 @@ exports.init = function (ssb, config) { const metafeedFinder = new MetafeedFinder(ssb, opts) const requestManager = new RequestManager(ssb, opts, metafeedFinder) - // Note: ssb.ebt.request and ssb.ebt.block are idempotent operations, - // so it's safe to call these methods redundantly, which is most likely - // true in most cases. These three blocks below may sometimes overlap, but - // that's okay, as long as we cover *all* cases. - // Replicate myself ASAP, without request manager ssb.ebt.request(ssb.id, true) @@ -45,13 +40,9 @@ exports.init = function (ssb, config) { pull.drain((graph) => { for (const source of Object.keys(graph)) { for (const dest of Object.keys(graph[source])) { - const value = graph[source][dest] - // Only if I am the `source` and `value >= 0`, request replication - if (source === ssb.id) { - requestManager.add(dest, value) - } // Compute every block edge unrelated to me if (source !== ssb.id && dest !== ssb.id) { + const value = graph[source][dest] ssb.ebt.block(source, dest, value === -1) } } diff --git a/test/unit/blocks.js b/test/unit/blocks.js index 93b4b77..c20d396 100644 --- a/test/unit/blocks.js +++ b/test/unit/blocks.js @@ -17,17 +17,15 @@ tape('listen to friends stream and ebt.blocks initial blocked peers', (t) => { init(sbot) { return { graphStream() { + return pull.empty() + }, + hopStream() { return pull.values([ { - [sbot.id]: { - [bobId]: -1, - }, + [bobId]: -1, }, ]) }, - hopStream() { - return pull.empty() - }, } }, }) @@ -79,22 +77,18 @@ tape('listen to friends stream ebt.blocks subsequent blocks', (t) => { init(sbot) { return { graphStream() { + return pull.empty() + }, + hopStream() { return pull.values([ { - [sbot.id]: { - [bobId]: -1, - }, + [bobId]: -1, }, { - [sbot.id]: { - [bobId]: -2, - }, + [bobId]: -2, }, ]) }, - hopStream() { - return pull.empty() - }, } }, }) diff --git a/test/unit/follows.js b/test/unit/follows.js index 0b37f1e..7a6a968 100644 --- a/test/unit/follows.js +++ b/test/unit/follows.js @@ -16,17 +16,15 @@ tape('listen to friends stream and replicates initial follows', (t) => { init(sbot) { return { graphStream() { + return pull.empty() + }, + hopStream() { return pull.values([ { - [sbot.id]: { - [bobId]: 1, - }, + [bobId]: 1, }, ]) }, - hopStream() { - return pull.empty() - }, } }, }) @@ -70,17 +68,15 @@ tape('listen to friends stream and replicates subsequent follows', (t) => { init(sbot) { return { graphStream() { + return pull.empty() + }, + hopStream() { return pull.values([ { - [sbot.id]: { - [bobId]: 1, - }, + [bobId]: 1, }, ]) }, - hopStream() { - return pull.empty() - }, } }, }) From 04661a907aed22fa208044857550019d0877bc0c Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 17:55:03 +0300 Subject: [PATCH 60/94] hack block3 test for now --- test/integration/block3.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/test/integration/block3.js b/test/integration/block3.js index 0ef7c8b..b288f9d 100644 --- a/test/integration/block3.js +++ b/test/integration/block3.js @@ -4,7 +4,6 @@ const tape = require('tape') const crypto = require('crypto') -const ssbKeys = require('ssb-keys') const pify = require('promisify-4loc') const sleep = require('util').promisify(setTimeout) const SecretStack = require('secret-stack') @@ -32,7 +31,7 @@ const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT const alice = createSsbServer({ temp: 'test-block3-alice', timeout: CONNECTION_TIMEOUT, - keys: ssbKeys.generate(), + keys: u.keysFor('alice'), replicationScheduler: { debouncePeriod: 0, }, @@ -41,7 +40,8 @@ const alice = createSsbServer({ const bob = createSsbServer({ temp: 'test-block3-bob', timeout: CONNECTION_TIMEOUT, - keys: ssbKeys.generate(), + keys: u.keysFor('bob'), + ebt: {logging: true}, replicationScheduler: { debouncePeriod: 0, }, @@ -50,7 +50,7 @@ const bob = createSsbServer({ const carol = createSsbServer({ temp: 'test-block3-carol', timeout: CONNECTION_TIMEOUT, - keys: ssbKeys.generate(), + keys: u.keysFor('carol'), replicationScheduler: { debouncePeriod: 0, }, @@ -66,7 +66,7 @@ tape('alice blocks bob and both are connected to carla', async (t) => { pify(carol.publish)(u.follow(alice.id)), ]) - await Promise.all([ + const [bc, ca] = await Promise.all([ pify(bob.connect)(carol.getAddress()), pify(carol.connect)(alice.getAddress()), ]) @@ -79,8 +79,15 @@ tape('alice blocks bob and both are connected to carla', async (t) => { const clockBob = await pify(bob.getVectorClock)() + // FIXME: this disconnection followed by a re-connection is a hack + // to bypass a race condition in ssb-ebt where it doesn't wait for + // ssb-friends to compute changes to the social graph, + await pify(bc.close)(true) + await pify(alice.publish)(u.block(bob.id)) + await pify(bob.connect)(carol.getAddress()) + await sleep(REPLICATION_TIMEOUT) const clockBob2 = await pify(bob.getVectorClock)() From c27f01671d2b0d493198fd234c406dd61a41f966 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 23 Sep 2021 18:17:09 +0300 Subject: [PATCH 61/94] remove some logging --- test/integration/block3.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/block3.js b/test/integration/block3.js index b288f9d..5eb31ce 100644 --- a/test/integration/block3.js +++ b/test/integration/block3.js @@ -41,7 +41,6 @@ const bob = createSsbServer({ temp: 'test-block3-bob', timeout: CONNECTION_TIMEOUT, keys: u.keysFor('bob'), - ebt: {logging: true}, replicationScheduler: { debouncePeriod: 0, }, From 0fe59646f6c4e861218b4fa8a02c59a3b3728e86 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 24 Sep 2021 14:07:21 +0300 Subject: [PATCH 62/94] improve test/integration/partial.js --- test/integration/partial.js | 59 +++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/test/integration/partial.js b/test/integration/partial.js index dae2cd9..bea29c8 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -108,6 +108,7 @@ tape('alice writes index feeds and bob replicates them', async (t) => { 0: { subfeeds: [ { feedpurpose: 'main' }, + { feedpurpose: 'mygame' }, { feedpurpose: 'indexes', subfeeds: [ @@ -241,6 +242,7 @@ tape('carol acts as an intermediate between alice and bob', async (t) => { 0: null, 1: { subfeeds: [ + { feedpurpose: 'mygame' }, { feedpurpose: 'indexes', subfeeds: [ @@ -286,7 +288,7 @@ tape('carol acts as an intermediate between alice and bob', async (t) => { t.end() }) -tape('bob reconfigures to replicate everything from alice', async (t) => { +tape('bob reconfigures to replicate all indexes from alice', async (t) => { bob.replicationScheduler.reconfigure({ partialReplication: { 0: null, @@ -323,6 +325,8 @@ tape('bob reconfigures to replicate everything from alice', async (t) => { t.end() }) +let gameFeed + tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { await pify(bob.db.publish)(u.block(alice.id)) t.pass('bob blocked alice') @@ -331,7 +335,7 @@ tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { t.pass('alice published a new post') const aliceRootMF = await pify(alice.metafeeds.find)() - await pify(alice.metafeeds.create)(aliceRootMF, { + gameFeed = await pify(alice.metafeeds.create)(aliceRootMF, { feedpurpose: 'mygame', feedformat: 'classic', metadata: { @@ -341,6 +345,12 @@ tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { }) t.pass('alice created a game subfeed') + await pify(alice.db.publishAs)(gameFeed.keys, { + type: 'game', + move: { x: 1, y: 0 }, + }) + t.pass('alice published something on the game subfeed') + await sleep(REPLICATION_TIMEOUT) t.pass('replication period is over') @@ -376,6 +386,12 @@ tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { 'bob replicated 4 bendybutt msgs' ) + t.equals( + await bob.db.query(where(type('game')), count(), toPromise()), + 0, + 'bob has not replicated the game subfeed' + ) + t.end() }) @@ -404,6 +420,45 @@ tape('once bob unblocks alice, he replicates her subfeeds', async (t) => { t.end() }) +tape('bob reconfigures to replicate a game feed from alice', async (t) => { + bob.replicationScheduler.reconfigure({ + partialReplication: { + 0: null, + 1: { + subfeeds: [ + { + feedpurpose: 'mygame', + }, + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + $format: 'indexed', + }, + ], + }, + ], + }, + }, + }) + t.pass('reconfigure bob') + + await sleep(REPLICATION_TIMEOUT) + t.pass('replication period is over') + + t.equals( + await bob.db.query(where(type('game')), count(), toPromise()), + 1, + 'bob has replicated the game subfeed' + ) + + const bobClock = await pify(bob.getVectorClock)() + t.equals(bobClock[gameFeed.keys.id], 1, 'bob\'s clock has the game feed') + + t.end() +}) + tape('teardown', async (t) => { await Promise.all([ pify(alice.close)(true), From 1ea63f07f1be8ce5a5d0a00b49cb9d255f8c90d5 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 24 Sep 2021 14:07:47 +0300 Subject: [PATCH 63/94] improve test/integration/block3.js --- test/integration/block3.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/integration/block3.js b/test/integration/block3.js index 5eb31ce..b820a87 100644 --- a/test/integration/block3.js +++ b/test/integration/block3.js @@ -65,7 +65,7 @@ tape('alice blocks bob and both are connected to carla', async (t) => { pify(carol.publish)(u.follow(alice.id)), ]) - const [bc, ca] = await Promise.all([ + await Promise.all([ pify(bob.connect)(carol.getAddress()), pify(carol.connect)(alice.getAddress()), ]) @@ -78,15 +78,8 @@ tape('alice blocks bob and both are connected to carla', async (t) => { const clockBob = await pify(bob.getVectorClock)() - // FIXME: this disconnection followed by a re-connection is a hack - // to bypass a race condition in ssb-ebt where it doesn't wait for - // ssb-friends to compute changes to the social graph, - await pify(bc.close)(true) - await pify(alice.publish)(u.block(bob.id)) - await pify(bob.connect)(carol.getAddress()) - await sleep(REPLICATION_TIMEOUT) const clockBob2 = await pify(bob.getVectorClock)() From 2f6173b3741fb6a7227ab3e5a2cc7777a2155c74 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 27 Sep 2021 11:32:32 +0300 Subject: [PATCH 64/94] switch from full to partial if found metafeed/announce --- metafeed-finder.js | 30 ++++++++--- req-manager.js | 61 ++++++++++++++++------ test/integration/partial.js | 100 +++++++++++++++++++++++++++++++++--- 3 files changed, 159 insertions(+), 32 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index f075bca..02342e9 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -1,7 +1,7 @@ const pull = require('pull-stream') const debug = require('debug')('ssb:replication-scheduler') const detectSsbNetworkErrorSeverity = require('ssb-network-errors') -const { where, type, toPullStream } = require('ssb-db2/operators') +const { where, type, live, toPullStream } = require('ssb-db2/operators') const { validateMetafeedAnnounce } = require('ssb-meta-feeds/validate') const DEFAULT_PERIOD = 500 @@ -22,6 +22,11 @@ module.exports = class MetafeedFinder { this._opts.partialReplication && Object.values(this._opts.partialReplication).some((templ) => !!templ) ) { + if (!this._ssb.db || !this._ssb.db.query) { + throw new Error( + 'ssb-replication-scheduler expects ssb-db2 to be installed, to use partial replication' + ) + } this._loadAllFromLog() } } @@ -57,13 +62,24 @@ module.exports = class MetafeedFinder { return this._inverseMap.get(metaFeedId) } - _loadAllFromLog() { - if (!this._ssb.db || !this._ssb.db.query) { - throw new Error( - 'ssb-replication-scheduler expects ssb-db2 to be installed, to use partial replication' - ) - } + liveStream() { + return pull( + this._ssb.db.query( + where(type('metafeed/announce')), + live(), + toPullStream() + ), + pull.filter(this._validateMetafeedAnnounce), + pull.map((msg) => { + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg.value) + this._map.set(mainFeedId, metaFeedId) + this._inverseMap.set(metaFeedId, mainFeedId) + return [mainFeedId, metaFeedId] + }) + ) + } + _loadAllFromLog() { pull( this._ssb.db.query(where(type('metafeed/announce')), toPullStream()), pull.filter(this._validateMetafeedAnnounce), diff --git a/req-manager.js b/req-manager.js index 9b704ed..9e07a46 100644 --- a/req-manager.js +++ b/req-manager.js @@ -24,6 +24,8 @@ module.exports = class RequestManager { this._wantsMoreFlushing = false this._latestAdd = 0 this._timer = null + this._liveDrainer = null + this._liveDrainer2 = null this._hasCloseHook = false this._templates = this._setupTemplates(this._opts.partialReplication) @@ -75,6 +77,8 @@ module.exports = class RequestManager { } if (this._liveDrainer) this._liveDrainer.abort() this._liveDrainer = null + if (this._liveDrainer2) this._liveDrainer2.abort() + this._liveDrainer2 = null this._flush() } @@ -112,17 +116,20 @@ module.exports = class RequestManager { const that = this this._ssb.close.hook(function (fn, args) { if (that._liveDrainer) that._liveDrainer.abort() + if (that._liveDrainer2) that._liveDrainer2.abort() fn.apply(this, args) }) } _getCurrentHops(feedId) { - return ( - this._requestables.get(feedId) || - this._requested.get(feedId) || - this._requestedPartially.get(feedId) || - null - ) + let h + h = this._requestables.get(feedId) + if (typeof h === 'number') return h + h = this._requested.get(feedId) + if (typeof h === 'number') return h + h = this._requestedPartially.get(feedId) + if (typeof h === 'number') return h + return null } _scanBendyButtFeeds() { @@ -138,6 +145,27 @@ module.exports = class RequestManager { })) ) + // Automatically switch to partial replication if (while replicating fully) + // we bump into a metafeed/announce msg + pull( + this._metafeedFinder.liveStream(), + (this._liveDrainer2 = pull.drain(([mainFeedId, metaFeedId]) => { + if ( + this._requested.has(mainFeedId) && + !this._requestedPartially.has(metaFeedId) + ) { + debug( + 'switch from full replication to partial replication for %s', + mainFeedId + ) + const hops = this._requested.get(mainFeedId) + this._unrequest(mainFeedId) + this._requestables.set(mainFeedId, hops) + this._requestPartially(mainFeedId) + } + })) + ) + // FIXME: pull tombstoneBranchStream and do ssb.ebt.request(tombId, false) } @@ -155,7 +183,7 @@ module.exports = class RequestManager { const hops = this._requestedPartially.get(mainFeedId) const matchedNode = this._matchBranchWith(hops, branch, mainFeedId) if (!matchedNode) return - this._request(subfeed, matchedNode['$format']) + this._request(subfeed, hops, matchedNode['$format']) return } @@ -186,14 +214,14 @@ module.exports = class RequestManager { } } - _fetchAndRequestMetafeed(feedId) { + _fetchAndRequestMetafeed(feedId, hops) { this._metafeedFinder.fetch(feedId, (err, metafeedId) => { if (err) { console.error(err) } else if (!metafeedId) { console.error('cannot partially replicate ' + feedId) } else { - this._request(metafeedId) + this._request(metafeedId, hops) } }) } @@ -202,7 +230,7 @@ module.exports = class RequestManager { if (this._requestedPartially.has(feedId)) return if (!this._requestables.has(feedId)) return - const hops = this._requestables.get(feedId) + const hops = this._getCurrentHops(feedId) this._requestedPartially.set(feedId, hops) this._requestables.delete(feedId) @@ -219,7 +247,7 @@ module.exports = class RequestManager { }, () => { if (branchesFound === false) { - this._fetchAndRequestMetafeed(feedId) + this._fetchAndRequestMetafeed(feedId, hops) } } ) @@ -227,20 +255,19 @@ module.exports = class RequestManager { } // Fetch metafeedId for this (main) feedId for the first time else { - this._fetchAndRequestMetafeed(feedId) + this._fetchAndRequestMetafeed(feedId, hops) } } - _request(feedId, ebtFormat = undefined) { + _request(feedId, hops = null, ebtFormat = undefined) { if (this._requested.has(feedId)) return if (this._requestables.has(feedId)) { - const hops = this._requestables.get(feedId) - this._requested.set(feedId, hops) + hops = this._requestables.get(feedId) this._requestables.delete(feedId) - } else { - this._requested.set(feedId, null) } + this._requested.set(feedId, hops) + this._ssb.ebt.block(this._ssb.id, feedId, false, ebtFormat) this._ssb.ebt.request(feedId, true, ebtFormat) } diff --git a/test/integration/partial.js b/test/integration/partial.js index bea29c8..6173598 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -29,7 +29,7 @@ const createSsbServer = SecretStack({ caps }) const CONNECTION_TIMEOUT = 1e3 const INACTIVITY_TIMEOUT = 60e3 -const REPLICATION_TIMEOUT = 2e3 +const REPLICATION_TIMEOUT = 4e3 const INDEX_WRITING_TIMEOUT = 3e3 const aliceKeys = u.keysFor('alice') @@ -264,8 +264,11 @@ tape('carol acts as an intermediate between alice and bob', async (t) => { await pify(carol.connect)(alice.getAddress()) t.pass('carol is connected to alice') - await pify(carol.db.publish)(u.follow(alice.id)) - t.pass('carol follows alice') + await Promise.all([ + pify(carol.db.publish)(u.follow(alice.id)), + pify(carol.db.publish)(u.follow(bob.id)), + ]) + t.pass('carol follows alice and bob') await sleep(REPLICATION_TIMEOUT) t.pass('replication period is over') @@ -423,12 +426,25 @@ tape('once bob unblocks alice, he replicates her subfeeds', async (t) => { tape('bob reconfigures to replicate a game feed from alice', async (t) => { bob.replicationScheduler.reconfigure({ partialReplication: { - 0: null, - 1: { + 0: { subfeeds: [ + { feedpurpose: 'main' }, { - feedpurpose: 'mygame', + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + $format: 'indexed', + }, + ], }, + ], + }, + /**/ + + 1: { + subfeeds: [ + { feedpurpose: 'mygame' }, { feedpurpose: 'indexes', subfeeds: [ @@ -444,7 +460,7 @@ tape('bob reconfigures to replicate a game feed from alice', async (t) => { }) t.pass('reconfigure bob') - await sleep(REPLICATION_TIMEOUT) + await sleep(3 * REPLICATION_TIMEOUT) t.pass('replication period is over') t.equals( @@ -454,7 +470,75 @@ tape('bob reconfigures to replicate a game feed from alice', async (t) => { ) const bobClock = await pify(bob.getVectorClock)() - t.equals(bobClock[gameFeed.keys.id], 1, 'bob\'s clock has the game feed') + t.equals(bobClock[gameFeed.keys.id], 1, "bob's clock has the game feed") + + t.end() +}) + +tape('bob starts a root meta feed and indexes, alice replicates', async (t) => { + alice.replicationScheduler.reconfigure({ + partialReplication: { + 0: { + subfeeds: [ + { feedpurpose: 'main' }, + { feedpurpose: 'mygame' }, + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + $format: 'indexed', + }, + ], + }, + ], + }, + + 1: { + subfeeds: [ + { + feedpurpose: 'mygame', + }, + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + $format: 'indexed', + }, + ], + }, + ], + }, + }, + }) + t.pass('reconfigure alice to partially replicate friends') + + // wait a bit so that alice is still replicating bob fully + await sleep(1000) + + await pify(bob.indexFeedWriter.start)({ + author: bob.id, + type: 'post', + private: false, + }) + await pify(bob.indexFeedWriter.start)({ + author: bob.id, + type: 'contact', + private: false, + }) + + await sleep(INDEX_WRITING_TIMEOUT) + t.pass('waited for Bob to publish meta feed msgs') + + await sleep(REPLICATION_TIMEOUT * 2) + t.pass('replication period is over') + + t.equals( + await alice.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 9, // 5 + add main + add indexes + add post index + add contact index + 'alice replicated 9 bendybutt msgs' + ) t.end() }) From f5500900314173e433708dec195886e378069ed5 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 27 Sep 2021 11:37:55 +0300 Subject: [PATCH 65/94] add debug in req-manager.js --- req-manager.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/req-manager.js b/req-manager.js index 9e07a46..9501122 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,4 +1,5 @@ const pull = require('pull-stream') +const debug = require('debug')('ssb:replication-scheduler') const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') const indexedEBTFormat = require('ssb-ebt/formats/indexed') const Template = require('./template') @@ -229,6 +230,7 @@ module.exports = class RequestManager { _requestPartially(feedId) { if (this._requestedPartially.has(feedId)) return if (!this._requestables.has(feedId)) return + debug('will process %s for partial replication', feedId) const hops = this._getCurrentHops(feedId) this._requestedPartially.set(feedId, hops) @@ -261,19 +263,21 @@ module.exports = class RequestManager { _request(feedId, hops = null, ebtFormat = undefined) { if (this._requested.has(feedId)) return + debug('will replicate %s', feedId) if (this._requestables.has(feedId)) { hops = this._requestables.get(feedId) this._requestables.delete(feedId) } this._requested.set(feedId, hops) - this._ssb.ebt.block(this._ssb.id, feedId, false, ebtFormat) this._ssb.ebt.request(feedId, true, ebtFormat) } _unrequest(feedId, ebtFormat = undefined) { if (this._unrequested.has(feedId)) return + debug('will stop replicating %s', feedId) + const hops = this._getCurrentHops(feedId) this._requestables.delete(feedId) this._requested.delete(feedId) @@ -296,6 +300,8 @@ module.exports = class RequestManager { _block(feedId, ebtFormat = undefined) { if (this._blocked.has(feedId)) return + debug('will block replication of %s', feedId) + const hops = this._getCurrentHops(feedId) this._requestables.delete(feedId) this._requested.delete(feedId) From 9ec50d4e60350a4899395637c7cdf9a6acfcd9a8 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 27 Sep 2021 11:43:37 +0300 Subject: [PATCH 66/94] fix flushing boolean in req-manager.js --- req-manager.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/req-manager.js b/req-manager.js index 9501122..c2b9e01 100644 --- a/req-manager.js +++ b/req-manager.js @@ -380,9 +380,9 @@ module.exports = class RequestManager { } }, (err) => { + this._flushing = false if (err) console.error(err) if (this._templates) this._scanBendyButtFeeds() - this._flushing = false if (this._wantsMoreFlushing) { this._scheduleDebouncedFlush() } From 9ae9085b29c9c7e5bbed6db390458d4e83283f90 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 27 Sep 2021 15:51:19 +0300 Subject: [PATCH 67/94] refactor internally MetafeedFinder if conditions --- metafeed-finder.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 02342e9..ffeb2e3 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -35,14 +35,11 @@ module.exports = class MetafeedFinder { if (this._map.has(mainFeedId)) { const metaFeedId = this._map.get(mainFeedId) cb(null, metaFeedId) - return } else if (mainFeedId === this._ssb.id) { this._ssb.metafeeds.find((err, rootMF) => { - if (err) { - return cb(err) - } else if (!rootMF) { - cb(null, null) - } else { + if (err) cb(err) + else if (!rootMF) cb(null, null) + else { const metaFeedId = rootMF.keys.id this._map.set(mainFeedId, metaFeedId) this._inverseMap.set(metaFeedId, mainFeedId) From ab9da47eb288272dc3f449fcd8cadf6053200563 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 27 Sep 2021 15:56:15 +0300 Subject: [PATCH 68/94] rename some variables in RequestManager --- req-manager.js | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/req-manager.js b/req-manager.js index c2b9e01..3b4da98 100644 --- a/req-manager.js +++ b/req-manager.js @@ -25,8 +25,8 @@ module.exports = class RequestManager { this._wantsMoreFlushing = false this._latestAdd = 0 this._timer = null - this._liveDrainer = null - this._liveDrainer2 = null + this._liveBranchDrainer = null + this._liveFinderDrainer = null this._hasCloseHook = false this._templates = this._setupTemplates(this._opts.partialReplication) @@ -76,10 +76,10 @@ module.exports = class RequestManager { this._requestables.set(feedId, hops) this._requestedPartially.delete(feedId) } - if (this._liveDrainer) this._liveDrainer.abort() - this._liveDrainer = null - if (this._liveDrainer2) this._liveDrainer2.abort() - this._liveDrainer2 = null + if (this._liveBranchDrainer) this._liveBranchDrainer.abort() + this._liveBranchDrainer = null + if (this._liveFinderDrainer) this._liveFinderDrainer.abort() + this._liveFinderDrainer = null this._flush() } @@ -116,8 +116,8 @@ module.exports = class RequestManager { this._hasCloseHook = true const that = this this._ssb.close.hook(function (fn, args) { - if (that._liveDrainer) that._liveDrainer.abort() - if (that._liveDrainer2) that._liveDrainer2.abort() + if (that._liveBranchDrainer) that._liveBranchDrainer.abort() + if (that._liveFinderDrainer) that._liveFinderDrainer.abort() fn.apply(this, args) }) } @@ -133,13 +133,13 @@ module.exports = class RequestManager { return null } - _scanBendyButtFeeds() { - if (this._liveDrainer) return + _drainLiveStreams() { + if (this._liveBranchDrainer) return if (!this._hasCloseHook) this._setupCloseHook() pull( this._ssb.metafeeds.branchStream({ old: true, live: true }), - (this._liveDrainer = pull.drain((branch) => { + (this._liveBranchDrainer = pull.drain((branch) => { const metaFeedId = branch[0][0] const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) this._handleBranch(branch, mainFeedId) @@ -150,7 +150,7 @@ module.exports = class RequestManager { // we bump into a metafeed/announce msg pull( this._metafeedFinder.liveStream(), - (this._liveDrainer2 = pull.drain(([mainFeedId, metaFeedId]) => { + (this._liveFinderDrainer = pull.drain(([mainFeedId, metaFeedId]) => { if ( this._requested.has(mainFeedId) && !this._requestedPartially.has(metaFeedId) @@ -382,7 +382,7 @@ module.exports = class RequestManager { (err) => { this._flushing = false if (err) console.error(err) - if (this._templates) this._scanBendyButtFeeds() + if (this._templates) this._drainLiveStreams() if (this._wantsMoreFlushing) { this._scheduleDebouncedFlush() } From 0286fd31798d5a72da9c4d0400d0b42ca5d9a867 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Mon, 27 Sep 2021 17:38:12 +0300 Subject: [PATCH 69/94] fix treatment of live MetafeedFinder stream --- metafeed-finder.js | 46 +++++++++++++++----------- package.json | 1 + req-manager.js | 81 +++++++++++++++++++++++++++------------------- 3 files changed, 76 insertions(+), 52 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index ffeb2e3..110a405 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -1,5 +1,6 @@ const pull = require('pull-stream') const debug = require('debug')('ssb:replication-scheduler') +const pushable = require('pull-pushable') const detectSsbNetworkErrorSeverity = require('ssb-network-errors') const { where, type, live, toPullStream } = require('ssb-db2/operators') const { validateMetafeedAnnounce } = require('ssb-meta-feeds/validate') @@ -16,6 +17,7 @@ module.exports = class MetafeedFinder { this._requestsByMainfeedId = new Map() // mainFeedId => Array this._latestRequestTime = 0 this._timer = null + this._liveStream = pushable() // If at least one hops template is configured, then load if ( @@ -60,20 +62,7 @@ module.exports = class MetafeedFinder { } liveStream() { - return pull( - this._ssb.db.query( - where(type('metafeed/announce')), - live(), - toPullStream() - ), - pull.filter(this._validateMetafeedAnnounce), - pull.map((msg) => { - const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg.value) - this._map.set(mainFeedId, metaFeedId) - this._inverseMap.set(metaFeedId, mainFeedId) - return [mainFeedId, metaFeedId] - }) - ) + return this._liveStream } _loadAllFromLog() { @@ -82,20 +71,34 @@ module.exports = class MetafeedFinder { pull.filter(this._validateMetafeedAnnounce), pull.drain( (msg) => { - const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msg.value) - this._map.set(mainFeedId, metaFeedId) - this._inverseMap.set(metaFeedId, mainFeedId) + this._updateMapsFromMsgValue(msg.value) }, () => { debug( 'loaded Map of all known main=>rootMF from disk, total %d', this._map.size ) + this._startLiveStream() } ) ) } + _startLiveStream() { + pull( + this._ssb.db.query( + where(type('metafeed/announce')), + live(), + toPullStream() + ), + pull.filter(this._validateMetafeedAnnounce), + pull.drain((msg) => { + this._updateMapsFromMsgValue(msg.value) + this._liveStream.push(this._pluckFromAnnounceMsg(msg.value)) + }) + ) + } + _validateMetafeedAnnounce(msg) { const err = validateMetafeedAnnounce(msg) if (err) { @@ -112,6 +115,12 @@ module.exports = class MetafeedFinder { return [mainFeedId, metaFeedId] } + _updateMapsFromMsgValue(msgVal) { + const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) + this._map.set(mainFeedId, metaFeedId) + this._inverseMap.set(metaFeedId, mainFeedId) + } + _request(mainFeedId, cb) { const callbacks = this._requestsByMainfeedId.get(mainFeedId) || [] callbacks.push(cb) @@ -183,10 +192,9 @@ module.exports = class MetafeedFinder { pull.filter((value) => this._validateMetafeedAnnounce({ value })), (drainer = pull.drain( (msgVal) => { + this._updateMapsFromMsgValue(msgVal) const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) if (requests.has(mainFeedId)) { - this._map.set(mainFeedId, metaFeedId) - this._inverseMap.set(metaFeedId, mainFeedId) this._persist(msgVal) const callbacks = requests.get(mainFeedId) requests.delete(mainFeedId) diff --git a/package.json b/package.json index d3d4d13..3f2af71 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "dependencies": { "debug": "^4.3.2", + "pull-pushable": "^2.2.0", "pull-stream": "^3.6.0", "ssb-db2": "^2.5.2", "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", diff --git a/req-manager.js b/req-manager.js index 3b4da98..7bf3de5 100644 --- a/req-manager.js +++ b/req-manager.js @@ -25,6 +25,7 @@ module.exports = class RequestManager { this._wantsMoreFlushing = false this._latestAdd = 0 this._timer = null + this._oldBranchDrainer = null this._liveBranchDrainer = null this._liveFinderDrainer = null this._hasCloseHook = false @@ -76,10 +77,9 @@ module.exports = class RequestManager { this._requestables.set(feedId, hops) this._requestedPartially.delete(feedId) } - if (this._liveBranchDrainer) this._liveBranchDrainer.abort() - this._liveBranchDrainer = null - if (this._liveFinderDrainer) this._liveFinderDrainer.abort() - this._liveFinderDrainer = null + // Refresh only the old branches stream drainer, not the live streams + if (this._oldBranchDrainer) this._oldBranchDrainer.abort() + this._oldBranchDrainer = null this._flush() } @@ -116,6 +116,7 @@ module.exports = class RequestManager { this._hasCloseHook = true const that = this this._ssb.close.hook(function (fn, args) { + if (that._oldBranchDrainer) that._oldBranchDrainer.abort() if (that._liveBranchDrainer) that._liveBranchDrainer.abort() if (that._liveFinderDrainer) that._liveFinderDrainer.abort() fn.apply(this, args) @@ -133,39 +134,53 @@ module.exports = class RequestManager { return null } - _drainLiveStreams() { - if (this._liveBranchDrainer) return + _setupStreamDrainers() { if (!this._hasCloseHook) this._setupCloseHook() - pull( - this._ssb.metafeeds.branchStream({ old: true, live: true }), - (this._liveBranchDrainer = pull.drain((branch) => { - const metaFeedId = branch[0][0] - const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - this._handleBranch(branch, mainFeedId) - })) - ) + if (!this._oldBranchDrainer) { + pull( + this._ssb.metafeeds.branchStream({ old: true, live: false }), + (this._oldBranchDrainer = pull.drain((branch) => { + const metaFeedId = branch[0][0] + const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) + this._handleBranch(branch, mainFeedId) + })) + ) + } + + if (!this._liveBranchDrainer) { + pull( + this._ssb.metafeeds.branchStream({ old: false, live: true }), + (this._liveBranchDrainer = pull.drain((branch) => { + const metaFeedId = branch[0][0] + const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) + this._handleBranch(branch, mainFeedId) + })) + ) + } // Automatically switch to partial replication if (while replicating fully) // we bump into a metafeed/announce msg - pull( - this._metafeedFinder.liveStream(), - (this._liveFinderDrainer = pull.drain(([mainFeedId, metaFeedId]) => { - if ( - this._requested.has(mainFeedId) && - !this._requestedPartially.has(metaFeedId) - ) { - debug( - 'switch from full replication to partial replication for %s', - mainFeedId - ) - const hops = this._requested.get(mainFeedId) - this._unrequest(mainFeedId) - this._requestables.set(mainFeedId, hops) - this._requestPartially(mainFeedId) - } - })) - ) + if (!this._liveFinderDrainer) { + pull( + this._metafeedFinder.liveStream(), + (this._liveFinderDrainer = pull.drain(([mainFeedId, metaFeedId]) => { + if ( + this._requested.has(mainFeedId) && + !this._requestedPartially.has(metaFeedId) + ) { + debug( + 'switch from full replication to partial replication for %s', + mainFeedId + ) + const hops = this._requested.get(mainFeedId) + this._unrequest(mainFeedId) + this._requestables.set(mainFeedId, hops) + this._requestPartially(mainFeedId) + } + })) + ) + } // FIXME: pull tombstoneBranchStream and do ssb.ebt.request(tombId, false) } @@ -382,7 +397,7 @@ module.exports = class RequestManager { (err) => { this._flushing = false if (err) console.error(err) - if (this._templates) this._drainLiveStreams() + if (this._templates) this._setupStreamDrainers() if (this._wantsMoreFlushing) { this._scheduleDebouncedFlush() } From f41464d52cba6ab7666c521896580f5386ddc10b Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 29 Sep 2021 11:01:49 +0300 Subject: [PATCH 70/94] add start() API and autostart config --- README.md | 18 +++++++++++++-- index.js | 69 +++++++++++++++++++++++++++++++++---------------------- 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 708a50e..b0a8919 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,10 @@ depending whether the feed is friendly or blocked. - Replication is strictly disabled for: - Any feed you explicitly block +There are two APIs available in case you want to have more control over this +module: `start()` and `reconfigure()`. Read more about these at the bottom of +this file. + ## Configuration Some parameters and opinions can be configured by the user or by application @@ -67,6 +71,13 @@ object. The possible options are listed below: ```js { replicationScheduler: { + /** + * Whether the replication scheduler should start automatically as soon as + * the SSB app is initialized. When `false`, you have to call + * `ssb.replicationScheduler.start()` manually. Default is `true`. + */ + autostart: true, + /** * If `partialReplication` is an object, it tells the replication scheduler * to perform partial replication, whenever remote feeds support it. If @@ -285,9 +296,12 @@ partialReplication: { } ``` -## muxrpc APIs +## APIs + +### `ssb.replicationScheduler.start() => void` (sync) + -### `ssb.replicationScheduler.reconfigure(config) => void` +### `ssb.replicationScheduler.reconfigure(config) => void` (sync) At any point during the execution of your program, you can reconfigure the replication rules using this API. The configuration object passed to this API diff --git a/index.js b/index.js index 5441f49..3d8eb7c 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,7 @@ const RequestManager = require('./req-manager') const DEFAULT_OPTS = { partialReplication: null, + autostart: true, } exports.name = 'replicationScheduler' @@ -27,42 +28,54 @@ exports.init = function (ssb, config) { } const opts = config.replicationScheduler || DEFAULT_OPTS - const metafeedFinder = new MetafeedFinder(ssb, opts) const requestManager = new RequestManager(ssb, opts, metafeedFinder) + let started = false + + if (opts.autostart === true || typeof opts.autostart === 'undefined') { + start() + } - // Replicate myself ASAP, without request manager - ssb.ebt.request(ssb.id, true) + function start() { + if (started) return + started = true - // For each edge in the social graph, call either `request` or `block` - pull( - ssb.friends.graphStream({ old: true, live: true }), - pull.drain((graph) => { - for (const source of Object.keys(graph)) { - for (const dest of Object.keys(graph[source])) { - // Compute every block edge unrelated to me - if (source !== ssb.id && dest !== ssb.id) { - const value = graph[source][dest] - ssb.ebt.block(source, dest, value === -1) + // Replicate myself ASAP, without request manager + ssb.ebt.request(ssb.id, true) + + // For each edge in the social graph, call either `request` or `block` + pull( + ssb.friends.graphStream({ old: true, live: true }), + pull.drain((graph) => { + for (const source of Object.keys(graph)) { + for (const dest of Object.keys(graph[source])) { + // Compute every block edge unrelated to me + if (source !== ssb.id && dest !== ssb.id) { + const value = graph[source][dest] + ssb.ebt.block(source, dest, value === -1) + } } } - } - }) - ) + }) + ) + + // request/block nodes at a reachable distance (within hops config) from me + pull( + ssb.friends.hopStream({ old: true, live: true }), + pull.drain((hops) => { + for (const dest of Object.keys(hops)) { + requestManager.add(dest, hops[dest]) + } + }) + ) + } - // request/block nodes at a reachable distance (within hops config) from me - pull( - ssb.friends.hopStream({ old: true, live: true }), - pull.drain((hops) => { - for (const dest of Object.keys(hops)) { - requestManager.add(dest, hops[dest]) - } - }) - ) + function reconfigure(opts) { + requestManager.reconfigure(opts) + } return { - reconfigure(opts) { - requestManager.reconfigure(opts) - }, + start, + reconfigure, } } From c9b4517497f63b42d1454fab9468654d4b56f859 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 1 Oct 2021 15:26:48 +0300 Subject: [PATCH 71/94] update ssb-meta-feeds to 0.24.0 --- metafeed-finder.js | 2 +- package.json | 2 +- test/integration/partial.js | 24 ++++++++++++++---------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 110a405..ba7a42e 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -38,7 +38,7 @@ module.exports = class MetafeedFinder { const metaFeedId = this._map.get(mainFeedId) cb(null, metaFeedId) } else if (mainFeedId === this._ssb.id) { - this._ssb.metafeeds.find((err, rootMF) => { + this._ssb.metafeeds.getRoot((err, rootMF) => { if (err) cb(err) else if (!rootMF) cb(null, null) else { diff --git a/package.json b/package.json index 3f2af71..5528514 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "pull-stream": "^3.6.0", "ssb-db2": "^2.5.2", "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", - "ssb-meta-feeds": "~0.23.0", + "ssb-meta-feeds": "~0.24.0", "ssb-network-errors": "^1.0.0", "ssb-subset-ql": "~0.6.1" }, diff --git a/test/integration/partial.js b/test/integration/partial.js index 6173598..78d9798 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -337,16 +337,20 @@ tape('once bob blocks alice, he cant replicate subfeeds anymore', async (t) => { await pify(alice.db.publish)({ type: 'post', text: 'Whatever' }) t.pass('alice published a new post') - const aliceRootMF = await pify(alice.metafeeds.find)() - gameFeed = await pify(alice.metafeeds.create)(aliceRootMF, { - feedpurpose: 'mygame', - feedformat: 'classic', - metadata: { - score: 0, - whateverElse: true, - }, - }) - t.pass('alice created a game subfeed') + const aliceRootMF = await pify(alice.metafeeds.getRoot)() + gameFeed = await pify(alice.metafeeds.findOrCreate)( + aliceRootMF, + (f) => f.feedpurpose === 'chess', + { + feedpurpose: 'mygame', + feedformat: 'classic', + metadata: { + score: 0, + whateverElse: true, + }, + } + ) + t.pass('alice created a game subfeed ' + gameFeed.keys.id.slice(0, 20)) await pify(alice.db.publishAs)(gameFeed.keys, { type: 'game', From 8c69236c5cc7ccbb41dfa7f8c019a5247127719b Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 1 Oct 2021 15:29:41 +0300 Subject: [PATCH 72/94] protect MetafeedFinder forEachNeighborPeer against self ID --- metafeed-finder.js | 1 + 1 file changed, 1 insertion(+) diff --git a/metafeed-finder.js b/metafeed-finder.js index ba7a42e..49cd3d6 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -142,6 +142,7 @@ module.exports = class MetafeedFinder { async _forEachNeighborPeer(run) { for (const peerId of Object.keys(this._ssb.peers)) { + if (peerId === this._ssb.id) continue for (const rpc of this._ssb.peers[peerId]) { const goToNext = await new Promise((resolve) => run(rpc, resolve)) if (!goToNext) return From 1d283be37358faa09d71b9f10531cca850e7d531 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 6 Oct 2021 13:12:38 +0300 Subject: [PATCH 73/94] refactor RequestManager's _handleBranch() --- req-manager.js | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/req-manager.js b/req-manager.js index 7bf3de5..43b95c1 100644 --- a/req-manager.js +++ b/req-manager.js @@ -138,23 +138,21 @@ module.exports = class RequestManager { if (!this._hasCloseHook) this._setupCloseHook() if (!this._oldBranchDrainer) { + const opts = { old: true, live: false } pull( - this._ssb.metafeeds.branchStream({ old: true, live: false }), + this._ssb.metafeeds.branchStream(opts), (this._oldBranchDrainer = pull.drain((branch) => { - const metaFeedId = branch[0][0] - const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - this._handleBranch(branch, mainFeedId) + this._handleBranch(branch) })) ) } if (!this._liveBranchDrainer) { + const opts = { old: false, live: true } pull( - this._ssb.metafeeds.branchStream({ old: false, live: true }), + this._ssb.metafeeds.branchStream(opts), (this._liveBranchDrainer = pull.drain((branch) => { - const metaFeedId = branch[0][0] - const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - this._handleBranch(branch, mainFeedId) + this._handleBranch(branch) })) ) } @@ -191,8 +189,11 @@ module.exports = class RequestManager { return template.matchBranch(branch, mainFeedId) } - _handleBranch(branch, mainFeedId) { + _handleBranch(branch) { + const first = branch[0] const last = branch[branch.length - 1] + const metaFeedId = first[0] + const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) const subfeed = last[0] if (this._requestedPartially.has(mainFeedId)) { @@ -260,7 +261,7 @@ module.exports = class RequestManager { pull.drain( (branch) => { branchesFound = true - this._handleBranch(branch, feedId) + this._handleBranch(branch) }, () => { if (branchesFound === false) { @@ -307,7 +308,7 @@ module.exports = class RequestManager { pull( this._ssb.metafeeds.branchStream({ root, old: true, live: false }), pull.drain((branch) => { - this._handleBranch(branch, feedId) + this._handleBranch(branch) }) ) } @@ -331,7 +332,7 @@ module.exports = class RequestManager { pull( this._ssb.metafeeds.branchStream({ root, old: true, live: false }), pull.drain((branch) => { - this._handleBranch(branch, feedId) + this._handleBranch(branch) }) ) } From fbc7205e6693f09906877e81a81e2917b7d0b318 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Wed, 6 Oct 2021 17:02:07 +0300 Subject: [PATCH 74/94] tiny renaming in RequestManager _handleBranch --- req-manager.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/req-manager.js b/req-manager.js index 43b95c1..8c3fde8 100644 --- a/req-manager.js +++ b/req-manager.js @@ -190,11 +190,11 @@ module.exports = class RequestManager { } _handleBranch(branch) { - const first = branch[0] - const last = branch[branch.length - 1] - const metaFeedId = first[0] + const root = branch[0] + const leaf = branch[branch.length - 1] + const metaFeedId = root[0] const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) - const subfeed = last[0] + const subfeed = leaf[0] if (this._requestedPartially.has(mainFeedId)) { const hops = this._requestedPartially.get(mainFeedId) From 5be809134ca7cf1a3ead6d53a4a0d3d2ffd44362 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Oct 2021 11:08:31 +0300 Subject: [PATCH 75/94] detect tombstones and stop replicating those subfeeds --- package.json | 2 +- req-manager.js | 60 +++++++++++++++++++++++++++------- test/integration/partial.js | 65 ++++++++++++++++++++++++++++++++++++- 3 files changed, 113 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 5528514..b6b2a46 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "pull-stream": "^3.6.0", "ssb-db2": "^2.5.2", "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", - "ssb-meta-feeds": "~0.24.0", + "ssb-meta-feeds": "~0.26.0", "ssb-network-errors": "^1.0.0", "ssb-subset-ql": "~0.6.1" }, diff --git a/req-manager.js b/req-manager.js index 8c3fde8..b857b25 100644 --- a/req-manager.js +++ b/req-manager.js @@ -20,13 +20,14 @@ module.exports = class RequestManager { this._requestedPartially = new Map() // feedId => hops this._unrequested = new Map() // feedId => hops when it used to be requested this._blocked = new Map() // feedId => hops before it was blocked - // FIXME: how should we handle tombstoning? + this._tombstoned = new Set() // feedIds this._flushing = false this._wantsMoreFlushing = false this._latestAdd = 0 this._timer = null this._oldBranchDrainer = null this._liveBranchDrainer = null + this._tombstonedBranchDrainer = null this._liveFinderDrainer = null this._hasCloseHook = false this._templates = this._setupTemplates(this._opts.partialReplication) @@ -118,6 +119,7 @@ module.exports = class RequestManager { this._ssb.close.hook(function (fn, args) { if (that._oldBranchDrainer) that._oldBranchDrainer.abort() if (that._liveBranchDrainer) that._liveBranchDrainer.abort() + if (that._tombstonedBranchDrainer) that._tombstonedBranchDrainer.abort() if (that._liveFinderDrainer) that._liveFinderDrainer.abort() fn.apply(this, args) }) @@ -138,7 +140,7 @@ module.exports = class RequestManager { if (!this._hasCloseHook) this._setupCloseHook() if (!this._oldBranchDrainer) { - const opts = { old: true, live: false } + const opts = { tombstoned: false, old: true, live: false } pull( this._ssb.metafeeds.branchStream(opts), (this._oldBranchDrainer = pull.drain((branch) => { @@ -148,7 +150,7 @@ module.exports = class RequestManager { } if (!this._liveBranchDrainer) { - const opts = { old: false, live: true } + const opts = { tombstoned: false, old: false, live: true } pull( this._ssb.metafeeds.branchStream(opts), (this._liveBranchDrainer = pull.drain((branch) => { @@ -157,6 +159,17 @@ module.exports = class RequestManager { ) } + if (!this._tombstonedBranchDrainer) { + const opts = { tombstoned: true, old: true, live: true } + pull( + this._ssb.metafeeds.branchStream(opts), + (this._tombstonedBranchDrainer = pull.drain((branch) => { + const [leafId] = branch[branch.length - 1] + this._tombstone(leafId, true) + })) + ) + } + // Automatically switch to partial replication if (while replicating fully) // we bump into a metafeed/announce msg if (!this._liveFinderDrainer) { @@ -179,8 +192,6 @@ module.exports = class RequestManager { })) ) } - - // FIXME: pull tombstoneBranchStream and do ssb.ebt.request(tombId, false) } _matchBranchWith(hops, branch, mainFeedId) { @@ -256,8 +267,9 @@ module.exports = class RequestManager { const root = this._metafeedFinder.get(feedId) if (root) { let branchesFound = false + const opts = { root, tombstoned: false, old: true, live: false } pull( - this._ssb.metafeeds.branchStream({ root, old: true, live: false }), + this._ssb.metafeeds.branchStream(opts), pull.drain( (branch) => { branchesFound = true @@ -302,9 +314,9 @@ module.exports = class RequestManager { this._ssb.ebt.request(feedId, false, ebtFormat) this._ssb.ebt.block(this._ssb.id, feedId, false, ebtFormat) - // Weave through all of the subfeeds and unrequest them too - const root = this._metafeedFinder.get(feedId) - if (root) { + if (this._ssb.metafeeds) { + // Weave through all of the subfeeds and unrequest them too + const root = this._metafeedFinder.get(feedId) || feedId pull( this._ssb.metafeeds.branchStream({ root, old: true, live: false }), pull.drain((branch) => { @@ -326,9 +338,9 @@ module.exports = class RequestManager { this._ssb.ebt.request(feedId, false, ebtFormat) this._ssb.ebt.block(this._ssb.id, feedId, true, ebtFormat) - // Weave through all of the subfeeds and block them too - const root = this._metafeedFinder.get(feedId) - if (root) { + if (this._ssb.metafeeds) { + // Weave through all of the subfeeds and block them too + const root = this._metafeedFinder.get(feedId) || feedId pull( this._ssb.metafeeds.branchStream({ root, old: true, live: false }), pull.drain((branch) => { @@ -338,6 +350,30 @@ module.exports = class RequestManager { } } + _tombstone(feedId, shouldWeave = false) { + if (this._tombstoned.has(feedId)) return + debug('will stop replicating tombstoned %s', feedId) + + this._requestables.delete(feedId) + this._requested.delete(feedId) + this._requestedPartially.delete(feedId) + this._blocked.delete(feedId) + this._tombstoned.add(feedId) + this._ssb.ebt.request(feedId, false) + + // Weave through all of the subfeeds and tombstone them too + if (shouldWeave) { + const opts = { root: feedId, tombstoned: null, old: true, live: false } + pull( + this._ssb.metafeeds.branchStream(opts), + pull.drain((branch) => { + const [leafId] = branch[branch.length - 1] + this._tombstone(leafId, false) + }) + ) + } + } + _supportsPartialReplication(feedId, cb) { this._metafeedFinder.fetch(feedId, (err, metafeedId) => { if (err) cb(err) diff --git a/test/integration/partial.js b/test/integration/partial.js index 78d9798..d15cee3 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -35,14 +35,16 @@ const INDEX_WRITING_TIMEOUT = 3e3 const aliceKeys = u.keysFor('alice') const bobKeys = u.keysFor('bob') const carolKeys = u.keysFor('carol') +const davidKeys = u.keysFor('david') let alice let bob -let carol +let david tape('setup', async (t) => { rimraf.sync(path.join(os.tmpdir(), 'server-alice')) rimraf.sync(path.join(os.tmpdir(), 'server-bob')) rimraf.sync(path.join(os.tmpdir(), 'server-carol')) + rimraf.sync(path.join(os.tmpdir(), 'server-david')) alice = createSsbServer({ path: path.join(os.tmpdir(), 'server-alice'), @@ -547,11 +549,72 @@ tape('bob starts a root meta feed and indexes, alice replicates', async (t) => { t.end() }) +tape('alice tombstones a subfeed, and david cannot replicate it', async (t) => { + const aliceRootMF = await pify(alice.metafeeds.getRoot)() + await pify(alice.metafeeds.findAndTombstone)( + aliceRootMF, + (f) => f.feedpurpose === 'mygame', + 'This game is too good' + ) + + david = createSsbServer({ + path: path.join(os.tmpdir(), 'server-david'), + keys: davidKeys, + timeout: CONNECTION_TIMEOUT, + timers: { inactivity: INACTIVITY_TIMEOUT }, + friends: { hops: 2 }, + replicationScheduler: { + debouncePeriod: 1, + partialReplication: { + 0: null, + 1: { + subfeeds: [ + { feedpurpose: 'mygame' }, + { + feedpurpose: 'indexes', + subfeeds: [ + { + feedpurpose: 'index', + $format: 'indexed', + }, + ], + }, + ], + }, + }, + }, + }) + t.pass('david initialized') + + // This needs to happen before publishing follows, otherwise carol + // defaults to normal replication (which means she won't replicate meta feeds) + await pify(david.connect)(alice.getAddress()) + t.pass('david is connected to alice') + + await pify(david.db.publish)(u.follow(alice.id)) + t.pass('david follows alice') + + await sleep(REPLICATION_TIMEOUT) + t.pass('replication period is over') + + t.equals( + await david.db.query(where(authorIsBendyButtV1()), count(), toPromise()), + 6, // add main + add indexes + add post + add contact + add game + tombstone + 'david replicated 6 bendybutt msgs' + ) + + const davidClock = await pify(david.getVectorClock)() + t.notOk(davidClock[gameFeed.keys.id], "david's clock lacks the game feed") + + t.end() +}) + tape('teardown', async (t) => { await Promise.all([ pify(alice.close)(true), pify(bob.close)(true), pify(carol.close)(true), + pify(david.close)(true), ]) t.end() From 3e10ce08fac51fbf36653529afd0ec5e81d5cb2f Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Oct 2021 11:32:27 +0300 Subject: [PATCH 76/94] rename some variables in RequestManager --- req-manager.js | 64 +++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/req-manager.js b/req-manager.js index b857b25..48f0a11 100644 --- a/req-manager.js +++ b/req-manager.js @@ -17,7 +17,7 @@ module.exports = class RequestManager { : DEFAULT_PERIOD this._requestables = new Map() // feedId => hops this._requested = new Map() // feedId => hops - this._requestedPartially = new Map() // feedId => hops + this._requestedPartially = new Map() // mainFeedId => hops this._unrequested = new Map() // feedId => hops when it used to be requested this._blocked = new Map() // feedId => hops before it was blocked this._tombstoned = new Set() // feedIds @@ -41,23 +41,23 @@ module.exports = class RequestManager { } } - add(feedId, hops) { - if (hops === -1) return this._block(feedId) - if (hops < -1) return this._unrequest(feedId) + add(mainFeedId, hops) { + if (hops === -1) return this._block(mainFeedId) + if (hops < -1) return this._unrequest(mainFeedId) - const sameHops = hops === this._getCurrentHops(feedId) - if (sameHops && this._requestables.has(feedId)) return - if (sameHops && this._requested.has(feedId)) return - if (sameHops && this._requestedPartially.has(feedId)) return + const sameHops = hops === this._getCurrentHops(mainFeedId) + if (sameHops && this._requestables.has(mainFeedId)) return + if (sameHops && this._requested.has(mainFeedId)) return + if (sameHops && this._requestedPartially.has(mainFeedId)) return if (!sameHops) { - this._requestables.delete(feedId) - this._requested.delete(feedId) - this._requestedPartially.delete(feedId) - this._unrequested.delete(feedId) - this._blocked.delete(feedId) + this._requestables.delete(mainFeedId) + this._requested.delete(mainFeedId) + this._requestedPartially.delete(mainFeedId) + this._unrequested.delete(mainFeedId) + this._blocked.delete(mainFeedId) - this._requestables.set(feedId, hops) + this._requestables.set(mainFeedId, hops) this._latestAdd = Date.now() this._scheduleDebouncedFlush() } @@ -74,9 +74,9 @@ module.exports = class RequestManager { this._requestables.set(feedId, hops) this._requested.delete(feedId) } - for (const [feedId, hops] of this._requestedPartially) { - this._requestables.set(feedId, hops) - this._requestedPartially.delete(feedId) + for (const [mainFeedId, hops] of this._requestedPartially) { + this._requestables.set(mainFeedId, hops) + this._requestedPartially.delete(mainFeedId) } // Refresh only the old branches stream drainer, not the live streams if (this._oldBranchDrainer) this._oldBranchDrainer.abort() @@ -242,29 +242,29 @@ module.exports = class RequestManager { } } - _fetchAndRequestMetafeed(feedId, hops) { - this._metafeedFinder.fetch(feedId, (err, metafeedId) => { + _fetchAndRequestMetafeed(mainFeedId, hops) { + this._metafeedFinder.fetch(mainFeedId, (err, metafeedId) => { if (err) { console.error(err) } else if (!metafeedId) { - console.error('cannot partially replicate ' + feedId) + console.error('cannot partially replicate ' + mainFeedId) } else { this._request(metafeedId, hops) } }) } - _requestPartially(feedId) { - if (this._requestedPartially.has(feedId)) return - if (!this._requestables.has(feedId)) return - debug('will process %s for partial replication', feedId) + _requestPartially(mainFeedId) { + if (this._requestedPartially.has(mainFeedId)) return + if (!this._requestables.has(mainFeedId)) return + debug('will process %s for partial replication', mainFeedId) - const hops = this._getCurrentHops(feedId) - this._requestedPartially.set(feedId, hops) - this._requestables.delete(feedId) + const hops = this._getCurrentHops(mainFeedId) + this._requestedPartially.set(mainFeedId, hops) + this._requestables.delete(mainFeedId) // We may already have the meta feed, so continue replicating - const root = this._metafeedFinder.get(feedId) + const root = this._metafeedFinder.get(mainFeedId) if (root) { let branchesFound = false const opts = { root, tombstoned: false, old: true, live: false } @@ -277,7 +277,7 @@ module.exports = class RequestManager { }, () => { if (branchesFound === false) { - this._fetchAndRequestMetafeed(feedId, hops) + this._fetchAndRequestMetafeed(mainFeedId, hops) } } ) @@ -285,7 +285,7 @@ module.exports = class RequestManager { } // Fetch metafeedId for this (main) feedId for the first time else { - this._fetchAndRequestMetafeed(feedId, hops) + this._fetchAndRequestMetafeed(mainFeedId, hops) } } @@ -374,8 +374,8 @@ module.exports = class RequestManager { } } - _supportsPartialReplication(feedId, cb) { - this._metafeedFinder.fetch(feedId, (err, metafeedId) => { + _supportsPartialReplication(mainFeedId, cb) { + this._metafeedFinder.fetch(mainFeedId, (err, metafeedId) => { if (err) cb(err) else cb(null, !!metafeedId) }) From 3342f192014f5f8d91a497a55d023c2fafa295d8 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Oct 2021 11:32:44 +0300 Subject: [PATCH 77/94] tweak RequestManager _handleBranch logic --- req-manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/req-manager.js b/req-manager.js index 48f0a11..7214380 100644 --- a/req-manager.js +++ b/req-manager.js @@ -205,6 +205,7 @@ module.exports = class RequestManager { const leaf = branch[branch.length - 1] const metaFeedId = root[0] const mainFeedId = this._metafeedFinder.getInverse(metaFeedId) + if (!mainFeedId) return const subfeed = leaf[0] if (this._requestedPartially.has(mainFeedId)) { From 48e9df5cfc9ded4214812271dabdd6fa19356dc3 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Oct 2021 13:56:31 +0300 Subject: [PATCH 78/94] change how "partial replication enabled" is checked --- req-manager.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index 7214380..1376a70 100644 --- a/req-manager.js +++ b/req-manager.js @@ -315,7 +315,7 @@ module.exports = class RequestManager { this._ssb.ebt.request(feedId, false, ebtFormat) this._ssb.ebt.block(this._ssb.id, feedId, false, ebtFormat) - if (this._ssb.metafeeds) { + if (this._templates) { // Weave through all of the subfeeds and unrequest them too const root = this._metafeedFinder.get(feedId) || feedId pull( @@ -339,7 +339,7 @@ module.exports = class RequestManager { this._ssb.ebt.request(feedId, false, ebtFormat) this._ssb.ebt.block(this._ssb.id, feedId, true, ebtFormat) - if (this._ssb.metafeeds) { + if (this._templates) { // Weave through all of the subfeeds and block them too const root = this._metafeedFinder.get(feedId) || feedId pull( From 2ee7063dbe5732c10f651a24e71cb7d5cf2b54d3 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Oct 2021 18:10:18 +0300 Subject: [PATCH 79/94] fix RequestManager _findTemplateForHops() corner case logic --- req-manager.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/req-manager.js b/req-manager.js index 1376a70..05c40ed 100644 --- a/req-manager.js +++ b/req-manager.js @@ -108,8 +108,12 @@ module.exports = class RequestManager { _findTemplateForHops(hops) { if (!this._templates) return null - const eligibleHopsArr = [...this._templates.keys()].filter((h) => h >= hops) - const pickedHops = Math.min(...eligibleHopsArr) + const templateKeys = [...this._templates.keys()] + const eligibleHopsArr = templateKeys.filter((h) => h >= hops) + const pickedHops = + eligibleHopsArr.length > 0 + ? Math.min(...eligibleHopsArr) + : Math.max(...templateKeys) return this._templates.get(pickedHops) } From 4a8225227768adf417cb8c52157b72f345363ae9 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 7 Oct 2021 18:19:55 +0300 Subject: [PATCH 80/94] fix tests for tombstoning --- test/integration/partial.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/partial.js b/test/integration/partial.js index d15cee3..0219d6e 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -599,8 +599,10 @@ tape('alice tombstones a subfeed, and david cannot replicate it', async (t) => { t.equals( await david.db.query(where(authorIsBendyButtV1()), count(), toPromise()), - 6, // add main + add indexes + add post + add contact + add game + tombstone - 'david replicated 6 bendybutt msgs' + // ALICE: main + indexes + post idx + contact idx + add game + tombstone + // BOB: main + indexes + post idx + contact idx + 10, + 'david replicated 10 bendybutt msgs' ) const davidClock = await pify(david.getVectorClock)() From 175b70f552089121297ec3e870acf57956f96a59 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 8 Oct 2021 12:05:01 +0300 Subject: [PATCH 81/94] add a small timer.unref() --- metafeed-finder.js | 1 + 1 file changed, 1 insertion(+) diff --git a/metafeed-finder.js b/metafeed-finder.js index 49cd3d6..bdd854f 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -162,6 +162,7 @@ module.exports = class MetafeedFinder { else if (Date.now() - this._latestRequestTime > this._period) this._flush() }, this._period * 0.5) + if (this._timer.unref) this._timer.unref() } _makeQL1(map) { From 2338e68aa36ec91529d25fbacbe2a1246aaec081 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 8 Oct 2021 12:19:09 +0300 Subject: [PATCH 82/94] improve pull.drain in MetafeedFinder _flush --- metafeed-finder.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index bdd854f..37b895d 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -183,7 +183,6 @@ module.exports = class MetafeedFinder { } async _flush() { - let drainer const requests = new Map(this._requestsByMainfeedId) this._requestsByMainfeedId.clear() @@ -192,21 +191,22 @@ module.exports = class MetafeedFinder { pull( rpc.getSubset(this._makeQL1(requests), { querylang: 'ssb-ql-1' }), pull.filter((value) => this._validateMetafeedAnnounce({ value })), - (drainer = pull.drain( + pull.drain( (msgVal) => { this._updateMapsFromMsgValue(msgVal) const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) + this._liveStream.push([mainFeedId, metaFeedId]) + this._persist(msgVal) + if (requests.has(mainFeedId)) { - this._persist(msgVal) const callbacks = requests.get(mainFeedId) requests.delete(mainFeedId) for (const cb of callbacks) cb(null, metaFeedId) - if (requests.size === 0) { - drainer.abort() - goToNextNeighbor(false) - } else { - goToNextNeighbor(true) - } + } + + if (requests.size === 0) { + goToNextNeighbor(false) + return false // abort this drain } }, (err) => { @@ -219,7 +219,7 @@ module.exports = class MetafeedFinder { } goToNextNeighbor(true) } - )) + ) ) }) From efd7f43f3a040cb5ffb709a71b3fd6205614f149 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 8 Oct 2021 12:38:35 +0300 Subject: [PATCH 83/94] cosmetic refactor of MetafeedFinder period arg --- metafeed-finder.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 37b895d..ee1fe01 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -5,13 +5,11 @@ const detectSsbNetworkErrorSeverity = require('ssb-network-errors') const { where, type, live, toPullStream } = require('ssb-db2/operators') const { validateMetafeedAnnounce } = require('ssb-meta-feeds/validate') -const DEFAULT_PERIOD = 500 - module.exports = class MetafeedFinder { - constructor(ssb, opts, period) { + constructor(ssb, opts, period = 500) { this._ssb = ssb this._opts = opts - this._period = period || DEFAULT_PERIOD + this._period = period this._map = new Map() // mainFeedId => rootMetaFeedId this._inverseMap = new Map() // rootMetaFeedId => mainFeedId this._requestsByMainfeedId = new Map() // mainFeedId => Array From f547a552054d57ca2aab8c35db2810e56a19360c Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 8 Oct 2021 12:41:28 +0300 Subject: [PATCH 84/94] MetafeedFinder calls getSubset on newly connected peers --- metafeed-finder.js | 54 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index ee1fe01..6d98415 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -13,6 +13,7 @@ module.exports = class MetafeedFinder { this._map = new Map() // mainFeedId => rootMetaFeedId this._inverseMap = new Map() // rootMetaFeedId => mainFeedId this._requestsByMainfeedId = new Map() // mainFeedId => Array + this._retryables = new Set() // mainFeedIds this._latestRequestTime = 0 this._timer = null this._liveStream = pushable() @@ -28,6 +29,7 @@ module.exports = class MetafeedFinder { ) } this._loadAllFromLog() + this._monitorConnectedPeers() } } @@ -97,6 +99,26 @@ module.exports = class MetafeedFinder { ) } + _monitorConnectedPeers() { + if (!this._ssb.conn) { + console.warn( + 'No ssb-conn installed, ssb-replication-scheduler will ' + + ' miss some useful data from connected peers regarding metafeeds' + ) + return + } + + pull( + this._ssb.conn.hub().listen(), + pull.filter((ev) => ev.type === 'connected'), + pull.drain((ev) => { + if (this._retryables.size > 0) { + this._retryWithPeer(ev.details.rpc) + } + }) + ) + } + _validateMetafeedAnnounce(msg) { const err = validateMetafeedAnnounce(msg) if (err) { @@ -117,6 +139,7 @@ module.exports = class MetafeedFinder { const [mainFeedId, metaFeedId] = this._pluckFromAnnounceMsg(msgVal) this._map.set(mainFeedId, metaFeedId) this._inverseMap.set(metaFeedId, mainFeedId) + this._retryables.delete(mainFeedId) } _request(mainFeedId, cb) { @@ -163,12 +186,12 @@ module.exports = class MetafeedFinder { if (this._timer.unref) this._timer.unref() } - _makeQL1(map) { + _makeQL1(mapOrSet) { const query = { op: 'or', args: [], } - for (const mainFeedId of map.keys()) { + for (const mainFeedId of mapOrSet.keys()) { query.args.push({ op: 'and', args: [ @@ -225,10 +248,35 @@ module.exports = class MetafeedFinder { // We couldn't find metaFeedIds for some mainFeedIds, so we assume there // none. Note, this may give false negatives depending on who you're // connected to! - for (const callbacks of requests.values()) { + for (const [mainFeedId, callbacks] of requests.entries()) { + this._retryables.add(mainFeedId) for (const cb of callbacks) cb(null, null) } requests.clear() } } + + _retryWithPeer(rpc) { + debug('"getSubset" retry on peer %s for metafeed/announce messages', rpc.id) + pull( + rpc.getSubset(this._makeQL1(this._retryables), { querylang: 'ssb-ql-1' }), + pull.filter((value) => this._validateMetafeedAnnounce({ value })), + pull.drain( + (msgVal) => { + this._updateMapsFromMsgValue(msgVal) + this._liveStream.push(this._pluckFromAnnounceMsg(msgVal)) + this._persist(msgVal) + }, + (err) => { + if (err && detectSsbNetworkErrorSeverity(err) >= 2) { + debug( + 'failed "getSubset" muxrpc retry at peer %s because: %s', + rpc.id, + err.message || err + ) + } + } + ) + ) + } } From afa3fad5cbc28d23ec98e4ae7a0128a9184efc54 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 8 Oct 2021 14:53:53 +0300 Subject: [PATCH 85/94] make this REUSE compliant --- metafeed-finder.js | 4 ++++ req-manager.js | 4 ++++ template.js | 4 ++++ test/integration/partial.js | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/metafeed-finder.js b/metafeed-finder.js index 6d98415..30ae4de 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros +// +// SPDX-License-Identifier: LGPL-3.0-only + const pull = require('pull-stream') const debug = require('debug')('ssb:replication-scheduler') const pushable = require('pull-pushable') diff --git a/req-manager.js b/req-manager.js index 05c40ed..c161caf 100644 --- a/req-manager.js +++ b/req-manager.js @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros +// +// SPDX-License-Identifier: LGPL-3.0-only + const pull = require('pull-stream') const debug = require('debug')('ssb:replication-scheduler') const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') diff --git a/template.js b/template.js index e34d019..e665273 100644 --- a/template.js +++ b/template.js @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros +// +// SPDX-License-Identifier: LGPL-3.0-only + const { QL0 } = require('ssb-subset-ql') /** diff --git a/test/integration/partial.js b/test/integration/partial.js index 0219d6e..f152cbf 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -1,3 +1,7 @@ +// SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros +// +// SPDX-License-Identifier: Unlicense + const tape = require('tape') const path = require('path') const os = require('os') From b3c153aed9ac50f0ca837e83d609eda2c89e8fdc Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Tue, 12 Oct 2021 17:11:31 +0300 Subject: [PATCH 86/94] use ssb-ebt 8.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b6b2a46..e423a8b 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "pull-pushable": "^2.2.0", "pull-stream": "^3.6.0", "ssb-db2": "^2.5.2", - "ssb-ebt": "ssbc/ssb-ebt#partial-replication-changes", + "ssb-ebt": "^8.0.0", "ssb-meta-feeds": "~0.26.0", "ssb-network-errors": "^1.0.0", "ssb-subset-ql": "~0.6.1" From 26f48fb170b0c6a08b62712a35a73b95adecbe8a Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Sun, 17 Oct 2021 16:53:01 +0300 Subject: [PATCH 87/94] MetafeedFinder with period zero behaves synchronously --- metafeed-finder.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metafeed-finder.js b/metafeed-finder.js index 30ae4de..ba07fce 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -176,6 +176,10 @@ module.exports = class MetafeedFinder { } _scheduleDebouncedFlush() { + if (this._period === 0) { + this._flush() + return + } if (this._timer) return // Timer is already enabled this._timer = setInterval(() => { // Turn off the timer if there is nothing to flush From 7d296c2d13cb98757d8959216b99fec0e5a0c821 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Sun, 17 Oct 2021 16:55:25 +0300 Subject: [PATCH 88/94] MetafeedFinder doesnt flush if there's nothing to flush --- metafeed-finder.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metafeed-finder.js b/metafeed-finder.js index ba07fce..8a608fa 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -212,6 +212,8 @@ module.exports = class MetafeedFinder { } async _flush() { + if (this._requestsByMainfeedId.size === 0) return + const requests = new Map(this._requestsByMainfeedId) this._requestsByMainfeedId.clear() From acb09f512097f0fdf97747fecf7e75e4c88a149f Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Thu, 17 Feb 2022 18:32:04 +0200 Subject: [PATCH 89/94] small refactors --- index.js | 7 ++----- req-manager.js | 5 +++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/index.js b/index.js index 3d8eb7c..895e6e5 100644 --- a/index.js +++ b/index.js @@ -3,7 +3,6 @@ // SPDX-License-Identifier: LGPL-3.0-only const pull = require('pull-stream') -const MetafeedFinder = require('./metafeed-finder') const RequestManager = require('./req-manager') const DEFAULT_OPTS = { @@ -28,8 +27,7 @@ exports.init = function (ssb, config) { } const opts = config.replicationScheduler || DEFAULT_OPTS - const metafeedFinder = new MetafeedFinder(ssb, opts) - const requestManager = new RequestManager(ssb, opts, metafeedFinder) + const requestManager = new RequestManager(ssb, opts) let started = false if (opts.autostart === true || typeof opts.autostart === 'undefined') { @@ -43,13 +41,12 @@ exports.init = function (ssb, config) { // Replicate myself ASAP, without request manager ssb.ebt.request(ssb.id, true) - // For each edge in the social graph, call either `request` or `block` + // Take every block or unblock into account, except if it's about me pull( ssb.friends.graphStream({ old: true, live: true }), pull.drain((graph) => { for (const source of Object.keys(graph)) { for (const dest of Object.keys(graph[source])) { - // Compute every block edge unrelated to me if (source !== ssb.id && dest !== ssb.id) { const value = graph[source][dest] ssb.ebt.block(source, dest, value === -1) diff --git a/req-manager.js b/req-manager.js index c161caf..c718856 100644 --- a/req-manager.js +++ b/req-manager.js @@ -7,14 +7,15 @@ const debug = require('debug')('ssb:replication-scheduler') const bendyButtEBTFormat = require('ssb-ebt/formats/bendy-butt') const indexedEBTFormat = require('ssb-ebt/formats/indexed') const Template = require('./template') +const MetafeedFinder = require('./metafeed-finder') const DEFAULT_PERIOD = 150 // ms module.exports = class RequestManager { - constructor(ssb, opts, metafeedFinder) { + constructor(ssb, opts) { this._ssb = ssb this._opts = opts - this._metafeedFinder = metafeedFinder + this._metafeedFinder = new MetafeedFinder(ssb, opts) this._period = typeof opts.debouncePeriod === 'number' ? opts.debouncePeriod From f238c489c065cb29c5375393f0dad49e973aee53 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 18 Feb 2022 14:36:08 +0200 Subject: [PATCH 90/94] batch metafeed-finder RPC calls --- metafeed-finder.js | 9 +++++++-- req-manager.js | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/metafeed-finder.js b/metafeed-finder.js index 8a608fa..27e6b12 100644 --- a/metafeed-finder.js +++ b/metafeed-finder.js @@ -10,10 +10,11 @@ const { where, type, live, toPullStream } = require('ssb-db2/operators') const { validateMetafeedAnnounce } = require('ssb-meta-feeds/validate') module.exports = class MetafeedFinder { - constructor(ssb, opts, period = 500) { + constructor(ssb, opts, batchLimit = 8, period = 500) { this._ssb = ssb this._opts = opts this._period = period + this._batchLimit = batchLimit this._map = new Map() // mainFeedId => rootMetaFeedId this._inverseMap = new Map() // rootMetaFeedId => mainFeedId this._requestsByMainfeedId = new Map() // mainFeedId => Array @@ -151,7 +152,11 @@ module.exports = class MetafeedFinder { callbacks.push(cb) this._requestsByMainfeedId.set(mainFeedId, callbacks) this._latestRequestTime = Date.now() - this._scheduleDebouncedFlush() + if (this._requestsByMainfeedId.size >= this._batchLimit) { + this._flush() + } else { + this._scheduleDebouncedFlush() + } } _persist(msgVal) { diff --git a/req-manager.js b/req-manager.js index c718856..a7bec24 100644 --- a/req-manager.js +++ b/req-manager.js @@ -10,12 +10,13 @@ const Template = require('./template') const MetafeedFinder = require('./metafeed-finder') const DEFAULT_PERIOD = 150 // ms +const BATCH_LIMIT = 8 module.exports = class RequestManager { constructor(ssb, opts) { this._ssb = ssb this._opts = opts - this._metafeedFinder = new MetafeedFinder(ssb, opts) + this._metafeedFinder = new MetafeedFinder(ssb, opts, BATCH_LIMIT) this._period = typeof opts.debouncePeriod === 'number' ? opts.debouncePeriod From 4d4908b4cc84ed3f44ab0bd86048e046d1524c9c Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 18 Feb 2022 14:54:23 +0200 Subject: [PATCH 91/94] test: intermediary friend does not forward to blocked --- test/integration/block4.js | 110 +++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 test/integration/block4.js diff --git a/test/integration/block4.js b/test/integration/block4.js new file mode 100644 index 0000000..2e78516 --- /dev/null +++ b/test/integration/block4.js @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros +// +// SPDX-License-Identifier: Unlicense + +const tape = require('tape') +const crypto = require('crypto') +const pify = require('promisify-4loc') +const sleep = require('util').promisify(setTimeout) +const SecretStack = require('secret-stack') +const u = require('../misc/util') + +// alice, bob, and carol all follow each other, +// but then bob offends alice, and she blocks him. +// this means that: +// +// 1. when bob tries to connect to alice, she refuses. +// 2. alice never tries to connect to bob. (removed from peers) +// 3. carol will not give bob any, she will not give him any data from alice. + +const createSsbServer = SecretStack({ + caps: { shs: crypto.randomBytes(32).toString('base64') }, +}) + .use(require('ssb-db')) + .use(require('ssb-ebt')) + .use(require('ssb-friends')) + .use(require('../..')) + +const CONNECTION_TIMEOUT = 500 // ms +const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT + +const alice = createSsbServer({ + temp: 'test-block4-alice', + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('alice'), + replicationScheduler: { + debouncePeriod: 0, + }, + friends: { + hops: 4, + }, +}) + +const bob = createSsbServer({ + temp: 'test-block4-bob', + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('bob'), + replicationScheduler: { + debouncePeriod: 0, + }, + friends: { + hops: 4, + }, +}) + +const carol = createSsbServer({ + temp: 'test-block4-carol', + timeout: CONNECTION_TIMEOUT, + keys: u.keysFor('carol'), + replicationScheduler: { + debouncePeriod: 0, + }, + friends: { + hops: 4, + }, +}) + +tape('middle friend does not forward data to the blocked one', async (t) => { + t.plan(6) + + // Alice follows Carols, but Carol blocks Alice + await Promise.all([ + pify(alice.publish)(u.follow(carol.id)), + pify(carol.publish)(u.block(alice.id)), + ]) + + // Bob is mutual friends with all others + await Promise.all([ + pify(bob.publish)(u.follow(alice.id)), + pify(alice.publish)(u.follow(bob.id)), + pify(bob.publish)(u.follow(carol.id)), + pify(carol.publish)(u.follow(bob.id)), + ]) + + await Promise.all([ + pify(bob.connect)(alice.getAddress()), + pify(bob.connect)(carol.getAddress()), + ]) + + await sleep(REPLICATION_TIMEOUT) + + // Bob has all the data from everyone + const clockBob = await pify(bob.getVectorClock)() + t.equals(clockBob[alice.id], 2, 'bob has 2 messages from alice') + t.equals(clockBob[bob.id], 2, 'bob has 2 messages from bob') + t.equals(clockBob[carol.id], 2, 'bob has 2 messages from carol') + + // Alice has data from Bob but NOT carol + const clockAlice = await pify(alice.getVectorClock)() + t.equals(clockAlice[alice.id], 2, 'alice has 2 messages from alice') + t.equals(clockAlice[bob.id], 2, 'alice has 2 messages from bob') + t.equals(clockAlice[carol.id], undefined, 'alice has no messages from carol') + + await Promise.all([ + pify(alice.close)(true), + pify(bob.close)(true), + pify(carol.close)(true), + ]) + + t.end() +}) From 5338f449237d94e6e36a72ac8a5d15b1b587f6c7 Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 18 Feb 2022 15:01:07 +0200 Subject: [PATCH 92/94] remove wrong code comment --- test/integration/block4.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/test/integration/block4.js b/test/integration/block4.js index 2e78516..4e590d2 100644 --- a/test/integration/block4.js +++ b/test/integration/block4.js @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2021 Andre 'Staltz' Medeiros +// SPDX-FileCopyrightText: 2022 Andre 'Staltz' Medeiros // // SPDX-License-Identifier: Unlicense @@ -9,14 +9,6 @@ const sleep = require('util').promisify(setTimeout) const SecretStack = require('secret-stack') const u = require('../misc/util') -// alice, bob, and carol all follow each other, -// but then bob offends alice, and she blocks him. -// this means that: -// -// 1. when bob tries to connect to alice, she refuses. -// 2. alice never tries to connect to bob. (removed from peers) -// 3. carol will not give bob any, she will not give him any data from alice. - const createSsbServer = SecretStack({ caps: { shs: crypto.randomBytes(32).toString('base64') }, }) From 04c35aa1600c298a85df7b1c5205c47a85075c2f Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 18 Feb 2022 15:19:35 +0200 Subject: [PATCH 93/94] bump several dependencies --- package.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index e423a8b..90a9db4 100644 --- a/package.json +++ b/package.json @@ -22,11 +22,11 @@ "debug": "^4.3.2", "pull-pushable": "^2.2.0", "pull-stream": "^3.6.0", - "ssb-db2": "^2.5.2", + "ssb-db2": "^2.8.9", "ssb-ebt": "^8.0.0", - "ssb-meta-feeds": "~0.26.0", - "ssb-network-errors": "^1.0.0", - "ssb-subset-ql": "~0.6.1" + "ssb-meta-feeds": "~0.28.0", + "ssb-network-errors": "^1.0.1", + "ssb-subset-ql": "~0.6.2" }, "devDependencies": { "cat-names": "^3.0.0", @@ -43,13 +43,13 @@ "secret-stack": "^6.4.0", "ssb-caps": "^1.1.0", "ssb-db": "^20.4.0", - "ssb-fixtures": "^2.5.3", + "ssb-fixtures": "^3.0.4", "ssb-friends": "^5.1.0", - "ssb-index-feed-writer": "~0.6.0", + "ssb-index-feed-writer": "~0.7.0", "ssb-keys": "^8.2.0", - "ssb-subset-rpc": "~0.3.0", + "ssb-subset-rpc": "~0.3.1", "tap-spec": "^5.0.0", - "tape": "^5.2.2" + "tape": "^5.5.2" }, "scripts": { "test": "tape 'test/@(unit|integration)/*.js' | tap-spec", From 0a451af9f5cf2bb596a00b31a8a2d4fbf58180cd Mon Sep 17 00:00:00 2001 From: Andre Staltz Date: Fri, 18 Feb 2022 16:48:00 +0200 Subject: [PATCH 94/94] tweak tests to show "catch all" replication rule --- test/integration/partial.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/integration/partial.js b/test/integration/partial.js index f152cbf..845cb2f 100644 --- a/test/integration/partial.js +++ b/test/integration/partial.js @@ -113,8 +113,6 @@ tape('alice writes index feeds and bob replicates them', async (t) => { partialReplication: { 0: { subfeeds: [ - { feedpurpose: 'main' }, - { feedpurpose: 'mygame' }, { feedpurpose: 'indexes', subfeeds: [ @@ -124,6 +122,10 @@ tape('alice writes index feeds and bob replicates them', async (t) => { }, ], }, + // Empty object to signal "replicate anything else". + // Note that order is important. This more general rule has the come + // after the more specific rule for indexed subfeeds. + {} ], }, 1: null, @@ -450,7 +452,6 @@ tape('bob reconfigures to replicate a game feed from alice', async (t) => { }, ], }, - /**/ 1: { subfeeds: [