diff --git a/api.js b/api.js index c200a20..1b3d009 100644 --- a/api.js +++ b/api.js @@ -4,8 +4,17 @@ const run = require('promisify-tuple') const debug = require('debug')('ssb:meta-feeds') +const pickShard = require('./pick-shard') const alwaysTrue = () => true +const BB1 = 'bendybutt-v1' +const v1Details = { feedpurpose: 'v1', feedformat: BB1 } +const v1Visit = detailsToVisit(v1Details) +function detailsToVisit(details) { + return (feed) => + feed.feedpurpose === details.feedpurpose && + feed.feedformat === details.feedformat +} exports.init = function (sbot, config) { function filter(metafeed, visit, maybeCB) { @@ -210,11 +219,39 @@ exports.init = function (sbot, config) { } } + function commonFindOrCreate(details, cb) { + if (!details.feedformat) details.feedformat = 'classic' + + findOrCreate((err, rootFeed) => { + if (err) return cb(err) + + findOrCreate(rootFeed, v1Visit, v1Details, (err, v1Feed) => { + if (err) return cb(err) + + const shardDetails = { + feedpurpose: pickShard(rootFeed.keys.id, details.feedpurpose), + feedformat: BB1, + } + const shardVisit = detailsToVisit(shardDetails) + + findOrCreate(v1Feed, shardVisit, shardDetails, (err, shardFeed) => { + if (err) return cb(err) + + findOrCreate(shardFeed, detailsToVisit(details), details, cb) + }) + }) + }) + } + return { - getRoot, - findOrCreate, - findAndTombstone, - findById, branchStream, + findOrCreate: commonFindOrCreate, + + advanced: { + getRoot, + findOrCreate, + findAndTombstone, + findById, + }, } } diff --git a/index.js b/index.js index 68fcb06..2f28f7e 100644 --- a/index.js +++ b/index.js @@ -2,19 +2,19 @@ // // SPDX-License-Identifier: LGPL-3.0-only +const API = require('./api') const Keys = require('./keys') const Messages = require('./messages') +const Lookup = require('./lookup') const Query = require('./query') -const API = require('./api') const Validate = require('./validate') -const FeedsLookup = require('./feeds-lookup') exports.name = 'metafeeds' exports.init = function (sbot, config) { const messages = Messages.init(sbot, config) const query = Query.init(sbot, config) - const lookup = FeedsLookup.init(sbot, config) + const lookup = Lookup.init(sbot, config) const api = API.init(sbot, config) return { diff --git a/feeds-lookup.js b/lookup.js similarity index 100% rename from feeds-lookup.js rename to lookup.js diff --git a/messages.js b/messages.js index 02a0c42..67a1054 100644 --- a/messages.js +++ b/messages.js @@ -39,7 +39,7 @@ exports.init = function init(sbot) { return { feedFormat: 'bendybutt-v1', keys: mfKeys, - contentKeys: feedKeys, + contentKeys: feedKeys, // see ssb-bendy-butt/format.js content, encryptionFormat: 'box2', // in case metadata.recps is set } diff --git a/package.json b/package.json index 28d0ca6..1d71164 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,8 @@ "rimraf": "^3.0.2", "secret-stack": "^6.4.0", "ssb-bendy-butt": "^1.0.1", - "ssb-db2": "^6.1.1", "ssb-caps": "^1.1.0", + "ssb-db2": "^6.1.1", "tap-arc": "^0.3.5", "tape": "^5.6.0" }, diff --git a/pick-shard.js b/pick-shard.js new file mode 100644 index 0000000..3f24aed --- /dev/null +++ b/pick-shard.js @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2022 Mix Irving +// +// SPDX-License-Identifier: LGPL-3.0-only + +const bfe = require('ssb-bfe') +const crypto = require('crypto') + +module.exports = function pickShard(rootFeedId, idString) { + const buf = Buffer.concat([bfe.encode(rootFeedId), bfe.encode(idString)]) + + const hash = crypto.createHash('sha256') + hash.update(buf) + + return hash.digest('hex')[0] +} diff --git a/test/api.test.js b/test/api.test.js index 508bd78..e01edd7 100644 --- a/test/api.test.js +++ b/test/api.test.js @@ -31,16 +31,16 @@ function testReadAndPersisted(t, sbot, testRead) { /* Tests */ -test('getRoot() when there is nothing', (t) => { +test('advanced.getRoot() when there is nothing', (t) => { const sbot = Testbot() - sbot.metafeeds.getRoot((err, found) => { + sbot.metafeeds.advanced.getRoot((err, found) => { t.error(err, 'no err for find()') t.notOk(found, 'nothing found') sbot.close(true, t.end) }) }) -test('findOrCreate(null, null, null, cb)', (t) => { +test('advanced.findOrCreate(null, null, null, cb)', (t) => { const sbot = Testbot() sbot.db.query( where(author(sbot.id)), @@ -48,7 +48,7 @@ test('findOrCreate(null, null, null, cb)', (t) => { if (err) throw err t.equals(msgs.length, 0, 'empty db') - sbot.metafeeds.findOrCreate(null, null, null, (err, mf) => { + sbot.metafeeds.advanced.findOrCreate(null, null, null, (err, mf) => { t.error(err, 'no err for findOrCreate()') // t.equals(mf.feeds.length, 1, '1 sub feed in the root metafeed') // t.equals(mf.feeds[0].feedpurpose, 'main', 'it is the main feed') @@ -60,10 +60,10 @@ test('findOrCreate(null, null, null, cb)', (t) => { ) }) -test('findOrCreate(cb)', (t) => { +test('advanced.findOrCreate(cb)', (t) => { const sbot = Testbot() - sbot.metafeeds.findOrCreate((err, mf) => { + sbot.metafeeds.advanced.findOrCreate((err, mf) => { t.error(err, 'no err for findOrCreate()') // t.equals(mf.feeds.length, 1, '1 sub feed in the root metafeed') // t.equals(mf.feeds[0].feedpurpose, 'main', 'it is the main feed') @@ -73,19 +73,19 @@ test('findOrCreate(cb)', (t) => { }) }) -test('findOrCreate is idempotent', (t) => { +test('advanced.findOrCreate is idempotent', (t) => { const sbot = Testbot() - sbot.metafeeds.findOrCreate(null, null, null, (err, mf) => { + sbot.metafeeds.advanced.findOrCreate(null, null, null, (err, mf) => { t.error(err, 'no err for findOrCreate()') t.ok(mf, 'got a metafeed') - sbot.metafeeds.getRoot((err, mf) => { + sbot.metafeeds.advanced.getRoot((err, mf) => { t.error(err, 'no err for getRoot()') t.equals(mf.seed.toString('hex').length, 64, 'seed length is okay') t.equals(typeof mf.keys.id, 'string', 'key seems okay') const originalSeed = mf.seed.toString('hex') const originalID = mf.keys.id - sbot.metafeeds.findOrCreate((err, mf) => { + sbot.metafeeds.advanced.findOrCreate((err, mf) => { t.error(err, 'no err for findOrCreate(null, ...)') t.equals(mf.seed.toString('hex'), originalSeed, 'same seed') t.equals(mf.keys.id, originalID, 'same ID') @@ -96,14 +96,14 @@ test('findOrCreate is idempotent', (t) => { }) }) -test('findOrCreate() a sub feed', (t) => { +test('advanced.findOrCreate() a sub feed', (t) => { const sbot = Testbot() - sbot.metafeeds.findOrCreate(null, null, null, (err, mf) => { - sbot.metafeeds.getRoot((err, mf) => { + sbot.metafeeds.advanced.findOrCreate(null, null, null, (err, mf) => { + sbot.metafeeds.advanced.getRoot((err, mf) => { t.error(err, 'gets rootFeed') // lets create a new chess feed - sbot.metafeeds.findOrCreate( + sbot.metafeeds.advanced.findOrCreate( mf, (f) => f.feedpurpose === 'chess', { @@ -123,11 +123,11 @@ test('findOrCreate() a sub feed', (t) => { test('all FeedDetails have same format', (t) => { const sbot = Testbot() - sbot.metafeeds.findOrCreate(null, null, null, (err, mf) => { + sbot.metafeeds.advanced.findOrCreate(null, null, null, (err, mf) => { if (err) throw err - sbot.metafeeds.getRoot((err, mf) => { + sbot.metafeeds.advanced.getRoot((err, mf) => { if (err) throw err - sbot.metafeeds.findOrCreate( + sbot.metafeeds.advanced.findOrCreate( null, () => true, {}, @@ -140,7 +140,7 @@ test('all FeedDetails have same format', (t) => { 'getRoot and findOrCreate return the same root FeedDetails' ) - sbot.metafeeds.findOrCreate( + sbot.metafeeds.advanced.findOrCreate( mf, (f) => f.feedpurpose === 'chess', { @@ -163,10 +163,10 @@ test('all FeedDetails have same format', (t) => { }) }) -test('findOrCreate() a subfeed under a sub meta feed', (t) => { +test('advanced.findOrCreate() a subfeed under a sub meta feed', (t) => { const sbot = Testbot() - sbot.metafeeds.findOrCreate(null, null, null, (err, rootMF) => { - sbot.metafeeds.findOrCreate( + sbot.metafeeds.advanced.findOrCreate(null, null, null, (err, rootMF) => { + sbot.metafeeds.advanced.findOrCreate( rootMF, (f) => f.feedpurpose === 'indexes', { feedpurpose: 'indexes', feedformat: 'bendybutt-v1' }, @@ -178,7 +178,7 @@ test('findOrCreate() a subfeed under a sub meta feed', (t) => { 'has a bendy butt SSB URI' ) - sbot.metafeeds.findOrCreate( + sbot.metafeeds.advanced.findOrCreate( indexesMF, (f) => f.feedpurpose === 'index', { @@ -209,8 +209,8 @@ test('findOrCreate() a subfeed under a sub meta feed', (t) => { // - indexes // - about async function setupTree(sbot) { - const rootMF = await p(sbot.metafeeds.findOrCreate)() - const chessF = await p(sbot.metafeeds.findOrCreate)( + const rootMF = await p(sbot.metafeeds.advanced.findOrCreate)() + const chessF = await p(sbot.metafeeds.advanced.findOrCreate)( rootMF, (f) => f.feedpurpose === 'chess', { @@ -219,12 +219,12 @@ async function setupTree(sbot) { metadata: { score: 0 }, } ) - const indexesMF = await p(sbot.metafeeds.findOrCreate)( + const indexesMF = await p(sbot.metafeeds.advanced.findOrCreate)( rootMF, (f) => f.feedpurpose === 'indexes', { feedpurpose: 'indexes', feedformat: 'bendybutt-v1' } ) - const indexF = await p(sbot.metafeeds.findOrCreate)( + const indexF = await p(sbot.metafeeds.advanced.findOrCreate)( indexesMF, (f) => f.feedpurpose === 'index', { @@ -237,16 +237,16 @@ async function setupTree(sbot) { return { rootMF, chessF, indexesMF, indexF } } -test('findById', (t) => { +test('advanced.findById', (t) => { const sbot = Testbot() setupTree(sbot).then(({ indexF, indexesMF }) => { - sbot.metafeeds.findById(null, (err, details) => { + sbot.metafeeds.advanced.findById(null, (err, details) => { t.match(err.message, /feedId should be provided/, 'error about feedId') t.notOk(details) testReadAndPersisted(t, sbot, (t, sbot, cb) => { - sbot.metafeeds.findById(indexF.keys.id, (err, details) => { + sbot.metafeeds.advanced.findById(indexF.keys.id, (err, details) => { if (err) return cb(err) t.deepEquals(Object.keys(details), [ @@ -303,7 +303,7 @@ test('branchStream', (t) => { }) }) -test('findAndTombstone and tombstoning branchStream', (t) => { +test('advanced.findAndTombstone and tombstoning branchStream', (t) => { const sbot = Testbot() setupTree(sbot).then(({ rootMF }) => { @@ -368,7 +368,7 @@ test('findAndTombstone and tombstoning branchStream', (t) => { }) ) - sbot.metafeeds.findAndTombstone( + sbot.metafeeds.advanced.findAndTombstone( rootMF, (f) => f.feedpurpose === 'chess', 'This game is too good', @@ -379,19 +379,19 @@ test('findAndTombstone and tombstoning branchStream', (t) => { }) }) -test('findOrCreate() recps', (t) => { +test('advanced.findOrCreate (metadata.recps)', (t) => { const sbot = Testbot() - const testkey = Buffer.from( + const ownKey = Buffer.from( '30720d8f9cbf37f6d7062826f6decac93e308060a8aaaa77e6a4747f40ee1a76', 'hex' ) - sbot.box2.setOwnDMKey(testkey) + sbot.box2.setOwnDMKey(ownKey) testReadAndPersisted(t, sbot, (t, sbot, cb) => { - sbot.metafeeds.findOrCreate((err, mf) => { + sbot.metafeeds.advanced.findOrCreate((err, mf) => { if (err) return cb(err) - sbot.metafeeds.findOrCreate( + sbot.metafeeds.advanced.findOrCreate( mf, (f) => f.feedpurpose === 'private', { @@ -411,3 +411,71 @@ test('findOrCreate() recps', (t) => { }) }) }) + +// sugary top level API + +test('findOrCreate', (t) => { + const sbot = Testbot() + + const details = { + feedpurpose: 'chess', + // feedformat: 'classic', optional + } + + sbot.metafeeds.findOrCreate(details, (err, chessF) => { + if (err) throw err + t.equal(chessF.feedpurpose, details.feedpurpose, 'creates feed') + + sbot.metafeeds.findOrCreate(details, (err, chessF2) => { + if (err) throw err + t.deepEqual(chessF, chessF2, 'finds feed') + + pull( + sbot.metafeeds.branchStream({ root: null, old: true, live: false }), + pull.collect((err, branches) => { + if (err) throw err + // console.log(branches.map(branch => branch.map(f => f[1] && f[1].feedpurpose))) + + t.equal(branches.length, 5, 'correct number of feeds created') + // root, v1, shard, chess (AND MAIN) + + const purposePath = branches + .pop() + .map((f) => f[1] && f[1].feedpurpose) + t.deepEqual(purposePath, [null, 'v1', purposePath[2], 'chess']) + // TODO it would be nice for testing that we could deterministically know the shard + // but I don't know how to fix the "seed" that the root feed is derived from + + sbot.close() + t.end() + }) + ) + }) + }) +}) + +test('findOrCreate (metadata.recps)', (t) => { + const sbot = Testbot() + + const ownKey = Buffer.from( + '30720d8f9cbf37f6d7062826f6decac93e308060a8aaaa77e6a4747f40ee1a76', + 'hex' + ) + sbot.box2.setOwnDMKey(ownKey) + + const details = { + feedpurpose: 'chess', + // feedformat: 'classic', optional + metadata: { + recps: [sbot.id], + }, + } + + sbot.metafeeds.findOrCreate(details, (err, chessF) => { + if (err) throw err + + t.deepEqual(chessF.metadata.recps, [sbot.id], 'creates encrypted subfee') + sbot.close() + t.end() + }) +}) diff --git a/test/pick-shard.test.js b/test/pick-shard.test.js new file mode 100644 index 0000000..354af4a --- /dev/null +++ b/test/pick-shard.test.js @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2022 Mix Irving +// +// SPDX-License-Identifier: Unlicense + +const test = require('tape') +const Keys = require('ssb-keys') + +const pickShard = require('../pick-shard') + +test('pick-shard', (t) => { + const rootFeedId = Keys.generate(null, null, 'bendybutt-v1').id + + t.equal( + pickShard(rootFeedId, 'dog'), + pickShard(rootFeedId, 'dog'), + 'is deterministic' + ) + + const validShards = new Set('0123456789abcdef'.split('')) + let pass = true + // NOTE these are all Strings + for (let i = 0; i < 1600; i++) { + const shard = pickShard(rootFeedId, `test-${i}`) + if (!validShards.has(shard)) pass = false + } + t.equal(pass, true, 'picked shards are nibbles') + + t.end() +})