Squashed commit of the following:

commit fdf2c98
Merge: 1663678 fbe1e39
Author: mixmix <[email protected]>
Date:   Tue Sep 27 13:52:33 2022 +1300

    Merge branch 'auto-shard' of github.com:ssbc/ssb-meta-feeds into auto-shard

commit 1663678
Merge: fd31fb1 70527d5
Author: mixmix <[email protected]>
Date:   Tue Sep 27 13:52:11 2022 +1300

    Merge branch 'atomic-tests-cb-persist' into auto-shard

commit 70527d5
Merge: 375bd22 8c9f464
Author: mixmix <[email protected]>
Date:   Tue Sep 27 13:51:02 2022 +1300

    Merge branch 'atomic-tests-cb-testbot' into atomic-tests-cb-persist

commit 8c9f464
Merge: 8ef1d0e 0e7a624
Author: mixmix <[email protected]>
Date:   Tue Sep 27 13:49:38 2022 +1300

    Merge branch 'master' of github.com:ssbc/ssb-meta-feeds into atomic-tests-cb-testbot

commit fbe1e39
Author: Andre Staltz <[email protected]>
Date:   Wed Sep 21 14:18:03 2022 +0300

    remove a code comment from api.js

commit fe51f97
Author: Andre Staltz <[email protected]>
Date:   Wed Sep 21 14:12:15 2022 +0300

    remove a code comment from lookup.js

commit fd31fb1
Author: mixmix <[email protected]>
Date:   Wed Sep 21 16:18:40 2022 +1200

    tidy

commit 4cad7a8
Author: mixmix <[email protected]>
Date:   Wed Sep 21 14:13:56 2022 +1200

    cover recps case, update README

commit 8ef1d0e
Author: mixmix <[email protected]>
Date:   Wed Sep 21 12:25:43 2022 +1200

    use testbot everywhere, group non-atomic tests

commit de10af1
Merge: 4e5accf 375bd22
Author: mixmix <[email protected]>
Date:   Wed Sep 21 11:44:44 2022 +1200

    Merge branch 'atomic-tests-cb-persist' into auto-shard

commit 4e5accf
Author: mixmix <[email protected]>
Date:   Wed Sep 21 11:40:38 2022 +1200

    fixup

commit 375bd22
Author: mixmix <[email protected]>
Date:   Wed Sep 21 11:21:36 2022 +1200

    add persistence tests to api

commit 13a0f22
Author: Andre Staltz <[email protected]>
Date:   Tue Sep 20 11:15:33 2022 +0300

    revert promise tests to cb tests

commit 8267fa8
Author: mixmix <[email protected]>
Date:   Tue Sep 20 17:33:45 2022 +1200

    fixup

commit ed72484
Author: mixmix <[email protected]>
Date:   Tue Sep 20 17:28:01 2022 +1200

    split out Advanced API to unclutter the README

commit 4409661
Author: mixmix <[email protected]>
Date:   Tue Sep 20 17:18:29 2022 +1200

    fixups

commit 6d4313c
Author: mixmix <[email protected]>
Date:   Wed Sep 14 17:10:01 2022 +1200

    test that shard-feed is created in correct place

commit ba96d7f
Author: mixmix <[email protected]>
Date:   Wed Sep 14 16:44:11 2022 +1200

    add API which auto-shards

commit e4cb0c8
Author: mixmix <[email protected]>
Date:   Tue Sep 20 16:54:41 2022 +1200

    make tests atomic, add persistence tests

api.js

@@ -4,8 +4,17 @@
 
 const run = require('promisify-tuple')
 const debug = require('debug')('ssb:meta-feeds')
+const pickShard = require('./pick-shard')
 
 const alwaysTrue = () => true
+const BB1 = 'bendybutt-v1'
+const v1Details = { feedpurpose: 'v1', feedformat: BB1 }
+const v1Visit = detailsToVisit(v1Details)
+function detailsToVisit(details) {
+  return (feed) =>
+    feed.feedpurpose === details.feedpurpose &&
+    feed.feedformat === details.feedformat
+}
 
 exports.init = function (sbot, config) {
   function filter(metafeed, visit, maybeCB) {
@@ -210,11 +219,39 @@ exports.init = function (sbot, config) {
     }
   }
 
+  function commonFindOrCreate(details, cb) {
+    if (!details.feedformat) details.feedformat = 'classic'
+
+    findOrCreate((err, rootFeed) => {
+      if (err) return cb(err)
+
+      findOrCreate(rootFeed, v1Visit, v1Details, (err, v1Feed) => {
+        if (err) return cb(err)
+
+        const shardDetails = {
+          feedpurpose: pickShard(rootFeed.keys.id, details.feedpurpose),
+          feedformat: BB1,
+        }
+        const shardVisit = detailsToVisit(shardDetails)
+
+        findOrCreate(v1Feed, shardVisit, shardDetails, (err, shardFeed) => {
+          if (err) return cb(err)
+
+          findOrCreate(shardFeed, detailsToVisit(details), details, cb)
+        })
+      })
+    })
+  }
+
   return {
-    getRoot,
-    findOrCreate,
-    findAndTombstone,
-    findById,
     branchStream,
+    findOrCreate: commonFindOrCreate,
+
+    advanced: {
+      getRoot,
+      findOrCreate,
+      findAndTombstone,
+      findById,
+    },
   }
 }

index.js

@@ -2,19 +2,19 @@
 //
 // SPDX-License-Identifier: LGPL-3.0-only
 
+const API = require('./api')
 const Keys = require('./keys')
 const Messages = require('./messages')
+const Lookup = require('./lookup')
 const Query = require('./query')
-const API = require('./api')
 const Validate = require('./validate')
-const FeedsLookup = require('./feeds-lookup')
 
 exports.name = 'metafeeds'
 
 exports.init = function (sbot, config) {
   const messages = Messages.init(sbot, config)
   const query = Query.init(sbot, config)
-  const lookup = FeedsLookup.init(sbot, config)
+  const lookup = Lookup.init(sbot, config)
   const api = API.init(sbot, config)
 
   return {

messages.js

@@ -39,7 +39,7 @@ exports.init = function init(sbot) {
     return {
       feedFormat: 'bendybutt-v1',
       keys: mfKeys,
-      contentKeys: feedKeys,
+      contentKeys: feedKeys, // see ssb-bendy-butt/format.js
       content,
       encryptionFormat: 'box2', // in case metadata.recps is set
     }

package.json

@@ -38,8 +38,8 @@
     "rimraf": "^3.0.2",
     "secret-stack": "^6.4.0",
     "ssb-bendy-butt": "^1.0.1",
-    "ssb-db2": "^6.1.1",
     "ssb-caps": "^1.1.0",
+    "ssb-db2": "^6.1.1",
     "tap-arc": "^0.3.5",
     "tape": "^5.6.0"
   },

pick-shard.js

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: 2022 Mix Irving
+//
+// SPDX-License-Identifier: LGPL-3.0-only
+
+const bfe = require('ssb-bfe')
+const crypto = require('crypto')
+
+module.exports = function pickShard(rootFeedId, idString) {
+  const buf = Buffer.concat([bfe.encode(rootFeedId), bfe.encode(idString)])
+
+  const hash = crypto.createHash('sha256')
+  hash.update(buf)
+
+  return hash.digest('hex')[0]
+}