From 66af62c77ac16c3a6ed60500d2bca8448f5665cc Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Fri, 3 Sep 2021 16:06:07 +0200
Subject: [PATCH 01/50] Expose the clock over a RPC

---
 index.js      | 12 +++++++--
 test/clock.js | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 2 deletions(-)
 create mode 100644 test/clock.js

diff --git a/index.js b/index.js
index 3184a5e..70a03aa 100644
--- a/index.js
+++ b/index.js
@@ -23,7 +23,8 @@ exports.manifest = {
   replicate: 'duplex',
   request: 'sync',
   block: 'sync',
-  peerStatus: 'sync'
+  peerStatus: 'sync',
+  getClock: 'async'
 }
 
 exports.permissions = {
@@ -174,10 +175,17 @@ exports.init = function (sbot, config) {
     return data
   }
 
+  function clock(cb) {
+    initialized.once(() => {
+      cb(null, ebt.state.clock)
+    })
+  }
+
   return {
     request,
     block,
     replicate,
-    peerStatus
+    peerStatus,
+    clock
   }
 }
diff --git a/test/clock.js b/test/clock.js
new file mode 100644
index 0000000..2039361
--- /dev/null
+++ b/test/clock.js
@@ -0,0 +1,74 @@
+const tape = require('tape')
+const crypto = require('crypto')
+const SecretStack = require('secret-stack')
+const sleep = require('util').promisify(setTimeout)
+const pify = require('promisify-4loc')
+const u = require('./misc/util')
+
+const createSsbServer = SecretStack({
+  caps: { shs: crypto.randomBytes(32).toString('base64') }
+})
+  .use(require('ssb-db'))
+  .use(require('../'))
+
+const CONNECTION_TIMEOUT = 500 // ms
+const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
+
+const alice = createSsbServer({
+  temp: 'test-block-alice',
+  timeout: CONNECTION_TIMEOUT,
+  keys: u.keysFor('alice')
+})
+
+const bob = createSsbServer({
+  temp: 'test-block-bob',
+  timeout: CONNECTION_TIMEOUT,
+  keys: u.keysFor('bob')
+})
+
+tape('alice blocks bob, then unblocks', async (t) => {
+  await Promise.all([
+    pify(alice.publish)({ type: 'post', text: 'hello' }),
+    pify(bob.publish)({ type: 'post', text: 'hello' })
+  ])
+
+  const clockAlice = await pify(alice.ebt.clock)()
+  t.equal(clockAlice[alice.id], 1, 'self clock')
+
+  // Self replicate
+  alice.ebt.request(alice.id, true)
+  bob.ebt.request(bob.id, true)
+
+  alice.ebt.request(bob.id, true)
+  bob.ebt.request(alice.id, true)
+
+  await pify(bob.connect)(alice.getAddress())
+
+  await sleep(REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  const clockAliceAfter = await pify(alice.ebt.clock)()
+  t.equal(clockAliceAfter[alice.id], 1, 'clock ok')
+  t.equal(clockAliceAfter[bob.id], 1, 'clock ok')
+
+  const clockBobAfter = await pify(alice.ebt.clock)()
+  t.equal(clockBobAfter[alice.id], 1, 'clock ok')
+  t.equal(clockBobAfter[bob.id], 1, 'clock ok')
+  
+  await pify(alice.publish)({ type: 'post', text: 'hello again' }),
+
+  await sleep(REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  const clockAliceAfter2 = await pify(alice.ebt.clock)()
+  t.equal(clockAliceAfter2[alice.id], 2, 'clock ok')
+
+  const clockBobAfter2 = await pify(alice.ebt.clock)()
+  t.equal(clockBobAfter2[alice.id], 2, 'clock ok')
+  
+  await Promise.all([
+    pify(alice.close)(true),
+    pify(bob.close)(true)
+  ])
+  t.end()
+})

From a20e9d8a813f8a18e45233eee410162cfa367096 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Sat, 4 Sep 2021 21:12:55 +0200
Subject: [PATCH 02/50] Refactor to allow multiple ebts

---
 index.js | 181 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 128 insertions(+), 53 deletions(-)

diff --git a/index.js b/index.js
index 70a03aa..6880044 100644
--- a/index.js
+++ b/index.js
@@ -2,7 +2,7 @@ const path = require('path')
 const pull = require('pull-stream')
 const toPull = require('push-stream-to-pull-stream')
 const EBT = require('epidemic-broadcast-trees')
-const isFeed = require('ssb-ref').isFeed
+const ref = require('ssb-ref')
 const Store = require('lossy-store')
 const toUrlFriendly = require('base64-url').escape
 const getSeverity = require('ssb-network-errors')
@@ -36,7 +36,7 @@ exports.permissions = {
 // there was a bug that caused some peers
 // to request things that weren't feeds.
 // this is fixed, so just ignore anything that isn't a feed.
-function cleanClock (clock) {
+function cleanClock (clock, isFeed) {
   for (const k in clock) {
     if (!isFeed(k)) {
       delete clock[k]
@@ -45,48 +45,88 @@ function cleanClock (clock) {
 }
 
 exports.init = function (sbot, config) {
-  const dir = config.path ? path.join(config.path, 'ebt') : null
-  const store = Store(dir, null, toUrlFriendly)
-
-  const ebt = EBT({
-    logging: config.ebt && config.ebt.logging,
-    id: sbot.id,
-    getClock (id, cb) {
-      store.ensure(id, function () {
-        const clock = store.get(id) || {}
-        cleanClock(clock)
-        cb(null, clock)
-      })
-    },
-    setClock (id, clock) {
-      cleanClock(clock, 'non-feed key when saving clock')
-      store.set(id, clock)
-    },
-    getAt (pair, cb) {
-      sbot.getAtSequence([pair.id, pair.sequence], (err, data) => {
-        cb(err, data ? data.value : null)
-      })
-    },
-    append (msg, cb) {
-      sbot.add(msg, (err, msg) => {
-        cb(err && err.fatal ? err : null, msg)
-      })
-    },
-    isFeed: isFeed
-  })
+  const formats = {
+    'classic': {
+      isMsg(m) {
+        return Number.isInteger(m.sequence) && m.sequence > 0 &&
+          typeof m.author == 'string' && m.content
+      },
+      isFeed: ref.isFeed,
+      getMsgAuthor(msg) {
+        return msg.author
+      },
+      getMsgSequence(msg) {
+        return msg.sequence
+      }
+    }
+  }
+
+  const ebts = {}
+  function addEBT(formatName) {
+    const dirName = 'ebt' + (formatName === 'classic') ? '' : formatName
+    const dir = config.path ? path.join(config.path, dirName) : null
+    const store = Store(dir, null, toUrlFriendly)
+
+    const isFeed = formats[formatName].isFeed
+
+    const ebt = EBT(Object.assign({
+      logging: config.ebt && config.ebt.logging,
+      id: sbot.id,
+      getClock (id, cb) {
+        store.ensure(id, function () {
+          const clock = store.get(id) || {}
+          cleanClock(clock, isFeed)
+          cb(null, clock)
+        })
+      },
+      setClock (id, clock) {
+        cleanClock(clock, isFeed)
+        store.set(id, clock)
+      },
+      getAt (pair, cb) {
+        sbot.getAtSequence([pair.id, pair.sequence], (err, data) => {
+          cb(err, data ? data.value : null)
+        })
+      },
+      append (msg, cb) {
+        sbot.add(msg, (err, msg) => {
+          cb(err && err.fatal ? err : null, msg)
+        })
+      }
+    }, formats[formatName]))
+
+    ebts[formatName] = ebt
+  }
+
+  function getEBT(formatName) {
+    const ebt = ebts[formatName]
+    if (!ebt)
+      throw new Error('Unknown format' + formatName)
+
+    return ebt
+  }
+
+  addEBT('classic')
 
   const initialized = DeferredPromise()
 
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
-    ebt.state.clock = clock || {}
-    ebt.update()
+    for (let format in ebts) {
+      const ebt = ebts[format]
+      // FIXME: do we need to split the clock?
+      ebt.state.clock = clock || {}
+      ebt.update()
+    }
     initialized.resolve()
   })
 
   sbot.post((msg) => {
     initialized.promise.then(() => {
-      ebt.onAppend(msg.value)
+      for (let format in ebts) {
+        if (formats[format].isMsg(msg.value))
+          ebts[format].onAppend(msg.value)
+      }
     })
   })
 
@@ -95,6 +135,7 @@ exports.init = function (sbot, config) {
   if (sbot.progress) {
     hook(sbot.progress, function (fn) {
       const _progress = fn()
+      const ebt = ebts['classic']
       const ebtProg = ebt.progress()
       if (ebtProg.target) _progress.ebt = ebtProg
       return _progress
@@ -105,29 +146,43 @@ exports.init = function (sbot, config) {
     if (rpc.id === sbot.id) return // ssb-client connecting to ssb-server
     if (isClient) {
       initialized.promise.then(() => {
-        const opts = { version: 3 }
-        const local = toPull.duplex(ebt.createStream(rpc.id, opts.version, true))
-        const remote = rpc.ebt.replicate(opts, (networkError) => {
-          if (networkError && getSeverity(networkError) >= 3) {
-            console.error('rpc.ebt.replicate exception:', networkError)
-          }
-        })
-        pull(local, remote, local)
+        for (let format in ebts) {
+          const ebt = ebts[format]
+          const opts = { version: 3, format }
+          const local = toPull.duplex(ebt.createStream(rpc.id, opts.version, true))
+          const remote = rpc.ebt.replicate(opts, (networkError) => {
+            if (networkError && getSeverity(networkError) >= 3) {
+              console.error('rpc.ebt.replicate exception:', networkError)
+            }
+          })
+          pull(local, remote, local)
+        }
       })
     }
   })
 
-  function request (destFeedId, requesting) {
+  function request(destFeedId, requesting, formatName) {
     initialized.promise.then(() => {
-      if (!isFeed(destFeedId)) return
-      ebt.request(destFeedId, requesting)
+      formatName = formatName || 'classic'
+      const format = formats[formatName]
+
+      if (!(format && format.isFeed(destFeedId))) return
+      
+      ebts[formatName].request(destFeedId, requesting)
     })
   }
 
-  function block (origFeedId, destFeedId, blocking) {
+  function block(origFeedId, destFeedId, blocking, formatName) {
     initialized.promise.then(() => {
-      if (!isFeed(origFeedId)) return
-      if (!isFeed(destFeedId)) return
+      formatName = formatName || 'classic'
+      const format = formats[formatName]
+
+      if (!format) return
+      if (!format.isFeed(origFeedId)) return
+      if (!format.isFeed(destFeedId)) return
+
+      const ebt = ebts[formatName]
+
       if (blocking) {
         ebt.block(origFeedId, destFeedId, true)
       } else if (
@@ -145,6 +200,9 @@ exports.init = function (sbot, config) {
       throw new Error('expected ebt.replicate({version: 3})')
     }
 
+    let formatName = opts.format || 'classic'
+    const ebt = getEBT(formatName)
+
     var deferred = pullDefer.duplex()
     initialized.promise.then(() => {
       // `this` refers to the remote peer who called this muxrpc API
@@ -153,14 +211,18 @@ exports.init = function (sbot, config) {
     return deferred
   }
 
-  // get replication status for feeds for this id.
-  function peerStatus (id) {
+  // get replication status for feeds for this id
+  function peerStatus(id, formatName) {
     id = id || sbot.id
+    formatName = formatName || 'classic'
+    const ebt = getEBT(formatName)
+
     const data = {
       id: id,
       seq: ebt.state.clock[id],
       peers: {}
     }
+
     for (const k in ebt.state.peers) {
       const peer = ebt.state.peers[k]
       if (peer.clock[id] != null ||
@@ -172,20 +234,33 @@ exports.init = function (sbot, config) {
         }
       }
     }
+
     return data
   }
 
-  function clock(cb) {
-    initialized.once(() => {
+  function clock(formatName, cb) {
+    if (!cb) {
+      cb = formatName
+      formatName = 'classic'
+    }
+
+    initialized.promise.then(() => {
+      const ebt = getEBT(formatName)
       cb(null, ebt.state.clock)
     })
   }
 
+  function registerFormat(formatName, methods) {
+    formats[formatName] = methods
+    addEBT(formatName)
+  }
+
   return {
     request,
     block,
     replicate,
     peerStatus,
-    clock
+    clock,
+    registerFormat
   }
 }

From d96de1df9365899263f4ef4952c0150620c6c9fd Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Mon, 6 Sep 2021 11:10:06 +0200
Subject: [PATCH 03/50] Update test/clock.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: André Staltz <andre@staltz.com>
---
 test/clock.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/clock.js b/test/clock.js
index 2039361..9e9a673 100644
--- a/test/clock.js
+++ b/test/clock.js
@@ -63,7 +63,7 @@ tape('alice blocks bob, then unblocks', async (t) => {
   const clockAliceAfter2 = await pify(alice.ebt.clock)()
   t.equal(clockAliceAfter2[alice.id], 2, 'clock ok')
 
-  const clockBobAfter2 = await pify(alice.ebt.clock)()
+  const clockBobAfter2 = await pify(bob.ebt.clock)()
   t.equal(clockBobAfter2[alice.id], 2, 'clock ok')
   
   await Promise.all([

From 9dfd1464515aa9ac5cc9b99014c56981f37e55b7 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Mon, 6 Sep 2021 11:10:17 +0200
Subject: [PATCH 04/50] Update test/clock.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: André Staltz <andre@staltz.com>
---
 test/clock.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/clock.js b/test/clock.js
index 9e9a673..a158fd7 100644
--- a/test/clock.js
+++ b/test/clock.js
@@ -51,7 +51,7 @@ tape('alice blocks bob, then unblocks', async (t) => {
   t.equal(clockAliceAfter[alice.id], 1, 'clock ok')
   t.equal(clockAliceAfter[bob.id], 1, 'clock ok')
 
-  const clockBobAfter = await pify(alice.ebt.clock)()
+  const clockBobAfter = await pify(bob.ebt.clock)()
   t.equal(clockBobAfter[alice.id], 1, 'clock ok')
   t.equal(clockBobAfter[bob.id], 1, 'clock ok')
   

From 67991e8af0ed611a6a89ef1dc9409b5820e3acf9 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Mon, 6 Sep 2021 11:11:03 +0200
Subject: [PATCH 05/50] Refactor formats a bit more

---
 index.js      | 11 ++++++++++-
 test/clock.js |  6 +++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/index.js b/index.js
index 6880044..1ba4900 100644
--- a/index.js
+++ b/index.js
@@ -47,16 +47,25 @@ function cleanClock (clock, isFeed) {
 exports.init = function (sbot, config) {
   const formats = {
     'classic': {
+      // used in sbot.post & in ebt:stream to distinguish between
+      // messages and notes
       isMsg(m) {
         return Number.isInteger(m.sequence) && m.sequence > 0 &&
           typeof m.author == 'string' && m.content
       },
+      // used in request, block, cleanClock
       isFeed: ref.isFeed,
+      // used in ebt:events
       getMsgAuthor(msg) {
         return msg.author
       },
+      // used in ebt:events
       getMsgSequence(msg) {
         return msg.sequence
+      },
+      // used in getAt
+      getAtTransform(msg) {
+        return msg ? msg.value : null
       }
     }
   }
@@ -85,7 +94,7 @@ exports.init = function (sbot, config) {
       },
       getAt (pair, cb) {
         sbot.getAtSequence([pair.id, pair.sequence], (err, data) => {
-          cb(err, data ? data.value : null)
+          cb(err, formats[formatName].getAtTransform(data))
         })
       },
       append (msg, cb) {
diff --git a/test/clock.js b/test/clock.js
index a158fd7..0e5df1a 100644
--- a/test/clock.js
+++ b/test/clock.js
@@ -15,18 +15,18 @@ const CONNECTION_TIMEOUT = 500 // ms
 const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 
 const alice = createSsbServer({
-  temp: 'test-block-alice',
+  temp: 'test-clock-alice',
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('alice')
 })
 
 const bob = createSsbServer({
-  temp: 'test-block-bob',
+  temp: 'test-clock-bob',
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('bob')
 })
 
-tape('alice blocks bob, then unblocks', async (t) => {
+tape('clock works', async (t) => {
   await Promise.all([
     pify(alice.publish)({ type: 'post', text: 'hello' }),
     pify(bob.publish)({ type: 'post', text: 'hello' })

From 8f3bb4443a03c32c84b22db7e9aaf78da101738e Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Mon, 6 Sep 2021 23:25:20 +0200
Subject: [PATCH 06/50] Refactor formats a bit and add test

---
 index.js        |  53 +++++++-------
 package.json    |   3 +
 test/formats.js | 187 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 219 insertions(+), 24 deletions(-)
 create mode 100644 test/formats.js

diff --git a/index.js b/index.js
index 1ba4900..84094cb 100644
--- a/index.js
+++ b/index.js
@@ -47,62 +47,67 @@ function cleanClock (clock, isFeed) {
 exports.init = function (sbot, config) {
   const formats = {
     'classic': {
-      // used in sbot.post & in ebt:stream to distinguish between
-      // messages and notes
-      isMsg(m) {
-        return Number.isInteger(m.sequence) && m.sequence > 0 &&
-          typeof m.author == 'string' && m.content
-      },
-      // used in request, block, cleanClock
+      // used in request, block, cleanClock, sbot.post
       isFeed: ref.isFeed,
+      // used in getAt
+      fromDB(msg) {
+        return msg ? msg.value : null
+      },
+      // used in append
+      toDB(msgVal) {
+        return msgVal
+      },
+
+      // used in ebt:stream to distinguish between messages and notes
+      isMsg(msgVal) {
+        return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
+          typeof msgVal.author == 'string' && msgVal.content
+      },
       // used in ebt:events
-      getMsgAuthor(msg) {
-        return msg.author
+      getMsgAuthor(msgVal) {
+        return msgVal.author
       },
       // used in ebt:events
-      getMsgSequence(msg) {
-        return msg.sequence
+      getMsgSequence(msgVal) {
+        return msgVal.sequence
       },
-      // used in getAt
-      getAtTransform(msg) {
-        return msg ? msg.value : null
-      }
     }
   }
 
   const ebts = {}
   function addEBT(formatName) {
-    const dirName = 'ebt' + (formatName === 'classic') ? '' : formatName
+    const dirName = 'ebt' + (formatName === 'classic' ? '' : formatName)
     const dir = config.path ? path.join(config.path, dirName) : null
     const store = Store(dir, null, toUrlFriendly)
 
-    const isFeed = formats[formatName].isFeed
+    const format = formats[formatName]
 
     const ebt = EBT(Object.assign({
+      format: formatName,
       logging: config.ebt && config.ebt.logging,
       id: sbot.id,
       getClock (id, cb) {
         store.ensure(id, function () {
           const clock = store.get(id) || {}
-          cleanClock(clock, isFeed)
+          cleanClock(clock, format.isFeed)
           cb(null, clock)
         })
       },
       setClock (id, clock) {
-        cleanClock(clock, isFeed)
+        cleanClock(clock, format.isFeed)
         store.set(id, clock)
       },
       getAt (pair, cb) {
-        sbot.getAtSequence([pair.id, pair.sequence], (err, data) => {
-          cb(err, formats[formatName].getAtTransform(data))
+        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+          cb(err, format.fromDB(msg))
         })
       },
-      append (msg, cb) {
-        sbot.add(msg, (err, msg) => {
+      append (msgVal, cb) {
+        sbot.add(format.toDB(msgVal), (err, msg) => {
           cb(err && err.fatal ? err : null, msg)
         })
       }
-    }, formats[formatName]))
+    }, format))
 
     ebts[formatName] = ebt
   }
diff --git a/package.json b/package.json
index a3287cd..ed07e76 100644
--- a/package.json
+++ b/package.json
@@ -37,10 +37,13 @@
     "rimraf": "^2.7.1",
     "rng": "^0.2.2",
     "secret-stack": "^6.4.0",
+    "ssb-bendy-butt": "^0.12.2",
     "ssb-client": "^4.9.0",
     "ssb-db": "^19.2.0",
+    "ssb-db2": "github:ssb-ngi-pointer/ssb-db2#support-bendy-butt",
     "ssb-generate": "^1.0.1",
     "ssb-keys": "^8.1.0",
+    "ssb-uri2": "^1.5.2",
     "ssb-validate": "^4.1.4",
     "standardx": "^7.0.0",
     "tap-spec": "^5.0.0",
diff --git a/test/formats.js b/test/formats.js
new file mode 100644
index 0000000..822b86c
--- /dev/null
+++ b/test/formats.js
@@ -0,0 +1,187 @@
+const tape = require('tape')
+const crypto = require('crypto')
+const SecretStack = require('secret-stack')
+const sleep = require('util').promisify(setTimeout)
+const pify = require('promisify-4loc')
+const u = require('./misc/util')
+
+const caps = require('ssb-caps')
+const rimraf = require('rimraf')
+const mkdirp = require('mkdirp')
+const ssbKeys = require('ssb-keys')
+const SSBURI = require('ssb-uri2')
+const bendyButt = require('ssb-bendy-butt')
+
+function createSsbServer() {
+  return SecretStack({ appKey: caps.shs })
+    .use(require('ssb-db2'))
+    .use(require('ssb-db2/compat/ebt'))
+    .use(require('../'))
+}
+
+const CONNECTION_TIMEOUT = 500 // ms
+const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
+
+const aliceDir = '/tmp/test-format-alice'
+rimraf.sync(aliceDir)
+mkdirp.sync(aliceDir)
+
+const alice = createSsbServer().call(null, {
+  path: aliceDir,
+  timeout: CONNECTION_TIMEOUT,
+  keys: u.keysFor('alice'),
+  ebt: {
+    logging: false
+  }
+})
+
+const bobDir = '/tmp/test-format-bob'
+rimraf.sync(bobDir)
+mkdirp.sync(bobDir)
+
+const bob = createSsbServer().call(null, {
+  path: bobDir,
+  timeout: CONNECTION_TIMEOUT,
+  keys: u.keysFor('bob')
+})
+
+console.log("alice", alice.id)
+console.log("bob", bob.id)
+
+tape('multiple formats', async (t) => {
+  const bendyButtMethods = {
+    // used in request, block, cleanClock, sbot.post
+    isFeed: SSBURI.isBendyButtV1FeedSSBURI,
+    // used in getAt
+    fromDB(msg) {
+      return msg ? bendyButt.encode(msg.value) : null
+    },
+    // used in append
+    toDB(msgVal) {
+      return bendyButt.decode(msgVal)
+    },
+
+    // used in ebt:stream to distinguish between messages and notes
+    isMsg(bbVal) {
+      if (!Buffer.isBuffer(bbVal)) return false
+      
+      const msgVal = bendyButt.decode(bbVal)
+      return msgVal &&
+        Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
+        typeof msgVal.author == 'string' && msgVal.content
+    },
+    // used in ebt:events
+    getMsgAuthor(bbVal) {
+      //console.log("bb getMsgAuthor", Buffer.isBuffer(bbVal))
+      if (Buffer.isBuffer(bbVal))
+        return bendyButt.decode(bbVal).author
+      else
+        return bbVal.author
+    },
+    // used in ebt:events
+    getMsgSequence(bbVal) {
+      //console.log("bb getMsgSequence", Buffer.isBuffer(bbVal))
+      if (Buffer.isBuffer(bbVal))
+        return bendyButt.decode(bbVal).sequence
+      else
+        return bbVal.sequence
+    }
+  }
+  
+  alice.ebt.registerFormat('bendybutt', bendyButtMethods)
+  bob.ebt.registerFormat('bendybutt', bendyButtMethods)
+
+  // self replicate
+  alice.ebt.request(alice.id, true)
+  bob.ebt.request(bob.id, true)
+  
+  // publish normal messages
+  await Promise.all([
+    pify(alice.db.publish)({ type: 'post', text: 'hello' }),
+    pify(bob.db.publish)({ type: 'post', text: 'hello' })
+  ])
+
+  function getBBMsg(mainKeys) {
+    // fake some keys
+    const mfKeys = ssbKeys.generate()
+    const classicUri = SSBURI.fromFeedSigil(mfKeys.id)
+    const { type, /* format, */ data } = SSBURI.decompose(classicUri)
+    const bendybuttUri = SSBURI.compose({ type, format: 'bendybutt-v1', data })
+    mfKeys.id = bendybuttUri
+
+    const content = {
+      type: "metafeed/add",
+      feedpurpose: "main",
+      subfeed: mainKeys.id,
+      metafeed: mfKeys.id,
+      tangles: {
+        metafeed: {
+          root: null,
+          previous: null
+        }
+      }
+    }
+
+    const bbmsg = bendyButt.encodeNew(
+      content,
+      mainKeys,
+      mfKeys,
+      1,
+      null,
+      Date.now(),
+      null
+    )
+
+    return bendyButt.decode(bbmsg)
+  }
+
+  const aliceBBMsg = getBBMsg(alice.config.keys)
+  const bobBBMsg = getBBMsg(bob.config.keys)
+
+  const aliceMFId = aliceBBMsg.author
+  const bobMFId = bobBBMsg.author
+
+  // self replicate
+  alice.ebt.request(aliceMFId, true, 'bendybutt')
+  bob.ebt.request(bobMFId, true, 'bendybutt')
+  
+  await Promise.all([
+    pify(alice.add)(aliceBBMsg),
+    pify(bob.add)(bobBBMsg)
+  ])
+
+  alice.ebt.request(bob.id, true)
+  alice.ebt.request(bobMFId, true, 'bendybutt')
+
+  bob.ebt.request(alice.id, true)
+  bob.ebt.request(aliceMFId, true, 'bendybutt')
+
+  await pify(bob.connect)(alice.getAddress())
+
+  await sleep(REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  const clockAlice = await pify(alice.ebt.clock)('classic')
+  const bbClockAlice = await pify(alice.ebt.clock)('bendybutt')
+  
+  t.equal(clockAlice[alice.id], 1, 'clock ok')
+  t.equal(clockAlice[bob.id], 1, 'clock ok')
+
+  t.equal(bbClockAlice[aliceMFId], 1, 'clock ok')
+  t.equal(bbClockAlice[bobMFId], 1, 'clock ok')
+
+  const clockBob = await pify(bob.ebt.clock)('classic')
+  const bbClockBob = await pify(bob.ebt.clock)('bendybutt')
+
+  t.equal(clockBob[alice.id], 1, 'clock ok')
+  t.equal(clockBob[bob.id], 1, 'clock ok')
+
+  t.equal(bbClockBob[aliceMFId], 1, 'clock ok')
+  t.equal(bbClockBob[bobMFId], 1, 'clock ok')
+  
+  await Promise.all([
+    pify(alice.close)(true),
+    pify(bob.close)(true)
+  ])
+  t.end()
+})

From a3534bf186b51af7ca250e14b43047af6b2671ea Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 7 Sep 2021 14:03:51 +0200
Subject: [PATCH 07/50] More format tests, fix some bugs

---
 index.js        |  17 ++--
 test/formats.js | 233 ++++++++++++++++++++++++++++++------------------
 2 files changed, 155 insertions(+), 95 deletions(-)

diff --git a/index.js b/index.js
index 84094cb..65d458e 100644
--- a/index.js
+++ b/index.js
@@ -61,7 +61,7 @@ exports.init = function (sbot, config) {
       // used in ebt:stream to distinguish between messages and notes
       isMsg(msgVal) {
         return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
-          typeof msgVal.author == 'string' && msgVal.content
+          ref.isFeed(msgVal.author) && msgVal.content
       },
       // used in ebt:events
       getMsgAuthor(msgVal) {
@@ -83,7 +83,6 @@ exports.init = function (sbot, config) {
     const format = formats[formatName]
 
     const ebt = EBT(Object.assign({
-      format: formatName,
       logging: config.ebt && config.ebt.logging,
       id: sbot.id,
       getClock (id, cb) {
@@ -126,10 +125,14 @@ exports.init = function (sbot, config) {
 
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
-    for (let format in ebts) {
-      const ebt = ebts[format]
-      // FIXME: do we need to split the clock?
-      ebt.state.clock = clock || {}
+    for (let formatName in ebts) {
+      const format = formats[formatName]
+      const ebt = ebts[formatName]
+      validClock = {}
+      for (let k in clock)
+        if (format.isFeed(k))
+          validClock[k] = clock[k]
+      ebt.state.clock = validClock
       ebt.update()
     }
     initialized.resolve()
@@ -138,7 +141,7 @@ exports.init = function (sbot, config) {
   sbot.post((msg) => {
     initialized.promise.then(() => {
       for (let format in ebts) {
-        if (formats[format].isMsg(msg.value))
+        if (formats[format].isFeed(msg.value.author))
           ebts[format].onAppend(msg.value)
       }
     })
diff --git a/test/formats.js b/test/formats.js
index 822b86c..ad3c1bc 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -26,7 +26,7 @@ const aliceDir = '/tmp/test-format-alice'
 rimraf.sync(aliceDir)
 mkdirp.sync(aliceDir)
 
-const alice = createSsbServer().call(null, {
+let alice = createSsbServer().call(null, {
   path: aliceDir,
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('alice'),
@@ -39,55 +39,90 @@ const bobDir = '/tmp/test-format-bob'
 rimraf.sync(bobDir)
 mkdirp.sync(bobDir)
 
-const bob = createSsbServer().call(null, {
+let bob = createSsbServer().call(null, {
   path: bobDir,
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('bob')
 })
 
-console.log("alice", alice.id)
-console.log("bob", bob.id)
+const bendyButtMethods = {
+  // used in request, block, cleanClock, sbot.post
+  isFeed: SSBURI.isBendyButtV1FeedSSBURI,
+  // used in getAt
+  fromDB(msg) {
+    return msg ? bendyButt.encode(msg.value) : null
+  },
+  // used in append
+  toDB(msgVal) {
+    return bendyButt.decode(msgVal)
+  },
+
+  // used in ebt:stream to distinguish between messages and notes
+  isMsg(bbVal) {
+    if (!Buffer.isBuffer(bbVal)) return false
+
+    const msgVal = bendyButt.decode(bbVal)
+    return msgVal &&
+      Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
+      typeof msgVal.author == 'string' && msgVal.content
+  },
+  // used in ebt:events
+  getMsgAuthor(bbVal) {
+    //console.log("bb getMsgAuthor", Buffer.isBuffer(bbVal))
+    if (Buffer.isBuffer(bbVal))
+      return bendyButt.decode(bbVal).author
+    else
+      return bbVal.author
+  },
+  // used in ebt:events
+  getMsgSequence(bbVal) {
+    //console.log("bb getMsgSequence", Buffer.isBuffer(bbVal))
+    if (Buffer.isBuffer(bbVal))
+      return bendyButt.decode(bbVal).sequence
+    else
+      return bbVal.sequence
+  }
+}
 
-tape('multiple formats', async (t) => {
-  const bendyButtMethods = {
-    // used in request, block, cleanClock, sbot.post
-    isFeed: SSBURI.isBendyButtV1FeedSSBURI,
-    // used in getAt
-    fromDB(msg) {
-      return msg ? bendyButt.encode(msg.value) : null
-    },
-    // used in append
-    toDB(msgVal) {
-      return bendyButt.decode(msgVal)
-    },
-
-    // used in ebt:stream to distinguish between messages and notes
-    isMsg(bbVal) {
-      if (!Buffer.isBuffer(bbVal)) return false
-      
-      const msgVal = bendyButt.decode(bbVal)
-      return msgVal &&
-        Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
-        typeof msgVal.author == 'string' && msgVal.content
-    },
-    // used in ebt:events
-    getMsgAuthor(bbVal) {
-      //console.log("bb getMsgAuthor", Buffer.isBuffer(bbVal))
-      if (Buffer.isBuffer(bbVal))
-        return bendyButt.decode(bbVal).author
-      else
-        return bbVal.author
-    },
-    // used in ebt:events
-    getMsgSequence(bbVal) {
-      //console.log("bb getMsgSequence", Buffer.isBuffer(bbVal))
-      if (Buffer.isBuffer(bbVal))
-        return bendyButt.decode(bbVal).sequence
-      else
-        return bbVal.sequence
+function getBBMsg(mainKeys) {
+  // fake some keys
+  const mfKeys = ssbKeys.generate()
+  const classicUri = SSBURI.fromFeedSigil(mfKeys.id)
+  const { type, /* format, */ data } = SSBURI.decompose(classicUri)
+  const bendybuttUri = SSBURI.compose({ type, format: 'bendybutt-v1', data })
+  mfKeys.id = bendybuttUri
+
+  const content = {
+    type: "metafeed/add",
+    feedpurpose: "main",
+    subfeed: mainKeys.id,
+    metafeed: mfKeys.id,
+    tangles: {
+      metafeed: {
+        root: null,
+        previous: null
+      }
     }
   }
-  
+
+  const bbmsg = bendyButt.encodeNew(
+    content,
+    mainKeys,
+    mfKeys,
+    1,
+    null,
+    Date.now(),
+    null
+  )
+
+  return bendyButt.decode(bbmsg)
+}
+
+// need them later
+let aliceMFId
+let bobMFId
+
+tape('multiple formats', async (t) => {
   alice.ebt.registerFormat('bendybutt', bendyButtMethods)
   bob.ebt.registerFormat('bendybutt', bendyButtMethods)
 
@@ -101,45 +136,11 @@ tape('multiple formats', async (t) => {
     pify(bob.db.publish)({ type: 'post', text: 'hello' })
   ])
 
-  function getBBMsg(mainKeys) {
-    // fake some keys
-    const mfKeys = ssbKeys.generate()
-    const classicUri = SSBURI.fromFeedSigil(mfKeys.id)
-    const { type, /* format, */ data } = SSBURI.decompose(classicUri)
-    const bendybuttUri = SSBURI.compose({ type, format: 'bendybutt-v1', data })
-    mfKeys.id = bendybuttUri
-
-    const content = {
-      type: "metafeed/add",
-      feedpurpose: "main",
-      subfeed: mainKeys.id,
-      metafeed: mfKeys.id,
-      tangles: {
-        metafeed: {
-          root: null,
-          previous: null
-        }
-      }
-    }
-
-    const bbmsg = bendyButt.encodeNew(
-      content,
-      mainKeys,
-      mfKeys,
-      1,
-      null,
-      Date.now(),
-      null
-    )
-
-    return bendyButt.decode(bbmsg)
-  }
-
   const aliceBBMsg = getBBMsg(alice.config.keys)
   const bobBBMsg = getBBMsg(bob.config.keys)
 
-  const aliceMFId = aliceBBMsg.author
-  const bobMFId = bobBBMsg.author
+  aliceMFId = aliceBBMsg.author
+  bobMFId = bobBBMsg.author
 
   // self replicate
   alice.ebt.request(aliceMFId, true, 'bendybutt')
@@ -161,23 +162,79 @@ tape('multiple formats', async (t) => {
   await sleep(REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
+  const expectedClassicClock = {
+    [alice.id]: 1,
+    [bob.id]: 1
+  }
+  const expectedBBClock = {
+    [aliceMFId]: 1,
+    [bobMFId]: 1
+  }
+
   const clockAlice = await pify(alice.ebt.clock)('classic')
-  const bbClockAlice = await pify(alice.ebt.clock)('bendybutt')
-  
-  t.equal(clockAlice[alice.id], 1, 'clock ok')
-  t.equal(clockAlice[bob.id], 1, 'clock ok')
+  t.deepEqual(clockAlice, expectedClassicClock, 'alice correct classic clock')
 
-  t.equal(bbClockAlice[aliceMFId], 1, 'clock ok')
-  t.equal(bbClockAlice[bobMFId], 1, 'clock ok')
+  const bbClockAlice = await pify(alice.ebt.clock)('bendybutt')
+  t.deepEqual(bbClockAlice, expectedBBClock, 'alice correct bb clock')
 
   const clockBob = await pify(bob.ebt.clock)('classic')
+  t.deepEqual(clockBob, expectedClassicClock, 'bob correct classic clock')
+
   const bbClockBob = await pify(bob.ebt.clock)('bendybutt')
+  t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
+
+  await Promise.all([
+    pify(alice.close)(true),
+    pify(bob.close)(true)
+  ])
+  t.end()
+})
 
-  t.equal(clockBob[alice.id], 1, 'clock ok')
-  t.equal(clockBob[bob.id], 1, 'clock ok')
+tape('multiple formats restart', async (t) => {
+  alice = createSsbServer().call(null, {
+    path: aliceDir,
+    timeout: CONNECTION_TIMEOUT,
+    keys: u.keysFor('alice'),
+    ebt: {
+      logging: false
+    }
+  })
+
+  bob = createSsbServer().call(null, {
+    path: bobDir,
+    timeout: CONNECTION_TIMEOUT,
+    keys: u.keysFor('bob')
+  })
 
-  t.equal(bbClockBob[aliceMFId], 1, 'clock ok')
-  t.equal(bbClockBob[bobMFId], 1, 'clock ok')
+  alice.ebt.registerFormat('bendybutt', bendyButtMethods)
+  bob.ebt.registerFormat('bendybutt', bendyButtMethods)
+
+  // self replicate
+  alice.ebt.request(alice.id, true)
+  alice.ebt.request(aliceMFId, true, 'bendybutt')
+  bob.ebt.request(bob.id, true)
+  bob.ebt.request(bobMFId, true, 'bendybutt')
+
+  const expectedClassicClock = {
+    [alice.id]: 1,
+    [bob.id]: 1
+  }
+  const expectedBBClock = {
+    [aliceMFId]: 1,
+    [bobMFId]: 1
+  }
+
+  const clockAlice = await pify(alice.ebt.clock)('classic')
+  t.deepEqual(clockAlice, expectedClassicClock, 'alice correct classic clock')
+
+  const bbClockAlice = await pify(alice.ebt.clock)('bendybutt')
+  t.deepEqual(bbClockAlice, expectedBBClock, 'alice correct bb clock')
+
+  const clockBob = await pify(bob.ebt.clock)('classic')
+  t.deepEqual(clockBob, expectedClassicClock, 'bob correct classic clock')
+
+  const bbClockBob = await pify(bob.ebt.clock)('bendybutt')
+  t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
   
   await Promise.all([
     pify(alice.close)(true),

From 6ada5fe855346ada018987ce4a142dd01fc41c0c Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 7 Sep 2021 16:54:18 +0200
Subject: [PATCH 08/50] Use replicateFormat for newer feed formats, fix clock
 manifest

---
 index.js | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/index.js b/index.js
index 65d458e..eb53b15 100644
--- a/index.js
+++ b/index.js
@@ -21,15 +21,16 @@ exports.version = '1.0.0'
 
 exports.manifest = {
   replicate: 'duplex',
+  replicateFormat: 'duplex',
   request: 'sync',
   block: 'sync',
   peerStatus: 'sync',
-  getClock: 'async'
+  clock: 'async'
 }
 
 exports.permissions = {
   anonymous: {
-    allow: ['replicate']
+    allow: ['replicate', 'replicateFormat', 'clock']
   }
 }
 
@@ -167,7 +168,12 @@ exports.init = function (sbot, config) {
           const ebt = ebts[format]
           const opts = { version: 3, format }
           const local = toPull.duplex(ebt.createStream(rpc.id, opts.version, true))
-          const remote = rpc.ebt.replicate(opts, (networkError) => {
+
+          // for backwards compatibility we always replicate classic
+          // feeds using existing replicate RPC
+          const replicate = (format === 'classic' ? rpc.ebt.replicate : rpc.ebt.replicateFormat)
+
+          const remote = replicate(opts, (networkError) => {
             if (networkError && getSeverity(networkError) >= 3) {
               console.error('rpc.ebt.replicate exception:', networkError)
             }
@@ -212,7 +218,7 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function replicate (opts) {
+  function replicateFormat(opts) {
     if (opts.version !== 3) {
       throw new Error('expected ebt.replicate({version: 3})')
     }
@@ -275,7 +281,8 @@ exports.init = function (sbot, config) {
   return {
     request,
     block,
-    replicate,
+    replicate: replicateFormat,
+    replicateFormat,
     peerStatus,
     clock,
     registerFormat

From 2f609a4a0394a062218d7f3799cbbda7665090fe Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 8 Sep 2021 15:04:35 +0200
Subject: [PATCH 09/50] Index test

---
 index.js        |  27 ++++----
 test/formats.js | 176 ++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 177 insertions(+), 26 deletions(-)

diff --git a/index.js b/index.js
index eb53b15..135e6c3 100644
--- a/index.js
+++ b/index.js
@@ -50,13 +50,15 @@ exports.init = function (sbot, config) {
     'classic': {
       // used in request, block, cleanClock, sbot.post
       isFeed: ref.isFeed,
-      // used in getAt
-      fromDB(msg) {
-        return msg ? msg.value : null
+      getAtSequence(sbot, pair, cb) {
+        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+          cb(err, msg ? msg.value : null)
+        })
       },
-      // used in append
-      toDB(msgVal) {
-        return msgVal
+      appendMsg(sbot, msgVal, cb) {
+        sbot.add(msgVal, (err, msg) => {
+          cb(err && err.fatal ? err : null, msg)
+        })
       },
 
       // used in ebt:stream to distinguish between messages and notes
@@ -98,14 +100,10 @@ exports.init = function (sbot, config) {
         store.set(id, clock)
       },
       getAt (pair, cb) {
-        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
-          cb(err, format.fromDB(msg))
-        })
+        format.getAtSequence(sbot, pair, cb)
       },
       append (msgVal, cb) {
-        sbot.add(format.toDB(msgVal), (err, msg) => {
-          cb(err && err.fatal ? err : null, msg)
-        })
+        format.appendMsg(sbot, msgVal, cb)
       }
     }, format))
 
@@ -129,10 +127,12 @@ exports.init = function (sbot, config) {
     for (let formatName in ebts) {
       const format = formats[formatName]
       const ebt = ebts[formatName]
+
       validClock = {}
       for (let k in clock)
         if (format.isFeed(k))
           validClock[k] = clock[k]
+
       ebt.state.clock = validClock
       ebt.update()
     }
@@ -285,6 +285,7 @@ exports.init = function (sbot, config) {
     replicateFormat,
     peerStatus,
     clock,
-    registerFormat
+    registerFormat,
+    formats
   }
 }
diff --git a/test/formats.js b/test/formats.js
index ad3c1bc..3010508 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -30,9 +30,6 @@ let alice = createSsbServer().call(null, {
   path: aliceDir,
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('alice'),
-  ebt: {
-    logging: false
-  }
 })
 
 const bobDir = '/tmp/test-format-bob'
@@ -48,13 +45,15 @@ let bob = createSsbServer().call(null, {
 const bendyButtMethods = {
   // used in request, block, cleanClock, sbot.post
   isFeed: SSBURI.isBendyButtV1FeedSSBURI,
-  // used in getAt
-  fromDB(msg) {
-    return msg ? bendyButt.encode(msg.value) : null
+  getAtSequence(sbot, pair, cb) {
+    sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+      cb(err, msg ? bendyButt.encode(msg.value) : null)
+    })
   },
-  // used in append
-  toDB(msgVal) {
-    return bendyButt.decode(msgVal)
+  appendMsg(sbot, msgVal, cb) {
+    sbot.add(bendyButt.decode(msgVal), (err, msg) => {
+      cb(err && err.fatal ? err : null, msg)
+    })
   },
 
   // used in ebt:stream to distinguish between messages and notes
@@ -195,9 +194,6 @@ tape('multiple formats restart', async (t) => {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
     keys: u.keysFor('alice'),
-    ebt: {
-      logging: false
-    }
   })
 
   bob = createSsbServer().call(null, {
@@ -214,7 +210,7 @@ tape('multiple formats restart', async (t) => {
   alice.ebt.request(aliceMFId, true, 'bendybutt')
   bob.ebt.request(bob.id, true)
   bob.ebt.request(bobMFId, true, 'bendybutt')
-
+  
   const expectedClassicClock = {
     [alice.id]: 1,
     [bob.id]: 1
@@ -242,3 +238,157 @@ tape('multiple formats restart', async (t) => {
   ])
   t.end()
 })
+
+let indexedFeeds = []
+
+// first solution:
+// replicating index feeds using classic and indexed using a special
+// format does not work because ebt:events expects messages to come in
+// order. Furthermore getting the vector clock of this indexed ebt on
+// start is also a bit complicated
+
+// second solution:
+// - don't send indexes over classic unless you only replicate the index
+// - use a special indexed format that sends both index + indexed as 1
+//   message with indexed as payload
+// - this requires that we are able to add 2 messages in a
+//   transaction. Should be doable, but requires changes all the way
+//   down the stack
+
+const indexedFeedMethods = Object.assign(
+  {}, alice.ebt.formats['classic'], {
+    isFeed(author) {
+      return indexedFeeds.includes(author)
+    },
+    appendMsg(sbot, msgVal, cb) {
+      const payload = msgVal.content.indexed.payload
+      delete msgVal.content.indexed.payload
+      sbot.add(msgVal, (err, msg) => {
+        console.log("add index?", err)
+        if (err) return cb(err)
+        sbot.add(payload, (err, indexedMsg) => {
+          console.log("add payload?", err)
+          if (err) return cb(err)
+          else cb(null, msg)
+        })
+      })
+    },
+    getAtSequence(sbot, pair, cb) {
+      sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+        if (err) return cb(err)
+        const { author, sequence } = msg.value.content.indexed
+        sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
+          if (err) return cb(err)
+
+          // add referenced message as payload
+          msg.value.content.indexed.payload = indexedMsg.value
+
+          cb(null, msg.value)
+        })
+      })
+    }
+  }
+)
+
+tape('index format', async (t) => {
+  const aliceIndexKey = ssbKeys.generate()
+  const bobIndexKey = ssbKeys.generate()
+
+  indexedFeeds.push(aliceIndexKey.id)
+  indexedFeeds.push(bobIndexKey.id)
+
+  alice = createSsbServer().call(null, {
+    path: aliceDir,
+    timeout: CONNECTION_TIMEOUT,
+    keys: u.keysFor('alice'),
+    ebt: { logging: true }
+  })
+
+  bob = createSsbServer().call(null, {
+    path: bobDir,
+    timeout: CONNECTION_TIMEOUT,
+    keys: u.keysFor('bob')
+  })
+
+  alice.ebt.registerFormat('indexedfeed', indexedFeedMethods)
+  bob.ebt.registerFormat('indexedfeed', indexedFeedMethods)
+
+  // publish a few more messages
+  const res = await Promise.all([
+    pify(alice.db.publish)({ type: 'dog', name: 'Buff' }),
+    pify(alice.db.publish)({ type: 'post', text: 'hello 2' }),
+    pify(bob.db.publish)({ type: 'dog', name: 'Biff' }),
+    pify(bob.db.publish)({ type: 'post', text: 'hello 2' })
+  ])
+
+  // index the dog messages
+  await Promise.all([
+    pify(alice.db.publishAs)(aliceIndexKey, {
+      type: 'metafeed/index',
+      indexed: {
+        key: res[0].key,
+        author: alice.id,
+        sequence: res[0].value.sequence
+      }
+    }),
+    pify(bob.db.publishAs)(bobIndexKey, {
+      type: 'metafeed/index',
+      indexed: {
+        key: res[2].key,
+        author: bob.id,
+        sequence: res[2].value.sequence
+      }
+    })
+  ])
+
+  // self replicate
+  alice.ebt.request(alice.id, true)
+  alice.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+  bob.ebt.request(bob.id, true)
+  bob.ebt.request(bobIndexKey.id, true, 'indexedfeed')
+
+  // only replicate index feeds
+  alice.ebt.request(bobIndexKey.id, true, 'indexedfeed')
+  bob.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+
+  await pify(bob.connect)(alice.getAddress())
+
+  await sleep(2 * REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  // we should only get the dog message and not second post
+  const expectedAliceClassicClock = {
+    [alice.id]: 3,
+    [aliceIndexKey.id]: 1,
+    [bob.id]: 2,
+    [bobIndexKey.id]: 1
+  }
+  const expectedBobClassicClock = {
+    [alice.id]: 2,
+    [aliceIndexKey.id]: 1,
+    [bob.id]: 3,
+    [bobIndexKey.id]: 1
+  }
+  const expectedIndexClock = {
+    [aliceIndexKey.id]: 1,
+    [bobIndexKey.id]: 1
+  }
+
+  const clockAlice = await pify(alice.ebt.clock)('classic')
+  t.deepEqual(clockAlice, expectedAliceClassicClock, 'alice correct classic clock')
+
+  const indexClockAlice = await pify(alice.ebt.clock)('indexedfeed')
+  t.deepEqual(indexClockAlice, expectedIndexClock, 'alice correct index clock')
+
+  const clockBob = await pify(bob.ebt.clock)('classic')
+  t.deepEqual(clockBob, expectedBobClassicClock, 'bob correct classic clock')
+
+  const indexClockBob = await pify(bob.ebt.clock)('indexedfeed')
+  t.deepEqual(indexClockBob, expectedIndexClock, 'bob correct index clock')
+
+  await Promise.all([
+    pify(alice.close)(true),
+    pify(bob.close)(true)
+  ])
+  t.end()
+})

From c171049688d0b0e83c1ea504ba13f0f593bbd874 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 00:21:10 +0200
Subject: [PATCH 10/50] Add sliced index replication test

---
 index.js        |   8 +++
 test/formats.js | 158 +++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 130 insertions(+), 36 deletions(-)

diff --git a/index.js b/index.js
index 135e6c3..1f5d78b 100644
--- a/index.js
+++ b/index.js
@@ -273,6 +273,13 @@ exports.init = function (sbot, config) {
     })
   }
 
+  function setClockForSlicedReplication(format, feed, sequence) {
+    initialized.promise.then(() => {
+      const ebt = getEBT(format)
+      ebt.state.clock[feed] = sequence
+    })
+  }
+
   function registerFormat(formatName, methods) {
     formats[formatName] = methods
     addEBT(formatName)
@@ -285,6 +292,7 @@ exports.init = function (sbot, config) {
     replicateFormat,
     peerStatus,
     clock,
+    setClockForSlicedReplication,
     registerFormat,
     formats
   }
diff --git a/test/formats.js b/test/formats.js
index 3010508..6e3efe3 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -11,6 +11,7 @@ const mkdirp = require('mkdirp')
 const ssbKeys = require('ssb-keys')
 const SSBURI = require('ssb-uri2')
 const bendyButt = require('ssb-bendy-butt')
+const { where, author, toPromise } = require('ssb-db2/operators')
 
 function createSsbServer() {
   return SecretStack({ appKey: caps.shs })
@@ -128,7 +129,7 @@ tape('multiple formats', async (t) => {
   // self replicate
   alice.ebt.request(alice.id, true)
   bob.ebt.request(bob.id, true)
-  
+
   // publish normal messages
   await Promise.all([
     pify(alice.db.publish)({ type: 'post', text: 'hello' }),
@@ -144,7 +145,7 @@ tape('multiple formats', async (t) => {
   // self replicate
   alice.ebt.request(aliceMFId, true, 'bendybutt')
   bob.ebt.request(bobMFId, true, 'bendybutt')
-  
+
   await Promise.all([
     pify(alice.add)(aliceBBMsg),
     pify(bob.add)(bobBBMsg)
@@ -210,7 +211,7 @@ tape('multiple formats restart', async (t) => {
   alice.ebt.request(aliceMFId, true, 'bendybutt')
   bob.ebt.request(bob.id, true)
   bob.ebt.request(bobMFId, true, 'bendybutt')
-  
+
   const expectedClassicClock = {
     [alice.id]: 1,
     [bob.id]: 1
@@ -231,7 +232,7 @@ tape('multiple formats restart', async (t) => {
 
   const bbClockBob = await pify(bob.ebt.clock)('bendybutt')
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
-  
+
   await Promise.all([
     pify(alice.close)(true),
     pify(bob.close)(true)
@@ -263,11 +264,9 @@ const indexedFeedMethods = Object.assign(
     appendMsg(sbot, msgVal, cb) {
       const payload = msgVal.content.indexed.payload
       delete msgVal.content.indexed.payload
-      sbot.add(msgVal, (err, msg) => {
-        console.log("add index?", err)
+      sbot.db.add(msgVal, (err, msg) => {
         if (err) return cb(err)
-        sbot.add(payload, (err, indexedMsg) => {
-          console.log("add payload?", err)
+        sbot.db.addOOO(payload, (err, indexedMsg) => {
           if (err) return cb(err)
           else cb(null, msg)
         })
@@ -290,8 +289,9 @@ const indexedFeedMethods = Object.assign(
   }
 )
 
+const aliceIndexKey = ssbKeys.generate()
+
 tape('index format', async (t) => {
-  const aliceIndexKey = ssbKeys.generate()
   const bobIndexKey = ssbKeys.generate()
 
   indexedFeeds.push(aliceIndexKey.id)
@@ -300,8 +300,7 @@ tape('index format', async (t) => {
   alice = createSsbServer().call(null, {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('alice'),
-    ebt: { logging: true }
+    keys: u.keysFor('alice')
   })
 
   bob = createSsbServer().call(null, {
@@ -315,10 +314,10 @@ tape('index format', async (t) => {
 
   // publish a few more messages
   const res = await Promise.all([
-    pify(alice.db.publish)({ type: 'dog', name: 'Buff' }),
     pify(alice.db.publish)({ type: 'post', text: 'hello 2' }),
-    pify(bob.db.publish)({ type: 'dog', name: 'Biff' }),
-    pify(bob.db.publish)({ type: 'post', text: 'hello 2' })
+    pify(alice.db.publish)({ type: 'dog', name: 'Buff' }),
+    pify(bob.db.publish)({ type: 'post', text: 'hello 2' }),
+    pify(bob.db.publish)({ type: 'dog', name: 'Biff' })
   ])
 
   // index the dog messages
@@ -326,17 +325,17 @@ tape('index format', async (t) => {
     pify(alice.db.publishAs)(aliceIndexKey, {
       type: 'metafeed/index',
       indexed: {
-        key: res[0].key,
+        key: res[1].key,
         author: alice.id,
-        sequence: res[0].value.sequence
+        sequence: res[1].value.sequence
       }
     }),
     pify(bob.db.publishAs)(bobIndexKey, {
       type: 'metafeed/index',
       indexed: {
-        key: res[2].key,
+        key: res[3].key,
         author: bob.id,
-        sequence: res[2].value.sequence
+        sequence: res[3].value.sequence
       }
     })
   ])
@@ -357,32 +356,26 @@ tape('index format', async (t) => {
   t.pass('wait for replication to complete')
 
   // we should only get the dog message and not second post
-  const expectedAliceClassicClock = {
-    [alice.id]: 3,
-    [aliceIndexKey.id]: 1,
-    [bob.id]: 2,
-    [bobIndexKey.id]: 1
-  }
-  const expectedBobClassicClock = {
-    [alice.id]: 2,
-    [aliceIndexKey.id]: 1,
-    [bob.id]: 3,
-    [bobIndexKey.id]: 1
-  }
+  const aliceBobMessages = await alice.db.query(
+    where(author(bob.id)),
+    toPromise()
+  )
+  t.equal(aliceBobMessages.length, 2, 'alice correct messages from bob')
+
+  const bobAliceMessages = await bob.db.query(
+    where(author(alice.id)),
+    toPromise()
+  )
+  t.equal(bobAliceMessages.length, 2, 'bob correct messages from alice')
+
   const expectedIndexClock = {
     [aliceIndexKey.id]: 1,
     [bobIndexKey.id]: 1
   }
 
-  const clockAlice = await pify(alice.ebt.clock)('classic')
-  t.deepEqual(clockAlice, expectedAliceClassicClock, 'alice correct classic clock')
-
   const indexClockAlice = await pify(alice.ebt.clock)('indexedfeed')
   t.deepEqual(indexClockAlice, expectedIndexClock, 'alice correct index clock')
 
-  const clockBob = await pify(bob.ebt.clock)('classic')
-  t.deepEqual(clockBob, expectedBobClassicClock, 'bob correct classic clock')
-
   const indexClockBob = await pify(bob.ebt.clock)('indexedfeed')
   t.deepEqual(indexClockBob, expectedIndexClock, 'bob correct index clock')
 
@@ -392,3 +385,96 @@ tape('index format', async (t) => {
   ])
   t.end()
 })
+
+const sliceIndexedFeedMethods = Object.assign(
+  {}, alice.ebt.formats['classic'], {
+    isFeed(author) {
+      return indexedFeeds.includes(author)
+    },
+    appendMsg(sbot, msgVal, cb) {
+      const payload = msgVal.content.indexed.payload
+      delete msgVal.content.indexed.payload
+      sbot.db.addOOO(msgVal, (err, msg) => {
+        if (err) return cb(err)
+        sbot.db.addOOO(payload, (err, indexedMsg) => {
+          if (err) return cb(err)
+          else cb(null, msg)
+        })
+      })
+    },
+    getAtSequence(sbot, pair, cb) {
+      sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+        if (err) return cb(err)
+        const { author, sequence } = msg.value.content.indexed
+        sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
+          if (err) return cb(err)
+
+          // add referenced message as payload
+          msg.value.content.indexed.payload = indexedMsg.value
+
+          cb(null, msg.value)
+        })
+      })
+    }
+  }
+)
+
+tape('sliced index replication', async (t) => {
+  alice = createSsbServer().call(null, {
+    path: aliceDir,
+    timeout: CONNECTION_TIMEOUT,
+    keys: u.keysFor('alice')
+  })
+
+  const carolDir = '/tmp/test-format-carol'
+  rimraf.sync(carolDir)
+  mkdirp.sync(carolDir)
+
+  let carol = createSsbServer().call(null, {
+    path: carolDir,
+    timeout: CONNECTION_TIMEOUT,
+    keys: u.keysFor('carol')
+  })
+
+  alice.ebt.registerFormat('indexedfeed', sliceIndexedFeedMethods)
+  carol.ebt.registerFormat('indexedfeed', sliceIndexedFeedMethods)
+
+  // self replicate
+  alice.ebt.request(alice.id, true)
+  alice.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+  carol.ebt.request(carol.id, true)
+
+  // publish a few more messages
+  const res = await pify(alice.db.publish)({ type: 'dog', name: 'Buffy' })
+
+  // index the new dog message
+  await pify(alice.db.publishAs)(aliceIndexKey, {
+      type: 'metafeed/index',
+      indexed: {
+        key: res.key,
+        author: alice.id,
+        sequence: res.value.sequence
+      }
+  })
+
+  await pify(carol.connect)(alice.getAddress())
+
+  const clockAlice = await pify(alice.ebt.clock)('indexedfeed')
+  t.equal(clockAlice[aliceIndexKey.id], 2, 'alice correct index clock')
+
+  carol.ebt.setClockForSlicedReplication('indexedfeed', aliceIndexKey.id,
+                                         1)
+  carol.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+
+  await sleep(REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  const carolMessages = await carol.db.query(toPromise())
+  t.equal(carolMessages.length, 2, '1 index + 1 indexed message')
+
+  await Promise.all([
+    pify(alice.close)(true),
+    pify(carol.close)(true)
+  ])
+  t.end()
+})

From db0eccff7310800dffcc94c2c94b3bb6384d8b4d Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 00:23:08 +0200
Subject: [PATCH 11/50] Use released ssb-db2

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index ed07e76..d0850c4 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,7 @@
     "ssb-bendy-butt": "^0.12.2",
     "ssb-client": "^4.9.0",
     "ssb-db": "^19.2.0",
-    "ssb-db2": "github:ssb-ngi-pointer/ssb-db2#support-bendy-butt",
+    "ssb-db2": "^2.4.0",
     "ssb-generate": "^1.0.1",
     "ssb-keys": "^8.1.0",
     "ssb-uri2": "^1.5.2",

From 877bcc1dd5a2ca242cc1772dc644f70077e7c75e Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 00:29:07 +0200
Subject: [PATCH 12/50] Tweak replication timeout

---
 test/formats.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/formats.js b/test/formats.js
index 6e3efe3..6f6da27 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -466,7 +466,7 @@ tape('sliced index replication', async (t) => {
                                          1)
   carol.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
 
-  await sleep(REPLICATION_TIMEOUT)
+  await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
   const carolMessages = await carol.db.query(toPromise())

From e703ae0fdd2b093a66877a95ea9dbf2ec7ea6b25 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 20:38:07 +0200
Subject: [PATCH 13/50] Update index.js

Co-authored-by: Henry <111202+cryptix@users.noreply.github.com>
---
 index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.js b/index.js
index 1f5d78b..eff2f33 100644
--- a/index.js
+++ b/index.js
@@ -113,7 +113,7 @@ exports.init = function (sbot, config) {
   function getEBT(formatName) {
     const ebt = ebts[formatName]
     if (!ebt)
-      throw new Error('Unknown format' + formatName)
+      throw new Error('Unknown format: ' + formatName)
 
     return ebt
   }

From f8f4cee2923b1b7bbe420c18da9aa98359bc689a Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 20:54:21 +0200
Subject: [PATCH 14/50] Refactor peerStatus to deduce the format

---
 index.js        | 9 +++++++--
 test/formats.js | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/index.js b/index.js
index eff2f33..2470fea 100644
--- a/index.js
+++ b/index.js
@@ -235,9 +235,14 @@ exports.init = function (sbot, config) {
   }
 
   // get replication status for feeds for this id
-  function peerStatus(id, formatName) {
+  function peerStatus(id) {
     id = id || sbot.id
-    formatName = formatName || 'classic'
+
+    formatName = 'classic'
+    for (let format in ebts)
+      if (formats[format].isFeed(id))
+        formatName = format
+
     const ebt = getEBT(formatName)
 
     const data = {
diff --git a/test/formats.js b/test/formats.js
index 6f6da27..81cd867 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -463,7 +463,7 @@ tape('sliced index replication', async (t) => {
   t.equal(clockAlice[aliceIndexKey.id], 2, 'alice correct index clock')
 
   carol.ebt.setClockForSlicedReplication('indexedfeed', aliceIndexKey.id,
-                                         1)
+                                         clockAlice[aliceIndexKey.id] - 1)
   carol.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
 
   await sleep(2 * REPLICATION_TIMEOUT)

From cd7194efb73d09f08a0f11db33b9beda0fdc3019 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 21:09:06 +0200
Subject: [PATCH 15/50] Use opts in clock to make it consistent with other APIs
 and update README

---
 README.md       | 14 ++++++++++++++
 index.js        |  8 ++++----
 test/formats.js | 22 +++++++++++-----------
 3 files changed, 29 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index b441aa6..c69dce4 100644
--- a/README.md
+++ b/README.md
@@ -106,6 +106,20 @@ the peer who initiated the call (the client) should call this. You do not need
 to call this method, it is called automatically in ssb-ebt whenever our peer
 connects to a remote peer. `opts` is an object with one field: `version`.
 
+### (Internal) `ssb.ebt.replicateFormat(opts)` ("duplex" muxrpc API)
+
+Creates a duplex replication stream to the remote peer. This behaves
+similar to `replicate` except it takes an extra field `format`
+specifying what is transferred over this EBT stream. Classic feeds are
+still replicated using `replicate` while this will be used to
+replicate other feed formats.
+
+### (Internal) `ssb.ebt.clock(opts, cb)` ("async" muxrpc API)
+
+Gets the current vector clock of a remote peer. `opts` is an object
+with one field: `format` specifying what format to get the vector
+clock for. Defaults to 'classic'.
+
 ## Testing and debugging
 
 There are several scripts in `./debug` which can be used for testing EBT
diff --git a/index.js b/index.js
index 2470fea..a9cb913 100644
--- a/index.js
+++ b/index.js
@@ -266,14 +266,14 @@ exports.init = function (sbot, config) {
     return data
   }
 
-  function clock(formatName, cb) {
+  function clock(opts, cb) {
     if (!cb) {
-      cb = formatName
-      formatName = 'classic'
+      cb = opts
+      opts = { format: 'classic' }
     }
 
     initialized.promise.then(() => {
-      const ebt = getEBT(formatName)
+      const ebt = getEBT(opts.format)
       cb(null, ebt.state.clock)
     })
   }
diff --git a/test/formats.js b/test/formats.js
index 81cd867..41406c8 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -171,16 +171,16 @@ tape('multiple formats', async (t) => {
     [bobMFId]: 1
   }
 
-  const clockAlice = await pify(alice.ebt.clock)('classic')
+  const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockAlice, expectedClassicClock, 'alice correct classic clock')
 
-  const bbClockAlice = await pify(alice.ebt.clock)('bendybutt')
+  const bbClockAlice = await pify(alice.ebt.clock)({ format: 'bendybutt' })
   t.deepEqual(bbClockAlice, expectedBBClock, 'alice correct bb clock')
 
-  const clockBob = await pify(bob.ebt.clock)('classic')
+  const clockBob = await pify(bob.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockBob, expectedClassicClock, 'bob correct classic clock')
 
-  const bbClockBob = await pify(bob.ebt.clock)('bendybutt')
+  const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt' })
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
 
   await Promise.all([
@@ -221,16 +221,16 @@ tape('multiple formats restart', async (t) => {
     [bobMFId]: 1
   }
 
-  const clockAlice = await pify(alice.ebt.clock)('classic')
+  const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockAlice, expectedClassicClock, 'alice correct classic clock')
 
-  const bbClockAlice = await pify(alice.ebt.clock)('bendybutt')
+  const bbClockAlice = await pify(alice.ebt.clock)({ format: 'bendybutt' })
   t.deepEqual(bbClockAlice, expectedBBClock, 'alice correct bb clock')
 
-  const clockBob = await pify(bob.ebt.clock)('classic')
+  const clockBob = await pify(bob.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockBob, expectedClassicClock, 'bob correct classic clock')
 
-  const bbClockBob = await pify(bob.ebt.clock)('bendybutt')
+  const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt' })
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
 
   await Promise.all([
@@ -373,10 +373,10 @@ tape('index format', async (t) => {
     [bobIndexKey.id]: 1
   }
 
-  const indexClockAlice = await pify(alice.ebt.clock)('indexedfeed')
+  const indexClockAlice = await pify(alice.ebt.clock)({ format: 'indexedfeed' })
   t.deepEqual(indexClockAlice, expectedIndexClock, 'alice correct index clock')
 
-  const indexClockBob = await pify(bob.ebt.clock)('indexedfeed')
+  const indexClockBob = await pify(bob.ebt.clock)({ format: 'indexedfeed' })
   t.deepEqual(indexClockBob, expectedIndexClock, 'bob correct index clock')
 
   await Promise.all([
@@ -459,7 +459,7 @@ tape('sliced index replication', async (t) => {
 
   await pify(carol.connect)(alice.getAddress())
 
-  const clockAlice = await pify(alice.ebt.clock)('indexedfeed')
+  const clockAlice = await pify(alice.ebt.clock)({ format: 'indexedfeed' })
   t.equal(clockAlice[aliceIndexKey.id], 2, 'alice correct index clock')
 
   carol.ebt.setClockForSlicedReplication('indexedfeed', aliceIndexKey.id,

From 011be45c623229a475cea931bc7339493ca491de Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 21:45:36 +0200
Subject: [PATCH 16/50] Refactor request + block to deduce format

---
 index.js        | 16 ++++++++++++----
 test/formats.js | 24 ++++++++++++------------
 2 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/index.js b/index.js
index a9cb913..aaeb2e5 100644
--- a/index.js
+++ b/index.js
@@ -184,9 +184,13 @@ exports.init = function (sbot, config) {
     }
   })
 
-  function request(destFeedId, requesting, formatName) {
+  function request(destFeedId, requesting) {
     initialized.promise.then(() => {
-      formatName = formatName || 'classic'
+      formatName = 'classic'
+      for (let format in ebts)
+        if (formats[format].isFeed(destFeedId))
+          formatName = format
+
       const format = formats[formatName]
 
       if (!(format && format.isFeed(destFeedId))) return
@@ -195,9 +199,13 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function block(origFeedId, destFeedId, blocking, formatName) {
+  function block(origFeedId, destFeedId, blocking) {
     initialized.promise.then(() => {
-      formatName = formatName || 'classic'
+      formatName = 'classic'
+      for (let format in ebts)
+        if (formats[format].isFeed(origFeedId))
+          formatName = format
+
       const format = formats[formatName]
 
       if (!format) return
diff --git a/test/formats.js b/test/formats.js
index 41406c8..80e6113 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -143,8 +143,8 @@ tape('multiple formats', async (t) => {
   bobMFId = bobBBMsg.author
 
   // self replicate
-  alice.ebt.request(aliceMFId, true, 'bendybutt')
-  bob.ebt.request(bobMFId, true, 'bendybutt')
+  alice.ebt.request(aliceMFId, true)
+  bob.ebt.request(bobMFId, true)
 
   await Promise.all([
     pify(alice.add)(aliceBBMsg),
@@ -152,10 +152,10 @@ tape('multiple formats', async (t) => {
   ])
 
   alice.ebt.request(bob.id, true)
-  alice.ebt.request(bobMFId, true, 'bendybutt')
+  alice.ebt.request(bobMFId, true)
 
   bob.ebt.request(alice.id, true)
-  bob.ebt.request(aliceMFId, true, 'bendybutt')
+  bob.ebt.request(aliceMFId, true)
 
   await pify(bob.connect)(alice.getAddress())
 
@@ -208,9 +208,9 @@ tape('multiple formats restart', async (t) => {
 
   // self replicate
   alice.ebt.request(alice.id, true)
-  alice.ebt.request(aliceMFId, true, 'bendybutt')
+  alice.ebt.request(aliceMFId, true)
   bob.ebt.request(bob.id, true)
-  bob.ebt.request(bobMFId, true, 'bendybutt')
+  bob.ebt.request(bobMFId, true)
 
   const expectedClassicClock = {
     [alice.id]: 1,
@@ -342,13 +342,13 @@ tape('index format', async (t) => {
 
   // self replicate
   alice.ebt.request(alice.id, true)
-  alice.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+  alice.ebt.request(aliceIndexKey.id, true)
   bob.ebt.request(bob.id, true)
-  bob.ebt.request(bobIndexKey.id, true, 'indexedfeed')
+  bob.ebt.request(bobIndexKey.id, true)
 
   // only replicate index feeds
-  alice.ebt.request(bobIndexKey.id, true, 'indexedfeed')
-  bob.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+  alice.ebt.request(bobIndexKey.id, true)
+  bob.ebt.request(aliceIndexKey.id, true)
 
   await pify(bob.connect)(alice.getAddress())
 
@@ -441,7 +441,7 @@ tape('sliced index replication', async (t) => {
 
   // self replicate
   alice.ebt.request(alice.id, true)
-  alice.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+  alice.ebt.request(aliceIndexKey.id, true)
   carol.ebt.request(carol.id, true)
 
   // publish a few more messages
@@ -464,7 +464,7 @@ tape('sliced index replication', async (t) => {
 
   carol.ebt.setClockForSlicedReplication('indexedfeed', aliceIndexKey.id,
                                          clockAlice[aliceIndexKey.id] - 1)
-  carol.ebt.request(aliceIndexKey.id, true, 'indexedfeed')
+  carol.ebt.request(aliceIndexKey.id, true)
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')

From 0f1b1891a30f3453c9f8fc74db376d34803ccf40 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 22:02:14 +0200
Subject: [PATCH 17/50] Document registerFormat

---
 README.md | 42 ++++++++++++++++++++++++++++++++++++++++++
 index.js  |  4 ++--
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c69dce4..e991138 100644
--- a/README.md
+++ b/README.md
@@ -97,6 +97,48 @@ The output looks like this:
 }
 ```
 
+### `ssb.ebt.registerFormat(formatName, methods)` ("sync" muxrpc API)
+
+By registering a format you create a new EBT instance used for
+replicating feeds in that format. This means its own clock. Message
+will be replicated using the `replicateFormat` api. `formatName` must
+be a string and methods must implement the following functions. The
+example shows the 'classic' implementation.
+
+<details>
+<summary>CLICK HERE</summary>
+
+```js
+{
+  // used in request, block, cleanClock, sbot.post, vectorClock
+  isFeed: ref.isFeed,
+  getAtSequence(sbot, pair, cb) {
+    sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+      cb(err, msg ? msg.value : null)
+    })
+  },
+  appendMsg(sbot, msgVal, cb) {
+    sbot.add(msgVal, (err, msg) => {
+      cb(err && err.fatal ? err : null, msg)
+    })
+  },
+
+  // used in ebt:stream to distinguish between messages and notes
+  isMsg(msgVal) {
+    return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
+      ref.isFeed(msgVal.author) && msgVal.content
+  },
+  // used in ebt:events
+  getMsgAuthor(msgVal) {
+    return msgVal.author
+  },
+  // used in ebt:events
+  getMsgSequence(msgVal) {
+    return msgVal.sequence
+  }
+}
+```
+
 </details>
 
 ### (Internal) `ssb.ebt.replicate(opts)` ("duplex" muxrpc API)
diff --git a/index.js b/index.js
index aaeb2e5..51669a3 100644
--- a/index.js
+++ b/index.js
@@ -48,7 +48,7 @@ function cleanClock (clock, isFeed) {
 exports.init = function (sbot, config) {
   const formats = {
     'classic': {
-      // used in request, block, cleanClock, sbot.post
+      // used in request, block, cleanClock, sbot.post, vectorClock
       isFeed: ref.isFeed,
       getAtSequence(sbot, pair, cb) {
         sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
@@ -73,7 +73,7 @@ exports.init = function (sbot, config) {
       // used in ebt:events
       getMsgSequence(msgVal) {
         return msgVal.sequence
-      },
+      }
     }
   }
 

From 0d09d6c239d49324851e80e66fba88441a6f5ff5 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 22:12:58 +0200
Subject: [PATCH 18/50] Document setClockForSlicedReplication and simplify api

---
 README.md       |  9 ++++++++-
 index.js        | 11 ++++++++---
 test/formats.js |  2 +-
 3 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index e991138..e8d414d 100644
--- a/README.md
+++ b/README.md
@@ -138,9 +138,16 @@ example shows the 'classic' implementation.
   }
 }
 ```
-
 </details>
 
+### `ssb.ebt.setClockForSlicedReplication(feedId, sequence)` ("sync" muxrpc API)
+
+Sets the internal clock of a feed to a specific sequence. Note this
+does not start replicating the feed, it only updates the clock. By
+combining this with `clock` it is possible do to sliced replication
+with a remote peer where say only the latest 100 messages of a feed is
+replicated.
+
 ### (Internal) `ssb.ebt.replicate(opts)` ("duplex" muxrpc API)
 
 Creates a duplex replication stream to the remote peer. When two peers connect,
diff --git a/index.js b/index.js
index 51669a3..9fe1186 100644
--- a/index.js
+++ b/index.js
@@ -286,10 +286,15 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function setClockForSlicedReplication(format, feed, sequence) {
+  function setClockForSlicedReplication(feedId, sequence) {
+    formatName = 'classic'
+    for (let format in ebts)
+      if (formats[format].isFeed(feedId))
+        formatName = format
+
     initialized.promise.then(() => {
-      const ebt = getEBT(format)
-      ebt.state.clock[feed] = sequence
+      const ebt = getEBT(formatName)
+      ebt.state.clock[feedId] = sequence
     })
   }
 
diff --git a/test/formats.js b/test/formats.js
index 80e6113..f56364f 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -462,7 +462,7 @@ tape('sliced index replication', async (t) => {
   const clockAlice = await pify(alice.ebt.clock)({ format: 'indexedfeed' })
   t.equal(clockAlice[aliceIndexKey.id], 2, 'alice correct index clock')
 
-  carol.ebt.setClockForSlicedReplication('indexedfeed', aliceIndexKey.id,
+  carol.ebt.setClockForSlicedReplication(aliceIndexKey.id,
                                          clockAlice[aliceIndexKey.id] - 1)
   carol.ebt.request(aliceIndexKey.id, true)
 

From 0bca49ee7814aa279518f5a8ce278bf1b3b040cf Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 9 Sep 2021 22:15:10 +0200
Subject: [PATCH 19/50] Missing </details>

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index e8d414d..07faa0b 100644
--- a/README.md
+++ b/README.md
@@ -96,6 +96,7 @@ The output looks like this:
   }
 }
 ```
+</details>
 
 ### `ssb.ebt.registerFormat(formatName, methods)` ("sync" muxrpc API)
 

From 74231429fa569a3056cbdfef8ba79afdfb072ad4 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Sat, 11 Sep 2021 21:31:50 +0200
Subject: [PATCH 20/50] Bendy butt fixes from 8k testing

---
 index.js        | 11 ++++++++---
 test/formats.js | 15 +++++++++------
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/index.js b/index.js
index 9fe1186..f8cc3ff 100644
--- a/index.js
+++ b/index.js
@@ -60,6 +60,10 @@ exports.init = function (sbot, config) {
           cb(err && err.fatal ? err : null, msg)
         })
       },
+      // used in onAppend
+      convertMsg(msgVal) {
+        return msgVal
+      },
 
       // used in ebt:stream to distinguish between messages and notes
       isMsg(msgVal) {
@@ -141,9 +145,10 @@ exports.init = function (sbot, config) {
 
   sbot.post((msg) => {
     initialized.promise.then(() => {
-      for (let format in ebts) {
-        if (formats[format].isFeed(msg.value.author))
-          ebts[format].onAppend(msg.value)
+      for (let formatName in ebts) {
+        const format = formats[formatName]
+        if (format.isFeed(msg.value.author))
+          ebts[formatName].onAppend(format.convertMsg(msg.value))
       }
     })
   })
diff --git a/test/formats.js b/test/formats.js
index f56364f..ba8cd08 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -56,15 +56,18 @@ const bendyButtMethods = {
       cb(err && err.fatal ? err : null, msg)
     })
   },
+  convertMsg(msgVal) {
+    return bendyButt.encode(msgVal)
+  },
 
   // used in ebt:stream to distinguish between messages and notes
   isMsg(bbVal) {
-    if (!Buffer.isBuffer(bbVal)) return false
-
-    const msgVal = bendyButt.decode(bbVal)
-    return msgVal &&
-      Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
-      typeof msgVal.author == 'string' && msgVal.content
+    if (Buffer.isBuffer(bbVal)) {
+      const msgVal = bendyButt.decode(bbVal)
+      return msgVal && SSBURI.isBendyButtV1FeedSSBURI(msgVal.author)
+    } else {
+      return bbVal && SSBURI.isBendyButtV1FeedSSBURI(bbVal.author)
+    }
   },
   // used in ebt:events
   getMsgAuthor(bbVal) {

From 67623483305b0ca20148f82e8c360ce5461ae0ff Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 14 Sep 2021 09:58:02 +0200
Subject: [PATCH 21/50] Rewrite tests to use index writer + fix bendy butt name

---
 index.js        |  30 ++--
 package.json    |   2 +
 test/formats.js | 376 +++++++++++++++++++++++++-----------------------
 3 files changed, 216 insertions(+), 192 deletions(-)

diff --git a/index.js b/index.js
index f8cc3ff..f849f98 100644
--- a/index.js
+++ b/index.js
@@ -64,6 +64,10 @@ exports.init = function (sbot, config) {
       convertMsg(msgVal) {
         return msgVal
       },
+      // used in vectorClock
+      isReady(sbot) {
+        return Promise.resolve(true)
+      },
 
       // used in ebt:stream to distinguish between messages and notes
       isMsg(msgVal) {
@@ -128,19 +132,23 @@ exports.init = function (sbot, config) {
 
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
-    for (let formatName in ebts) {
-      const format = formats[formatName]
-      const ebt = ebts[formatName]
 
-      validClock = {}
-      for (let k in clock)
-        if (format.isFeed(k))
-          validClock[k] = clock[k]
+    const readies = Object.values(formats).map(f => f.isReady(sbot))
+    Promise.all(readies).then(() => {
+      for (let formatName in ebts) {
+        const format = formats[formatName]
+        const ebt = ebts[formatName]
 
-      ebt.state.clock = validClock
-      ebt.update()
-    }
-    initialized.resolve()
+        validClock = {}
+        for (let k in clock)
+          if (format.isFeed(k))
+            validClock[k] = clock[k]
+
+        ebt.state.clock = validClock
+        ebt.update()
+      }
+      initialized.resolve()
+    })
   })
 
   sbot.post((msg) => {
diff --git a/package.json b/package.json
index d0850c4..9b4da4d 100644
--- a/package.json
+++ b/package.json
@@ -42,7 +42,9 @@
     "ssb-db": "^19.2.0",
     "ssb-db2": "^2.4.0",
     "ssb-generate": "^1.0.1",
+    "ssb-index-feed-writer": "^0.5.1",
     "ssb-keys": "^8.1.0",
+    "ssb-meta-feeds": "^0.19.0",
     "ssb-uri2": "^1.5.2",
     "ssb-validate": "^4.1.4",
     "standardx": "^7.0.0",
diff --git a/test/formats.js b/test/formats.js
index ba8cd08..91953cc 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -11,40 +11,44 @@ const mkdirp = require('mkdirp')
 const ssbKeys = require('ssb-keys')
 const SSBURI = require('ssb-uri2')
 const bendyButt = require('ssb-bendy-butt')
-const { where, author, toPromise } = require('ssb-db2/operators')
+const { where, author, type, toPromise } = require('ssb-db2/operators')
 
-function createSsbServer() {
+function createSSBServer() {
   return SecretStack({ appKey: caps.shs })
     .use(require('ssb-db2'))
     .use(require('ssb-db2/compat/ebt'))
+    .use(require('ssb-meta-feeds'))
+    .use(require('ssb-index-feed-writer'))
     .use(require('../'))
 }
 
 const CONNECTION_TIMEOUT = 500 // ms
 const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 
-const aliceDir = '/tmp/test-format-alice'
-rimraf.sync(aliceDir)
-mkdirp.sync(aliceDir)
+function getFreshDir(name) {
+  const dir = '/tmp/test-format-' + name
+  rimraf.sync(dir)
+  mkdirp.sync(dir)
+  return dir
+}
 
-let alice = createSsbServer().call(null, {
+const aliceDir = getFreshDir('alice')
+let alice = createSSBServer().call(null, {
   path: aliceDir,
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('alice'),
 })
 
-const bobDir = '/tmp/test-format-bob'
-rimraf.sync(bobDir)
-mkdirp.sync(bobDir)
-
-let bob = createSsbServer().call(null, {
+const bobDir = getFreshDir('bob')
+let bob = createSSBServer().call(null, {
   path: bobDir,
   timeout: CONNECTION_TIMEOUT,
   keys: u.keysFor('bob')
 })
 
+// FIXME: this should be somewhere else
 const bendyButtMethods = {
-  // used in request, block, cleanClock, sbot.post
+  // used in request, block, cleanClock, sbot.post, vectorClock
   isFeed: SSBURI.isBendyButtV1FeedSSBURI,
   getAtSequence(sbot, pair, cb) {
     sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
@@ -59,6 +63,10 @@ const bendyButtMethods = {
   convertMsg(msgVal) {
     return bendyButt.encode(msgVal)
   },
+  // used in vectorClock
+  isReady(sbot) {
+    return Promise.resolve(true)
+  },
 
   // used in ebt:stream to distinguish between messages and notes
   isMsg(bbVal) {
@@ -71,7 +79,6 @@ const bendyButtMethods = {
   },
   // used in ebt:events
   getMsgAuthor(bbVal) {
-    //console.log("bb getMsgAuthor", Buffer.isBuffer(bbVal))
     if (Buffer.isBuffer(bbVal))
       return bendyButt.decode(bbVal).author
     else
@@ -79,7 +86,6 @@ const bendyButtMethods = {
   },
   // used in ebt:events
   getMsgSequence(bbVal) {
-    //console.log("bb getMsgSequence", Buffer.isBuffer(bbVal))
     if (Buffer.isBuffer(bbVal))
       return bendyButt.decode(bbVal).sequence
     else
@@ -96,7 +102,7 @@ function getBBMsg(mainKeys) {
   mfKeys.id = bendybuttUri
 
   const content = {
-    type: "metafeed/add",
+    type: "metafeed/add/existing",
     feedpurpose: "main",
     subfeed: mainKeys.id,
     metafeed: mfKeys.id,
@@ -126,8 +132,8 @@ let aliceMFId
 let bobMFId
 
 tape('multiple formats', async (t) => {
-  alice.ebt.registerFormat('bendybutt', bendyButtMethods)
-  bob.ebt.registerFormat('bendybutt', bendyButtMethods)
+  alice.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
+  bob.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
 
   // self replicate
   alice.ebt.request(alice.id, true)
@@ -177,13 +183,13 @@ tape('multiple formats', async (t) => {
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockAlice, expectedClassicClock, 'alice correct classic clock')
 
-  const bbClockAlice = await pify(alice.ebt.clock)({ format: 'bendybutt' })
+  const bbClockAlice = await pify(alice.ebt.clock)({ format: 'bendybutt-v1' })
   t.deepEqual(bbClockAlice, expectedBBClock, 'alice correct bb clock')
 
   const clockBob = await pify(bob.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockBob, expectedClassicClock, 'bob correct classic clock')
 
-  const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt' })
+  const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt-v1' })
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
 
   await Promise.all([
@@ -194,20 +200,20 @@ tape('multiple formats', async (t) => {
 })
 
 tape('multiple formats restart', async (t) => {
-  alice = createSsbServer().call(null, {
+  alice = createSSBServer().call(null, {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
     keys: u.keysFor('alice'),
   })
 
-  bob = createSsbServer().call(null, {
+  bob = createSSBServer().call(null, {
     path: bobDir,
     timeout: CONNECTION_TIMEOUT,
     keys: u.keysFor('bob')
   })
 
-  alice.ebt.registerFormat('bendybutt', bendyButtMethods)
-  bob.ebt.registerFormat('bendybutt', bendyButtMethods)
+  alice.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
+  bob.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
 
   // self replicate
   alice.ebt.request(alice.id, true)
@@ -227,13 +233,13 @@ tape('multiple formats restart', async (t) => {
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockAlice, expectedClassicClock, 'alice correct classic clock')
 
-  const bbClockAlice = await pify(alice.ebt.clock)({ format: 'bendybutt' })
+  const bbClockAlice = await pify(alice.ebt.clock)({ format: 'bendybutt-v1' })
   t.deepEqual(bbClockAlice, expectedBBClock, 'alice correct bb clock')
 
   const clockBob = await pify(bob.ebt.clock)({ format: 'classic' })
   t.deepEqual(clockBob, expectedClassicClock, 'bob correct classic clock')
 
-  const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt' })
+  const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt-v1' })
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
 
   await Promise.all([
@@ -243,237 +249,245 @@ tape('multiple formats restart', async (t) => {
   t.end()
 })
 
-let indexedFeeds = []
-
-// first solution:
-// replicating index feeds using classic and indexed using a special
-// format does not work because ebt:events expects messages to come in
-// order. Furthermore getting the vector clock of this indexed ebt on
-// start is also a bit complicated
-
-// second solution:
-// - don't send indexes over classic unless you only replicate the index
-// - use a special indexed format that sends both index + indexed as 1
-//   message with indexed as payload
-// - this requires that we are able to add 2 messages in a
-//   transaction. Should be doable, but requires changes all the way
-//   down the stack
 
-const indexedFeedMethods = Object.assign(
-  {}, alice.ebt.formats['classic'], {
-    isFeed(author) {
-      return indexedFeeds.includes(author)
-    },
-    appendMsg(sbot, msgVal, cb) {
-      const payload = msgVal.content.indexed.payload
-      delete msgVal.content.indexed.payload
-      sbot.db.add(msgVal, (err, msg) => {
-        if (err) return cb(err)
-        sbot.db.addOOO(payload, (err, indexedMsg) => {
+// FIXME: this needs the ability to add 2 messages in a transaction
+function indexedFeedMethods(sbot) {
+  return Object.assign(
+    {}, alice.ebt.formats['classic'], {
+      isFeed(author) {
+        const info = sbot.metafeeds.findByIdSync(author)
+        return info && info.feedpurpose === 'index'
+      },
+      appendMsg(sbot, msgVal, cb) {
+        const payload = msgVal.content.indexed.payload
+        delete msgVal.content.indexed.payload
+        sbot.db.add(msgVal, (err, msg) => {
           if (err) return cb(err)
-          else cb(null, msg)
+          sbot.db.addOOO(payload, (err, indexedMsg) => {
+            if (err) return cb(err)
+            else cb(null, msg)
+          })
         })
-      })
-    },
-    getAtSequence(sbot, pair, cb) {
-      sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
-        if (err) return cb(err)
-        const { author, sequence } = msg.value.content.indexed
-        sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
+      },
+      getAtSequence(sbot, pair, cb) {
+        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
           if (err) return cb(err)
-
-          // add referenced message as payload
-          msg.value.content.indexed.payload = indexedMsg.value
-
-          cb(null, msg.value)
+          const { sequence } = msg.value.content.indexed
+          const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
+          if (!authorInfo) return cb(new Error("Unknown author", msg.value.author))
+          const { author } = JSON.parse(authorInfo.metadata.query)
+          sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
+            if (err) return cb(err)
+
+            // add referenced message as payload
+            msg.value.content.indexed.payload = indexedMsg.value
+
+            cb(null, msg.value)
+          })
         })
-      })
+      },
+      isReady(sbot) {
+        return pify(sbot.metafeeds.loadState)()
+      },
     }
-  }
-)
+  )
+}
 
-const aliceIndexKey = ssbKeys.generate()
+const carolDir = getFreshDir('carol')
 
 tape('index format', async (t) => {
-  const bobIndexKey = ssbKeys.generate()
-
-  indexedFeeds.push(aliceIndexKey.id)
-  indexedFeeds.push(bobIndexKey.id)
-
-  alice = createSsbServer().call(null, {
-    path: aliceDir,
+  const carol = createSSBServer().call(null, {
+    path: carolDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('alice')
+    keys: u.keysFor('carol')
   })
 
-  bob = createSsbServer().call(null, {
-    path: bobDir,
+  const daveDir = getFreshDir('dave')
+  const dave = createSSBServer().call(null, {
+    path: daveDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('bob')
+    keys: u.keysFor('dave')
   })
 
-  alice.ebt.registerFormat('indexedfeed', indexedFeedMethods)
-  bob.ebt.registerFormat('indexedfeed', indexedFeedMethods)
+  carol.ebt.registerFormat('indexedfeed', indexedFeedMethods(carol))
+  carol.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
+  dave.ebt.registerFormat('indexedfeed', indexedFeedMethods(dave))
+  dave.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
 
-  // publish a few more messages
+  const carolIndexId = (await pify(carol.indexFeedWriter.start)({ author: carol.id, type: 'dog', private: false })).subfeed
+  const daveIndexId = (await pify(dave.indexFeedWriter.start)({ author: dave.id, type: 'dog', private: false })).subfeed
+
+  // publish some messages
   const res = await Promise.all([
-    pify(alice.db.publish)({ type: 'post', text: 'hello 2' }),
-    pify(alice.db.publish)({ type: 'dog', name: 'Buff' }),
-    pify(bob.db.publish)({ type: 'post', text: 'hello 2' }),
-    pify(bob.db.publish)({ type: 'dog', name: 'Biff' })
+    pify(carol.db.publish)({ type: 'post', text: 'hello 2' }),
+    pify(carol.db.publish)({ type: 'dog', name: 'Buff' }),
+    pify(dave.db.publish)({ type: 'post', text: 'hello 2' }),
+    pify(dave.db.publish)({ type: 'dog', name: 'Biff' })
   ])
 
-  // index the dog messages
-  await Promise.all([
-    pify(alice.db.publishAs)(aliceIndexKey, {
-      type: 'metafeed/index',
-      indexed: {
-        key: res[1].key,
-        author: alice.id,
-        sequence: res[1].value.sequence
-      }
-    }),
-    pify(bob.db.publishAs)(bobIndexKey, {
-      type: 'metafeed/index',
-      indexed: {
-        key: res[3].key,
-        author: bob.id,
-        sequence: res[3].value.sequence
-      }
-    })
-  ])
+  await sleep(2 * REPLICATION_TIMEOUT)
+  t.pass('wait for index to complete')
+
+  // get meta feed id to replicate
+  const carolMetaMessages = await carol.db.query(
+    where(type('metafeed/announce')),
+    toPromise()
+  )
+
+  // get meta index feed
+  const carolMetaIndexMessages = await carol.db.query(
+    where(type('metafeed/add/derived')),
+    toPromise()
+  )
+
+  // get meta feed id to replicate
+  const daveMetaMessages = await dave.db.query(
+    where(type('metafeed/announce')),
+    toPromise()
+  )
+  
+  // get meta index feed
+  const daveMetaIndexMessages = await dave.db.query(
+    where(type('metafeed/add/derived')),
+    toPromise()
+  )
 
+  const carolMetaId = carolMetaMessages[0].value.content.metafeed
+  const carolMetaIndexId = carolMetaIndexMessages[0].value.content.subfeed
+  
+  const daveMetaId = daveMetaMessages[0].value.content.metafeed
+  const daveMetaIndexId = daveMetaIndexMessages[0].value.content.subfeed
+  
   // self replicate
-  alice.ebt.request(alice.id, true)
-  alice.ebt.request(aliceIndexKey.id, true)
-  bob.ebt.request(bob.id, true)
-  bob.ebt.request(bobIndexKey.id, true)
+  carol.ebt.request(carol.id, true)
+  carol.ebt.request(carolMetaId, true)
+  carol.ebt.request(carolMetaIndexId, true)
+  carol.ebt.request(carolIndexId, true)
 
-  // only replicate index feeds
-  alice.ebt.request(bobIndexKey.id, true)
-  bob.ebt.request(aliceIndexKey.id, true)
+  dave.ebt.request(dave.id, true)
+  dave.ebt.request(daveMetaId, true)
+  dave.ebt.request(daveMetaIndexId, true)
+  dave.ebt.request(daveIndexId, true)
 
-  await pify(bob.connect)(alice.getAddress())
+  // replication
+  carol.ebt.request(daveMetaId, true)
+  carol.ebt.request(daveMetaIndexId, true)
+
+  dave.ebt.request(carolMetaId, true)
+  dave.ebt.request(carolMetaIndexId, true)
+  
+  await pify(dave.connect)(carol.getAddress())
+
+  await sleep(2 * REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  // debugging
+  const carolIndexMessages = await carol.db.query(
+    where(author(daveMetaIndexId)),
+    toPromise()
+  )
+  t.equal(carolIndexMessages.length, 1, 'carol has dave meta index')
+
+  const daveIndexMessages = await dave.db.query(
+    where(author(carolMetaIndexId)),
+    toPromise()
+  )
+  t.equal(daveIndexMessages.length, 1, 'dave has carol meta index')
+  
+  // now that we have meta feeds from the other peer we can replicate
+  // index feeds
+
+  carol.ebt.request(daveIndexId, true)
+  dave.ebt.request(carolIndexId, true)
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
   // we should only get the dog message and not second post
-  const aliceBobMessages = await alice.db.query(
-    where(author(bob.id)),
+  const carolDaveMessages = await carol.db.query(
+    where(author(dave.id)),
     toPromise()
   )
-  t.equal(aliceBobMessages.length, 2, 'alice correct messages from bob')
+  t.equal(carolDaveMessages.length, 1, 'carol got dog message from dave')
 
-  const bobAliceMessages = await bob.db.query(
-    where(author(alice.id)),
+  const daveCarolMessages = await dave.db.query(
+    where(author(carol.id)),
     toPromise()
   )
-  t.equal(bobAliceMessages.length, 2, 'bob correct messages from alice')
+  t.equal(daveCarolMessages.length, 1, 'dave got dob message from carol')
 
   const expectedIndexClock = {
-    [aliceIndexKey.id]: 1,
-    [bobIndexKey.id]: 1
+    [carolIndexId]: 1,
+    [daveIndexId]: 1
   }
 
-  const indexClockAlice = await pify(alice.ebt.clock)({ format: 'indexedfeed' })
-  t.deepEqual(indexClockAlice, expectedIndexClock, 'alice correct index clock')
+  const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexedfeed' })
+  t.deepEqual(indexClockCarol, expectedIndexClock, 'carol correct index clock')
 
-  const indexClockBob = await pify(bob.ebt.clock)({ format: 'indexedfeed' })
-  t.deepEqual(indexClockBob, expectedIndexClock, 'bob correct index clock')
+  const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexedfeed' })
+  t.deepEqual(indexClockDave, expectedIndexClock, 'dave correct index clock')
 
   await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
+    pify(carol.close)(true),
+    pify(dave.close)(true)
   ])
   t.end()
 })
 
-const sliceIndexedFeedMethods = Object.assign(
+const slicedReplication = Object.assign(
   {}, alice.ebt.formats['classic'], {
-    isFeed(author) {
-      return indexedFeeds.includes(author)
-    },
     appendMsg(sbot, msgVal, cb) {
-      const payload = msgVal.content.indexed.payload
-      delete msgVal.content.indexed.payload
       sbot.db.addOOO(msgVal, (err, msg) => {
         if (err) return cb(err)
-        sbot.db.addOOO(payload, (err, indexedMsg) => {
-          if (err) return cb(err)
-          else cb(null, msg)
-        })
-      })
-    },
-    getAtSequence(sbot, pair, cb) {
-      sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
-        if (err) return cb(err)
-        const { author, sequence } = msg.value.content.indexed
-        sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
-          if (err) return cb(err)
-
-          // add referenced message as payload
-          msg.value.content.indexed.payload = indexedMsg.value
-
-          cb(null, msg.value)
-        })
+        else cb(null, msg)
       })
     }
   }
 )
 
-tape('sliced index replication', async (t) => {
-  alice = createSsbServer().call(null, {
+// FIXME: this needs to be as before
+tape('sliced replication', async (t) => {
+  alice = createSSBServer().call(null, {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
     keys: u.keysFor('alice')
   })
 
-  const carolDir = '/tmp/test-format-carol'
-  rimraf.sync(carolDir)
-  mkdirp.sync(carolDir)
-
-  let carol = createSsbServer().call(null, {
+  let carol = createSSBServer().call(null, {
     path: carolDir,
     timeout: CONNECTION_TIMEOUT,
     keys: u.keysFor('carol')
   })
 
-  alice.ebt.registerFormat('indexedfeed', sliceIndexedFeedMethods)
-  carol.ebt.registerFormat('indexedfeed', sliceIndexedFeedMethods)
+  await Promise.all([
+    pify(alice.db.publish)({ type: 'post', text: 'hello2' }),
+    pify(alice.db.publish)({ type: 'post', text: 'hello3' }),
+  ])
+  
+  alice.ebt.registerFormat('slicedreplication', slicedReplication)
+  carol.ebt.registerFormat('slicedreplication', slicedReplication)
 
   // self replicate
   alice.ebt.request(alice.id, true)
-  alice.ebt.request(aliceIndexKey.id, true)
   carol.ebt.request(carol.id, true)
 
-  // publish a few more messages
-  const res = await pify(alice.db.publish)({ type: 'dog', name: 'Buffy' })
-
-  // index the new dog message
-  await pify(alice.db.publishAs)(aliceIndexKey, {
-      type: 'metafeed/index',
-      indexed: {
-        key: res.key,
-        author: alice.id,
-        sequence: res.value.sequence
-      }
-  })
-
   await pify(carol.connect)(alice.getAddress())
 
-  const clockAlice = await pify(alice.ebt.clock)({ format: 'indexedfeed' })
-  t.equal(clockAlice[aliceIndexKey.id], 2, 'alice correct index clock')
+  const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
+  t.equal(clockAlice[alice.id], 3, 'alice correct index clock')
 
-  carol.ebt.setClockForSlicedReplication(aliceIndexKey.id,
-                                         clockAlice[aliceIndexKey.id] - 1)
-  carol.ebt.request(aliceIndexKey.id, true)
+  carol.ebt.setClockForSlicedReplication(alice.id,
+                                         clockAlice[alice.id] - 2)
+  carol.ebt.request(alice.id, true)
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
-  const carolMessages = await carol.db.query(toPromise())
-  t.equal(carolMessages.length, 2, '1 index + 1 indexed message')
+  const carolMessages = await carol.db.query(
+    where(author(alice.id)),
+    toPromise()
+  )
+  t.equal(carolMessages.length, 2, 'latest 2 messages from alice')
 
   await Promise.all([
     pify(alice.close)(true),

From bb992d3bdb85e4f4b3b79d1315f71c25617af2b3 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 14 Sep 2021 11:08:34 +0200
Subject: [PATCH 22/50] Bump ssb-index-feed-writer to fix tests

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 9b4da4d..4abada9 100644
--- a/package.json
+++ b/package.json
@@ -42,7 +42,7 @@
     "ssb-db": "^19.2.0",
     "ssb-db2": "^2.4.0",
     "ssb-generate": "^1.0.1",
-    "ssb-index-feed-writer": "^0.5.1",
+    "ssb-index-feed-writer": "^0.6.0",
     "ssb-keys": "^8.1.0",
     "ssb-meta-feeds": "^0.19.0",
     "ssb-uri2": "^1.5.2",

From 5fe8343db1779140b40ce8e181f9d062f9b3bbd9 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 14 Sep 2021 11:14:52 +0200
Subject: [PATCH 23/50] Try increasing the wait for replication for tests

---
 test/formats.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/formats.js b/test/formats.js
index 91953cc..12a984a 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -378,7 +378,7 @@ tape('index format', async (t) => {
   
   await pify(dave.connect)(carol.getAddress())
 
-  await sleep(2 * REPLICATION_TIMEOUT)
+  await sleep(3 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
   // debugging
@@ -414,7 +414,7 @@ tape('index format', async (t) => {
     where(author(carol.id)),
     toPromise()
   )
-  t.equal(daveCarolMessages.length, 1, 'dave got dob message from carol')
+  t.equal(daveCarolMessages.length, 1, 'dave got dog message from carol')
 
   const expectedIndexClock = {
     [carolIndexId]: 1,

From 2208d43f3378a8177f1ba73a1ca7208d3aa84fa3 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 14 Sep 2021 11:15:55 +0200
Subject: [PATCH 24/50] Gargh, wrong test result

---
 test/formats.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/formats.js b/test/formats.js
index 12a984a..84b1470 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -378,7 +378,7 @@ tape('index format', async (t) => {
   
   await pify(dave.connect)(carol.getAddress())
 
-  await sleep(3 * REPLICATION_TIMEOUT)
+  await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
   // debugging

From 532f787e2b14439b933c83a674bdbdee024a9211 Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Tue, 14 Sep 2021 19:36:16 +0300
Subject: [PATCH 25/50] add missing ssb-caps dev dep

---
 package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package.json b/package.json
index 4abada9..5f851f6 100644
--- a/package.json
+++ b/package.json
@@ -38,6 +38,7 @@
     "rng": "^0.2.2",
     "secret-stack": "^6.4.0",
     "ssb-bendy-butt": "^0.12.2",
+    "ssb-caps": "^1.1.0",
     "ssb-client": "^4.9.0",
     "ssb-db": "^19.2.0",
     "ssb-db2": "^2.4.0",

From ae757fa3095d7025b299c55743316ae14bf218e4 Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Tue, 14 Sep 2021 19:36:46 +0300
Subject: [PATCH 26/50] add missing mkdirp dev dep

---
 package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package.json b/package.json
index 5f851f6..93e942c 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,7 @@
   "devDependencies": {
     "cat-names": "^3.0.0",
     "dog-names": "^2.0.0",
+    "mkdirp": "^1.0.4",
     "nyc": "^15.1.0",
     "promisify-4loc": "^1.0.0",
     "pull-paramap": "^1.2.2",

From 3c823eb13436f4f81128d2b72f12a10907344b1d Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Tue, 14 Sep 2021 21:38:40 +0200
Subject: [PATCH 27/50] Split formats into a folder

---
 README.md             |  18 ++++++-
 formats/bendy-butt.js |  50 ++++++++++++++++++
 formats/classic.js    |  40 ++++++++++++++
 formats/indexed.js    |  45 ++++++++++++++++
 formats/sliced.js     |  12 +++++
 index.js              |  40 ++------------
 package.json          |  10 ++--
 test/formats.js       | 119 ++++--------------------------------------
 8 files changed, 182 insertions(+), 152 deletions(-)
 create mode 100644 formats/bendy-butt.js
 create mode 100644 formats/classic.js
 create mode 100644 formats/indexed.js
 create mode 100644 formats/sliced.js

diff --git a/README.md b/README.md
index 07faa0b..c4cc91c 100644
--- a/README.md
+++ b/README.md
@@ -100,8 +100,12 @@ The output looks like this:
 
 ### `ssb.ebt.registerFormat(formatName, methods)` ("sync" muxrpc API)
 
+Register a new format for replication. Note this does not have to be a
+new feed format, it could also be indexed replication or sliced
+replication. See `formats` folder for examples.
+
 By registering a format you create a new EBT instance used for
-replicating feeds in that format. This means its own clock. Message
+replicating feeds using that format. This means its own clock. Message
 will be replicated using the `replicateFormat` api. `formatName` must
 be a string and methods must implement the following functions. The
 example shows the 'classic' implementation.
@@ -112,7 +116,9 @@ example shows the 'classic' implementation.
 ```js
 {
   // used in request, block, cleanClock, sbot.post, vectorClock
-  isFeed: ref.isFeed,
+  sbotIsFeed(sbot, feedId) {
+    return ref.isFeed(feedId)
+  },
   getAtSequence(sbot, pair, cb) {
     sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
       cb(err, msg ? msg.value : null)
@@ -123,6 +129,14 @@ example shows the 'classic' implementation.
       cb(err && err.fatal ? err : null, msg)
     })
   },
+  // used in onAppend
+  convertMsg(msgVal) {
+    return msgVal
+  },
+  // used in vectorClock
+  isReady(sbot) {
+    return Promise.resolve(true)
+  },
 
   // used in ebt:stream to distinguish between messages and notes
   isMsg(msgVal) {
diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js
new file mode 100644
index 0000000..a40ad0c
--- /dev/null
+++ b/formats/bendy-butt.js
@@ -0,0 +1,50 @@
+const SSBURI = require('ssb-uri2')
+const bendyButt = require('ssb-bendy-butt')
+
+module.exports = {
+  // used in request, block, cleanClock, sbot.post, vectorClock
+  sbotIsFeed(sbot, feedId) {
+    return SSBURI.isBendyButtV1FeedSSBURI(feedId)
+  },
+  getAtSequence(sbot, pair, cb) {
+    sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+      cb(err, msg ? bendyButt.encode(msg.value) : null)
+    })
+  },
+  appendMsg(sbot, msgVal, cb) {
+    sbot.add(bendyButt.decode(msgVal), (err, msg) => {
+      cb(err && err.fatal ? err : null, msg)
+    })
+  },
+  convertMsg(msgVal) {
+    return bendyButt.encode(msgVal)
+  },
+  // used in vectorClock
+  isReady(sbot) {
+    return Promise.resolve(true)
+  },
+
+  // used in ebt:stream to distinguish between messages and notes
+  isMsg(bbVal) {
+    if (Buffer.isBuffer(bbVal)) {
+      const msgVal = bendyButt.decode(bbVal)
+      return msgVal && SSBURI.isBendyButtV1FeedSSBURI(msgVal.author)
+    } else {
+      return bbVal && SSBURI.isBendyButtV1FeedSSBURI(bbVal.author)
+    }
+  },
+  // used in ebt:events
+  getMsgAuthor(bbVal) {
+    if (Buffer.isBuffer(bbVal))
+      return bendyButt.decode(bbVal).author
+    else
+      return bbVal.author
+  },
+  // used in ebt:events
+  getMsgSequence(bbVal) {
+    if (Buffer.isBuffer(bbVal))
+      return bendyButt.decode(bbVal).sequence
+    else
+      return bbVal.sequence
+  }
+}
diff --git a/formats/classic.js b/formats/classic.js
new file mode 100644
index 0000000..9cd029d
--- /dev/null
+++ b/formats/classic.js
@@ -0,0 +1,40 @@
+const ref = require('ssb-ref')
+
+module.exports = {
+  // used in request, block, cleanClock, sbot.post, vectorClock
+  sbotIsFeed(sbot, feedId) {
+    return ref.isFeed(feedId)
+  },
+  getAtSequence(sbot, pair, cb) {
+    sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+      cb(err, msg ? msg.value : null)
+    })
+  },
+  appendMsg(sbot, msgVal, cb) {
+    sbot.add(msgVal, (err, msg) => {
+      cb(err && err.fatal ? err : null, msg)
+    })
+  },
+  // used in onAppend
+  convertMsg(msgVal) {
+    return msgVal
+  },
+  // used in vectorClock
+  isReady(sbot) {
+    return Promise.resolve(true)
+  },
+
+  // used in ebt:stream to distinguish between messages and notes
+  isMsg(msgVal) {
+    return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
+      ref.isFeed(msgVal.author) && msgVal.content
+  },
+  // used in ebt:events
+  getMsgAuthor(msgVal) {
+    return msgVal.author
+  },
+  // used in ebt:events
+  getMsgSequence(msgVal) {
+    return msgVal.sequence
+  }
+}
diff --git a/formats/indexed.js b/formats/indexed.js
new file mode 100644
index 0000000..910a01f
--- /dev/null
+++ b/formats/indexed.js
@@ -0,0 +1,45 @@
+const classic = require('./classic')
+const pify = require('promisify-4loc')
+
+module.exports = function () {
+  return Object.assign(
+    {}, classic, {
+      sbotIsFeed(sbot, author) {
+        const info = sbot.metafeeds.findByIdSync(author)
+        return info && info.feedpurpose === 'index'
+      },
+      appendMsg(sbot, msgVal, cb) {
+        // FIXME: this needs the ability to add 2 messages in a transaction
+        const payload = msgVal.content.indexed.payload
+        delete msgVal.content.indexed.payload
+        sbot.db.add(msgVal, (err, msg) => {
+          if (err) return cb(err)
+          sbot.db.addOOO(payload, (err, indexedMsg) => {
+            if (err) return cb(err)
+            else cb(null, msg)
+          })
+        })
+      },
+      getAtSequence(sbot, pair, cb) {
+        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+          if (err) return cb(err)
+          const { sequence } = msg.value.content.indexed
+          const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
+          if (!authorInfo) return cb(new Error("Unknown author", msg.value.author))
+          const { author } = JSON.parse(authorInfo.metadata.query)
+          sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
+            if (err) return cb(err)
+
+            // add referenced message as payload
+            msg.value.content.indexed.payload = indexedMsg.value
+
+            cb(null, msg.value)
+          })
+        })
+      },
+      isReady(sbot) {
+        return pify(sbot.metafeeds.loadState)()
+      }
+    }
+  )
+}
diff --git a/formats/sliced.js b/formats/sliced.js
new file mode 100644
index 0000000..d99c3e1
--- /dev/null
+++ b/formats/sliced.js
@@ -0,0 +1,12 @@
+const classic = require('./classic')
+
+module.exports = Object.assign(
+  {}, classic, {
+    appendMsg(sbot, msgVal, cb) {
+      sbot.db.addOOO(msgVal, (err, msg) => {
+        if (err) return cb(err)
+        else cb(null, msg)
+      })
+    }
+  }
+)
diff --git a/index.js b/index.js
index f849f98..139b192 100644
--- a/index.js
+++ b/index.js
@@ -2,7 +2,6 @@ const path = require('path')
 const pull = require('pull-stream')
 const toPull = require('push-stream-to-pull-stream')
 const EBT = require('epidemic-broadcast-trees')
-const ref = require('ssb-ref')
 const Store = require('lossy-store')
 const toUrlFriendly = require('base64-url').escape
 const getSeverity = require('ssb-network-errors')
@@ -47,42 +46,7 @@ function cleanClock (clock, isFeed) {
 
 exports.init = function (sbot, config) {
   const formats = {
-    'classic': {
-      // used in request, block, cleanClock, sbot.post, vectorClock
-      isFeed: ref.isFeed,
-      getAtSequence(sbot, pair, cb) {
-        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
-          cb(err, msg ? msg.value : null)
-        })
-      },
-      appendMsg(sbot, msgVal, cb) {
-        sbot.add(msgVal, (err, msg) => {
-          cb(err && err.fatal ? err : null, msg)
-        })
-      },
-      // used in onAppend
-      convertMsg(msgVal) {
-        return msgVal
-      },
-      // used in vectorClock
-      isReady(sbot) {
-        return Promise.resolve(true)
-      },
-
-      // used in ebt:stream to distinguish between messages and notes
-      isMsg(msgVal) {
-        return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
-          ref.isFeed(msgVal.author) && msgVal.content
-      },
-      // used in ebt:events
-      getMsgAuthor(msgVal) {
-        return msgVal.author
-      },
-      // used in ebt:events
-      getMsgSequence(msgVal) {
-        return msgVal.sequence
-      }
-    }
+    'classic': require('./formats/classic')
   }
 
   const ebts = {}
@@ -92,6 +56,8 @@ exports.init = function (sbot, config) {
     const store = Store(dir, null, toUrlFriendly)
 
     const format = formats[formatName]
+    // EBT expects a function of only feedId so we bind sbot here
+    format.isFeed = (feedId) => format.sbotIsFeed(sbot, feedId)
 
     const ebt = EBT(Object.assign({
       logging: config.ebt && config.ebt.logging,
diff --git a/package.json b/package.json
index 93e942c..db4ee9b 100644
--- a/package.json
+++ b/package.json
@@ -22,23 +22,24 @@
     "epidemic-broadcast-trees": "^8.0.4",
     "lossy-store": "^1.2.3",
     "p-defer": "^3.0.0",
+    "promisify-4loc": "^1.0.0",
     "pull-defer": "^0.2.3",
     "pull-stream": "^3.6.0",
     "push-stream-to-pull-stream": "^1.0.4",
+    "ssb-bendy-butt": "^0.12.2",
     "ssb-network-errors": "^1.0.0",
-    "ssb-ref": "^2.13.0"
+    "ssb-ref": "^2.13.0",
+    "ssb-uri2": "^1.5.2"
   },
   "devDependencies": {
     "cat-names": "^3.0.0",
     "dog-names": "^2.0.0",
     "mkdirp": "^1.0.4",
     "nyc": "^15.1.0",
-    "promisify-4loc": "^1.0.0",
     "pull-paramap": "^1.2.2",
     "rimraf": "^2.7.1",
     "rng": "^0.2.2",
     "secret-stack": "^6.4.0",
-    "ssb-bendy-butt": "^0.12.2",
     "ssb-caps": "^1.1.0",
     "ssb-client": "^4.9.0",
     "ssb-db": "^19.2.0",
@@ -46,8 +47,7 @@
     "ssb-generate": "^1.0.1",
     "ssb-index-feed-writer": "^0.6.0",
     "ssb-keys": "^8.1.0",
-    "ssb-meta-feeds": "^0.19.0",
-    "ssb-uri2": "^1.5.2",
+    "ssb-meta-feeds": "^0.21.0",
     "ssb-validate": "^4.1.4",
     "standardx": "^7.0.0",
     "tap-spec": "^5.0.0",
diff --git a/test/formats.js b/test/formats.js
index 84b1470..8dfb5ee 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -46,53 +46,6 @@ let bob = createSSBServer().call(null, {
   keys: u.keysFor('bob')
 })
 
-// FIXME: this should be somewhere else
-const bendyButtMethods = {
-  // used in request, block, cleanClock, sbot.post, vectorClock
-  isFeed: SSBURI.isBendyButtV1FeedSSBURI,
-  getAtSequence(sbot, pair, cb) {
-    sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
-      cb(err, msg ? bendyButt.encode(msg.value) : null)
-    })
-  },
-  appendMsg(sbot, msgVal, cb) {
-    sbot.add(bendyButt.decode(msgVal), (err, msg) => {
-      cb(err && err.fatal ? err : null, msg)
-    })
-  },
-  convertMsg(msgVal) {
-    return bendyButt.encode(msgVal)
-  },
-  // used in vectorClock
-  isReady(sbot) {
-    return Promise.resolve(true)
-  },
-
-  // used in ebt:stream to distinguish between messages and notes
-  isMsg(bbVal) {
-    if (Buffer.isBuffer(bbVal)) {
-      const msgVal = bendyButt.decode(bbVal)
-      return msgVal && SSBURI.isBendyButtV1FeedSSBURI(msgVal.author)
-    } else {
-      return bbVal && SSBURI.isBendyButtV1FeedSSBURI(bbVal.author)
-    }
-  },
-  // used in ebt:events
-  getMsgAuthor(bbVal) {
-    if (Buffer.isBuffer(bbVal))
-      return bendyButt.decode(bbVal).author
-    else
-      return bbVal.author
-  },
-  // used in ebt:events
-  getMsgSequence(bbVal) {
-    if (Buffer.isBuffer(bbVal))
-      return bendyButt.decode(bbVal).sequence
-    else
-      return bbVal.sequence
-  }
-}
-
 function getBBMsg(mainKeys) {
   // fake some keys
   const mfKeys = ssbKeys.generate()
@@ -127,6 +80,8 @@ function getBBMsg(mainKeys) {
   return bendyButt.decode(bbmsg)
 }
 
+const bendyButtMethods = require('../formats/bendy-butt')
+
 // need them later
 let aliceMFId
 let bobMFId
@@ -249,50 +204,6 @@ tape('multiple formats restart', async (t) => {
   t.end()
 })
 
-
-// FIXME: this needs the ability to add 2 messages in a transaction
-function indexedFeedMethods(sbot) {
-  return Object.assign(
-    {}, alice.ebt.formats['classic'], {
-      isFeed(author) {
-        const info = sbot.metafeeds.findByIdSync(author)
-        return info && info.feedpurpose === 'index'
-      },
-      appendMsg(sbot, msgVal, cb) {
-        const payload = msgVal.content.indexed.payload
-        delete msgVal.content.indexed.payload
-        sbot.db.add(msgVal, (err, msg) => {
-          if (err) return cb(err)
-          sbot.db.addOOO(payload, (err, indexedMsg) => {
-            if (err) return cb(err)
-            else cb(null, msg)
-          })
-        })
-      },
-      getAtSequence(sbot, pair, cb) {
-        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
-          if (err) return cb(err)
-          const { sequence } = msg.value.content.indexed
-          const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
-          if (!authorInfo) return cb(new Error("Unknown author", msg.value.author))
-          const { author } = JSON.parse(authorInfo.metadata.query)
-          sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
-            if (err) return cb(err)
-
-            // add referenced message as payload
-            msg.value.content.indexed.payload = indexedMsg.value
-
-            cb(null, msg.value)
-          })
-        })
-      },
-      isReady(sbot) {
-        return pify(sbot.metafeeds.loadState)()
-      },
-    }
-  )
-}
-
 const carolDir = getFreshDir('carol')
 
 tape('index format', async (t) => {
@@ -309,9 +220,12 @@ tape('index format', async (t) => {
     keys: u.keysFor('dave')
   })
 
-  carol.ebt.registerFormat('indexedfeed', indexedFeedMethods(carol))
+  const carolIndexedMethods = require('../formats/indexed.js')()
+  const daveIndexedMethods = require('../formats/indexed.js')()
+
+  carol.ebt.registerFormat('indexedfeed', carolIndexedMethods)
   carol.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
-  dave.ebt.registerFormat('indexedfeed', indexedFeedMethods(dave))
+  dave.ebt.registerFormat('indexedfeed', daveIndexedMethods)
   dave.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
 
   const carolIndexId = (await pify(carol.indexFeedWriter.start)({ author: carol.id, type: 'dog', private: false })).subfeed
@@ -434,18 +348,6 @@ tape('index format', async (t) => {
   t.end()
 })
 
-const slicedReplication = Object.assign(
-  {}, alice.ebt.formats['classic'], {
-    appendMsg(sbot, msgVal, cb) {
-      sbot.db.addOOO(msgVal, (err, msg) => {
-        if (err) return cb(err)
-        else cb(null, msg)
-      })
-    }
-  }
-)
-
-// FIXME: this needs to be as before
 tape('sliced replication', async (t) => {
   alice = createSSBServer().call(null, {
     path: aliceDir,
@@ -463,9 +365,10 @@ tape('sliced replication', async (t) => {
     pify(alice.db.publish)({ type: 'post', text: 'hello2' }),
     pify(alice.db.publish)({ type: 'post', text: 'hello3' }),
   ])
-  
-  alice.ebt.registerFormat('slicedreplication', slicedReplication)
-  carol.ebt.registerFormat('slicedreplication', slicedReplication)
+
+  const slicedMethods = require('../formats/sliced')
+  alice.ebt.registerFormat('slicedreplication', slicedMethods)
+  carol.ebt.registerFormat('slicedreplication', slicedMethods)
 
   // self replicate
   alice.ebt.request(alice.id, true)

From 5de319d166b6a4c8e867e1ed008a53c0990f4c1a Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 12:05:18 +0200
Subject: [PATCH 28/50] Use spread operator instead of object.assign

---
 formats/indexed.js | 65 +++++++++++++++++++++++-----------------------
 formats/sliced.js  | 17 ++++++------
 2 files changed, 40 insertions(+), 42 deletions(-)

diff --git a/formats/indexed.js b/formats/indexed.js
index 910a01f..433f301 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -2,44 +2,43 @@ const classic = require('./classic')
 const pify = require('promisify-4loc')
 
 module.exports = function () {
-  return Object.assign(
-    {}, classic, {
-      sbotIsFeed(sbot, author) {
-        const info = sbot.metafeeds.findByIdSync(author)
-        return info && info.feedpurpose === 'index'
-      },
-      appendMsg(sbot, msgVal, cb) {
-        // FIXME: this needs the ability to add 2 messages in a transaction
-        const payload = msgVal.content.indexed.payload
-        delete msgVal.content.indexed.payload
-        sbot.db.add(msgVal, (err, msg) => {
+  return {
+    ...classic, 
+    sbotIsFeed(sbot, author) {
+      const info = sbot.metafeeds.findByIdSync(author)
+      return info && info.feedpurpose === 'index'
+    },
+    appendMsg(sbot, msgVal, cb) {
+      // FIXME: this needs the ability to add 2 messages in a transaction
+      const payload = msgVal.content.indexed.payload
+      delete msgVal.content.indexed.payload
+      sbot.db.add(msgVal, (err, msg) => {
+        if (err) return cb(err)
+        sbot.db.addOOO(payload, (err, indexedMsg) => {
           if (err) return cb(err)
-          sbot.db.addOOO(payload, (err, indexedMsg) => {
-            if (err) return cb(err)
-            else cb(null, msg)
-          })
+          else cb(null, msg)
         })
-      },
-      getAtSequence(sbot, pair, cb) {
-        sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+      })
+    },
+    getAtSequence(sbot, pair, cb) {
+      sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+        if (err) return cb(err)
+        const { sequence } = msg.value.content.indexed
+        const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
+        if (!authorInfo) return cb(new Error("Unknown author", msg.value.author))
+        const { author } = JSON.parse(authorInfo.metadata.query)
+        sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
           if (err) return cb(err)
-          const { sequence } = msg.value.content.indexed
-          const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
-          if (!authorInfo) return cb(new Error("Unknown author", msg.value.author))
-          const { author } = JSON.parse(authorInfo.metadata.query)
-          sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
-            if (err) return cb(err)
 
-            // add referenced message as payload
-            msg.value.content.indexed.payload = indexedMsg.value
+          // add referenced message as payload
+          msg.value.content.indexed.payload = indexedMsg.value
 
-            cb(null, msg.value)
-          })
+          cb(null, msg.value)
         })
-      },
-      isReady(sbot) {
-        return pify(sbot.metafeeds.loadState)()
-      }
+      })
+    },
+    isReady(sbot) {
+      return pify(sbot.metafeeds.loadState)()
     }
-  )
+  }
 }
diff --git a/formats/sliced.js b/formats/sliced.js
index d99c3e1..a69fe8c 100644
--- a/formats/sliced.js
+++ b/formats/sliced.js
@@ -1,12 +1,11 @@
 const classic = require('./classic')
 
-module.exports = Object.assign(
-  {}, classic, {
-    appendMsg(sbot, msgVal, cb) {
-      sbot.db.addOOO(msgVal, (err, msg) => {
-        if (err) return cb(err)
-        else cb(null, msg)
-      })
-    }
+module.exports = {
+  ...classic,
+  appendMsg(sbot, msgVal, cb) {
+    sbot.db.addOOO(msgVal, (err, msg) => {
+      if (err) return cb(err)
+      else cb(null, msg)
+    })
   }
-)
+}

From 38f96dbae6f3465f0d04be095363f3f6059edaac Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 12:07:43 +0200
Subject: [PATCH 29/50] Minor review fixes

---
 formats/indexed.js | 5 +++--
 package.json       | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/formats/indexed.js b/formats/indexed.js
index 433f301..d3b30c1 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -1,5 +1,6 @@
 const classic = require('./classic')
 const pify = require('promisify-4loc')
+const { QL0 } = require('ssb-subset-ql')
 
 module.exports = function () {
   return {
@@ -25,8 +26,8 @@ module.exports = function () {
         if (err) return cb(err)
         const { sequence } = msg.value.content.indexed
         const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
-        if (!authorInfo) return cb(new Error("Unknown author", msg.value.author))
-        const { author } = JSON.parse(authorInfo.metadata.query)
+        if (!authorInfo) return cb(new Error("Unknown author:" + msg.value.author))
+        const { author } = QL0.parse(authorInfo.metadata.query)
         sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
           if (err) return cb(err)
 
diff --git a/package.json b/package.json
index db4ee9b..cd03f35 100644
--- a/package.json
+++ b/package.json
@@ -29,6 +29,7 @@
     "ssb-bendy-butt": "^0.12.2",
     "ssb-network-errors": "^1.0.0",
     "ssb-ref": "^2.13.0",
+    "ssb-subset-ql": "^0.6.1",
     "ssb-uri2": "^1.5.2"
   },
   "devDependencies": {

From 213fc0be30d3ad128a5e7a0960a109d8a5b25b9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Staltz?= <andre@staltz.com>
Date: Wed, 15 Sep 2021 13:59:26 +0300
Subject: [PATCH 30/50] Fix "API" typo in README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index c4cc91c..ca255c9 100644
--- a/README.md
+++ b/README.md
@@ -106,7 +106,7 @@ replication. See `formats` folder for examples.
 
 By registering a format you create a new EBT instance used for
 replicating feeds using that format. This means its own clock. Message
-will be replicated using the `replicateFormat` api. `formatName` must
+will be replicated using the `replicateFormat` API. `formatName` must
 be a string and methods must implement the following functions. The
 example shows the 'classic' implementation.
 

From 71fdda61df3ba06aa9ca97e1a4e564eb47ce3190 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Staltz?= <andre@staltz.com>
Date: Wed, 15 Sep 2021 13:59:51 +0300
Subject: [PATCH 31/50] Tweak README.md text about format example

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ca255c9..2b2c340 100644
--- a/README.md
+++ b/README.md
@@ -108,7 +108,7 @@ By registering a format you create a new EBT instance used for
 replicating feeds using that format. This means its own clock. Message
 will be replicated using the `replicateFormat` API. `formatName` must
 be a string and methods must implement the following functions. The
-example shows the 'classic' implementation.
+example below shows the implementation for 'classic' ed25519 SSB feeds.
 
 <details>
 <summary>CLICK HERE</summary>

From ebc87a32fbc8f95581d2a5c8792106da0908785d Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 13:22:30 +0200
Subject: [PATCH 32/50] Update index.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: André Staltz <andre@staltz.com>
---
 index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.js b/index.js
index 139b192..d0dd42e 100644
--- a/index.js
+++ b/index.js
@@ -210,7 +210,7 @@ exports.init = function (sbot, config) {
       throw new Error('expected ebt.replicate({version: 3})')
     }
 
-    let formatName = opts.format || 'classic'
+    const formatName = opts.format || 'classic'
     const ebt = getEBT(formatName)
 
     var deferred = pullDefer.duplex()

From ce7843722eac8996abc3b4b1ad392c71848b0149 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 13:34:58 +0200
Subject: [PATCH 33/50] Review fixes

---
 index.js | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/index.js b/index.js
index d0dd42e..c3e3e06 100644
--- a/index.js
+++ b/index.js
@@ -45,11 +45,10 @@ function cleanClock (clock, isFeed) {
 }
 
 exports.init = function (sbot, config) {
-  const formats = {
-    'classic': require('./formats/classic')
-  }
-
   const ebts = {}
+  const formats = {}
+  registerFormat('classic', require('./formats/classic'))
+
   function addEBT(formatName) {
     const dirName = 'ebt' + (formatName === 'classic' ? '' : formatName)
     const dir = config.path ? path.join(config.path, dirName) : null
@@ -57,9 +56,9 @@ exports.init = function (sbot, config) {
 
     const format = formats[formatName]
     // EBT expects a function of only feedId so we bind sbot here
-    format.isFeed = (feedId) => format.sbotIsFeed(sbot, feedId)
+    format.isFeed = format.sbotIsFeed.bind(format, sbot)
 
-    const ebt = EBT(Object.assign({
+    const ebt = EBT({
       logging: config.ebt && config.ebt.logging,
       id: sbot.id,
       getClock (id, cb) {
@@ -78,8 +77,9 @@ exports.init = function (sbot, config) {
       },
       append (msgVal, cb) {
         format.appendMsg(sbot, msgVal, cb)
-      }
-    }, format))
+      },
+      ...format
+    })
 
     ebts[formatName] = ebt
   }
@@ -92,8 +92,6 @@ exports.init = function (sbot, config) {
     return ebt
   }
 
-  addEBT('classic')
-
   const initialized = DeferredPromise()
 
   sbot.getVectorClock((err, clock) => {

From 8cb3b9c0b69a71066079f1f4ba6823f1efaba1f7 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 14:39:46 +0200
Subject: [PATCH 34/50] Add formats to package.json

---
 package.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index cd03f35..074becf 100644
--- a/package.json
+++ b/package.json
@@ -12,7 +12,8 @@
   "main": "index.js",
   "files": [
     "*.js",
-    "debug/*.js"
+    "debug/*.js",
+    "formats/*.js"
   ],
   "engines": {
     "node": ">=10"

From dee03d0678517945948cf08773118994b4145163 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 16:33:46 +0200
Subject: [PATCH 35/50] Refactor to not use formats

---
 formats/indexed.js | 64 +++++++++++++++---------------
 index.js           | 97 +++++++++++++++++++---------------------------
 test/formats.js    |  7 ++--
 3 files changed, 73 insertions(+), 95 deletions(-)

diff --git a/formats/indexed.js b/formats/indexed.js
index d3b30c1..4b5c13f 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -2,44 +2,42 @@ const classic = require('./classic')
 const pify = require('promisify-4loc')
 const { QL0 } = require('ssb-subset-ql')
 
-module.exports = function () {
-  return {
-    ...classic, 
-    sbotIsFeed(sbot, author) {
-      const info = sbot.metafeeds.findByIdSync(author)
-      return info && info.feedpurpose === 'index'
-    },
-    appendMsg(sbot, msgVal, cb) {
-      // FIXME: this needs the ability to add 2 messages in a transaction
-      const payload = msgVal.content.indexed.payload
-      delete msgVal.content.indexed.payload
-      sbot.db.add(msgVal, (err, msg) => {
+module.exports = {
+  ...classic, 
+  sbotIsFeed(sbot, author) {
+    const info = sbot.metafeeds.findByIdSync(author)
+    return info && info.feedpurpose === 'index'
+  },
+  appendMsg(sbot, msgVal, cb) {
+    // FIXME: this needs the ability to add 2 messages in a transaction
+    const payload = msgVal.content.indexed.payload
+    delete msgVal.content.indexed.payload
+    sbot.db.add(msgVal, (err, msg) => {
+      if (err) return cb(err)
+      sbot.db.addOOO(payload, (err, indexedMsg) => {
         if (err) return cb(err)
-        sbot.db.addOOO(payload, (err, indexedMsg) => {
-          if (err) return cb(err)
-          else cb(null, msg)
-        })
+        else cb(null, msg)
       })
-    },
-    getAtSequence(sbot, pair, cb) {
-      sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+    })
+  },
+  getAtSequence(sbot, pair, cb) {
+    sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
+      if (err) return cb(err)
+      const { sequence } = msg.value.content.indexed
+      const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
+      if (!authorInfo) return cb(new Error("Unknown author:" + msg.value.author))
+      const { author } = QL0.parse(authorInfo.metadata.query)
+      sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
         if (err) return cb(err)
-        const { sequence } = msg.value.content.indexed
-        const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
-        if (!authorInfo) return cb(new Error("Unknown author:" + msg.value.author))
-        const { author } = QL0.parse(authorInfo.metadata.query)
-        sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
-          if (err) return cb(err)
 
-          // add referenced message as payload
-          msg.value.content.indexed.payload = indexedMsg.value
+        // add referenced message as payload
+        msg.value.content.indexed.payload = indexedMsg.value
 
-          cb(null, msg.value)
-        })
+        cb(null, msg.value)
       })
-    },
-    isReady(sbot) {
-      return pify(sbot.metafeeds.loadState)()
-    }
+    })
+  },
+  isReady(sbot) {
+    return pify(sbot.metafeeds.loadState)()
   }
 }
diff --git a/index.js b/index.js
index c3e3e06..000891e 100644
--- a/index.js
+++ b/index.js
@@ -46,17 +46,16 @@ function cleanClock (clock, isFeed) {
 
 exports.init = function (sbot, config) {
   const ebts = {}
-  const formats = {}
   registerFormat('classic', require('./formats/classic'))
 
-  function addEBT(formatName) {
+  function registerFormat(formatName, format) {
     const dirName = 'ebt' + (formatName === 'classic' ? '' : formatName)
     const dir = config.path ? path.join(config.path, dirName) : null
     const store = Store(dir, null, toUrlFriendly)
 
-    const format = formats[formatName]
     // EBT expects a function of only feedId so we bind sbot here
-    format.isFeed = format.sbotIsFeed.bind(format, sbot)
+    const isFeed = format.sbotIsFeed.bind(format, sbot)
+    const { isMsg, getMsgAuthor, getMsgSequence } = format
 
     const ebt = EBT({
       logging: config.ebt && config.ebt.logging,
@@ -64,12 +63,12 @@ exports.init = function (sbot, config) {
       getClock (id, cb) {
         store.ensure(id, function () {
           const clock = store.get(id) || {}
-          cleanClock(clock, format.isFeed)
+          cleanClock(clock, isFeed)
           cb(null, clock)
         })
       },
       setClock (id, clock) {
-        cleanClock(clock, format.isFeed)
+        cleanClock(clock, isFeed)
         store.set(id, clock)
       },
       getAt (pair, cb) {
@@ -78,9 +77,18 @@ exports.init = function (sbot, config) {
       append (msgVal, cb) {
         format.appendMsg(sbot, msgVal, cb)
       },
-      ...format
+
+      isFeed,
+      isMsg,
+      getMsgAuthor,
+      getMsgSequence
     })
 
+    // attach a few methods we need in this module
+    ebt.convertMsg = format.convertMsg
+    ebt.isReady = format.isReady.bind(format, sbot)
+    ebt.isFeed = isFeed
+
     ebts[formatName] = ebt
   }
 
@@ -97,31 +105,27 @@ exports.init = function (sbot, config) {
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
 
-    const readies = Object.values(formats).map(f => f.isReady(sbot))
+    const readies = Object.values(ebts).map(ebt => ebt.isReady())
     Promise.all(readies).then(() => {
-      for (let formatName in ebts) {
-        const format = formats[formatName]
-        const ebt = ebts[formatName]
-
-        validClock = {}
+      Object.values(ebts).forEach(ebt => {
+        const validClock = {}
         for (let k in clock)
-          if (format.isFeed(k))
+          if (ebt.isFeed(k))
             validClock[k] = clock[k]
 
         ebt.state.clock = validClock
         ebt.update()
-      }
+      })
       initialized.resolve()
     })
   })
 
   sbot.post((msg) => {
     initialized.promise.then(() => {
-      for (let formatName in ebts) {
-        const format = formats[formatName]
-        if (format.isFeed(msg.value.author))
-          ebts[formatName].onAppend(format.convertMsg(msg.value))
-      }
+      Object.values(ebts).forEach(ebt => {
+        if (ebt.isFeed(msg.value.author))
+          ebt.onAppend(ebt.convertMsg(msg.value))
+      })
     })
   })
 
@@ -161,35 +165,28 @@ exports.init = function (sbot, config) {
     }
   })
 
+  function findEBTForFeed(feedId) {
+    let ebt = Object.values(ebts).reverse().find(ebt => ebt.isFeed(feedId))
+    if (ebt) return ebt
+    else return ebts['classic']
+  }
+
   function request(destFeedId, requesting) {
     initialized.promise.then(() => {
-      formatName = 'classic'
-      for (let format in ebts)
-        if (formats[format].isFeed(destFeedId))
-          formatName = format
+      const ebt = findEBTForFeed(destFeedId)
 
-      const format = formats[formatName]
-
-      if (!(format && format.isFeed(destFeedId))) return
+      if (!ebt.isFeed(destFeedId)) return
       
-      ebts[formatName].request(destFeedId, requesting)
+      ebt.request(destFeedId, requesting)
     })
   }
 
   function block(origFeedId, destFeedId, blocking) {
     initialized.promise.then(() => {
-      formatName = 'classic'
-      for (let format in ebts)
-        if (formats[format].isFeed(origFeedId))
-          formatName = format
-
-      const format = formats[formatName]
+      const ebt = findEBTForFeed(origFeedId)
 
-      if (!format) return
-      if (!format.isFeed(origFeedId)) return
-      if (!format.isFeed(destFeedId)) return
-
-      const ebt = ebts[formatName]
+      if (!ebt.isFeed(origFeedId)) return
+      if (!ebt.isFeed(destFeedId)) return
 
       if (blocking) {
         ebt.block(origFeedId, destFeedId, true)
@@ -223,12 +220,7 @@ exports.init = function (sbot, config) {
   function peerStatus(id) {
     id = id || sbot.id
 
-    formatName = 'classic'
-    for (let format in ebts)
-      if (formats[format].isFeed(id))
-        formatName = format
-
-    const ebt = getEBT(formatName)
+    const ebt = findEBTForFeed(id)
 
     const data = {
       id: id,
@@ -264,22 +256,12 @@ exports.init = function (sbot, config) {
   }
 
   function setClockForSlicedReplication(feedId, sequence) {
-    formatName = 'classic'
-    for (let format in ebts)
-      if (formats[format].isFeed(feedId))
-        formatName = format
-
     initialized.promise.then(() => {
-      const ebt = getEBT(formatName)
+      const ebt = findEBTForFeed(feedId)
       ebt.state.clock[feedId] = sequence
     })
   }
 
-  function registerFormat(formatName, methods) {
-    formats[formatName] = methods
-    addEBT(formatName)
-  }
-
   return {
     request,
     block,
@@ -288,7 +270,6 @@ exports.init = function (sbot, config) {
     peerStatus,
     clock,
     setClockForSlicedReplication,
-    registerFormat,
-    formats
+    registerFormat
   }
 }
diff --git a/test/formats.js b/test/formats.js
index 8dfb5ee..71f7acc 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -220,12 +220,11 @@ tape('index format', async (t) => {
     keys: u.keysFor('dave')
   })
 
-  const carolIndexedMethods = require('../formats/indexed.js')()
-  const daveIndexedMethods = require('../formats/indexed.js')()
+  const indexedMethods = require('../formats/indexed.js')
 
-  carol.ebt.registerFormat('indexedfeed', carolIndexedMethods)
+  carol.ebt.registerFormat('indexedfeed', indexedMethods)
   carol.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
-  dave.ebt.registerFormat('indexedfeed', daveIndexedMethods)
+  dave.ebt.registerFormat('indexedfeed', indexedMethods)
   dave.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
 
   const carolIndexId = (await pify(carol.indexFeedWriter.start)({ author: carol.id, type: 'dog', private: false })).subfeed

From 7c520b89c09314371e4cc9ad122d6b55c31ea846 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 21:19:21 +0200
Subject: [PATCH 36/50] Allow one to specify the format in request/block

---
 README.md       | 11 ++++++++---
 index.js        | 21 +++++++++++++--------
 test/formats.js | 21 ++++++++++++---------
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/README.md b/README.md
index 2b2c340..4150ab1 100644
--- a/README.md
+++ b/README.md
@@ -38,11 +38,13 @@ installed, instead, you need to call its API methods yourself (primarily
 `request` or `block`), or use a scheduler module such as
 [ssb-replication-scheduler](https://github.com/ssb-ngi-pointer/ssb-replication-scheduler).
 
-### `ssb.ebt.request(destination, replicating)` ("sync" muxrpc API)
+### `ssb.ebt.request(destination, replicating, formatName)` ("sync" muxrpc API)
 
 Request that the SSB feed ID `destination` be replicated. `replication` is a
 boolean, where `true` indicates we want to replicate the destination. If set to
-`false`, replication is stopped.
+`false`, replication is stopped. `formatName` is optional and used to specify 
+the specific EBT instance, otherwise the first where isFeed is `true` for 
+`destination` is used.
 
 Returns undefined, always.
 
@@ -56,6 +58,9 @@ them.
 the SSB feed ID of the peer being blocked, and `blocking` is a boolean that
 indicates whether to enable the block (`true`) or to unblock (`false`).
 
+`formatName` is optional and used to specify the specific EBT instance, 
+otherwise the first where isFeed is `true` for `origin` is used.
+
 Returns undefined, always.
 
 ### `ssb.ebt.peerStatus(id)` ("sync" muxrpc API)
@@ -155,7 +160,7 @@ example below shows the implementation for 'classic' ed25519 SSB feeds.
 ```
 </details>
 
-### `ssb.ebt.setClockForSlicedReplication(feedId, sequence)` ("sync" muxrpc API)
+### `ssb.ebt.setClockForSlicedReplication(feedId, sequence, formatName)` ("sync" muxrpc API)
 
 Sets the internal clock of a feed to a specific sequence. Note this
 does not start replicating the feed, it only updates the clock. By
diff --git a/index.js b/index.js
index 000891e..e6dbbad 100644
--- a/index.js
+++ b/index.js
@@ -165,15 +165,19 @@ exports.init = function (sbot, config) {
     }
   })
 
-  function findEBTForFeed(feedId) {
-    let ebt = Object.values(ebts).reverse().find(ebt => ebt.isFeed(feedId))
+  function findEBTForFeed(feedId, formatName) {
+    let ebt
+    if (formatName)
+      ebt = ebts[formatName]
+    else
+      ebt = Object.values(ebts).find(ebt => ebt.isFeed(feedId))
     if (ebt) return ebt
     else return ebts['classic']
   }
 
-  function request(destFeedId, requesting) {
+  function request(destFeedId, requesting, formatName) {
     initialized.promise.then(() => {
-      const ebt = findEBTForFeed(destFeedId)
+      const ebt = findEBTForFeed(destFeedId, formatName)
 
       if (!ebt.isFeed(destFeedId)) return
       
@@ -181,9 +185,9 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function block(origFeedId, destFeedId, blocking) {
+  function block(origFeedId, destFeedId, blocking, formatName) {
     initialized.promise.then(() => {
-      const ebt = findEBTForFeed(origFeedId)
+      const ebt = findEBTForFeed(origFeedId, formatName)
 
       if (!ebt.isFeed(origFeedId)) return
       if (!ebt.isFeed(destFeedId)) return
@@ -255,9 +259,10 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function setClockForSlicedReplication(feedId, sequence) {
+  function setClockForSlicedReplication(feedId, sequence, formatName) {
     initialized.promise.then(() => {
-      const ebt = findEBTForFeed(feedId)
+      const ebt = findEBTForFeed(feedId, formatName)
+
       ebt.state.clock[feedId] = sequence
     })
   }
diff --git a/test/formats.js b/test/formats.js
index 71f7acc..ffa3d9d 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -227,8 +227,10 @@ tape('index format', async (t) => {
   dave.ebt.registerFormat('indexedfeed', indexedMethods)
   dave.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
 
-  const carolIndexId = (await pify(carol.indexFeedWriter.start)({ author: carol.id, type: 'dog', private: false })).subfeed
-  const daveIndexId = (await pify(dave.indexFeedWriter.start)({ author: dave.id, type: 'dog', private: false })).subfeed
+  const carolIndexId = (await pify(carol.indexFeedWriter.start)({
+    author: carol.id, type: 'dog', private: false })).subfeed
+  const daveIndexId = (await pify(dave.indexFeedWriter.start)({
+    author: dave.id, type: 'dog', private: false })).subfeed
 
   // publish some messages
   const res = await Promise.all([
@@ -275,17 +277,16 @@ tape('index format', async (t) => {
   carol.ebt.request(carol.id, true)
   carol.ebt.request(carolMetaId, true)
   carol.ebt.request(carolMetaIndexId, true)
-  carol.ebt.request(carolIndexId, true)
+  carol.ebt.request(carolIndexId, true, "indexedfeed")
 
   dave.ebt.request(dave.id, true)
   dave.ebt.request(daveMetaId, true)
   dave.ebt.request(daveMetaIndexId, true)
-  dave.ebt.request(daveIndexId, true)
+  dave.ebt.request(daveIndexId, true, "indexedfeed")
 
   // replication
   carol.ebt.request(daveMetaId, true)
   carol.ebt.request(daveMetaIndexId, true)
-
   dave.ebt.request(carolMetaId, true)
   dave.ebt.request(carolMetaIndexId, true)
   
@@ -310,8 +311,8 @@ tape('index format', async (t) => {
   // now that we have meta feeds from the other peer we can replicate
   // index feeds
 
-  carol.ebt.request(daveIndexId, true)
-  dave.ebt.request(carolIndexId, true)
+  carol.ebt.request(daveIndexId, true, "indexedfeed")
+  dave.ebt.request(carolIndexId, true, "indexedfeed")
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
@@ -371,6 +372,7 @@ tape('sliced replication', async (t) => {
 
   // self replicate
   alice.ebt.request(alice.id, true)
+  alice.ebt.request(alice.id, true, "slicedreplication")
   carol.ebt.request(carol.id, true)
 
   await pify(carol.connect)(alice.getAddress())
@@ -379,8 +381,9 @@ tape('sliced replication', async (t) => {
   t.equal(clockAlice[alice.id], 3, 'alice correct index clock')
 
   carol.ebt.setClockForSlicedReplication(alice.id,
-                                         clockAlice[alice.id] - 2)
-  carol.ebt.request(alice.id, true)
+                                         clockAlice[alice.id] - 2,
+                                        "slicedreplication")
+  carol.ebt.request(alice.id, true, "slicedreplication")
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')

From a28bce9426af2a950f91808595663deeb183d633 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 21:47:14 +0200
Subject: [PATCH 37/50] Better sliced example

---
 formats/sliced.js | 11 -----------
 test/formats.js   | 46 ++++++++++++++++++++++++++++++++++++----------
 2 files changed, 36 insertions(+), 21 deletions(-)
 delete mode 100644 formats/sliced.js

diff --git a/formats/sliced.js b/formats/sliced.js
deleted file mode 100644
index a69fe8c..0000000
--- a/formats/sliced.js
+++ /dev/null
@@ -1,11 +0,0 @@
-const classic = require('./classic')
-
-module.exports = {
-  ...classic,
-  appendMsg(sbot, msgVal, cb) {
-    sbot.db.addOOO(msgVal, (err, msg) => {
-      if (err) return cb(err)
-      else cb(null, msg)
-    })
-  }
-}
diff --git a/test/formats.js b/test/formats.js
index ffa3d9d..c7ba041 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -366,33 +366,59 @@ tape('sliced replication', async (t) => {
     pify(alice.db.publish)({ type: 'post', text: 'hello3' }),
   ])
 
-  const slicedMethods = require('../formats/sliced')
-  alice.ebt.registerFormat('slicedreplication', slicedMethods)
-  carol.ebt.registerFormat('slicedreplication', slicedMethods)
+  // carol wants to slice replicate some things, so she overwrites
+  // classic for this purpose
+
+  const sliced = [alice.id]
+
+  const slicedMethods = {
+    ...require('../formats/classic'),
+    appendMsg(sbot, msgVal, cb) {
+      let append = sbot.add
+      if (sliced.includes(msgVal.author))
+        append = sbot.db.addOOO
+
+      append(msgVal, (err, msg) => {
+        if (err) return cb(err)
+        else cb(null, msg)
+      })
+    }
+  }
+
+  carol.ebt.registerFormat('classic', slicedMethods)
+
+  const bobId = u.keysFor('bob').id
 
   // self replicate
   alice.ebt.request(alice.id, true)
-  alice.ebt.request(alice.id, true, "slicedreplication")
+  alice.ebt.request(bob.id, true)
   carol.ebt.request(carol.id, true)
 
   await pify(carol.connect)(alice.getAddress())
 
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.equal(clockAlice[alice.id], 3, 'alice correct index clock')
-
+  
   carol.ebt.setClockForSlicedReplication(alice.id,
-                                         clockAlice[alice.id] - 2,
-                                        "slicedreplication")
-  carol.ebt.request(alice.id, true, "slicedreplication")
+                                         clockAlice[alice.id] - 2)
+  carol.ebt.request(alice.id, true)
+
+  carol.ebt.request(bobId, true) // in full
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
-  const carolMessages = await carol.db.query(
+  const carolAMessages = await carol.db.query(
     where(author(alice.id)),
     toPromise()
   )
-  t.equal(carolMessages.length, 2, 'latest 2 messages from alice')
+  t.equal(carolAMessages.length, 2, 'latest 2 messages from alice')
+
+  const carolBMessages = await carol.db.query(
+    where(author(bob.id)),
+    toPromise()
+  )
+  t.equal(carolBMessages.length, 1, 'all messages from bob')
 
   await Promise.all([
     pify(alice.close)(true),

From e5f7c1e425516a0fb39cfa6be73ec36571629386 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 15 Sep 2021 22:24:28 +0200
Subject: [PATCH 38/50] Refactor to simplify formats

---
 README.md             |  9 ++++----
 formats/bendy-butt.js |  1 +
 formats/classic.js    |  1 +
 formats/indexed.js    |  1 +
 index.js              | 48 +++++++++++++++++++++++++++----------------
 test/formats.js       | 30 +++++++++++++--------------
 6 files changed, 53 insertions(+), 37 deletions(-)

diff --git a/README.md b/README.md
index 4150ab1..8e56733 100644
--- a/README.md
+++ b/README.md
@@ -103,7 +103,7 @@ The output looks like this:
 ```
 </details>
 
-### `ssb.ebt.registerFormat(formatName, methods)` ("sync" muxrpc API)
+### `ssb.ebt.registerFormat(methods)` ("sync" muxrpc API)
 
 Register a new format for replication. Note this does not have to be a
 new feed format, it could also be indexed replication or sliced
@@ -111,15 +111,16 @@ replication. See `formats` folder for examples.
 
 By registering a format you create a new EBT instance used for
 replicating feeds using that format. This means its own clock. Message
-will be replicated using the `replicateFormat` API. `formatName` must
-be a string and methods must implement the following functions. The
-example below shows the implementation for 'classic' ed25519 SSB feeds.
+will be replicated using the `replicateFormat` API. The `methods`
+argument must implement the following functions. The example below
+shows the implementation for 'classic' ed25519 SSB feeds.
 
 <details>
 <summary>CLICK HERE</summary>
 
 ```js
 {
+  name: 'classic',
   // used in request, block, cleanClock, sbot.post, vectorClock
   sbotIsFeed(sbot, feedId) {
     return ref.isFeed(feedId)
diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js
index a40ad0c..124fd7d 100644
--- a/formats/bendy-butt.js
+++ b/formats/bendy-butt.js
@@ -2,6 +2,7 @@ const SSBURI = require('ssb-uri2')
 const bendyButt = require('ssb-bendy-butt')
 
 module.exports = {
+  name: 'bendybutt-v1',
   // used in request, block, cleanClock, sbot.post, vectorClock
   sbotIsFeed(sbot, feedId) {
     return SSBURI.isBendyButtV1FeedSSBURI(feedId)
diff --git a/formats/classic.js b/formats/classic.js
index 9cd029d..12c452c 100644
--- a/formats/classic.js
+++ b/formats/classic.js
@@ -1,6 +1,7 @@
 const ref = require('ssb-ref')
 
 module.exports = {
+  name: 'classic',
   // used in request, block, cleanClock, sbot.post, vectorClock
   sbotIsFeed(sbot, feedId) {
     return ref.isFeed(feedId)
diff --git a/formats/indexed.js b/formats/indexed.js
index 4b5c13f..6ac0f44 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -4,6 +4,7 @@ const { QL0 } = require('ssb-subset-ql')
 
 module.exports = {
   ...classic, 
+  name: 'indexed',
   sbotIsFeed(sbot, author) {
     const info = sbot.metafeeds.findByIdSync(author)
     return info && info.feedpurpose === 'index'
diff --git a/index.js b/index.js
index e6dbbad..3510c2a 100644
--- a/index.js
+++ b/index.js
@@ -45,11 +45,13 @@ function cleanClock (clock, isFeed) {
 }
 
 exports.init = function (sbot, config) {
-  const ebts = {}
-  registerFormat('classic', require('./formats/classic'))
+  const ebts = []
+  registerFormat(require('./formats/classic'))
 
-  function registerFormat(formatName, format) {
-    const dirName = 'ebt' + (formatName === 'classic' ? '' : formatName)
+  function registerFormat(format) {
+    if (!format.name) throw new Error('format must have a name')
+
+    const dirName = 'ebt' + (format.name === 'classic' ? '' : format.name)
     const dir = config.path ? path.join(config.path, dirName) : null
     const store = Store(dir, null, toUrlFriendly)
 
@@ -88,14 +90,21 @@ exports.init = function (sbot, config) {
     ebt.convertMsg = format.convertMsg
     ebt.isReady = format.isReady.bind(format, sbot)
     ebt.isFeed = isFeed
+    ebt.name = format.name
 
-    ebts[formatName] = ebt
+    const existingId = ebts.findIndex(e => e.name === format.name)
+    if (existingId != -1)
+      ebts[existingId] = ebt
+    else
+      ebts.push(ebt)
   }
 
   function getEBT(formatName) {
-    const ebt = ebts[formatName]
-    if (!ebt)
+    const ebt = ebts.find(ebt => ebt.name === formatName)
+    if (!ebt) {
+      console.log(ebts)
       throw new Error('Unknown format: ' + formatName)
+    }
 
     return ebt
   }
@@ -105,9 +114,9 @@ exports.init = function (sbot, config) {
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
 
-    const readies = Object.values(ebts).map(ebt => ebt.isReady())
+    const readies = ebts.map(ebt => ebt.isReady())
     Promise.all(readies).then(() => {
-      Object.values(ebts).forEach(ebt => {
+      ebts.forEach(ebt => {
         const validClock = {}
         for (let k in clock)
           if (ebt.isFeed(k))
@@ -122,7 +131,7 @@ exports.init = function (sbot, config) {
 
   sbot.post((msg) => {
     initialized.promise.then(() => {
-      Object.values(ebts).forEach(ebt => {
+      ebts.forEach(ebt => {
         if (ebt.isFeed(msg.value.author))
           ebt.onAppend(ebt.convertMsg(msg.value))
       })
@@ -134,7 +143,7 @@ exports.init = function (sbot, config) {
   if (sbot.progress) {
     hook(sbot.progress, function (fn) {
       const _progress = fn()
-      const ebt = ebts['classic']
+      const ebt = ebts.find(ebt => ebt.name === 'classic')
       const ebtProg = ebt.progress()
       if (ebtProg.target) _progress.ebt = ebtProg
       return _progress
@@ -145,8 +154,8 @@ exports.init = function (sbot, config) {
     if (rpc.id === sbot.id) return // ssb-client connecting to ssb-server
     if (isClient) {
       initialized.promise.then(() => {
-        for (let format in ebts) {
-          const ebt = ebts[format]
+        ebts.forEach(ebt => {
+          const format = ebt.name
           const opts = { version: 3, format }
           const local = toPull.duplex(ebt.createStream(rpc.id, opts.version, true))
 
@@ -160,7 +169,7 @@ exports.init = function (sbot, config) {
             }
           })
           pull(local, remote, local)
-        }
+        })
       })
     }
   })
@@ -168,11 +177,14 @@ exports.init = function (sbot, config) {
   function findEBTForFeed(feedId, formatName) {
     let ebt
     if (formatName)
-      ebt = ebts[formatName]
+      ebt = ebts.find(ebt => ebt.name === formatName)
     else
-      ebt = Object.values(ebts).find(ebt => ebt.isFeed(feedId))
-    if (ebt) return ebt
-    else return ebts['classic']
+      ebt = ebts.find(ebt => ebt.isFeed(feedId))
+
+    if (!ebt)
+      ebt = ebts.find(ebt => ebt.name === 'classic')
+
+    return ebt
   }
 
   function request(destFeedId, requesting, formatName) {
diff --git a/test/formats.js b/test/formats.js
index c7ba041..5aa0926 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -87,8 +87,8 @@ let aliceMFId
 let bobMFId
 
 tape('multiple formats', async (t) => {
-  alice.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
-  bob.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
+  alice.ebt.registerFormat(bendyButtMethods)
+  bob.ebt.registerFormat(bendyButtMethods)
 
   // self replicate
   alice.ebt.request(alice.id, true)
@@ -167,8 +167,8 @@ tape('multiple formats restart', async (t) => {
     keys: u.keysFor('bob')
   })
 
-  alice.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
-  bob.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
+  alice.ebt.registerFormat(bendyButtMethods)
+  bob.ebt.registerFormat(bendyButtMethods)
 
   // self replicate
   alice.ebt.request(alice.id, true)
@@ -222,10 +222,10 @@ tape('index format', async (t) => {
 
   const indexedMethods = require('../formats/indexed.js')
 
-  carol.ebt.registerFormat('indexedfeed', indexedMethods)
-  carol.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
-  dave.ebt.registerFormat('indexedfeed', indexedMethods)
-  dave.ebt.registerFormat('bendybutt-v1', bendyButtMethods)
+  carol.ebt.registerFormat(indexedMethods)
+  carol.ebt.registerFormat(bendyButtMethods)
+  dave.ebt.registerFormat(indexedMethods)
+  dave.ebt.registerFormat(bendyButtMethods)
 
   const carolIndexId = (await pify(carol.indexFeedWriter.start)({
     author: carol.id, type: 'dog', private: false })).subfeed
@@ -277,12 +277,12 @@ tape('index format', async (t) => {
   carol.ebt.request(carol.id, true)
   carol.ebt.request(carolMetaId, true)
   carol.ebt.request(carolMetaIndexId, true)
-  carol.ebt.request(carolIndexId, true, "indexedfeed")
+  carol.ebt.request(carolIndexId, true, "indexed")
 
   dave.ebt.request(dave.id, true)
   dave.ebt.request(daveMetaId, true)
   dave.ebt.request(daveMetaIndexId, true)
-  dave.ebt.request(daveIndexId, true, "indexedfeed")
+  dave.ebt.request(daveIndexId, true, "indexed")
 
   // replication
   carol.ebt.request(daveMetaId, true)
@@ -311,8 +311,8 @@ tape('index format', async (t) => {
   // now that we have meta feeds from the other peer we can replicate
   // index feeds
 
-  carol.ebt.request(daveIndexId, true, "indexedfeed")
-  dave.ebt.request(carolIndexId, true, "indexedfeed")
+  carol.ebt.request(daveIndexId, true, "indexed")
+  dave.ebt.request(carolIndexId, true, "indexed")
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
@@ -335,10 +335,10 @@ tape('index format', async (t) => {
     [daveIndexId]: 1
   }
 
-  const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexedfeed' })
+  const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexed' })
   t.deepEqual(indexClockCarol, expectedIndexClock, 'carol correct index clock')
 
-  const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexedfeed' })
+  const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed' })
   t.deepEqual(indexClockDave, expectedIndexClock, 'dave correct index clock')
 
   await Promise.all([
@@ -385,7 +385,7 @@ tape('sliced replication', async (t) => {
     }
   }
 
-  carol.ebt.registerFormat('classic', slicedMethods)
+  carol.ebt.registerFormat(slicedMethods)
 
   const bobId = u.keysFor('bob').id
 

From 55c2372649a2479e9a28d4aeb858f51f4891c88c Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 16 Sep 2021 11:28:18 +0200
Subject: [PATCH 39/50] Update index.js
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: André Staltz <andre@staltz.com>
---
 index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/index.js b/index.js
index 3510c2a..f497df0 100644
--- a/index.js
+++ b/index.js
@@ -93,7 +93,7 @@ exports.init = function (sbot, config) {
     ebt.name = format.name
 
     const existingId = ebts.findIndex(e => e.name === format.name)
-    if (existingId != -1)
+    if (existingId !== -1)
       ebts[existingId] = ebt
     else
       ebts.push(ebt)

From 8a0e3bb010fe9f7c5e19f9ad14d4e78a6cdd3f8d Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 16 Sep 2021 11:30:12 +0200
Subject: [PATCH 40/50] Review fixes

---
 formats/bendy-butt.js | 2 +-
 formats/classic.js    | 2 +-
 formats/indexed.js    | 2 +-
 index.js              | 5 +++--
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js
index 124fd7d..525d464 100644
--- a/formats/bendy-butt.js
+++ b/formats/bendy-butt.js
@@ -4,7 +4,7 @@ const bendyButt = require('ssb-bendy-butt')
 module.exports = {
   name: 'bendybutt-v1',
   // used in request, block, cleanClock, sbot.post, vectorClock
-  sbotIsFeed(sbot, feedId) {
+  isFeed(sbot, feedId) {
     return SSBURI.isBendyButtV1FeedSSBURI(feedId)
   },
   getAtSequence(sbot, pair, cb) {
diff --git a/formats/classic.js b/formats/classic.js
index 12c452c..9fc23d7 100644
--- a/formats/classic.js
+++ b/formats/classic.js
@@ -3,7 +3,7 @@ const ref = require('ssb-ref')
 module.exports = {
   name: 'classic',
   // used in request, block, cleanClock, sbot.post, vectorClock
-  sbotIsFeed(sbot, feedId) {
+  isFeed(sbot, feedId) {
     return ref.isFeed(feedId)
   },
   getAtSequence(sbot, pair, cb) {
diff --git a/formats/indexed.js b/formats/indexed.js
index 6ac0f44..cd5d0c4 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -5,7 +5,7 @@ const { QL0 } = require('ssb-subset-ql')
 module.exports = {
   ...classic, 
   name: 'indexed',
-  sbotIsFeed(sbot, author) {
+  isFeed(sbot, author) {
     const info = sbot.metafeeds.findByIdSync(author)
     return info && info.feedpurpose === 'index'
   },
diff --git a/index.js b/index.js
index f497df0..03dbfa9 100644
--- a/index.js
+++ b/index.js
@@ -7,6 +7,7 @@ const toUrlFriendly = require('base64-url').escape
 const getSeverity = require('ssb-network-errors')
 const DeferredPromise = require('p-defer')
 const pullDefer = require('pull-defer')
+const classicMethods = require('./formats/classic')
 
 function hook (hookable, fn) {
   if (typeof hookable === 'function' && hookable.hook) {
@@ -46,7 +47,7 @@ function cleanClock (clock, isFeed) {
 
 exports.init = function (sbot, config) {
   const ebts = []
-  registerFormat(require('./formats/classic'))
+  registerFormat(classicMethods)
 
   function registerFormat(format) {
     if (!format.name) throw new Error('format must have a name')
@@ -56,7 +57,7 @@ exports.init = function (sbot, config) {
     const store = Store(dir, null, toUrlFriendly)
 
     // EBT expects a function of only feedId so we bind sbot here
-    const isFeed = format.sbotIsFeed.bind(format, sbot)
+    const isFeed = format.isFeed.bind(format, sbot)
     const { isMsg, getMsgAuthor, getMsgSequence } = format
 
     const ebt = EBT({

From 6e64d0d4ead860f91b75f46d7b924c634cd62736 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 16 Sep 2021 11:32:34 +0200
Subject: [PATCH 41/50] lint it

---
 formats/bendy-butt.js | 26 ++++++++-------------
 formats/classic.js    | 16 ++++++-------
 formats/indexed.js    | 12 +++++-----
 index.js              | 44 +++++++++++++++--------------------
 test/clock.js         |  4 ++--
 test/formats.js       | 54 ++++++++++++++++++++-----------------------
 test/race.js          |  7 +++---
 7 files changed, 72 insertions(+), 91 deletions(-)

diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js
index 525d464..5e2605b 100644
--- a/formats/bendy-butt.js
+++ b/formats/bendy-butt.js
@@ -4,29 +4,29 @@ const bendyButt = require('ssb-bendy-butt')
 module.exports = {
   name: 'bendybutt-v1',
   // used in request, block, cleanClock, sbot.post, vectorClock
-  isFeed(sbot, feedId) {
+  isFeed (sbot, feedId) {
     return SSBURI.isBendyButtV1FeedSSBURI(feedId)
   },
-  getAtSequence(sbot, pair, cb) {
+  getAtSequence (sbot, pair, cb) {
     sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
       cb(err, msg ? bendyButt.encode(msg.value) : null)
     })
   },
-  appendMsg(sbot, msgVal, cb) {
+  appendMsg (sbot, msgVal, cb) {
     sbot.add(bendyButt.decode(msgVal), (err, msg) => {
       cb(err && err.fatal ? err : null, msg)
     })
   },
-  convertMsg(msgVal) {
+  convertMsg (msgVal) {
     return bendyButt.encode(msgVal)
   },
   // used in vectorClock
-  isReady(sbot) {
+  isReady (sbot) {
     return Promise.resolve(true)
   },
 
   // used in ebt:stream to distinguish between messages and notes
-  isMsg(bbVal) {
+  isMsg (bbVal) {
     if (Buffer.isBuffer(bbVal)) {
       const msgVal = bendyButt.decode(bbVal)
       return msgVal && SSBURI.isBendyButtV1FeedSSBURI(msgVal.author)
@@ -35,17 +35,11 @@ module.exports = {
     }
   },
   // used in ebt:events
-  getMsgAuthor(bbVal) {
-    if (Buffer.isBuffer(bbVal))
-      return bendyButt.decode(bbVal).author
-    else
-      return bbVal.author
+  getMsgAuthor (bbVal) {
+    if (Buffer.isBuffer(bbVal)) { return bendyButt.decode(bbVal).author } else { return bbVal.author }
   },
   // used in ebt:events
-  getMsgSequence(bbVal) {
-    if (Buffer.isBuffer(bbVal))
-      return bendyButt.decode(bbVal).sequence
-    else
-      return bbVal.sequence
+  getMsgSequence (bbVal) {
+    if (Buffer.isBuffer(bbVal)) { return bendyButt.decode(bbVal).sequence } else { return bbVal.sequence }
   }
 }
diff --git a/formats/classic.js b/formats/classic.js
index 9fc23d7..2890c70 100644
--- a/formats/classic.js
+++ b/formats/classic.js
@@ -3,39 +3,39 @@ const ref = require('ssb-ref')
 module.exports = {
   name: 'classic',
   // used in request, block, cleanClock, sbot.post, vectorClock
-  isFeed(sbot, feedId) {
+  isFeed (sbot, feedId) {
     return ref.isFeed(feedId)
   },
-  getAtSequence(sbot, pair, cb) {
+  getAtSequence (sbot, pair, cb) {
     sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
       cb(err, msg ? msg.value : null)
     })
   },
-  appendMsg(sbot, msgVal, cb) {
+  appendMsg (sbot, msgVal, cb) {
     sbot.add(msgVal, (err, msg) => {
       cb(err && err.fatal ? err : null, msg)
     })
   },
   // used in onAppend
-  convertMsg(msgVal) {
+  convertMsg (msgVal) {
     return msgVal
   },
   // used in vectorClock
-  isReady(sbot) {
+  isReady (sbot) {
     return Promise.resolve(true)
   },
 
   // used in ebt:stream to distinguish between messages and notes
-  isMsg(msgVal) {
+  isMsg (msgVal) {
     return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
       ref.isFeed(msgVal.author) && msgVal.content
   },
   // used in ebt:events
-  getMsgAuthor(msgVal) {
+  getMsgAuthor (msgVal) {
     return msgVal.author
   },
   // used in ebt:events
-  getMsgSequence(msgVal) {
+  getMsgSequence (msgVal) {
     return msgVal.sequence
   }
 }
diff --git a/formats/indexed.js b/formats/indexed.js
index cd5d0c4..ecd2135 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -3,13 +3,13 @@ const pify = require('promisify-4loc')
 const { QL0 } = require('ssb-subset-ql')
 
 module.exports = {
-  ...classic, 
+  ...classic,
   name: 'indexed',
-  isFeed(sbot, author) {
+  isFeed (sbot, author) {
     const info = sbot.metafeeds.findByIdSync(author)
     return info && info.feedpurpose === 'index'
   },
-  appendMsg(sbot, msgVal, cb) {
+  appendMsg (sbot, msgVal, cb) {
     // FIXME: this needs the ability to add 2 messages in a transaction
     const payload = msgVal.content.indexed.payload
     delete msgVal.content.indexed.payload
@@ -21,12 +21,12 @@ module.exports = {
       })
     })
   },
-  getAtSequence(sbot, pair, cb) {
+  getAtSequence (sbot, pair, cb) {
     sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
       if (err) return cb(err)
       const { sequence } = msg.value.content.indexed
       const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
-      if (!authorInfo) return cb(new Error("Unknown author:" + msg.value.author))
+      if (!authorInfo) return cb(new Error('Unknown author:' + msg.value.author))
       const { author } = QL0.parse(authorInfo.metadata.query)
       sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
         if (err) return cb(err)
@@ -38,7 +38,7 @@ module.exports = {
       })
     })
   },
-  isReady(sbot) {
+  isReady (sbot) {
     return pify(sbot.metafeeds.loadState)()
   }
 }
diff --git a/index.js b/index.js
index 03dbfa9..057144c 100644
--- a/index.js
+++ b/index.js
@@ -49,7 +49,7 @@ exports.init = function (sbot, config) {
   const ebts = []
   registerFormat(classicMethods)
 
-  function registerFormat(format) {
+  function registerFormat (format) {
     if (!format.name) throw new Error('format must have a name')
 
     const dirName = 'ebt' + (format.name === 'classic' ? '' : format.name)
@@ -94,13 +94,10 @@ exports.init = function (sbot, config) {
     ebt.name = format.name
 
     const existingId = ebts.findIndex(e => e.name === format.name)
-    if (existingId !== -1)
-      ebts[existingId] = ebt
-    else
-      ebts.push(ebt)
+    if (existingId !== -1) { ebts[existingId] = ebt } else { ebts.push(ebt) }
   }
 
-  function getEBT(formatName) {
+  function getEBT (formatName) {
     const ebt = ebts.find(ebt => ebt.name === formatName)
     if (!ebt) {
       console.log(ebts)
@@ -119,9 +116,9 @@ exports.init = function (sbot, config) {
     Promise.all(readies).then(() => {
       ebts.forEach(ebt => {
         const validClock = {}
-        for (let k in clock)
-          if (ebt.isFeed(k))
-            validClock[k] = clock[k]
+        for (const k in clock) {
+          if (ebt.isFeed(k)) { validClock[k] = clock[k] }
+        }
 
         ebt.state.clock = validClock
         ebt.update()
@@ -133,8 +130,7 @@ exports.init = function (sbot, config) {
   sbot.post((msg) => {
     initialized.promise.then(() => {
       ebts.forEach(ebt => {
-        if (ebt.isFeed(msg.value.author))
-          ebt.onAppend(ebt.convertMsg(msg.value))
+        if (ebt.isFeed(msg.value.author)) { ebt.onAppend(ebt.convertMsg(msg.value)) }
       })
     })
   })
@@ -175,30 +171,26 @@ exports.init = function (sbot, config) {
     }
   })
 
-  function findEBTForFeed(feedId, formatName) {
+  function findEBTForFeed (feedId, formatName) {
     let ebt
-    if (formatName)
-      ebt = ebts.find(ebt => ebt.name === formatName)
-    else
-      ebt = ebts.find(ebt => ebt.isFeed(feedId))
+    if (formatName) { ebt = ebts.find(ebt => ebt.name === formatName) } else { ebt = ebts.find(ebt => ebt.isFeed(feedId)) }
 
-    if (!ebt)
-      ebt = ebts.find(ebt => ebt.name === 'classic')
+    if (!ebt) { ebt = ebts.find(ebt => ebt.name === 'classic') }
 
     return ebt
   }
 
-  function request(destFeedId, requesting, formatName) {
+  function request (destFeedId, requesting, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(destFeedId, formatName)
 
       if (!ebt.isFeed(destFeedId)) return
-      
+
       ebt.request(destFeedId, requesting)
     })
   }
 
-  function block(origFeedId, destFeedId, blocking, formatName) {
+  function block (origFeedId, destFeedId, blocking, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(origFeedId, formatName)
 
@@ -217,7 +209,7 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function replicateFormat(opts) {
+  function replicateFormat (opts) {
     if (opts.version !== 3) {
       throw new Error('expected ebt.replicate({version: 3})')
     }
@@ -225,7 +217,7 @@ exports.init = function (sbot, config) {
     const formatName = opts.format || 'classic'
     const ebt = getEBT(formatName)
 
-    var deferred = pullDefer.duplex()
+    const deferred = pullDefer.duplex()
     initialized.promise.then(() => {
       // `this` refers to the remote peer who called this muxrpc API
       deferred.resolve(toPull.duplex(ebt.createStream(this.id, opts.version, false)))
@@ -234,7 +226,7 @@ exports.init = function (sbot, config) {
   }
 
   // get replication status for feeds for this id
-  function peerStatus(id) {
+  function peerStatus (id) {
     id = id || sbot.id
 
     const ebt = findEBTForFeed(id)
@@ -260,7 +252,7 @@ exports.init = function (sbot, config) {
     return data
   }
 
-  function clock(opts, cb) {
+  function clock (opts, cb) {
     if (!cb) {
       cb = opts
       opts = { format: 'classic' }
@@ -272,7 +264,7 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function setClockForSlicedReplication(feedId, sequence, formatName) {
+  function setClockForSlicedReplication (feedId, sequence, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(feedId, formatName)
 
diff --git a/test/clock.js b/test/clock.js
index 0e5df1a..179fb6f 100644
--- a/test/clock.js
+++ b/test/clock.js
@@ -54,7 +54,7 @@ tape('clock works', async (t) => {
   const clockBobAfter = await pify(bob.ebt.clock)()
   t.equal(clockBobAfter[alice.id], 1, 'clock ok')
   t.equal(clockBobAfter[bob.id], 1, 'clock ok')
-  
+
   await pify(alice.publish)({ type: 'post', text: 'hello again' }),
 
   await sleep(REPLICATION_TIMEOUT)
@@ -65,7 +65,7 @@ tape('clock works', async (t) => {
 
   const clockBobAfter2 = await pify(bob.ebt.clock)()
   t.equal(clockBobAfter2[alice.id], 2, 'clock ok')
-  
+
   await Promise.all([
     pify(alice.close)(true),
     pify(bob.close)(true)
diff --git a/test/formats.js b/test/formats.js
index 5aa0926..cca998e 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -1,5 +1,4 @@
 const tape = require('tape')
-const crypto = require('crypto')
 const SecretStack = require('secret-stack')
 const sleep = require('util').promisify(setTimeout)
 const pify = require('promisify-4loc')
@@ -13,7 +12,7 @@ const SSBURI = require('ssb-uri2')
 const bendyButt = require('ssb-bendy-butt')
 const { where, author, type, toPromise } = require('ssb-db2/operators')
 
-function createSSBServer() {
+function createSSBServer () {
   return SecretStack({ appKey: caps.shs })
     .use(require('ssb-db2'))
     .use(require('ssb-db2/compat/ebt'))
@@ -25,7 +24,7 @@ function createSSBServer() {
 const CONNECTION_TIMEOUT = 500 // ms
 const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 
-function getFreshDir(name) {
+function getFreshDir (name) {
   const dir = '/tmp/test-format-' + name
   rimraf.sync(dir)
   mkdirp.sync(dir)
@@ -36,7 +35,7 @@ const aliceDir = getFreshDir('alice')
 let alice = createSSBServer().call(null, {
   path: aliceDir,
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('alice'),
+  keys: u.keysFor('alice')
 })
 
 const bobDir = getFreshDir('bob')
@@ -46,7 +45,7 @@ let bob = createSSBServer().call(null, {
   keys: u.keysFor('bob')
 })
 
-function getBBMsg(mainKeys) {
+function getBBMsg (mainKeys) {
   // fake some keys
   const mfKeys = ssbKeys.generate()
   const classicUri = SSBURI.fromFeedSigil(mfKeys.id)
@@ -55,8 +54,8 @@ function getBBMsg(mainKeys) {
   mfKeys.id = bendybuttUri
 
   const content = {
-    type: "metafeed/add/existing",
-    feedpurpose: "main",
+    type: 'metafeed/add/existing',
+    feedpurpose: 'main',
     subfeed: mainKeys.id,
     metafeed: mfKeys.id,
     tangles: {
@@ -158,7 +157,7 @@ tape('multiple formats restart', async (t) => {
   alice = createSSBServer().call(null, {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('alice'),
+    keys: u.keysFor('alice')
   })
 
   bob = createSSBServer().call(null, {
@@ -227,13 +226,11 @@ tape('index format', async (t) => {
   dave.ebt.registerFormat(indexedMethods)
   dave.ebt.registerFormat(bendyButtMethods)
 
-  const carolIndexId = (await pify(carol.indexFeedWriter.start)({
-    author: carol.id, type: 'dog', private: false })).subfeed
-  const daveIndexId = (await pify(dave.indexFeedWriter.start)({
-    author: dave.id, type: 'dog', private: false })).subfeed
+  const carolIndexId = (await pify(carol.indexFeedWriter.start)({ author: carol.id, type: 'dog', private: false })).subfeed
+  const daveIndexId = (await pify(dave.indexFeedWriter.start)({ author: dave.id, type: 'dog', private: false })).subfeed
 
   // publish some messages
-  const res = await Promise.all([
+  await Promise.all([
     pify(carol.db.publish)({ type: 'post', text: 'hello 2' }),
     pify(carol.db.publish)({ type: 'dog', name: 'Buff' }),
     pify(dave.db.publish)({ type: 'post', text: 'hello 2' }),
@@ -260,7 +257,7 @@ tape('index format', async (t) => {
     where(type('metafeed/announce')),
     toPromise()
   )
-  
+
   // get meta index feed
   const daveMetaIndexMessages = await dave.db.query(
     where(type('metafeed/add/derived')),
@@ -269,27 +266,27 @@ tape('index format', async (t) => {
 
   const carolMetaId = carolMetaMessages[0].value.content.metafeed
   const carolMetaIndexId = carolMetaIndexMessages[0].value.content.subfeed
-  
+
   const daveMetaId = daveMetaMessages[0].value.content.metafeed
   const daveMetaIndexId = daveMetaIndexMessages[0].value.content.subfeed
-  
+
   // self replicate
   carol.ebt.request(carol.id, true)
   carol.ebt.request(carolMetaId, true)
   carol.ebt.request(carolMetaIndexId, true)
-  carol.ebt.request(carolIndexId, true, "indexed")
+  carol.ebt.request(carolIndexId, true, 'indexed')
 
   dave.ebt.request(dave.id, true)
   dave.ebt.request(daveMetaId, true)
   dave.ebt.request(daveMetaIndexId, true)
-  dave.ebt.request(daveIndexId, true, "indexed")
+  dave.ebt.request(daveIndexId, true, 'indexed')
 
   // replication
   carol.ebt.request(daveMetaId, true)
   carol.ebt.request(daveMetaIndexId, true)
   dave.ebt.request(carolMetaId, true)
   dave.ebt.request(carolMetaIndexId, true)
-  
+
   await pify(dave.connect)(carol.getAddress())
 
   await sleep(2 * REPLICATION_TIMEOUT)
@@ -307,12 +304,12 @@ tape('index format', async (t) => {
     toPromise()
   )
   t.equal(daveIndexMessages.length, 1, 'dave has carol meta index')
-  
+
   // now that we have meta feeds from the other peer we can replicate
   // index feeds
 
-  carol.ebt.request(daveIndexId, true, "indexed")
-  dave.ebt.request(carolIndexId, true, "indexed")
+  carol.ebt.request(daveIndexId, true, 'indexed')
+  dave.ebt.request(carolIndexId, true, 'indexed')
 
   await sleep(2 * REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
@@ -355,7 +352,7 @@ tape('sliced replication', async (t) => {
     keys: u.keysFor('alice')
   })
 
-  let carol = createSSBServer().call(null, {
+  const carol = createSSBServer().call(null, {
     path: carolDir,
     timeout: CONNECTION_TIMEOUT,
     keys: u.keysFor('carol')
@@ -363,7 +360,7 @@ tape('sliced replication', async (t) => {
 
   await Promise.all([
     pify(alice.db.publish)({ type: 'post', text: 'hello2' }),
-    pify(alice.db.publish)({ type: 'post', text: 'hello3' }),
+    pify(alice.db.publish)({ type: 'post', text: 'hello3' })
   ])
 
   // carol wants to slice replicate some things, so she overwrites
@@ -373,10 +370,9 @@ tape('sliced replication', async (t) => {
 
   const slicedMethods = {
     ...require('../formats/classic'),
-    appendMsg(sbot, msgVal, cb) {
+    appendMsg (sbot, msgVal, cb) {
       let append = sbot.add
-      if (sliced.includes(msgVal.author))
-        append = sbot.db.addOOO
+      if (sliced.includes(msgVal.author)) { append = sbot.db.addOOO }
 
       append(msgVal, (err, msg) => {
         if (err) return cb(err)
@@ -398,9 +394,9 @@ tape('sliced replication', async (t) => {
 
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.equal(clockAlice[alice.id], 3, 'alice correct index clock')
-  
+
   carol.ebt.setClockForSlicedReplication(alice.id,
-                                         clockAlice[alice.id] - 2)
+    clockAlice[alice.id] - 2)
   carol.ebt.request(alice.id, true)
 
   carol.ebt.request(bobId, true) // in full
diff --git a/test/race.js b/test/race.js
index 0d4e2ec..afcfef6 100644
--- a/test/race.js
+++ b/test/race.js
@@ -6,7 +6,7 @@ const pify = require('promisify-4loc')
 const sleep = require('util').promisify(setTimeout)
 const u = require('./misc/util')
 
-function delayedVectorClock(sbot, config) {
+function delayedVectorClock (sbot, config) {
   const realGetVectorClock = sbot.getVectorClock
   sbot.getVectorClock = (cb) => {
     setTimeout(() => realGetVectorClock(cb), 1000)
@@ -20,7 +20,6 @@ const createSbot = require('secret-stack')({
   .use(delayedVectorClock)
   .use(require('../'))
 
-
 const CONNECTION_TIMEOUT = 500 // ms
 const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 
@@ -44,7 +43,7 @@ tape('we wait for vectorclock being available before doing ebt', async (t) => {
 
   await Promise.all([
     pify(alice.publish)({ type: 'post', text: 'hello world' }),
-    pify(bob.publish)({ type: 'post', text: 'hello world' }),
+    pify(bob.publish)({ type: 'post', text: 'hello world' })
   ])
   t.pass('all peers have posted "hello world"')
 
@@ -69,7 +68,7 @@ tape('we wait for vectorclock being available before doing ebt', async (t) => {
     timeout: CONNECTION_TIMEOUT,
     keys: bobKeys
   })
-  
+
   // self replicate
   alice.ebt.request(alice.id, true)
   bob.ebt.request(bob.id, true)

From 417f523bb2e97ab34835955bd2e1e8acbe3b5938 Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Thu, 16 Sep 2021 14:07:42 +0300
Subject: [PATCH 42/50] setup prettier like in ssb-db2

---
 .prettierrc      |  4 ++
 index.js         | 98 ++++++++++++++++++++++++++++--------------------
 package.json     | 14 +++++--
 test/block.js    | 23 +++++-------
 test/clock.js    | 16 +++-----
 test/formats.js  | 96 +++++++++++++++++++++++------------------------
 test/index.js    | 20 +++++-----
 test/legacy.js   |  4 +-
 test/race.js     | 38 ++++++++++---------
 test/realtime.js | 13 +++----
 test/resync.js   | 28 ++++++++------
 test/server.js   | 26 ++++++-------
 test/unit.js     | 27 ++++++-------
 13 files changed, 215 insertions(+), 192 deletions(-)
 create mode 100644 .prettierrc

diff --git a/.prettierrc b/.prettierrc
new file mode 100644
index 0000000..b2095be
--- /dev/null
+++ b/.prettierrc
@@ -0,0 +1,4 @@
+{
+  "semi": false,
+  "singleQuote": true
+}
diff --git a/index.js b/index.js
index 057144c..6e4ce7f 100644
--- a/index.js
+++ b/index.js
@@ -9,7 +9,7 @@ const DeferredPromise = require('p-defer')
 const pullDefer = require('pull-defer')
 const classicMethods = require('./formats/classic')
 
-function hook (hookable, fn) {
+function hook(hookable, fn) {
   if (typeof hookable === 'function' && hookable.hook) {
     hookable.hook(fn)
   }
@@ -25,19 +25,19 @@ exports.manifest = {
   request: 'sync',
   block: 'sync',
   peerStatus: 'sync',
-  clock: 'async'
+  clock: 'async',
 }
 
 exports.permissions = {
   anonymous: {
-    allow: ['replicate', 'replicateFormat', 'clock']
-  }
+    allow: ['replicate', 'replicateFormat', 'clock'],
+  },
 }
 
 // there was a bug that caused some peers
 // to request things that weren't feeds.
 // this is fixed, so just ignore anything that isn't a feed.
-function cleanClock (clock, isFeed) {
+function cleanClock(clock, isFeed) {
   for (const k in clock) {
     if (!isFeed(k)) {
       delete clock[k]
@@ -49,7 +49,7 @@ exports.init = function (sbot, config) {
   const ebts = []
   registerFormat(classicMethods)
 
-  function registerFormat (format) {
+  function registerFormat(format) {
     if (!format.name) throw new Error('format must have a name')
 
     const dirName = 'ebt' + (format.name === 'classic' ? '' : format.name)
@@ -63,28 +63,28 @@ exports.init = function (sbot, config) {
     const ebt = EBT({
       logging: config.ebt && config.ebt.logging,
       id: sbot.id,
-      getClock (id, cb) {
+      getClock(id, cb) {
         store.ensure(id, function () {
           const clock = store.get(id) || {}
           cleanClock(clock, isFeed)
           cb(null, clock)
         })
       },
-      setClock (id, clock) {
+      setClock(id, clock) {
         cleanClock(clock, isFeed)
         store.set(id, clock)
       },
-      getAt (pair, cb) {
+      getAt(pair, cb) {
         format.getAtSequence(sbot, pair, cb)
       },
-      append (msgVal, cb) {
+      append(msgVal, cb) {
         format.appendMsg(sbot, msgVal, cb)
       },
 
       isFeed,
       isMsg,
       getMsgAuthor,
-      getMsgSequence
+      getMsgSequence,
     })
 
     // attach a few methods we need in this module
@@ -93,12 +93,13 @@ exports.init = function (sbot, config) {
     ebt.isFeed = isFeed
     ebt.name = format.name
 
-    const existingId = ebts.findIndex(e => e.name === format.name)
-    if (existingId !== -1) { ebts[existingId] = ebt } else { ebts.push(ebt) }
+    const existingId = ebts.findIndex((e) => e.name === format.name)
+    if (existingId !== -1) ebts[existingId] = ebt
+    else ebts.push(ebt)
   }
 
-  function getEBT (formatName) {
-    const ebt = ebts.find(ebt => ebt.name === formatName)
+  function getEBT(formatName) {
+    const ebt = ebts.find((ebt) => ebt.name === formatName)
     if (!ebt) {
       console.log(ebts)
       throw new Error('Unknown format: ' + formatName)
@@ -112,12 +113,14 @@ exports.init = function (sbot, config) {
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
 
-    const readies = ebts.map(ebt => ebt.isReady())
+    const readies = ebts.map((ebt) => ebt.isReady())
     Promise.all(readies).then(() => {
-      ebts.forEach(ebt => {
+      ebts.forEach((ebt) => {
         const validClock = {}
         for (const k in clock) {
-          if (ebt.isFeed(k)) { validClock[k] = clock[k] }
+          if (ebt.isFeed(k)) {
+            validClock[k] = clock[k]
+          }
         }
 
         ebt.state.clock = validClock
@@ -129,8 +132,10 @@ exports.init = function (sbot, config) {
 
   sbot.post((msg) => {
     initialized.promise.then(() => {
-      ebts.forEach(ebt => {
-        if (ebt.isFeed(msg.value.author)) { ebt.onAppend(ebt.convertMsg(msg.value)) }
+      ebts.forEach((ebt) => {
+        if (ebt.isFeed(msg.value.author)) {
+          ebt.onAppend(ebt.convertMsg(msg.value))
+        }
       })
     })
   })
@@ -140,7 +145,7 @@ exports.init = function (sbot, config) {
   if (sbot.progress) {
     hook(sbot.progress, function (fn) {
       const _progress = fn()
-      const ebt = ebts.find(ebt => ebt.name === 'classic')
+      const ebt = ebts.find((ebt) => ebt.name === 'classic')
       const ebtProg = ebt.progress()
       if (ebtProg.target) _progress.ebt = ebtProg
       return _progress
@@ -151,14 +156,17 @@ exports.init = function (sbot, config) {
     if (rpc.id === sbot.id) return // ssb-client connecting to ssb-server
     if (isClient) {
       initialized.promise.then(() => {
-        ebts.forEach(ebt => {
+        ebts.forEach((ebt) => {
           const format = ebt.name
           const opts = { version: 3, format }
-          const local = toPull.duplex(ebt.createStream(rpc.id, opts.version, true))
+          const local = toPull.duplex(
+            ebt.createStream(rpc.id, opts.version, true)
+          )
 
           // for backwards compatibility we always replicate classic
           // feeds using existing replicate RPC
-          const replicate = (format === 'classic' ? rpc.ebt.replicate : rpc.ebt.replicateFormat)
+          const replicate =
+            format === 'classic' ? rpc.ebt.replicate : rpc.ebt.replicateFormat
 
           const remote = replicate(opts, (networkError) => {
             if (networkError && getSeverity(networkError) >= 3) {
@@ -171,16 +179,22 @@ exports.init = function (sbot, config) {
     }
   })
 
-  function findEBTForFeed (feedId, formatName) {
+  function findEBTForFeed(feedId, formatName) {
     let ebt
-    if (formatName) { ebt = ebts.find(ebt => ebt.name === formatName) } else { ebt = ebts.find(ebt => ebt.isFeed(feedId)) }
+    if (formatName) {
+      ebt = ebts.find((ebt) => ebt.name === formatName)
+    } else {
+      ebt = ebts.find((ebt) => ebt.isFeed(feedId))
+    }
 
-    if (!ebt) { ebt = ebts.find(ebt => ebt.name === 'classic') }
+    if (!ebt) {
+      ebt = ebts.find((ebt) => ebt.name === 'classic')
+    }
 
     return ebt
   }
 
-  function request (destFeedId, requesting, formatName) {
+  function request(destFeedId, requesting, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(destFeedId, formatName)
 
@@ -190,7 +204,7 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function block (origFeedId, destFeedId, blocking, formatName) {
+  function block(origFeedId, destFeedId, blocking, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(origFeedId, formatName)
 
@@ -201,7 +215,7 @@ exports.init = function (sbot, config) {
         ebt.block(origFeedId, destFeedId, true)
       } else if (
         ebt.state.blocks[origFeedId] &&
-          ebt.state.blocks[origFeedId][destFeedId]
+        ebt.state.blocks[origFeedId][destFeedId]
       ) {
         // only update unblock if they were already blocked
         ebt.block(origFeedId, destFeedId, false)
@@ -209,7 +223,7 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function replicateFormat (opts) {
+  function replicateFormat(opts) {
     if (opts.version !== 3) {
       throw new Error('expected ebt.replicate({version: 3})')
     }
@@ -220,13 +234,15 @@ exports.init = function (sbot, config) {
     const deferred = pullDefer.duplex()
     initialized.promise.then(() => {
       // `this` refers to the remote peer who called this muxrpc API
-      deferred.resolve(toPull.duplex(ebt.createStream(this.id, opts.version, false)))
+      deferred.resolve(
+        toPull.duplex(ebt.createStream(this.id, opts.version, false))
+      )
     })
     return deferred
   }
 
   // get replication status for feeds for this id
-  function peerStatus (id) {
+  function peerStatus(id) {
     id = id || sbot.id
 
     const ebt = findEBTForFeed(id)
@@ -234,17 +250,19 @@ exports.init = function (sbot, config) {
     const data = {
       id: id,
       seq: ebt.state.clock[id],
-      peers: {}
+      peers: {},
     }
 
     for (const k in ebt.state.peers) {
       const peer = ebt.state.peers[k]
-      if (peer.clock[id] != null ||
-          (peer.replicating && peer.replicating[id] != null)) {
+      if (
+        peer.clock[id] != null ||
+        (peer.replicating && peer.replicating[id] != null)
+      ) {
         const rep = peer.replicating && peer.replicating[id]
         data.peers[k] = {
           seq: peer.clock[id],
-          replicating: rep
+          replicating: rep,
         }
       }
     }
@@ -252,7 +270,7 @@ exports.init = function (sbot, config) {
     return data
   }
 
-  function clock (opts, cb) {
+  function clock(opts, cb) {
     if (!cb) {
       cb = opts
       opts = { format: 'classic' }
@@ -264,7 +282,7 @@ exports.init = function (sbot, config) {
     })
   }
 
-  function setClockForSlicedReplication (feedId, sequence, formatName) {
+  function setClockForSlicedReplication(feedId, sequence, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(feedId, formatName)
 
@@ -280,6 +298,6 @@ exports.init = function (sbot, config) {
     peerStatus,
     clock,
     setClockForSlicedReplication,
-    registerFormat
+    registerFormat,
   }
 }
diff --git a/package.json b/package.json
index 074becf..1639cc2 100644
--- a/package.json
+++ b/package.json
@@ -36,8 +36,11 @@
   "devDependencies": {
     "cat-names": "^3.0.0",
     "dog-names": "^2.0.0",
+    "husky": "^4.3.0",
     "mkdirp": "^1.0.4",
     "nyc": "^15.1.0",
+    "prettier": "^2.1.2",
+    "pretty-quick": "^3.1.0",
     "pull-paramap": "^1.2.2",
     "rimraf": "^2.7.1",
     "rng": "^0.2.2",
@@ -51,15 +54,20 @@
     "ssb-keys": "^8.1.0",
     "ssb-meta-feeds": "^0.21.0",
     "ssb-validate": "^4.1.4",
-    "standardx": "^7.0.0",
     "tap-spec": "^5.0.0",
     "tape": "^5.2.2"
   },
   "scripts": {
-    "lint": "standardx --fix '**/*.js'",
     "test": "set -e; for t in test/*.js; do echo \"#Test: $t\"; tape $t | tap-spec; done",
     "test-verbose": "TEST_VERBOSE=1 npm test",
-    "coverage": "nyc --reporter=lcov npm test"
+    "coverage": "nyc --reporter=lcov npm test",
+    "format-code": "prettier --write \"*.js\" \"test/*.js\"",
+    "format-code-staged": "pretty-quick --staged --pattern \"*.js\" --pattern \"test/*.js\""
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "npm run format-code-staged"
+    }
   },
   "nyc": {
     "exclude": [
diff --git a/test/block.js b/test/block.js
index 70b2688..415e512 100644
--- a/test/block.js
+++ b/test/block.js
@@ -5,7 +5,7 @@ const pify = require('promisify-4loc')
 const u = require('./misc/util')
 
 const createSsbServer = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('../'))
@@ -16,19 +16,19 @@ const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 const alice = createSsbServer({
   temp: 'test-block-alice',
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('alice')
+  keys: u.keysFor('alice'),
 })
 
 const bob = createSsbServer({
   temp: 'test-block-bob',
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('bob')
+  keys: u.keysFor('bob'),
 })
 
 tape('alice blocks bob, then unblocks', async (t) => {
   await Promise.all([
     pify(alice.publish)({ type: 'post', text: 'hello' }),
-    pify(bob.publish)({ type: 'post', text: 'hello' })
+    pify(bob.publish)({ type: 'post', text: 'hello' }),
   ])
 
   // Self replicate
@@ -37,17 +37,17 @@ tape('alice blocks bob, then unblocks', async (t) => {
 
   alice.ebt.request(bob.id, false)
   alice.ebt.block(alice.id, bob.id, true)
-  t.pass('alice does not want bob\'s data')
+  t.pass("alice does not want bob's data")
 
   bob.ebt.request(alice.id, true)
   bob.ebt.block(bob.id, alice.id, false)
-  t.pass('bob wants alice\'s data')
+  t.pass("bob wants alice's data")
 
   try {
     await Promise.all([
       pify(bob.connect)(alice.getAddress()),
       u.readOnceFromDB(bob, REPLICATION_TIMEOUT),
-      u.readOnceFromDB(alice, REPLICATION_TIMEOUT)
+      u.readOnceFromDB(alice, REPLICATION_TIMEOUT),
     ])
     t.fail('replication should not succeed')
   } catch (err) {
@@ -63,14 +63,14 @@ tape('alice blocks bob, then unblocks', async (t) => {
 
   alice.ebt.request(bob.id, true)
   alice.ebt.block(alice.id, bob.id, false)
-  t.pass('alice now wants bob\'s data')
+  t.pass("alice now wants bob's data")
 
   // Silly idempotent operation just to increase test coverage
   alice.ebt.block(alice.id, bob.id, false)
 
   const [rpcBobToAlice, msgAtAlice] = await Promise.all([
     pify(bob.connect)(alice.getAddress()),
-    u.readOnceFromDB(alice, REPLICATION_TIMEOUT)
+    u.readOnceFromDB(alice, REPLICATION_TIMEOUT),
   ])
 
   t.equals(msgAtAlice.value.author, bob.id, 'alice has a msg from bob')
@@ -80,9 +80,6 @@ tape('alice blocks bob, then unblocks', async (t) => {
 
   await pify(rpcBobToAlice.close)(true)
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
diff --git a/test/clock.js b/test/clock.js
index 179fb6f..0cde7f4 100644
--- a/test/clock.js
+++ b/test/clock.js
@@ -6,7 +6,7 @@ const pify = require('promisify-4loc')
 const u = require('./misc/util')
 
 const createSsbServer = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('../'))
@@ -17,19 +17,19 @@ const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 const alice = createSsbServer({
   temp: 'test-clock-alice',
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('alice')
+  keys: u.keysFor('alice'),
 })
 
 const bob = createSsbServer({
   temp: 'test-clock-bob',
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('bob')
+  keys: u.keysFor('bob'),
 })
 
 tape('clock works', async (t) => {
   await Promise.all([
     pify(alice.publish)({ type: 'post', text: 'hello' }),
-    pify(bob.publish)({ type: 'post', text: 'hello' })
+    pify(bob.publish)({ type: 'post', text: 'hello' }),
   ])
 
   const clockAlice = await pify(alice.ebt.clock)()
@@ -56,8 +56,7 @@ tape('clock works', async (t) => {
   t.equal(clockBobAfter[bob.id], 1, 'clock ok')
 
   await pify(alice.publish)({ type: 'post', text: 'hello again' }),
-
-  await sleep(REPLICATION_TIMEOUT)
+    await sleep(REPLICATION_TIMEOUT)
   t.pass('wait for replication to complete')
 
   const clockAliceAfter2 = await pify(alice.ebt.clock)()
@@ -66,9 +65,6 @@ tape('clock works', async (t) => {
   const clockBobAfter2 = await pify(bob.ebt.clock)()
   t.equal(clockBobAfter2[alice.id], 2, 'clock ok')
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
diff --git a/test/formats.js b/test/formats.js
index cca998e..f721e90 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -12,7 +12,7 @@ const SSBURI = require('ssb-uri2')
 const bendyButt = require('ssb-bendy-butt')
 const { where, author, type, toPromise } = require('ssb-db2/operators')
 
-function createSSBServer () {
+function createSSBServer() {
   return SecretStack({ appKey: caps.shs })
     .use(require('ssb-db2'))
     .use(require('ssb-db2/compat/ebt'))
@@ -24,7 +24,7 @@ function createSSBServer () {
 const CONNECTION_TIMEOUT = 500 // ms
 const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 
-function getFreshDir (name) {
+function getFreshDir(name) {
   const dir = '/tmp/test-format-' + name
   rimraf.sync(dir)
   mkdirp.sync(dir)
@@ -35,17 +35,17 @@ const aliceDir = getFreshDir('alice')
 let alice = createSSBServer().call(null, {
   path: aliceDir,
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('alice')
+  keys: u.keysFor('alice'),
 })
 
 const bobDir = getFreshDir('bob')
 let bob = createSSBServer().call(null, {
   path: bobDir,
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('bob')
+  keys: u.keysFor('bob'),
 })
 
-function getBBMsg (mainKeys) {
+function getBBMsg(mainKeys) {
   // fake some keys
   const mfKeys = ssbKeys.generate()
   const classicUri = SSBURI.fromFeedSigil(mfKeys.id)
@@ -61,9 +61,9 @@ function getBBMsg (mainKeys) {
     tangles: {
       metafeed: {
         root: null,
-        previous: null
-      }
-    }
+        previous: null,
+      },
+    },
   }
 
   const bbmsg = bendyButt.encodeNew(
@@ -96,7 +96,7 @@ tape('multiple formats', async (t) => {
   // publish normal messages
   await Promise.all([
     pify(alice.db.publish)({ type: 'post', text: 'hello' }),
-    pify(bob.db.publish)({ type: 'post', text: 'hello' })
+    pify(bob.db.publish)({ type: 'post', text: 'hello' }),
   ])
 
   const aliceBBMsg = getBBMsg(alice.config.keys)
@@ -109,10 +109,7 @@ tape('multiple formats', async (t) => {
   alice.ebt.request(aliceMFId, true)
   bob.ebt.request(bobMFId, true)
 
-  await Promise.all([
-    pify(alice.add)(aliceBBMsg),
-    pify(bob.add)(bobBBMsg)
-  ])
+  await Promise.all([pify(alice.add)(aliceBBMsg), pify(bob.add)(bobBBMsg)])
 
   alice.ebt.request(bob.id, true)
   alice.ebt.request(bobMFId, true)
@@ -127,11 +124,11 @@ tape('multiple formats', async (t) => {
 
   const expectedClassicClock = {
     [alice.id]: 1,
-    [bob.id]: 1
+    [bob.id]: 1,
   }
   const expectedBBClock = {
     [aliceMFId]: 1,
-    [bobMFId]: 1
+    [bobMFId]: 1,
   }
 
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
@@ -146,10 +143,7 @@ tape('multiple formats', async (t) => {
   const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt-v1' })
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
 
@@ -157,13 +151,13 @@ tape('multiple formats restart', async (t) => {
   alice = createSSBServer().call(null, {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('alice')
+    keys: u.keysFor('alice'),
   })
 
   bob = createSSBServer().call(null, {
     path: bobDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('bob')
+    keys: u.keysFor('bob'),
   })
 
   alice.ebt.registerFormat(bendyButtMethods)
@@ -177,11 +171,11 @@ tape('multiple formats restart', async (t) => {
 
   const expectedClassicClock = {
     [alice.id]: 1,
-    [bob.id]: 1
+    [bob.id]: 1,
   }
   const expectedBBClock = {
     [aliceMFId]: 1,
-    [bobMFId]: 1
+    [bobMFId]: 1,
   }
 
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
@@ -196,10 +190,7 @@ tape('multiple formats restart', async (t) => {
   const bbClockBob = await pify(bob.ebt.clock)({ format: 'bendybutt-v1' })
   t.deepEqual(bbClockBob, expectedBBClock, 'bob correct bb clock')
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
 
@@ -209,14 +200,14 @@ tape('index format', async (t) => {
   const carol = createSSBServer().call(null, {
     path: carolDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('carol')
+    keys: u.keysFor('carol'),
   })
 
   const daveDir = getFreshDir('dave')
   const dave = createSSBServer().call(null, {
     path: daveDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('dave')
+    keys: u.keysFor('dave'),
   })
 
   const indexedMethods = require('../formats/indexed.js')
@@ -226,15 +217,27 @@ tape('index format', async (t) => {
   dave.ebt.registerFormat(indexedMethods)
   dave.ebt.registerFormat(bendyButtMethods)
 
-  const carolIndexId = (await pify(carol.indexFeedWriter.start)({ author: carol.id, type: 'dog', private: false })).subfeed
-  const daveIndexId = (await pify(dave.indexFeedWriter.start)({ author: dave.id, type: 'dog', private: false })).subfeed
+  const carolIndexId = (
+    await pify(carol.indexFeedWriter.start)({
+      author: carol.id,
+      type: 'dog',
+      private: false,
+    })
+  ).subfeed
+  const daveIndexId = (
+    await pify(dave.indexFeedWriter.start)({
+      author: dave.id,
+      type: 'dog',
+      private: false,
+    })
+  ).subfeed
 
   // publish some messages
   await Promise.all([
     pify(carol.db.publish)({ type: 'post', text: 'hello 2' }),
     pify(carol.db.publish)({ type: 'dog', name: 'Buff' }),
     pify(dave.db.publish)({ type: 'post', text: 'hello 2' }),
-    pify(dave.db.publish)({ type: 'dog', name: 'Biff' })
+    pify(dave.db.publish)({ type: 'dog', name: 'Biff' }),
   ])
 
   await sleep(2 * REPLICATION_TIMEOUT)
@@ -329,7 +332,7 @@ tape('index format', async (t) => {
 
   const expectedIndexClock = {
     [carolIndexId]: 1,
-    [daveIndexId]: 1
+    [daveIndexId]: 1,
   }
 
   const indexClockCarol = await pify(carol.ebt.clock)({ format: 'indexed' })
@@ -338,10 +341,7 @@ tape('index format', async (t) => {
   const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed' })
   t.deepEqual(indexClockDave, expectedIndexClock, 'dave correct index clock')
 
-  await Promise.all([
-    pify(carol.close)(true),
-    pify(dave.close)(true)
-  ])
+  await Promise.all([pify(carol.close)(true), pify(dave.close)(true)])
   t.end()
 })
 
@@ -349,18 +349,18 @@ tape('sliced replication', async (t) => {
   alice = createSSBServer().call(null, {
     path: aliceDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('alice')
+    keys: u.keysFor('alice'),
   })
 
   const carol = createSSBServer().call(null, {
     path: carolDir,
     timeout: CONNECTION_TIMEOUT,
-    keys: u.keysFor('carol')
+    keys: u.keysFor('carol'),
   })
 
   await Promise.all([
     pify(alice.db.publish)({ type: 'post', text: 'hello2' }),
-    pify(alice.db.publish)({ type: 'post', text: 'hello3' })
+    pify(alice.db.publish)({ type: 'post', text: 'hello3' }),
   ])
 
   // carol wants to slice replicate some things, so she overwrites
@@ -370,15 +370,17 @@ tape('sliced replication', async (t) => {
 
   const slicedMethods = {
     ...require('../formats/classic'),
-    appendMsg (sbot, msgVal, cb) {
+    appendMsg(sbot, msgVal, cb) {
       let append = sbot.add
-      if (sliced.includes(msgVal.author)) { append = sbot.db.addOOO }
+      if (sliced.includes(msgVal.author)) {
+        append = sbot.db.addOOO
+      }
 
       append(msgVal, (err, msg) => {
         if (err) return cb(err)
         else cb(null, msg)
       })
-    }
+    },
   }
 
   carol.ebt.registerFormat(slicedMethods)
@@ -395,8 +397,7 @@ tape('sliced replication', async (t) => {
   const clockAlice = await pify(alice.ebt.clock)({ format: 'classic' })
   t.equal(clockAlice[alice.id], 3, 'alice correct index clock')
 
-  carol.ebt.setClockForSlicedReplication(alice.id,
-    clockAlice[alice.id] - 2)
+  carol.ebt.setClockForSlicedReplication(alice.id, clockAlice[alice.id] - 2)
   carol.ebt.request(alice.id, true)
 
   carol.ebt.request(bobId, true) // in full
@@ -416,9 +417,6 @@ tape('sliced replication', async (t) => {
   )
   t.equal(carolBMessages.length, 1, 'all messages from bob')
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(carol.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(carol.close)(true)])
   t.end()
 })
diff --git a/test/index.js b/test/index.js
index dfff013..7f498e6 100644
--- a/test/index.js
+++ b/test/index.js
@@ -7,7 +7,7 @@ const SecretStack = require('secret-stack')
 const u = require('./misc/util')
 
 const createSbot = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('../')) // EBT
@@ -18,25 +18,25 @@ const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 const alice = createSbot({
   temp: 'random-animals_alice',
   timeout: CONNECTION_TIMEOUT,
-  keys: ssbKeys.generate()
+  keys: ssbKeys.generate(),
 })
 
 const bob = createSbot({
   temp: 'random-animals_bob',
   timeout: CONNECTION_TIMEOUT,
-  keys: ssbKeys.generate()
+  keys: ssbKeys.generate(),
 })
 
 const charles = createSbot({
   temp: 'random-animals_charles',
   timeout: CONNECTION_TIMEOUT,
-  keys: ssbKeys.generate()
+  keys: ssbKeys.generate(),
 })
 
 const names = {
   [alice.id]: 'alice',
   [bob.id]: 'bob',
-  [charles.id]: 'charles'
+  [charles.id]: 'charles',
 }
 
 tape('three peers replicate everything between each other', async (t) => {
@@ -59,10 +59,10 @@ tape('three peers replicate everything between each other', async (t) => {
   const recv = {
     alice: new Set(),
     bob: new Set(),
-    charles: new Set()
+    charles: new Set(),
   }
 
-  function consistent (name) {
+  function consistent(name) {
     if (!name) throw new Error('name must be provided')
     return function (msg) {
       u.log(name, 'received', msg.value.content, 'by', names[msg.value.author])
@@ -78,14 +78,14 @@ tape('three peers replicate everything between each other', async (t) => {
   await Promise.all([
     pify(alice.publish)({ type: 'post', text: 'hello world' }),
     pify(bob.publish)({ type: 'post', text: 'hello world' }),
-    pify(charles.publish)({ type: 'post', text: 'hello world' })
+    pify(charles.publish)({ type: 'post', text: 'hello world' }),
   ])
   t.pass('all peers have posted "hello world"')
 
   await Promise.all([
     pify(alice.connect)(bob.getAddress()),
     pify(alice.connect)(charles.getAddress()),
-    pify(charles.connect)(bob.getAddress())
+    pify(charles.connect)(bob.getAddress()),
   ])
   t.pass('the three peers are connected to each other as a triangle')
 
@@ -110,7 +110,7 @@ tape('three peers replicate everything between each other', async (t) => {
   await Promise.all([
     pify(alice.close)(true),
     pify(bob.close)(true),
-    pify(charles.close)(true)
+    pify(charles.close)(true),
   ])
   t.end()
 })
diff --git a/test/legacy.js b/test/legacy.js
index d3f05aa..d4ae43a 100644
--- a/test/legacy.js
+++ b/test/legacy.js
@@ -6,7 +6,7 @@ const sleep = require('util').promisify(setTimeout)
 const SecretStack = require('secret-stack')
 
 const createSbot = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('../')) // EBT
@@ -18,7 +18,7 @@ const bobKeys = ssbKeys.generate()
 const alice = createSbot({
   temp: 'random-animals',
   timeout: CONNECTION_TIMEOUT,
-  keys: ssbKeys.generate()
+  keys: ssbKeys.generate(),
 })
 
 tape('legacy (version 1) is unsupported', async (t) => {
diff --git a/test/race.js b/test/race.js
index afcfef6..5efaf29 100644
--- a/test/race.js
+++ b/test/race.js
@@ -6,7 +6,7 @@ const pify = require('promisify-4loc')
 const sleep = require('util').promisify(setTimeout)
 const u = require('./misc/util')
 
-function delayedVectorClock (sbot, config) {
+function delayedVectorClock(sbot, config) {
   const realGetVectorClock = sbot.getVectorClock
   sbot.getVectorClock = (cb) => {
     setTimeout(() => realGetVectorClock(cb), 1000)
@@ -14,7 +14,7 @@ function delayedVectorClock (sbot, config) {
 }
 
 const createSbot = require('secret-stack')({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(delayedVectorClock)
@@ -31,19 +31,19 @@ tape('we wait for vectorclock being available before doing ebt', async (t) => {
   let alice = createSbot({
     path: '/tmp/alice',
     timeout: CONNECTION_TIMEOUT,
-    keys: aliceKeys
+    keys: aliceKeys,
   })
 
   const bobKeys = ssbKeys.generate()
   let bob = createSbot({
     path: '/tmp/bob',
     timeout: CONNECTION_TIMEOUT,
-    keys: bobKeys
+    keys: bobKeys,
   })
 
   await Promise.all([
     pify(alice.publish)({ type: 'post', text: 'hello world' }),
-    pify(bob.publish)({ type: 'post', text: 'hello world' })
+    pify(bob.publish)({ type: 'post', text: 'hello world' }),
   ])
   t.pass('all peers have posted "hello world"')
 
@@ -52,21 +52,18 @@ tape('we wait for vectorclock being available before doing ebt', async (t) => {
 
   t.pass('restarting')
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
 
   alice = createSbot({
     path: '/tmp/alice',
     timeout: CONNECTION_TIMEOUT,
-    keys: aliceKeys
+    keys: aliceKeys,
   })
 
   bob = createSbot({
     path: '/tmp/bob',
     timeout: CONNECTION_TIMEOUT,
-    keys: bobKeys
+    keys: bobKeys,
   })
 
   // self replicate
@@ -86,12 +83,17 @@ tape('we wait for vectorclock being available before doing ebt', async (t) => {
 
   u.log('A', u.countClock(clockAlice), 'B', u.countClock(clockBob))
 
-  t.deepEqual(u.countClock(clockAlice), { total: 2, sum: 2 }, 'alice has both feeds')
-  t.deepEqual(u.countClock(clockBob), { total: 1, sum: 1 }, 'bob only has own feed')
-
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  t.deepEqual(
+    u.countClock(clockAlice),
+    { total: 2, sum: 2 },
+    'alice has both feeds'
+  )
+  t.deepEqual(
+    u.countClock(clockBob),
+    { total: 1, sum: 1 },
+    'bob only has own feed'
+  )
+
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
diff --git a/test/realtime.js b/test/realtime.js
index afaa60c..6b09584 100644
--- a/test/realtime.js
+++ b/test/realtime.js
@@ -7,12 +7,12 @@ const pify = require('promisify-4loc')
 const sleep = require('util').promisify(setTimeout)
 
 const createSsbServer = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('..'))
 
-function createHistoryStream (sbot, opts) {
+function createHistoryStream(sbot, opts) {
   return pull(
     sbot.createLogStream({ keys: false, live: opts.live }),
     pull.filter((msg) => msg.author === opts.id)
@@ -26,13 +26,13 @@ tape('replicate between 2 peers', async (t) => {
   const alice = createSsbServer({
     temp: 'test-alice',
     timeout: CONNECTION_TIMEOUT,
-    keys: ssbKeys.generate()
+    keys: ssbKeys.generate(),
   })
 
   const bob = createSsbServer({
     temp: 'test-bob',
     timeout: CONNECTION_TIMEOUT,
-    keys: ssbKeys.generate()
+    keys: ssbKeys.generate(),
   })
 
   // Self replicate
@@ -78,9 +78,6 @@ tape('replicate between 2 peers', async (t) => {
   t.equal(coldMsgs.length, 10)
   t.deepEqual(hotMsgs, coldMsgs)
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
diff --git a/test/resync.js b/test/resync.js
index 18be4f4..8e86e50 100644
--- a/test/resync.js
+++ b/test/resync.js
@@ -7,7 +7,7 @@ const sleep = require('util').promisify(setTimeout)
 const u = require('./misc/util')
 
 const createSbot = require('secret-stack')({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('../'))
@@ -19,13 +19,13 @@ tape('peer can recover and resync its content from a friend', async (t) => {
   const alice = createSbot({
     temp: 'alice',
     timeout: CONNECTION_TIMEOUT,
-    keys: ssbKeys.generate()
+    keys: ssbKeys.generate(),
   })
 
   const bob = createSbot({
     temp: 'bob',
     timeout: CONNECTION_TIMEOUT,
-    keys: ssbKeys.generate()
+    keys: ssbKeys.generate(),
   })
 
   t.ok(alice.getAddress(), 'alice has an address')
@@ -44,7 +44,7 @@ tape('peer can recover and resync its content from a friend', async (t) => {
   // in this test, bob's feed is on alice,
   // because bob's database corrupted (but had key backup)
   peers.push(alice.createFeed(bob.keys))
-  t.pass('alice has bob\'s content')
+  t.pass("alice has bob's content")
 
   await pify(gen.messages)(
     function (n) {
@@ -52,14 +52,14 @@ tape('peer can recover and resync its content from a friend', async (t) => {
         return {
           type: 'contact',
           contact: u.randary(peers).id,
-          following: true
+          following: true,
         }
       }
       return {
         type: 'test',
         ts: Date.now(),
         random: Math.random(),
-        value: u.randbytes(u.randint(1024)).toString('base64')
+        value: u.randbytes(u.randint(1024)).toString('base64'),
       }
     },
     peers,
@@ -84,13 +84,19 @@ tape('peer can recover and resync its content from a friend', async (t) => {
     }
   }
 
-  u.log('A', u.countClock(clockAlice), 'B', u.countClock(clockBob), 'diff', diff, 'common', commonCount)
+  u.log(
+    'A',
+    u.countClock(clockAlice),
+    'B',
+    u.countClock(clockBob),
+    'diff',
+    diff,
+    'common',
+    commonCount
+  )
   t.equals(diff, 0, 'no diff between alice and bob')
   t.equals(commonCount > 0, true, 'bob has some content')
 
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })
diff --git a/test/server.js b/test/server.js
index b9f026e..1839055 100644
--- a/test/server.js
+++ b/test/server.js
@@ -10,7 +10,7 @@ const sleep = require('util').promisify(setTimeout)
 // and get them to follow each other...
 
 const createSsbServer = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('..'))
@@ -25,19 +25,19 @@ tape('replicate between 3 peers', async (t) => {
     temp: 'server-alice',
     keys: ssbKeys.generate(),
     timeout: CONNECTION_TIMEOUT,
-    level: 'info'
+    level: 'info',
   })
   const bob = createSsbServer({
     temp: 'server-bob',
     keys: ssbKeys.generate(),
     timeout: CONNECTION_TIMEOUT,
-    level: 'info'
+    level: 'info',
   })
   const carol = createSsbServer({
     temp: 'server-carol',
     keys: ssbKeys.generate(),
     timeout: CONNECTION_TIMEOUT,
-    level: 'info'
+    level: 'info',
   })
 
   // Wait for all bots to be ready
@@ -52,7 +52,7 @@ tape('replicate between 3 peers', async (t) => {
     pify(bob.publish)({ type: 'post', text: 'world' }),
 
     pify(carol.publish)({ type: 'post', text: 'hello' }),
-    pify(carol.publish)({ type: 'post', text: 'world' })
+    pify(carol.publish)({ type: 'post', text: 'world' }),
   ])
 
   // Self replicate
@@ -81,13 +81,13 @@ tape('replicate between 3 peers', async (t) => {
   const [connectionBA, connectionBC, connectionCA] = await Promise.all([
     pify(bob.connect)(alice.getAddress()),
     pify(bob.connect)(carol.getAddress()),
-    pify(carol.connect)(alice.getAddress())
+    pify(carol.connect)(alice.getAddress()),
   ])
 
   const expectedClock = {
     [alice.id]: 2,
     [bob.id]: 2,
-    [carol.id]: 2
+    [carol.id]: 2,
   }
 
   await sleep(REPLICATION_TIMEOUT)
@@ -95,23 +95,23 @@ tape('replicate between 3 peers', async (t) => {
   const [clockAlice, clockBob, clockCarol] = await Promise.all([
     pify(alice.getVectorClock)(),
     pify(bob.getVectorClock)(),
-    pify(carol.getVectorClock)()
+    pify(carol.getVectorClock)(),
   ])
 
-  t.deepEqual(clockAlice, expectedClock, 'alice\'s clock is correct')
-  t.deepEqual(clockBob, expectedClock, 'bob\'s clock is correct')
-  t.deepEqual(clockCarol, expectedClock, 'carol\'s clock is correct')
+  t.deepEqual(clockAlice, expectedClock, "alice's clock is correct")
+  t.deepEqual(clockBob, expectedClock, "bob's clock is correct")
+  t.deepEqual(clockCarol, expectedClock, "carol's clock is correct")
 
   await Promise.all([
     pify(connectionBA.close)(true),
     pify(connectionBC.close)(true),
-    pify(connectionCA.close)(true)
+    pify(connectionCA.close)(true),
   ])
 
   await Promise.all([
     pify(alice.close)(true),
     pify(bob.close)(true),
-    pify(carol.close)(true)
+    pify(carol.close)(true),
   ])
 
   t.end()
diff --git a/test/unit.js b/test/unit.js
index 6985c33..d0a07e4 100644
--- a/test/unit.js
+++ b/test/unit.js
@@ -5,7 +5,7 @@ const pify = require('promisify-4loc')
 const u = require('./misc/util')
 
 const createSsbServer = SecretStack({
-  caps: { shs: crypto.randomBytes(32).toString('base64') }
+  caps: { shs: crypto.randomBytes(32).toString('base64') },
 })
   .use(require('ssb-db'))
   .use(require('../'))
@@ -16,13 +16,13 @@ const REPLICATION_TIMEOUT = 2 * CONNECTION_TIMEOUT
 const alice = createSsbServer({
   temp: 'test-block-alice',
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('alice')
+  keys: u.keysFor('alice'),
 })
 
 const bob = createSsbServer({
   temp: 'test-block-bob',
   timeout: CONNECTION_TIMEOUT,
-  keys: u.keysFor('bob')
+  keys: u.keysFor('bob'),
 })
 
 tape('alice replicates bob', async (t) => {
@@ -30,7 +30,7 @@ tape('alice replicates bob', async (t) => {
     pify(alice.publish)({ type: 'post', text: 'hello' }),
     pify(alice.publish)({ type: 'post', text: 'world' }),
     pify(bob.publish)({ type: 'post', text: 'hello' }),
-    pify(bob.publish)({ type: 'post', text: 'world' })
+    pify(bob.publish)({ type: 'post', text: 'world' }),
   ])
 
   // Self replicate
@@ -39,16 +39,16 @@ tape('alice replicates bob', async (t) => {
 
   alice.ebt.request(bob.id, true)
   alice.ebt.block(alice.id, bob.id, false)
-  t.pass('alice wants bob\'s data')
+  t.pass("alice wants bob's data")
 
   bob.ebt.request(alice.id, true)
   bob.ebt.block(bob.id, alice.id, false)
-  t.pass('bob wants alice\'s data')
+  t.pass("bob wants alice's data")
 
   const [rpcBobToAlice, msgAtBob, msgAtAlice] = await Promise.all([
     pify(bob.connect)(alice.getAddress()),
     u.readOnceFromDB(bob, REPLICATION_TIMEOUT),
-    u.readOnceFromDB(alice, REPLICATION_TIMEOUT)
+    u.readOnceFromDB(alice, REPLICATION_TIMEOUT),
   ])
 
   t.equals(msgAtAlice.value.author, bob.id, 'alice has a msg from bob')
@@ -57,7 +57,7 @@ tape('alice replicates bob', async (t) => {
   await pify(rpcBobToAlice.close)(true)
 })
 
-tape('ssb.progress', t => {
+tape('ssb.progress', (t) => {
   const p = alice.progress()
   t.ok(p, 'progress')
   t.ok(p.ebt, 'progress.ebt')
@@ -67,7 +67,7 @@ tape('ssb.progress', t => {
   t.end()
 })
 
-tape('ssb.ebt.peerStatus', t => {
+tape('ssb.ebt.peerStatus', (t) => {
   const s = alice.ebt.peerStatus(bob.id)
   t.ok(s, 'response is an object')
   t.equals(s.id, bob.id, 'response.id is correct')
@@ -75,14 +75,14 @@ tape('ssb.ebt.peerStatus', t => {
   t.end()
 })
 
-tape('silly ssb.ebt.request', t => {
+tape('silly ssb.ebt.request', (t) => {
   t.doesNotThrow(() => {
     alice.ebt.request('not a feed id', true)
   })
   t.end()
 })
 
-tape('silly ssb.ebt.block', t => {
+tape('silly ssb.ebt.block', (t) => {
   t.doesNotThrow(() => {
     alice.ebt.block(bob.id, 'not a feed id', true)
   })
@@ -93,9 +93,6 @@ tape('silly ssb.ebt.block', t => {
 })
 
 tape('teardown', async (t) => {
-  await Promise.all([
-    pify(alice.close)(true),
-    pify(bob.close)(true)
-  ])
+  await Promise.all([pify(alice.close)(true), pify(bob.close)(true)])
   t.end()
 })

From 14e04e75b8746c45fce96931800f1e8cc3174fd7 Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Thu, 16 Sep 2021 17:13:23 +0300
Subject: [PATCH 43/50] introduce format.prepareForIsFeed()

---
 README.md             | 12 ++++++++----
 formats/bendy-butt.js |  3 +++
 formats/classic.js    |  3 +++
 formats/indexed.js    |  3 +++
 index.js              | 45 ++++++++++++++++++++++++++-----------------
 5 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index 8e56733..202977d 100644
--- a/README.md
+++ b/README.md
@@ -42,8 +42,8 @@ installed, instead, you need to call its API methods yourself (primarily
 
 Request that the SSB feed ID `destination` be replicated. `replication` is a
 boolean, where `true` indicates we want to replicate the destination. If set to
-`false`, replication is stopped. `formatName` is optional and used to specify 
-the specific EBT instance, otherwise the first where isFeed is `true` for 
+`false`, replication is stopped. `formatName` is optional and used to specify
+the specific EBT instance, otherwise the first where isFeed is `true` for
 `destination` is used.
 
 Returns undefined, always.
@@ -58,7 +58,7 @@ them.
 the SSB feed ID of the peer being blocked, and `blocking` is a boolean that
 indicates whether to enable the block (`true`) or to unblock (`false`).
 
-`formatName` is optional and used to specify the specific EBT instance, 
+`formatName` is optional and used to specify the specific EBT instance,
 otherwise the first where isFeed is `true` for `origin` is used.
 
 Returns undefined, always.
@@ -121,8 +121,12 @@ shows the implementation for 'classic' ed25519 SSB feeds.
 ```js
 {
   name: 'classic',
+  // In case `isFeed` needs to load some state asynchronously
+  prepareForIsFeed(sbot, feedId, cb) {
+    cb()
+  },
   // used in request, block, cleanClock, sbot.post, vectorClock
-  sbotIsFeed(sbot, feedId) {
+  isFeed(sbot, feedId) {
     return ref.isFeed(feedId)
   },
   getAtSequence(sbot, pair, cb) {
diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js
index 5e2605b..fe5ab17 100644
--- a/formats/bendy-butt.js
+++ b/formats/bendy-butt.js
@@ -3,6 +3,9 @@ const bendyButt = require('ssb-bendy-butt')
 
 module.exports = {
   name: 'bendybutt-v1',
+  prepareForIsFeed(sbot, feedId, cb) {
+    cb()
+  },
   // used in request, block, cleanClock, sbot.post, vectorClock
   isFeed (sbot, feedId) {
     return SSBURI.isBendyButtV1FeedSSBURI(feedId)
diff --git a/formats/classic.js b/formats/classic.js
index 2890c70..cbee9c7 100644
--- a/formats/classic.js
+++ b/formats/classic.js
@@ -2,6 +2,9 @@ const ref = require('ssb-ref')
 
 module.exports = {
   name: 'classic',
+  prepareForIsFeed(sbot, feedId, cb) {
+    cb()
+  },
   // used in request, block, cleanClock, sbot.post, vectorClock
   isFeed (sbot, feedId) {
     return ref.isFeed(feedId)
diff --git a/formats/indexed.js b/formats/indexed.js
index ecd2135..d193b91 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -5,6 +5,9 @@ const { QL0 } = require('ssb-subset-ql')
 module.exports = {
   ...classic,
   name: 'indexed',
+  prepareForIsFeed(sbot, feedId, cb) {
+    sbot.metafeeds.ensureLoaded(feedId, cb)
+  },
   isFeed (sbot, author) {
     const info = sbot.metafeeds.findByIdSync(author)
     return info && info.feedpurpose === 'index'
diff --git a/index.js b/index.js
index 6e4ce7f..a3bba6c 100644
--- a/index.js
+++ b/index.js
@@ -92,6 +92,7 @@ exports.init = function (sbot, config) {
     ebt.isReady = format.isReady.bind(format, sbot)
     ebt.isFeed = isFeed
     ebt.name = format.name
+    ebt.prepareForIsFeed = format.prepareForIsFeed.bind(format, sbot)
 
     const existingId = ebts.findIndex((e) => e.name === format.name)
     if (existingId !== -1) ebts[existingId] = ebt
@@ -196,30 +197,38 @@ exports.init = function (sbot, config) {
 
   function request(destFeedId, requesting, formatName) {
     initialized.promise.then(() => {
-      const ebt = findEBTForFeed(destFeedId, formatName)
-
-      if (!ebt.isFeed(destFeedId)) return
-
-      ebt.request(destFeedId, requesting)
+      if (requesting) {
+        const ebt = findEBTForFeed(destFeedId, formatName)
+        ebt.prepareForIsFeed(destFeedId, () => {
+          if (!ebt.isFeed(destFeedId)) return
+          ebt.request(destFeedId, true)
+        })
+      } else {
+        // If we don't want a destFeedId, make sure it's not registered anywhere
+        ebts.forEach((ebt) => {
+          ebt.request(destFeedId, false)
+        })
+      }
     })
   }
 
   function block(origFeedId, destFeedId, blocking, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(origFeedId, formatName)
-
-      if (!ebt.isFeed(origFeedId)) return
-      if (!ebt.isFeed(destFeedId)) return
-
-      if (blocking) {
-        ebt.block(origFeedId, destFeedId, true)
-      } else if (
-        ebt.state.blocks[origFeedId] &&
-        ebt.state.blocks[origFeedId][destFeedId]
-      ) {
-        // only update unblock if they were already blocked
-        ebt.block(origFeedId, destFeedId, false)
-      }
+      ebt.prepareForIsFeed(() => {
+        if (!ebt.isFeed(origFeedId)) return
+        if (!ebt.isFeed(destFeedId)) return
+
+        if (blocking) {
+          ebt.block(origFeedId, destFeedId, true)
+        } else if (
+          ebt.state.blocks[origFeedId] &&
+          ebt.state.blocks[origFeedId][destFeedId]
+        ) {
+          // only update unblock if they were already blocked
+          ebt.block(origFeedId, destFeedId, false)
+        }
+      })
     })
   }
 

From 2fe87910ca37a8dc50d6ad7458617885a3f51e48 Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Thu, 16 Sep 2021 17:24:15 +0300
Subject: [PATCH 44/50] update ssb-meta-feeds and fix usage

---
 index.js     | 2 +-
 package.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/index.js b/index.js
index a3bba6c..75e86b0 100644
--- a/index.js
+++ b/index.js
@@ -215,7 +215,7 @@ exports.init = function (sbot, config) {
   function block(origFeedId, destFeedId, blocking, formatName) {
     initialized.promise.then(() => {
       const ebt = findEBTForFeed(origFeedId, formatName)
-      ebt.prepareForIsFeed(() => {
+      ebt.prepareForIsFeed(destFeedId, () => {
         if (!ebt.isFeed(origFeedId)) return
         if (!ebt.isFeed(destFeedId)) return
 
diff --git a/package.json b/package.json
index 1639cc2..1a29fe2 100644
--- a/package.json
+++ b/package.json
@@ -52,7 +52,7 @@
     "ssb-generate": "^1.0.1",
     "ssb-index-feed-writer": "^0.6.0",
     "ssb-keys": "^8.1.0",
-    "ssb-meta-feeds": "^0.21.0",
+    "ssb-meta-feeds": "^0.22.0",
     "ssb-validate": "^4.1.4",
     "tap-spec": "^5.0.0",
     "tape": "^5.2.2"

From 0e398b8f734c92ee42ac99e6a769bd20fb6e695b Mon Sep 17 00:00:00 2001
From: Andre Staltz <andre@staltz.com>
Date: Wed, 22 Sep 2021 14:45:18 +0300
Subject: [PATCH 45/50] improve test for index feeds

---
 test/formats.js | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/test/formats.js b/test/formats.js
index f721e90..0b12551 100644
--- a/test/formats.js
+++ b/test/formats.js
@@ -341,6 +341,17 @@ tape('index format', async (t) => {
   const indexClockDave = await pify(dave.ebt.clock)({ format: 'indexed' })
   t.deepEqual(indexClockDave, expectedIndexClock, 'dave correct index clock')
 
+  await pify(carol.db.publish)({ type: 'dog', text: 'woof woof' })
+
+  await sleep(2 * REPLICATION_TIMEOUT)
+  t.pass('wait for replication to complete')
+
+  const daveCarolMessages2 = await dave.db.query(
+    where(author(carol.id)),
+    toPromise()
+  )
+  t.equal(daveCarolMessages2.length, 2, 'dave got 2 dog messages from carol')
+
   await Promise.all([pify(carol.close)(true), pify(dave.close)(true)])
   t.end()
 })

From b4e06a3609061838466813dd50a12ec0804d40d4 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Wed, 22 Sep 2021 23:39:18 +0200
Subject: [PATCH 46/50] Use transactions for indexed and fix tests

---
 formats/bendy-butt.js |  4 ++--
 formats/classic.js    |  4 ++--
 formats/indexed.js    | 38 ++++++++++++++++++++------------------
 index.js              |  6 ++++--
 package.json          |  2 +-
 5 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/formats/bendy-butt.js b/formats/bendy-butt.js
index fe5ab17..90c02d8 100644
--- a/formats/bendy-butt.js
+++ b/formats/bendy-butt.js
@@ -20,8 +20,8 @@ module.exports = {
       cb(err && err.fatal ? err : null, msg)
     })
   },
-  convertMsg (msgVal) {
-    return bendyButt.encode(msgVal)
+  convertMsg (sbot, msgVal, cb) {
+    cb(null, bendyButt.encode(msgVal))
   },
   // used in vectorClock
   isReady (sbot) {
diff --git a/formats/classic.js b/formats/classic.js
index cbee9c7..c3fd4fa 100644
--- a/formats/classic.js
+++ b/formats/classic.js
@@ -20,8 +20,8 @@ module.exports = {
     })
   },
   // used in onAppend
-  convertMsg (msgVal) {
-    return msgVal
+  convertMsg (sbot, msgVal, cb) {
+    cb(null, msgVal)
   },
   // used in vectorClock
   isReady (sbot) {
diff --git a/formats/indexed.js b/formats/indexed.js
index d193b91..19d9271 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -13,32 +13,34 @@ module.exports = {
     return info && info.feedpurpose === 'index'
   },
   appendMsg (sbot, msgVal, cb) {
-    // FIXME: this needs the ability to add 2 messages in a transaction
     const payload = msgVal.content.indexed.payload
     delete msgVal.content.indexed.payload
-    sbot.db.add(msgVal, (err, msg) => {
-      if (err) return cb(err)
-      sbot.db.addOOO(payload, (err, indexedMsg) => {
-        if (err) return cb(err)
-        else cb(null, msg)
-      })
-    })
+    sbot.db.addTransaction([msgVal], [payload], cb)
   },
   getAtSequence (sbot, pair, cb) {
     sbot.getAtSequence([pair.id, pair.sequence], (err, msg) => {
       if (err) return cb(err)
-      const { sequence } = msg.value.content.indexed
-      const authorInfo = sbot.metafeeds.findByIdSync(msg.value.author)
-      if (!authorInfo) return cb(new Error('Unknown author:' + msg.value.author))
-      const { author } = QL0.parse(authorInfo.metadata.query)
-      sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
-        if (err) return cb(err)
 
-        // add referenced message as payload
-        msg.value.content.indexed.payload = indexedMsg.value
+      module.exports.convertMsg(sbot, msg.value, cb)
+    })
+  },
+  convertMsg (sbot, msgVal, cb) {
+    const { sequence } = msgVal.content.indexed
+    const authorInfo = sbot.metafeeds.findByIdSync(msgVal.author)
+    if (!authorInfo) return cb(new Error('Unknown author:' + msgVal.author))
+    const { author } = QL0.parse(authorInfo.metadata.query)
+    sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
+      if (err) return cb(err)
+
+      // add referenced message as payload
+      const msgValWithPayload = { ...msgVal }
+      msgValWithPayload.content = { ...msgValWithPayload.content }
+      msgValWithPayload.content.indexed = {
+        ...msgValWithPayload.content.indexed,
+        payload: indexedMsg.value
+      }
 
-        cb(null, msg.value)
-      })
+      cb(null, msgValWithPayload)
     })
   },
   isReady (sbot) {
diff --git a/index.js b/index.js
index 75e86b0..e1c9465 100644
--- a/index.js
+++ b/index.js
@@ -88,7 +88,7 @@ exports.init = function (sbot, config) {
     })
 
     // attach a few methods we need in this module
-    ebt.convertMsg = format.convertMsg
+    ebt.convertMsg = format.convertMsg.bind(format, sbot)
     ebt.isReady = format.isReady.bind(format, sbot)
     ebt.isFeed = isFeed
     ebt.name = format.name
@@ -135,7 +135,9 @@ exports.init = function (sbot, config) {
     initialized.promise.then(() => {
       ebts.forEach((ebt) => {
         if (ebt.isFeed(msg.value.author)) {
-          ebt.onAppend(ebt.convertMsg(msg.value))
+          ebt.convertMsg(msg.value, (err, converted) => {
+            ebt.onAppend(converted)
+          })
         }
       })
     })
diff --git a/package.json b/package.json
index 1a29fe2..efbf6d1 100644
--- a/package.json
+++ b/package.json
@@ -48,7 +48,7 @@
     "ssb-caps": "^1.1.0",
     "ssb-client": "^4.9.0",
     "ssb-db": "^19.2.0",
-    "ssb-db2": "^2.4.0",
+    "ssb-db2": "^2.6.0",
     "ssb-generate": "^1.0.1",
     "ssb-index-feed-writer": "^0.6.0",
     "ssb-keys": "^8.1.0",

From 39e40d81e4362e59a5fa8235f3ecc6d409056d7c Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 23 Sep 2021 08:23:51 +0200
Subject: [PATCH 47/50] Send indexes as tuples instead

---
 formats/indexed.js | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/formats/indexed.js b/formats/indexed.js
index 19d9271..2c0bf23 100644
--- a/formats/indexed.js
+++ b/formats/indexed.js
@@ -1,9 +1,8 @@
-const classic = require('./classic')
 const pify = require('promisify-4loc')
+const ref = require('ssb-ref')
 const { QL0 } = require('ssb-subset-ql')
 
 module.exports = {
-  ...classic,
   name: 'indexed',
   prepareForIsFeed(sbot, feedId, cb) {
     sbot.metafeeds.ensureLoaded(feedId, cb)
@@ -12,9 +11,8 @@ module.exports = {
     const info = sbot.metafeeds.findByIdSync(author)
     return info && info.feedpurpose === 'index'
   },
-  appendMsg (sbot, msgVal, cb) {
-    const payload = msgVal.content.indexed.payload
-    delete msgVal.content.indexed.payload
+  appendMsg (sbot, msgTuple, cb) {
+    const [msgVal, payload] = msgTuple
     sbot.db.addTransaction([msgVal], [payload], cb)
   },
   getAtSequence (sbot, pair, cb) {
@@ -32,18 +30,24 @@ module.exports = {
     sbot.getAtSequence([author, sequence], (err, indexedMsg) => {
       if (err) return cb(err)
 
-      // add referenced message as payload
-      const msgValWithPayload = { ...msgVal }
-      msgValWithPayload.content = { ...msgValWithPayload.content }
-      msgValWithPayload.content.indexed = {
-        ...msgValWithPayload.content.indexed,
-        payload: indexedMsg.value
-      }
-
-      cb(null, msgValWithPayload)
+      cb(null, [msgVal, indexedMsg.value])
     })
   },
   isReady (sbot) {
     return pify(sbot.metafeeds.loadState)()
+  },
+  isMsg (msgTuple) {
+    if (Array.isArray(msgTuple) && msgTuple.length === 2) {
+      const [msgVal, payload] = msgTuple
+      return Number.isInteger(msgVal.sequence) && msgVal.sequence > 0 &&
+        ref.isFeed(msgVal.author) && msgVal.content
+    } else
+      return false
+  },
+  getMsgAuthor (msgTuple) {
+    return msgTuple[0].author
+  },
+  getMsgSequence (msgTuple) {
+    return msgTuple[0].sequence
   }
 }

From c488bd40cfc2b947d3598ff951587bbb53a050fa Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Thu, 23 Sep 2021 20:14:44 +0200
Subject: [PATCH 48/50] Change p-defer to a simple waiting queue system

---
 index.js     | 27 +++++++++++++++++----------
 package.json |  1 -
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/index.js b/index.js
index e1c9465..f9f0b35 100644
--- a/index.js
+++ b/index.js
@@ -5,7 +5,6 @@ const EBT = require('epidemic-broadcast-trees')
 const Store = require('lossy-store')
 const toUrlFriendly = require('base64-url').escape
 const getSeverity = require('ssb-network-errors')
-const DeferredPromise = require('p-defer')
 const pullDefer = require('pull-defer')
 const classicMethods = require('./formats/classic')
 
@@ -109,7 +108,12 @@ exports.init = function (sbot, config) {
     return ebt
   }
 
-  const initialized = DeferredPromise()
+  let isReady = false
+  let waiting = []
+  function onReady(fn) {
+    if (isReady) fn()
+    else waiting.push(fn)
+  }
 
   sbot.getVectorClock((err, clock) => {
     if (err) console.warn('Failed to getVectorClock in ssb-ebt because:', err)
@@ -127,12 +131,15 @@ exports.init = function (sbot, config) {
         ebt.state.clock = validClock
         ebt.update()
       })
-      initialized.resolve()
+
+      isReady = true
+      for (let i = 0; i < waiting.length; ++i) waiting[i]()
+      waiting = []
     })
   })
 
   sbot.post((msg) => {
-    initialized.promise.then(() => {
+    onReady(() => {
       ebts.forEach((ebt) => {
         if (ebt.isFeed(msg.value.author)) {
           ebt.convertMsg(msg.value, (err, converted) => {
@@ -158,7 +165,7 @@ exports.init = function (sbot, config) {
   sbot.on('rpc:connect', function (rpc, isClient) {
     if (rpc.id === sbot.id) return // ssb-client connecting to ssb-server
     if (isClient) {
-      initialized.promise.then(() => {
+      onReady(() => {
         ebts.forEach((ebt) => {
           const format = ebt.name
           const opts = { version: 3, format }
@@ -198,7 +205,7 @@ exports.init = function (sbot, config) {
   }
 
   function request(destFeedId, requesting, formatName) {
-    initialized.promise.then(() => {
+    onReady(() => {
       if (requesting) {
         const ebt = findEBTForFeed(destFeedId, formatName)
         ebt.prepareForIsFeed(destFeedId, () => {
@@ -215,7 +222,7 @@ exports.init = function (sbot, config) {
   }
 
   function block(origFeedId, destFeedId, blocking, formatName) {
-    initialized.promise.then(() => {
+    onReady(() => {
       const ebt = findEBTForFeed(origFeedId, formatName)
       ebt.prepareForIsFeed(destFeedId, () => {
         if (!ebt.isFeed(origFeedId)) return
@@ -243,7 +250,7 @@ exports.init = function (sbot, config) {
     const ebt = getEBT(formatName)
 
     const deferred = pullDefer.duplex()
-    initialized.promise.then(() => {
+    onReady(() => {
       // `this` refers to the remote peer who called this muxrpc API
       deferred.resolve(
         toPull.duplex(ebt.createStream(this.id, opts.version, false))
@@ -287,14 +294,14 @@ exports.init = function (sbot, config) {
       opts = { format: 'classic' }
     }
 
-    initialized.promise.then(() => {
+    onReady(() => {
       const ebt = getEBT(opts.format)
       cb(null, ebt.state.clock)
     })
   }
 
   function setClockForSlicedReplication(feedId, sequence, formatName) {
-    initialized.promise.then(() => {
+    onReady(() => {
       const ebt = findEBTForFeed(feedId, formatName)
 
       ebt.state.clock[feedId] = sequence
diff --git a/package.json b/package.json
index efbf6d1..e457ef2 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,6 @@
     "base64-url": "^2.2.0",
     "epidemic-broadcast-trees": "^8.0.4",
     "lossy-store": "^1.2.3",
-    "p-defer": "^3.0.0",
     "promisify-4loc": "^1.0.0",
     "pull-defer": "^0.2.3",
     "pull-stream": "^3.6.0",

From a8fd93870164ccab34c734c90f222e967d18a149 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Fri, 24 Sep 2021 12:04:29 +0200
Subject: [PATCH 49/50] Bump EBT

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index e457ef2..ed2af43 100644
--- a/package.json
+++ b/package.json
@@ -20,7 +20,7 @@
   },
   "dependencies": {
     "base64-url": "^2.2.0",
-    "epidemic-broadcast-trees": "^8.0.4",
+    "epidemic-broadcast-trees": "^9.0.0",
     "lossy-store": "^1.2.3",
     "promisify-4loc": "^1.0.0",
     "pull-defer": "^0.2.3",

From e03caffc31ef48171f20e4ba00a04a78c40dd691 Mon Sep 17 00:00:00 2001
From: Anders Rune Jensen <arj03@protonmail.ch>
Date: Fri, 8 Oct 2021 13:39:30 +0200
Subject: [PATCH 50/50] Better error handling

---
 index.js | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/index.js b/index.js
index f9f0b35..41705c6 100644
--- a/index.js
+++ b/index.js
@@ -100,10 +100,7 @@ exports.init = function (sbot, config) {
 
   function getEBT(formatName) {
     const ebt = ebts.find((ebt) => ebt.name === formatName)
-    if (!ebt) {
-      console.log(ebts)
-      throw new Error('Unknown format: ' + formatName)
-    }
+    if (!ebt) throw new Error('Unknown format: ' + formatName)
 
     return ebt
   }
@@ -143,7 +140,9 @@ exports.init = function (sbot, config) {
       ebts.forEach((ebt) => {
         if (ebt.isFeed(msg.value.author)) {
           ebt.convertMsg(msg.value, (err, converted) => {
-            ebt.onAppend(converted)
+            if (err)
+              console.warn('Failed to convert msg in ssb-ebt because:', err)
+            else ebt.onAppend(converted)
           })
         }
       })