Skip to content

Commit

Permalink
Merge branch 'master' of github.com:ssbc/ssb-meta-feeds into auto-shard
Browse files Browse the repository at this point in the history
  • Loading branch information
@mixmix
mixmix committed Sep 19, 2022
2 parents 7f898c1 + 7395acd commit 3d3e54f
Show file tree
Hide file tree
Showing 10 changed files with 561 additions and 161 deletions.
39 changes: 16 additions & 23 deletions api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,18 +50,6 @@ exports.init = function (sbot, config) {
sbot.metafeeds.lookup.findById(feedId, cb)
}

function loadState(cb) {
sbot.metafeeds.lookup.loadState(cb)
}

function ensureLoaded(feedId, cb) {
sbot.metafeeds.lookup.ensureLoaded(feedId, cb)
}

function findByIdSync(feedId) {
return sbot.metafeeds.lookup.findByIdSync(feedId)
}

function branchStream(opts) {
return sbot.metafeeds.lookup.branchStream(opts)
}
Expand Down Expand Up @@ -153,11 +141,7 @@ exports.init = function (sbot, config) {
sbot.metafeeds.query.getSeed((err, seed) => {
if (err) return cb(err)
if (!seed) return cb(null, null)
const metafeed = {
seed,
keys: sbot.metafeeds.keys.deriveRootMetaFeedKeyFromSeed(seed),
}
cb(null, metafeed)
cb(null, buildRootFeedDetails(seed))
})
}

Expand All @@ -181,11 +165,10 @@ exports.init = function (sbot, config) {
const opts = optsForSeed(mfKeys, sbot.id, seed)
const [err2] = await run(sbot.db.create)(opts)
if (err2) return cb(err2)
mf = { seed, keys: mfKeys }
mf = buildRootFeedDetails(seed)
} else {
debug('loaded seed')
const mfKeys = deriveRootMetaFeedKeyFromSeed(loadedSeed)
mf = { seed: loadedSeed, keys: mfKeys }
mf = buildRootFeedDetails(loadedSeed)
}

// Ensure root meta feed announcement exists on the main feed
Expand Down Expand Up @@ -248,6 +231,19 @@ exports.init = function (sbot, config) {
})
}

function buildRootFeedDetails(seed) {
const keys = sbot.metafeeds.keys.deriveRootMetaFeedKeyFromSeed(seed)
return {
metafeed: null,
subfeed: keys.id,
feedpurpose: 'root',
feedformat: 'bendybutt-v1',
seed,
keys,
metadata: {},
}
}

return {
branchStream,
findOrCreate: v1FindOrCreate,
Expand All @@ -257,9 +253,6 @@ exports.init = function (sbot, config) {
findOrCreate,
findAndTombstone,
findById,
findByIdSync,
loadState,
ensureLoaded,

// TODO add API for whether to encrypt the announce message for a subfeed
// for sympathetic replication?
Expand Down
7 changes: 4 additions & 3 deletions keys.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,17 @@ const keys = {

/**
* Derive a new feed key from a seed. Label must be either `metafeed` for the
* top level meta feed or a base64 encoded nonce. Feedformat can be either
* `bendybutt-v1` for a meta feed or `classic`.
* top level meta feed or a base64 encoded nonce. Feedformat can be
* `bendybutt-v1` for a meta feed or `classic` or `indexed-v1`.
*
* ```js
* const seed = sbot.metafeeds.keys.generateSeed()
* const mfKey = sbot.metafeeds.keys.deriveFeedKeyFromSeed(seed, 'metafeed')
* ```
* @param {Buffer} seed
* @param {string} label
* @param {'bendybutt-v1' | 'gabbygrove-v1' | 'classic'} format default is 'classic'
* @param {'bendybutt-v1' | 'indexed-v1' | 'gabbygrove-v1' | 'classic'} format
* default is 'classic'
*/
deriveFeedKeyFromSeed(seed, label, format = 'classic') {
if (!label) throw new Error('label was not supplied')
Expand Down
169 changes: 72 additions & 97 deletions lookup.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const { seekKey } = require('bipf')
const pull = require('pull-stream')
const cat = require('pull-cat')
const Notify = require('pull-notify')
const SSBURI = require('ssb-uri2')
const Defer = require('pull-defer')
const DeferredPromise = require('p-defer')
const {
where,
Expand All @@ -23,6 +23,7 @@ const validate = require('./validate')
const SUBFEED_PREFIX_OFFSET = Math.max(
'@'.length,
'ssb:feed/bendybutt-v1/'.length,
'ssb:feed/indexed-v1/'.length,
'ssb:feed/gabbygrove-v1/'.length
)

Expand All @@ -48,7 +49,6 @@ function subfeed(feedId) {
}

exports.init = function (sbot, config) {
let stateLoaded = false
const stateLoadedP = DeferredPromise()
let loadStateRequested = false
let liveDrainer = null
Expand Down Expand Up @@ -81,23 +81,10 @@ exports.init = function (sbot, config) {
}
}

function detectFeedFormat(feedId) {
if (feedId.startsWith('@') || SSBURI.isClassicFeedSSBURI(feedId)) {
return 'classic'
} else if (SSBURI.isBendyButtV1FeedSSBURI(feedId)) {
return 'bendybutt-v1'
} else if (SSBURI.isGabbyGroveV1FeedSSBURI(feedId)) {
return 'gabbygrove-v1'
} else {
console.warn('Unknown feed format: ' + feedId)
return null
}
}

function msgToDetails(prevDetails, msg) {
const content = msg.value.content
const details = { ...prevDetails }
details.feedformat = detectFeedFormat(content.subfeed)
details.feedformat = validate.detectFeedFormat(content.subfeed)
details.feedpurpose = content.feedpurpose || details.feedpurpose
details.metafeed = content.metafeed || details.metafeed
details.metadata = {} || details.metafeed
Expand Down Expand Up @@ -164,7 +151,6 @@ exports.init = function (sbot, config) {
pull.drain(updateLookup, (err) => {
if (err) return console.error(err)

stateLoaded = true
stateLoadedP.resolve()

sbot.close.hook(function (fn, args) {
Expand Down Expand Up @@ -244,94 +230,83 @@ exports.init = function (sbot, config) {
}
}

return {
loadState(cb) {
if (!loadStateRequested) loadState()

if (cb) stateLoadedP.promise.then(cb)
},
function findById(feedId, cb) {
try {
assertFeedId(feedId)
if (!validate.detectFeedFormat(feedId)) throw Error('Invalid feedId')
} catch (err) {
return cb(err)
}

ensureLoaded(feedId, cb) {
if (!loadStateRequested) loadState()
sbot.db.query(
where(subfeed(feedId)),
toCallback((err, msgs) => {
if (err) return cb(err)

if (detailsLookup.has(feedId)) cb()
else ensureQueue.add(feedId, cb)
},
msgs = msgs.filter((msg) => validate.isValid(msg))
if (msgs.find((m) => m.value.content.type === 'metafeed/tombstone')) {
return cb(null, null)
}
msgs = msgs.filter((m) =>
m.value.content.type.startsWith('metafeed/add/')
)
if (msgs.length === 0) {
return cb(null, null)
}

findByIdSync(feedId) {
if (!stateLoaded) {
throw new Error('Please call loadState() before using findByIdSync()')
}
assertFeedId(feedId)
const details = msgToDetails(undefined, msgs[0])
cb(null, details)
})
)
}

return detailsLookup.get(feedId)
},
function branchStream(opts) {
if (!loadStateRequested) {
loadState()
const deferred = Defer.source()
stateLoadedP.promise.then(() => {
deferred.resolve(branchStream(opts))
})
return deferred
}

findById(feedId, cb) {
try {
assertFeedId(feedId)
detectFeedFormat(feedId)
} catch (err) {
return cb(err)
const {
live = true,
old = false,
root = null,
tombstoned = null,
} = opts || {}

const filterTombstoneOrNot = (branch) => {
const [, leafDetails] = branch[branch.length - 1]
if (tombstoned === null) {
// Anything goes
return true
} else if (tombstoned === false) {
// All nodes in the branch must be non-tombstoned
return branch.every(([, details]) => !details || !details.tombstoned)
} else if (tombstoned === true) {
// The leaf must be tombstoned for this branch to be interesting to us
return leafDetails && !!leafDetails.tombstoned
}
}

sbot.db.query(
where(subfeed(feedId)),
toCallback((err, msgs) => {
if (err) return cb(err)

msgs = msgs.filter((msg) => validate.isValid(msg))
if (msgs.find((m) => m.value.content.type === 'metafeed/tombstone')) {
return cb(null, null)
}
msgs = msgs.filter((m) =>
m.value.content.type.startsWith('metafeed/add/')
)
if (msgs.length === 0) {
return cb(null, null)
}

const details = msgToDetails(undefined, msgs[0])
cb(null, details)
})
if (old && live) {
return pull(
cat([branchStreamOld(root), branchStreamLive(root)]),
pull.filter(filterTombstoneOrNot)
)
},

branchStream(opts) {
if (!loadStateRequested) loadState()
const {
live = true,
old = false,
root = null,
tombstoned = null,
} = opts || {}

const filterTombstoneOrNot = (branch) => {
const [, leafDetails] = branch[branch.length - 1]
if (tombstoned === null) {
// Anything goes
return true
} else if (tombstoned === false) {
// All nodes in the branch must be non-tombstoned
return branch.every(([, details]) => !details || !details.tombstoned)
} else if (tombstoned === true) {
// The leaf must be tombstoned for this branch to be interesting to us
return leafDetails && !!leafDetails.tombstoned
}
}
} else if (old) {
return pull(branchStreamOld(root), pull.filter(filterTombstoneOrNot))
} else if (live) {
return pull(branchStreamLive(root), pull.filter(filterTombstoneOrNot))
} else {
return pull.empty()
}
}

if (old && live) {
return pull(
cat([branchStreamOld(root), branchStreamLive(root)]),
pull.filter(filterTombstoneOrNot)
)
} else if (old) {
return pull(branchStreamOld(root), pull.filter(filterTombstoneOrNot))
} else if (live) {
return pull(branchStreamLive(root), pull.filter(filterTombstoneOrNot))
} else {
return pull.empty()
}
},
return {
findById,
branchStream,
}
}
9 changes: 6 additions & 3 deletions messages.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,19 @@ exports.init = function init(sbot) {
return crypto.randomBytes(32)
}

const supportedFormats = ['bendybutt-v1', 'classic', 'indexed-v1']

return {
/**
* Generate opts for "ssb.db.create" to create a message to be posted on
* a metafeed linking to a new feed.
* Similar to in `deriveFeedKeyFromSeed`, `feedformat` can be either
* `bendybutt-v1` for a metafeed or `classic`. `metadata` is an optional
* object to be included (object spread) in the message `content`.
* `bendybutt-v1` for a metafeed or then `classic` or `indexed-v1`.
* `metadata` is an optional object to be included (object spread) in the
* message `content`.
*/
optsForAddDerived(mfKeys, feedpurpose, seed, feedFormat, metadata) {
if (feedFormat !== 'classic' && feedFormat !== 'bendybutt-v1') {
if (!supportedFormats.includes(feedFormat)) {
throw new Error('Unknown feed format: ' + feedFormat)
}
const nonce = getNonce()
Expand Down
13 changes: 7 additions & 6 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "ssb-meta-feeds",
"description": "Manage meta feeds and its sub feeds, as a secret-stack plugin",
"version": "0.29.0",
"version": "0.31.0",
"homepage": "https://github.com/ssb-ngi-pointer/ssb-meta-feeds",
"repository": {
"type": "git",
Expand All @@ -21,13 +21,14 @@
"p-defer": "^3.0.0",
"promisify-tuple": "^1.2.0",
"pull-cat": "^1.1.11",
"pull-defer": "^0.2.3",
"pull-notify": "^0.1.1",
"pull-stream": "^3.6.14",
"ssb-bfe": "^3.3.0",
"ssb-db2": ">=3.0.0 <=5",
"ssb-keys": "^8.4.0",
"ssb-bfe": "^3.6.0",
"ssb-db2": ">=3.0.0 <=6",
"ssb-keys": "^8.5.0",
"ssb-ref": "^2.16.0",
"ssb-uri2": "^2.0.2"
"ssb-uri2": "^2.1.0"
},
"devDependencies": {
"c8": "^7.11.0",
Expand All @@ -37,8 +38,8 @@
"rimraf": "^3.0.2",
"secret-stack": "^6.4.0",
"ssb-bendy-butt": "^1.0.1",
"ssb-db2": "^6.1.1",
"ssb-caps": "^1.1.0",
"ssb-db2": "^5.1.0",
"tap-arc": "^0.3.5",
"tape": "^5.6.0"
},
Expand Down
Loading

0 comments on commit 3d3e54f

Please sign in to comment.