From 7a85ae29ca27e763b18b1b8653c4a70cf91cb913 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Thu, 21 Sep 2023 13:19:22 -0400 Subject: [PATCH] search: switch to nostrdb profile searching Changelog-Changed: Switch to nostrdb for @'s and user search --- damus/ContentView.swift | 4 +- damus/Models/DamusState.swift | 5 +- damus/Models/HomeModel.swift | 2 - damus/Nostr/Profiles.swift | 9 +- damus/TestData.swift | 4 +- damus/Views/Posting/UserSearch.swift | 3 +- damus/Views/Profile/ProfilePicView.swift | 3 +- damus/Views/SearchResultsView.swift | 16 +- nostrdb/Ndb.swift | 26 +++ nostrdb/nostrdb.c | 270 ++++++++++++++++++++++- nostrdb/nostrdb.h | 16 ++ 11 files changed, 325 insertions(+), 33 deletions(-) diff --git a/damus/ContentView.swift b/damus/ContentView.swift index ee1edb147..b360d0b93 100644 --- a/damus/ContentView.swift +++ b/damus/ContentView.swift @@ -629,7 +629,7 @@ struct ContentView: View { likes: EventCounter(our_pubkey: pubkey), boosts: EventCounter(our_pubkey: pubkey), contacts: Contacts(our_pubkey: pubkey), - profiles: Profiles(user_search_cache: user_search_cache, ndb: ndb), + profiles: Profiles(ndb: ndb), dms: home.dms, previews: PreviewCache(), zaps: Zaps(our_pubkey: pubkey), @@ -646,7 +646,6 @@ struct ContentView: View { muted_threads: MutedThreadsManager(keypair: keypair), wallet: WalletModel(settings: settings), nav: self.navigationCoordinator, - user_search_cache: user_search_cache, music: MusicController(onChange: music_changed), video: VideoController(), ndb: ndb @@ -919,7 +918,6 @@ func handle_unfollow(state: DamusState, unfollow: FollowRef) -> Bool { switch unfollow { case .pubkey(let pk): state.contacts.remove_friend(pk) - state.user_search_cache.updateOwnContactsPetnames(id: state.pubkey, oldEvent: old_contacts, newEvent: ev) case .hashtag: // nothing to handle here really break diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift index d8f5418b5..108ee3f5e 100644 --- a/damus/Models/DamusState.swift +++ b/damus/Models/DamusState.swift @@ -31,7 +31,6 @@ struct DamusState { let muted_threads: MutedThreadsManager let wallet: WalletModel let nav: NavigationCoordinator - let user_search_cache: UserSearchCache let music: MusicController? let video: VideoController let ndb: Ndb @@ -62,7 +61,6 @@ struct DamusState { } static var empty: DamusState { - let user_search_cache = UserSearchCache() let empty_pub: Pubkey = .empty let empty_sec: Privkey = .empty let kp = Keypair(pubkey: empty_pub, privkey: nil) @@ -73,7 +71,7 @@ struct DamusState { likes: EventCounter(our_pubkey: empty_pub), boosts: EventCounter(our_pubkey: empty_pub), contacts: Contacts(our_pubkey: empty_pub), - profiles: Profiles(user_search_cache: user_search_cache, ndb: .empty), + profiles: Profiles(ndb: .empty), dms: DirectMessagesModel(our_pubkey: empty_pub), previews: PreviewCache(), zaps: Zaps(our_pubkey: empty_pub), @@ -90,7 +88,6 @@ struct DamusState { muted_threads: MutedThreadsManager(keypair: kp), wallet: WalletModel(settings: UserSettingsStore()), nav: NavigationCoordinator(), - user_search_cache: user_search_cache, music: nil, video: VideoController(), ndb: .empty diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift index d2c285969..73afc11a5 100644 --- a/damus/Models/HomeModel.swift +++ b/damus/Models/HomeModel.swift @@ -734,8 +734,6 @@ func load_our_contacts(state: DamusState, m_old_ev: NostrEvent?, ev: NostrEvent) } } } - - state.user_search_cache.updateOwnContactsPetnames(id: contacts.our_pubkey, oldEvent: m_old_ev, newEvent: ev) } diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift index adb0a61e3..10dec04cb 100644 --- a/damus/Nostr/Profiles.swift +++ b/damus/Nostr/Profiles.swift @@ -38,10 +38,7 @@ class Profiles { @MainActor var nip05_pubkey: [String: Pubkey] = [:] - let user_search_cache: UserSearchCache - - init(user_search_cache: UserSearchCache, ndb: Ndb) { - self.user_search_cache = user_search_cache + init(ndb: Ndb) { self.ndb = ndb } @@ -84,6 +81,10 @@ class Profiles { return ndb.lookup_profile_by_key(key: key) } + func search(_ query: String, limit: Int, txn: NdbTxn) -> [Pubkey] { + return ndb.search_profile(query, limit: limit, txn: txn) + } + func lookup(id: Pubkey) -> NdbTxn { return ndb.lookup_profile(id).map({ pr in pr?.profile }) } diff --git a/damus/TestData.swift b/damus/TestData.swift index b152e8d4d..729b40c82 100644 --- a/damus/TestData.swift +++ b/damus/TestData.swift @@ -66,7 +66,6 @@ var test_damus_state: DamusState = ({ print("opening \(tempDir!)") let ndb = Ndb(path: tempDir)! let our_pubkey = test_pubkey - let user_search_cache = UserSearchCache() let pool = RelayPool(ndb: ndb) let settings = UserSettingsStore() let damus = DamusState(pool: pool, @@ -74,7 +73,7 @@ var test_damus_state: DamusState = ({ likes: .init(our_pubkey: our_pubkey), boosts: .init(our_pubkey: our_pubkey), contacts: .init(our_pubkey: our_pubkey), - profiles: .init(user_search_cache: user_search_cache, ndb: ndb), + profiles: .init(ndb: ndb), dms: .init(our_pubkey: our_pubkey), previews: .init(), zaps: .init(our_pubkey: our_pubkey), @@ -91,7 +90,6 @@ var test_damus_state: DamusState = ({ muted_threads: .init(keypair: test_keypair), wallet: .init(settings: settings), nav: .init(), - user_search_cache: user_search_cache, music: .init(onChange: {_ in }), video: .init(), ndb: ndb) diff --git a/damus/Views/Posting/UserSearch.swift b/damus/Views/Posting/UserSearch.swift index de05a897c..a593902ec 100644 --- a/damus/Views/Posting/UserSearch.swift +++ b/damus/Views/Posting/UserSearch.swift @@ -17,7 +17,8 @@ struct UserSearch: View { @EnvironmentObject var tagModel: TagModel var users: [Pubkey] { - return search_profiles(profiles: damus_state.profiles, search: search) + let txn = NdbTxn(ndb: damus_state.ndb) + return search_profiles(profiles: damus_state.profiles, search: search, txn: txn) } func on_user_tapped(pk: Pubkey) { diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift index 3d806d783..54afbbc53 100644 --- a/damus/Views/Profile/ProfilePicView.swift +++ b/damus/Views/Profile/ProfilePicView.swift @@ -113,8 +113,7 @@ func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> UR } func make_preview_profiles(_ pubkey: Pubkey) -> Profiles { - let user_search_cache = UserSearchCache() - let profiles = Profiles(user_search_cache: user_search_cache, ndb: test_damus_state.ndb) + let profiles = Profiles(ndb: test_damus_state.ndb) let picture = "http://cdn.jb55.com/img/red-me.jpg" let profile = Profile(name: "jb55", display_name: "William Casarin", about: "It's me", picture: picture, banner: "", website: "https://jb55.com", lud06: nil, lud16: nil, nip05: "jb55.com", damus_donation: nil) //let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) diff --git a/damus/Views/SearchResultsView.swift b/damus/Views/SearchResultsView.swift index 85db46a2b..b2748880c 100644 --- a/damus/Views/SearchResultsView.swift +++ b/damus/Views/SearchResultsView.swift @@ -108,10 +108,12 @@ struct SearchResultsView: View { } .frame(maxHeight: .infinity) .onAppear { - self.result = search_for_string(profiles: damus_state.profiles, search) + let txn = NdbTxn.init(ndb: damus_state.ndb) + self.result = search_for_string(profiles: damus_state.profiles, search: search, txn: txn) } .onChange(of: search) { new in - self.result = search_for_string(profiles: damus_state.profiles, new) + let txn = NdbTxn.init(ndb: damus_state.ndb) + self.result = search_for_string(profiles: damus_state.profiles, search: search, txn: txn) } } } @@ -125,7 +127,7 @@ struct SearchResultsView_Previews: PreviewProvider { */ -func search_for_string(profiles: Profiles, _ new: String) -> Search? { +func search_for_string(profiles: Profiles, search new: String, txn: NdbTxn) -> Search? { guard new.count != 0 else { return nil } @@ -154,7 +156,7 @@ func search_for_string(profiles: Profiles, _ new: String) -> Search? { return .note(NoteId(decoded.data)) } - let multisearch = MultiSearch(hashtag: make_hashtagable(new), profiles: search_profiles(profiles: profiles, search: new)) + let multisearch = MultiSearch(hashtag: make_hashtagable(new), profiles: search_profiles(profiles: profiles, search: new, txn: txn)) return .multi(multisearch) } @@ -171,7 +173,7 @@ func make_hashtagable(_ str: String) -> String { return String(new.filter{$0 != " "}) } -func search_profiles(profiles: Profiles, search: String) -> [Pubkey] { +func search_profiles(profiles: Profiles, search: String, txn: NdbTxn) -> [Pubkey] { // Search by hex pubkey. if let pubkey = hex_decode_pubkey(search), profiles.lookup_key_by_pubkey(pubkey) != nil @@ -189,5 +191,7 @@ func search_profiles(profiles: Profiles, search: String) -> [Pubkey] { } let new = search.lowercased() - return profiles.user_search_cache.search(key: new) + + return profiles.search(search, limit: 10, txn: txn) } + diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift index e64ad62ba..395ae3323 100644 --- a/nostrdb/Ndb.swift +++ b/nostrdb/Ndb.swift @@ -188,6 +188,32 @@ class Ndb { } } + func search_profile(_ search: String, limit: Int, txn: NdbTxn) -> [Pubkey] { + var pks = Array() + + return search.withCString { q in + var s = ndb_search() + guard ndb_search_profile(&txn.txn, &s, q) != 0 else { + return pks + } + + defer { ndb_search_profile_end(&s) } + pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32))) + + var n = limit + while n > 0 { + guard ndb_search_profile_next(&s) != 0 else { + return pks + } + pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32))) + + n -= 1 + } + + return pks + } + } + deinit { ndb_destroy(ndb.ndb) } diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c index b3f845713..8f7bbc84f 100644 --- a/nostrdb/nostrdb.c +++ b/nostrdb/nostrdb.c @@ -69,6 +69,7 @@ enum ndb_dbs { NDB_DB_NOTE_ID, NDB_DB_PROFILE_PK, NDB_DB_NDB_META, + NDB_DB_PROFILE_SEARCH, NDB_DBS, }; @@ -123,16 +124,149 @@ struct ndb_tsid { uint64_t timestamp; }; +static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id, + uint64_t timestamp, const char *search) +{ + memcpy(key->id, id, 32); + key->timestamp = timestamp; + strncpy(key->search, search, sizeof(key->search) - 1); + key->search[sizeof(key->search) - 1] = '\0'; +} + +static int ndb_write_profile_search_index(struct ndb_lmdb *lmdb, + MDB_txn *txn, + struct ndb_search_key *index_key, + uint64_t profile_key) +{ + int rc; + MDB_val key, val; + + key.mv_data = index_key; + key.mv_size = sizeof(*index_key); + val.mv_data = &profile_key; + val.mv_size = sizeof(profile_key); + + if ((rc = mdb_put(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], &key, &val, 0))) { + ndb_debug("ndb_write_profile_search_index failed: %s\n", + mdb_strerror(rc)); + return 0; + } + + return 1; +} + + +// map usernames and display names to profile keys for user searching +static int ndb_write_profile_search_indices(struct ndb_lmdb *lmdb, + MDB_txn *txn, + struct ndb_note *note, + uint64_t profile_key, + void *profile_root) +{ + struct ndb_search_key index; + NdbProfileRecord_table_t profile_record; + NdbProfile_table_t profile; + + profile_record = NdbProfileRecord_as_root(profile_root); + profile = NdbProfileRecord_profile_get(profile_record); + + const char *name = NdbProfile_name_get(profile); + const char *display_name = NdbProfile_display_name_get(profile); + + // words + pubkey + created + if (name) { + ndb_make_search_key(&index, note->pubkey, note->created_at, + name); + if (!ndb_write_profile_search_index(lmdb, txn, &index, + profile_key)) + return 0; + } + + if (display_name) { + // don't write the same name/display_name twice + if (name && !strcmp(display_name, name)) { + return 1; + } + ndb_make_search_key(&index, note->pubkey, note->created_at, + display_name); + if (!ndb_write_profile_search_index(lmdb, txn, &index, + profile_key)) + return 0; + } + + return 1; +} + +int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn) +{ + txn->ndb = ndb; + MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn; + return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0; +} + + // Migrations // static int ndb_migrate_user_search_indices(struct ndb *ndb) { + int rc; + MDB_cursor *cur; + MDB_val k, v; + void *profile_root; + NdbProfileRecord_table_t record; + struct ndb_txn txn; + struct ndb_note *note; + uint64_t note_key, profile_key; + size_t len; + int count; + + if (!ndb_begin_query(ndb, &txn)) { + fprintf(stderr, "ndb_migrate_user_search_indices: ndb_begin_query failed\n"); + return 0; + } + + if ((rc = mdb_cursor_open(txn.mdb_txn, ndb->lmdb.dbs[NDB_DB_PROFILE], &cur))) { + fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc); + return 0; + } + + count = 0; + + // loop through all profiles and write search indices + while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { + profile_root = v.mv_data; + profile_key = *((uint64_t*)k.mv_data); + record = NdbProfileRecord_as_root(profile_root); + note_key = NdbProfileRecord_note_key(record); + note = ndb_get_note_by_key(&txn, note_key, &len); + + if (note == NULL) { + fprintf(stderr, "ndb_migrate_user_search_indices: note lookup failed\n"); + return 0; + } + + if (!ndb_write_profile_search_indices(&ndb->lmdb, txn.mdb_txn, + note, profile_key, + profile_root)) { + + fprintf(stderr, "ndb_migrate_user_search_indices: ndb_write_profile_search_indices failed\n"); + return 0; + } + + count++; + } + + fprintf(stderr, "migrated %d profiles to include search indices\n", count); + + mdb_cursor_close(cur); + mdb_txn_commit(txn.mdb_txn); + return 1; } static struct ndb_migration MIGRATIONS[] = { - //{ .fn = ndb_migrate_user_search_indices } + { .fn = ndb_migrate_user_search_indices } }; @@ -187,6 +321,7 @@ static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b) { struct ndb_tsid *tsa, *tsb; MDB_val a2 = *a, b2 = *b; + a2.mv_size = sizeof(tsa->id); b2.mv_size = sizeof(tsb->id); @@ -271,13 +406,6 @@ struct ndb_writer_msg { }; }; -int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn) -{ - txn->ndb = ndb; - MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn; - return mdb_txn_begin(ndb->lmdb.env, NULL, 0, mdb_txn) == 0; -} - void ndb_end_query(struct ndb_txn *txn) { mdb_txn_abort(txn->mdb_txn); @@ -674,6 +802,113 @@ static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db) return *((uint64_t*)key.mv_data); } +// make a search key meant for user queries without any other note info +static void ndb_make_search_key_low(struct ndb_search_key *key, const char *search) +{ + memset(key->id, 0, sizeof(key->id)); + key->timestamp = 0; + strncpy(key->search, search, sizeof(key->search) - 1); + key->search[sizeof(key->search) - 1] = '\0'; +} + +int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query) +{ + int rc; + struct ndb_search_key s; + MDB_val k, v; + search->cursor = NULL; + + MDB_cursor **cursor = (MDB_cursor **)&search->cursor; + + ndb_make_search_key_low(&s, query); + + k.mv_data = &s; + k.mv_size = sizeof(s); + + if ((rc = mdb_cursor_open(txn->mdb_txn, + txn->ndb->lmdb.dbs[NDB_DB_PROFILE_SEARCH], + cursor))) { + printf("search_profile: cursor opened failed: %s\n", + mdb_strerror(rc)); + return 0; + } + + // Position cursor at the next key greater than or equal to the specified key + if (mdb_cursor_get(search->cursor, &k, &v, MDB_SET_RANGE)) { + printf("search_profile: cursor get failed\n"); + goto cleanup; + } else { + search->key = k.mv_data; + assert(v.mv_size == 8); + search->profile_key = *((uint64_t*)v.mv_data); + return 1; + } + +cleanup: + mdb_cursor_close(search->cursor); + search->cursor = NULL; + return 0; +} + +void ndb_search_profile_end(struct ndb_search *search) +{ + if (search->cursor) + mdb_cursor_close(search->cursor); +} + +int ndb_search_profile_next(struct ndb_search *search) +{ + int rc; + MDB_val k, v; + unsigned char *init_id; + + init_id = search->key->id; + k.mv_data = search->key; + k.mv_size = sizeof(*search->key); + +retry: + if ((rc = mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT))) { + ndb_debug("ndb_search_profile_next: %s\n", + mdb_strerror(rc)); + return 0; + } else { + search->key = k.mv_data; + assert(v.mv_size == 8); + search->profile_key = *((uint64_t*)v.mv_data); + + // skip duplicate pubkeys + if (!memcmp(init_id, search->key->id, 32)) + goto retry; + } + + return 1; +} + +static int ndb_search_key_cmp(const MDB_val *a, const MDB_val *b) +{ + int cmp; + struct ndb_search_key *ska, *skb; + + ska = a->mv_data; + skb = b->mv_data; + + MDB_val a2 = *a; + MDB_val b2 = *b; + + a2.mv_data = ska->search; + a2.mv_size = sizeof(ska->search) + sizeof(ska->id); + + cmp = mdb_cmp_memn(&a2, &b2); + if (cmp) return cmp; + + if (ska->timestamp < skb->timestamp) + return -1; + else if (ska->timestamp > skb->timestamp) + return 1; + + return 0; +} + static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, struct ndb_writer_profile *profile, uint64_t note_key) @@ -737,6 +972,13 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn, return 0; } + // write name, display_name profile search indices + if (!ndb_write_profile_search_indices(lmdb, txn, note, profile_key, + flatbuf)) { + ndb_debug("failed to write profile search indices\n"); + return 0; + } + return 1; } @@ -803,6 +1045,8 @@ static void ndb_write_version(struct ndb_lmdb *lmdb, MDB_txn *txn, uint64_t vers mdb_strerror(rc)); return; } + + fprintf(stderr, "writing version %" PRIu64 "\n", version); } static void *ndb_writer_thread(void *data) @@ -1090,6 +1334,13 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map return 0; } + // profile search db + if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) { + fprintf(stderr, "mdb_dbi_open profile_search failed, error %d\n", rc); + return 0; + } + mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], ndb_search_key_cmp); + // ndb metadata (db version, etc) if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) { fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc); @@ -1137,6 +1388,7 @@ static int ndb_run_migrations(struct ndb *ndb) latest_version = sizeof(MIGRATIONS) / sizeof(MIGRATIONS[0]); if ((version = ndb_db_version(ndb)) == -1) { + fprintf(stderr, "run_migrations: no version found, assuming new db\n"); version = latest_version; // no version found. fresh db? @@ -1146,6 +1398,8 @@ static int ndb_run_migrations(struct ndb *ndb) } return 1; + } else { + fprintf(stderr, "ndb: version %" PRIu64 " found\n", version); } if (version < latest_version) diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h index 1ae76b6c7..646df2136 100644 --- a/nostrdb/nostrdb.h +++ b/nostrdb/nostrdb.h @@ -25,6 +25,19 @@ struct ndb_t { struct ndb *ndb; }; +struct ndb_search_key +{ + char search[24]; + unsigned char id[32]; + uint64_t timestamp; +}; + +struct ndb_search { + struct ndb_search_key *key; + uint64_t profile_key; + void *cursor; // MDB_cursor * +}; + // required to keep a read struct ndb_txn { struct ndb *ndb; @@ -165,6 +178,9 @@ int ndb_db_version(struct ndb *ndb); int ndb_process_event(struct ndb *, const char *json, int len); int ndb_process_events(struct ndb *, const char *ldjson, size_t len); int ndb_begin_query(struct ndb *, struct ndb_txn *); +int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query); +int ndb_search_profile_next(struct ndb_search *search); +void ndb_search_profile_end(struct ndb_search *search); void ndb_end_query(struct ndb_txn *); void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);