From bae31966b477bee9b7947bff929f7579873c60b3 Mon Sep 17 00:00:00 2001 From: Sivan Ratson <89018301+Sivan22@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:52:00 +0200 Subject: [PATCH 1/3] fix: isolate in findBooks --- lib/models/app_model.dart | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/lib/models/app_model.dart b/lib/models/app_model.dart index 8c1617bd..97d55aa0 100644 --- a/lib/models/app_model.dart +++ b/lib/models/app_model.dart @@ -446,6 +446,8 @@ class AppModel with ChangeNotifier { if (showHebrewBooks.value) { books += await hebrewBooks; } + + // First filter the books var filteredBooks = books.where((book) { final title = book.title.toLowerCase(); return queryWords.every((word) => title.contains(word)); @@ -458,15 +460,28 @@ class AppModel with ChangeNotifier { .toList(); } - return Isolate.run(() { - filteredBooks.sort((a, b) { - final scoreA = ratio(query, a.title); - final scoreB = ratio(query, b.title); + // Create a simple data structure that can be sent to isolate + final searchData = + filteredBooks.asMap().map((index, book) => MapEntry(index, { + 'index': index, + 'title': book.title, + })); + + // Sort using isolate with the simplified data + final sortedIndices = await Isolate.run(() { + final entries = searchData.entries.toList(); + entries.sort((a, b) { + final scoreA = ratio(query, a.value['title'] as String); + final scoreB = ratio(query, b.value['title'] as String); return scoreB.compareTo(scoreA); }); - - return filteredBooks; + return entries.map((e) => e.value['index'] as int).toList(); }); + + // Reorder the original filtered books based on the sorted indices + final sortedBooks = + sortedIndices.map((index) => filteredBooks[index]).toList(); + return sortedBooks; } Future createRefsFromLibrary(int startIndex) async { From e55821c0494329ca2a4d87e5a70894627477b708 Mon Sep 17 00:00:00 2001 From: Sivan Ratson <89018301+Sivan22@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:23:27 +0200 Subject: [PATCH 2/3] fix: isolate for findbook --- lib/data/repository/data_repository.dart | 3 +- lib/models/app_model.dart | 76 +++++++++++-------- .../full_text_settings_screen.dart | 2 +- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/lib/data/repository/data_repository.dart b/lib/data/repository/data_repository.dart index f9ce8592..03b3dee6 100644 --- a/lib/data/repository/data_repository.dart +++ b/lib/data/repository/data_repository.dart @@ -50,7 +50,8 @@ class DataRepository { return _isarDataProvider.getNumberOfBooksWithRefs(); } - addAllTextsToMimir(Library library, {int start = 0, int end = 100000}) async { + addAllTextsToTantivy(Library library, + {int start = 0, int end = 100000}) async { _mimirDataProvider.addAllTBooksToTantivy(library, start: start, end: end); } } diff --git a/lib/models/app_model.dart b/lib/models/app_model.dart index 97d55aa0..61f183db 100644 --- a/lib/models/app_model.dart +++ b/lib/models/app_model.dart @@ -439,57 +439,54 @@ class AppModel with ChangeNotifier { Future> findBooks(String query, Category? category, {List? topics}) async { final queryWords = query.split(RegExp(r'\s+')); - var books = category?.getAllBooks() ?? (await library).getAllBooks(); + var allBooks = category?.getAllBooks() ?? (await library).getAllBooks(); if (showOtzarHachochma.value) { - books += await otzarBooks; + allBooks += await otzarBooks; } if (showHebrewBooks.value) { - books += await hebrewBooks; + allBooks += await hebrewBooks; } - // First filter the books - var filteredBooks = books.where((book) { + // First, filter books outside of isolate to get the working set + var filteredBooks = allBooks.where((book) { final title = book.title.toLowerCase(); - return queryWords.every((word) => title.contains(word)); + final bookTopics = book.topics.split(', '); + + bool matchesQuery = queryWords.every((word) => title.contains(word)); + bool matchesTopics = topics == null || + topics.isEmpty || + topics.every((t) => bookTopics.contains(t)); + + return matchesQuery && matchesTopics; }).toList(); - if (topics != null && topics.isNotEmpty) { - filteredBooks = filteredBooks - .where((book) => - topics.every((t) => book.topics.split(', ').contains(t))) - .toList(); + if (filteredBooks.isEmpty) { + return []; } - // Create a simple data structure that can be sent to isolate - final searchData = - filteredBooks.asMap().map((index, book) => MapEntry(index, { - 'index': index, + // Prepare data for isolate - only send what's needed for sorting + final List> sortData = filteredBooks + .asMap() + .map((i, book) => MapEntry(i, { + 'index': i, 'title': book.title, - })); - - // Sort using isolate with the simplified data - final sortedIndices = await Isolate.run(() { - final entries = searchData.entries.toList(); - entries.sort((a, b) { - final scoreA = ratio(query, a.value['title'] as String); - final scoreB = ratio(query, b.value['title'] as String); - return scoreB.compareTo(scoreA); - }); - return entries.map((e) => e.value['index'] as int).toList(); - }); + })) + .values + .toList(); - // Reorder the original filtered books based on the sorted indices - final sortedBooks = - sortedIndices.map((index) => filteredBooks[index]).toList(); - return sortedBooks; + // Sort indices in isolate + final sortedIndices = getSortedIndices(sortData, query); + + // Map sorted indices back to books + return (await sortedIndices).map((index) => filteredBooks[index]).toList(); } Future createRefsFromLibrary(int startIndex) async { data.createRefsFromLibrary(await library, startIndex); } - addAllTextsToMimir({int start = 0, int end = 100000}) async { - data.addAllTextsToMimir(await library, start: start, end: end); + addAllTextsToTantivy({int start = 0, int end = 100000}) async { + data.addAllTextsToTantivy(await library, start: start, end: end); } Future refreshLibrary() async { @@ -501,3 +498,16 @@ class AppModel with ChangeNotifier { /// An enum that represents the different screens in the application. enum Screens { library, find, reading, search, favorites, settings } + +Future> getSortedIndices( + List> data, String query) async { + return await Isolate.run(() { + List indices = List.generate(data.length, (i) => i); + indices.sort((a, b) { + final scoreA = ratio(query, data[a]['title'] as String); + final scoreB = ratio(query, data[b]['title'] as String); + return scoreB.compareTo(scoreA); + }); + return indices; + }); +} diff --git a/lib/screens/full_text_search/full_text_settings_screen.dart b/lib/screens/full_text_search/full_text_settings_screen.dart index 856e1d99..11543d07 100644 --- a/lib/screens/full_text_search/full_text_settings_screen.dart +++ b/lib/screens/full_text_search/full_text_settings_screen.dart @@ -86,7 +86,7 @@ class FullTextSettingsScreen extends StatelessWidget { ], )); if (await result == true) { - context.read().addAllTextsToMimir(); + context.read().addAllTextsToTantivy(); } }, child: const Text( From e5f3514fd4e536e6d8fd29250f7efd4247cf9f6e Mon Sep 17 00:00:00 2001 From: Sivan Ratson <89018301+Sivan22@users.noreply.github.com> Date: Thu, 21 Nov 2024 19:44:03 +0200 Subject: [PATCH 3/3] fix: isolate in getLineFromBook --- .../file_system_data_provider.dart | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/data/data_providers/file_system_data_provider.dart b/lib/data/data_providers/file_system_data_provider.dart index 82999882..d63d8384 100644 --- a/lib/data/data_providers/file_system_data_provider.dart +++ b/lib/data/data_providers/file_system_data_provider.dart @@ -273,7 +273,7 @@ class FileSystemData { /// Reads the file line by line and returns the content at the specified index. Future getLinkContent(Link link) async { String path = await _getBookPath(getTitleFromPath(link.path2)); - return Isolate.run(() async => await getLineFromFile(path, link.index2)); + return await getLineFromFile(path, link.index2); } /// Returns a list of all book paths in the library directory. @@ -304,14 +304,16 @@ class FileSystemData { /// Uses a stream to read the file line by line until the desired index /// is reached, then closes the stream to conserve resources. Future getLineFromFile(String path, int index) async { - File file = File(path); - final lines = file - .openRead() - .transform(utf8.decoder) - .transform(const LineSplitter()) - .take(index) - .toList(); - return (await lines).last; + return await Isolate.run(() async { + File file = File(path); + final lines = file + .openRead() + .transform(utf8.decoder) + .transform(const LineSplitter()) + .take(index) + .toList(); + return (await lines).last; + }); } /// Updates the mapping of book titles to their file system paths.