Performance improvements for large facilities :v3

-> Bug fixes when syncing
d-tree-org · Nov 26, 2024 · c15fa5f · c15fa5f
1 parent 4e2f7cd
commit c15fa5f
Show file tree

Hide file tree

Showing 9 changed files with 175 additions and 88 deletions.
diff --git a/...in/java/org/smartregister/fhircore/engine/data/local/syncStrategy/SyncStrategyCacheDao.kt b/...in/java/org/smartregister/fhircore/engine/data/local/syncStrategy/SyncStrategyCacheDao.kt
@@ -24,11 +24,40 @@ import androidx.room.Query
 @Dao
 abstract class SyncStrategyCacheDao {
 
+  suspend fun upsert(logicalId: String) =
+    with(get(logicalId)) {
+      if (this == null) {
+        insert(logicalId.toEntity())
+      } else {
+        update(this.logicalId)
+      }
+    }
+
+  suspend fun upsert(logicalIds: List<String>) =
+    logicalIds.onEach { logicalId ->
+      with(get(logicalId)) {
+        if (this == null) {
+          insert(logicalId.toEntity())
+        } else {
+          update(this.logicalId)
+        }
+      }
+    }
+
+  @Insert(onConflict = OnConflictStrategy.REPLACE)
+  abstract suspend fun insert(syncStrategyCacheEntity: List<SyncStrategyCacheEntity>)
+
   @Insert(onConflict = OnConflictStrategy.REPLACE)
-  abstract suspend fun upsert(syncStrategyCacheEntity: List<SyncStrategyCacheEntity>)
+  abstract suspend fun insert(syncStrategyCacheEntity: SyncStrategyCacheEntity)
 
   @Query("DELETE FROM syncstrategycacheentity") abstract suspend fun deleteAll()
 
-  @Query("SELECT * FROM syncstrategycacheentity")
+  @Query("SELECT * FROM syncstrategycacheentity WHERE shouldSync = 0")
   abstract suspend fun query(): List<SyncStrategyCacheEntity>
+
+  @Query("SELECT * FROM syncstrategycacheentity WHERE logicalId = :logicalId")
+  abstract suspend fun get(logicalId: String): SyncStrategyCacheEntity?
+
+  @Query("UPDATE syncstrategycacheentity SET shouldSync = 1 WHERE logicalId = :logicalId")
+  abstract suspend fun update(logicalId: String)
 }
diff --git a/...java/org/smartregister/fhircore/engine/data/local/syncStrategy/SyncStrategyCacheEntity.kt b/...java/org/smartregister/fhircore/engine/data/local/syncStrategy/SyncStrategyCacheEntity.kt
@@ -27,3 +27,5 @@ data class SyncStrategyCacheEntity(
 )
 
 fun List<String>.toEntity() = map { SyncStrategyCacheEntity(logicalId = it) }
+
+fun String.toEntity() = SyncStrategyCacheEntity(logicalId = this)
diff --git a/...rc/main/java/org/smartregister/fhircore/engine/data/remote/resource/syncStrategy/Utils.kt b/...rc/main/java/org/smartregister/fhircore/engine/data/remote/resource/syncStrategy/Utils.kt
@@ -16,45 +16,29 @@
 
 package org.smartregister.fhircore.engine.data.remote.resource.syncStrategy
 
+import android.content.Intent
+import androidx.localbroadcastmanager.content.LocalBroadcastManager
 import com.google.android.fhir.FhirEngine
 import com.google.android.fhir.search.search
 import java.util.concurrent.TimeUnit
-import kotlinx.coroutines.runBlocking
 import org.hl7.fhir.r4.model.ListResource
-import org.hl7.fhir.r4.model.Patient
 import org.smartregister.fhircore.engine.configuration.ConfigurationRegistry
 import org.smartregister.fhircore.engine.configuration.Item
 import org.smartregister.fhircore.engine.data.local.syncStrategy.SyncStrategyCacheDao
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.broadcast.SYNC_STATUS_BROADCAST_RECEIVER_KEY
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.broadcast.SyncStatusBroadcastReceiver
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.fhir.ParamSyncStatus
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.utils.SyncState
 import org.smartregister.fhircore.engine.util.SharedPreferenceKey
 import org.smartregister.fhircore.engine.util.SharedPreferenceKey.SYNC_STATUS
 import org.smartregister.fhircore.engine.util.SharedPreferencesHelper
 
-fun logicalIds(fhirEngine: FhirEngine) = runBlocking {
-  fhirEngine
-    .search<Patient> { filter(Patient.ACTIVE, { value = of(true) }) }
-    .map { it.resource.idPart }
-}
-
-fun List<String>.subListIds(syncStrategyCacheDao: SyncStrategyCacheDao): List<String> {
-  return runBlocking {
-    val cachedIds = syncStrategyCacheDao.query().map { it.logicalId }
-    ((cachedIds union this@subListIds) - (cachedIds intersect this@subListIds.toSet())).toList()
-  }
-}
+suspend fun logicalIds(syncStrategyCacheDao: SyncStrategyCacheDao) =
+  syncStrategyCacheDao.query().map { it.logicalId }
 
 fun hasCompletedInitialSync(sharedPreferencesHelper: SharedPreferencesHelper) =
   getSyncState(sharedPreferencesHelper) < SyncState.CompletedInitialSync.value
 
-fun setSubSequentSync(sharedPreferencesHelper: SharedPreferencesHelper) =
-  sharedPreferencesHelper.write(SYNC_STATUS.name, SyncState.SubSequentSync.value)
-
-fun isSubSequentSync(sharedPreferencesHelper: SharedPreferencesHelper) =
-  getSyncState(sharedPreferencesHelper) == SyncState.SubSequentSync.value
-
-fun isRunSyncNow(sharedPreferencesHelper: SharedPreferencesHelper) =
-  getSyncState(sharedPreferencesHelper) == SyncState.RunSyncNow.value
-
 fun getSyncState(sharedPreferencesHelper: SharedPreferencesHelper) =
   sharedPreferencesHelper.read(SYNC_STATUS.name, SyncState.InitialSync.value)
 
@@ -84,10 +68,30 @@ data class IdTimestamp(
   val timestamp: Long,
 )
 
-fun perOrgSyncConfig(
+fun onPerOrgSyncConfigItem(
   configurationRegistry: ConfigurationRegistry,
   sharedPreferencesHelper: SharedPreferencesHelper,
 ): Item? =
   configurationRegistry.getPerOrgSyncConfigs()?.items?.find {
     it.id == sharedPreferencesHelper.organisationCode()
   }
+
+fun syncConfigOfflineFirst(
+  configurationRegistry: ConfigurationRegistry,
+  sharedPreferencesHelper: SharedPreferencesHelper,
+): Boolean {
+  val configs =
+    onPerOrgSyncConfigItem(configurationRegistry, sharedPreferencesHelper) ?: return false
+  return !configs.offlineFirst
+}
+
+fun onSendBroadcast(
+  broadcaster: LocalBroadcastManager,
+  paramSyncStatus: ParamSyncStatus,
+) {
+  val broadcastIntent =
+    Intent(SyncStatusBroadcastReceiver::class.java.name).apply {
+      putExtra(SYNC_STATUS_BROADCAST_RECEIVER_KEY, paramSyncStatus)
+    }
+  broadcaster.sendBroadcast(broadcastIntent)
+}
diff --git a/...rtregister/fhircore/engine/data/remote/resource/syncStrategy/fhir/IdentifierSyncParams.kt b/...rtregister/fhircore/engine/data/remote/resource/syncStrategy/fhir/IdentifierSyncParams.kt
@@ -31,7 +31,7 @@ class IdentifierSyncParams(
 ) : DownloadWorkManager {
 
   private val urlOfTheNextPagesToDownloadForAResource = LinkedList<String>()
-  private val resourcesToDownloadWithSearchParams = LinkedList(identifiers.chunked(12))
+  private val resourcesToDownloadWithSearchParams = LinkedList(identifiers.chunked(32))
   private var patientPosition = 0
 
   override suspend fun getNextRequest(): DownloadRequest? {
@@ -62,11 +62,21 @@ class IdentifierSyncParams(
       .mapNotNull { it.resource as Bundle }
       .map { it.entry.map { it.resource } }
       .flatten()
-      .also { catchIds() }
+      .also(::catchIds)
   }
 
-  private fun catchIds() =
-    callback(ParamSyncStatus(identifiers.map { it.toString() }, identifiers.size, patientPosition))
+  private fun catchIds(resources: List<Resource>) =
+    resources
+      .filter { it.resourceType == ResourceType.Patient }
+      .also { patients ->
+        callback(
+          ParamSyncStatus(
+            logicalId = patients.map { it.idPart },
+            idsTotal = identifiers.size,
+            patientPositionAt = patientPosition,
+          ),
+        )
+      }
 
   private fun List<Int>.bundleOf(): Bundle {
     return Bundle().apply {

diff --git a/...gister/fhircore/engine/data/remote/resource/syncStrategy/fhir/LogicalIdSyncParamsBased.kt b/...gister/fhircore/engine/data/remote/resource/syncStrategy/fhir/LogicalIdSyncParamsBased.kt
@@ -62,10 +62,21 @@ class LogicalIdSyncParamsBased(
       .mapNotNull { it.resource as Bundle }
       .map { it.entry.map { it.resource } }
       .flatten()
-      .also { catchIds() }
+      .also(::catchIds)
   }
 
-  private fun catchIds() = callback(ParamSyncStatus(logicalIds, logicalIds.size, patientPosition))
+  private fun catchIds(resources: List<Resource>) =
+    resources
+      .filter { it.resourceType == ResourceType.Patient }
+      .also { patients ->
+        callback(
+          ParamSyncStatus(
+            logicalId = patients.map { it.idPart },
+            idsTotal = logicalIds.size,
+            patientPositionAt = patientPosition,
+          ),
+        )
+      }
 
   private fun List<String>.bundleOf(): Bundle {
     return Bundle().apply {

diff --git a/...ter/fhircore/engine/data/remote/resource/syncStrategy/fhir/ResourceParamsBasedDownload.kt b/...ter/fhircore/engine/data/remote/resource/syncStrategy/fhir/ResourceParamsBasedDownload.kt
@@ -37,6 +37,7 @@ typealias ResourceSearchParams = Map<ResourceType, ParamMap>
 class ResourceParamsBasedDownload(
   syncParams: ResourceSearchParams,
   val context: TimestampContext,
+  private val callback: (List<String>) -> Unit,
 ) : DownloadWorkManager {
   private val resourcesToDownloadWithSearchParams = LinkedList(syncParams.entries)
   private val urlOfTheNextPagesToDownloadForAResource = LinkedList<String>()
@@ -117,6 +118,7 @@ class ResourceParamsBasedDownload(
 
     return response.entry
       .map { it.resource }
+      .also(::catchIds)
       .also { resources ->
         resources
           .groupBy { it.resourceType }
@@ -133,6 +135,12 @@ class ResourceParamsBasedDownload(
           }
       }
   }
+
+  private fun catchIds(resources: List<Resource>) =
+    resources
+      .filter { it.resourceType == ResourceType.Patient }
+      .map { it.idPart }
+      .also { callback(it) }
 }
 
 interface TimestampContext {

diff --git a/android/engine/src/main/java/org/smartregister/fhircore/engine/sync/AppSyncWorker.kt b/android/engine/src/main/java/org/smartregister/fhircore/engine/sync/AppSyncWorker.kt
@@ -17,10 +17,11 @@
 package org.smartregister.fhircore.engine.sync
 
 import android.content.Context
-import android.content.Intent
 import androidx.hilt.work.HiltWorker
+import androidx.localbroadcastmanager.content.LocalBroadcastManager
 import androidx.work.WorkerParameters
 import com.google.android.fhir.FhirEngine
+import com.google.android.fhir.search.search
 import com.google.android.fhir.sync.AcceptLocalConflictResolver
 import com.google.android.fhir.sync.ConflictResolver
 import com.google.android.fhir.sync.DownloadWorkManager
@@ -30,22 +31,23 @@ import dagger.assisted.Assisted
 import dagger.assisted.AssistedInject
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
+import org.hl7.fhir.r4.model.ListResource
 import org.hl7.fhir.r4.model.ResourceType
 import org.smartregister.fhircore.engine.configuration.ConfigurationRegistry
 import org.smartregister.fhircore.engine.configuration.preferences.SyncUploadStrategy
 import org.smartregister.fhircore.engine.data.local.TingatheDatabase
 import org.smartregister.fhircore.engine.data.local.syncStrategy.toEntity
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.SyncParamStrategy
-import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.broadcast.SYNC_STATUS_BROADCAST_RECEIVER_KEY
-import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.broadcast.SyncStatusBroadcastReceiver
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.fhir.IdentifierSyncParams
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.fhir.LogicalIdSyncParamsBased
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.fhir.ResourceParamsBasedDownload
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.fhir.TimestampContext
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.getIdentifiers
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.hasCompletedInitialSync
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.logicalIds
-import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.perOrgSyncConfig
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.onSendBroadcast
 import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.saveLastUpdatedTimestamp
-import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.subListIds
+import org.smartregister.fhircore.engine.data.remote.resource.syncStrategy.syncConfigOfflineFirst
 import org.smartregister.fhircore.engine.ui.questionnaire.ContentCache
 import org.smartregister.fhircore.engine.util.AppDataStore
 import org.smartregister.fhircore.engine.util.DispatcherProvider
@@ -70,40 +72,65 @@ constructor(
 ) : FhirSyncWorker(appContext, workerParams) {
 
   private val syncStrategyCacheDao = database.syncStrategyCacheDao
-
-  private fun downloadWorkManager(): DownloadWorkManager {
-    return when {
-      hasCompletedInitialSync(preference) -> defaultDownloadManager()
-      else -> {
-        val subList = logicalIds(engine).subListIds(syncStrategyCacheDao)
-        return if (subList.isNotEmpty()) {
-          LogicalIdSyncParamsBased(subList) {
-            Timber.tag("TAG")
-              .e("downloadWorkManager: " + it.patientPositionAt + " of " + it.idsTotal)
+  private val broadcaster = LocalBroadcastManager.getInstance(dataStore.context.applicationContext)
+  private val listResourceTitle = "Patient Identifier List"
+
+  private fun downloadWorkManager(): DownloadWorkManager = runBlocking {
+    syncConfigOfflineFirst(configurationRegistry, preference)
+      .takeIf { it }
+      ?.let {
+        getIdentifiers(engine)?.let { item ->
+          return@runBlocking IdentifierSyncParams(item.data) {
             runBlocking {
-              syncStrategyCacheDao.upsert(it.logicalId.toEntity())
-              saveLastUpdatedTimestamp(dataStore)
-            }
-            val broadcastIntent =
-              Intent(SyncStatusBroadcastReceiver::class.java.name).apply {
-                putExtra(SYNC_STATUS_BROADCAST_RECEIVER_KEY, it)
+              onSendBroadcast(broadcaster, it)
+              syncStrategyCacheDao.insert(it.logicalId.toEntity())
+              if (it.patientPositionAt == item.data.size) {
+                purgeListResource()
               }
-            dataStore.context.sendBroadcast(broadcastIntent)
+            }
           }
-        } else {
-          defaultDownloadManager()
         }
       }
+
+    val logicalIds = logicalIds(syncStrategyCacheDao)
+
+    return@runBlocking if (logicalIds.isNotEmpty()) {
+      LogicalIdSyncParamsBased(logicalIds) {
+        Timber.e("${it.patientPositionAt} of ${logicalIds.size}")
+        runBlocking {
+          it.logicalId
+            .toEntity()
+            .map { catchEntity -> catchEntity.copy(shouldSync = true) }
+            .also { syncStrategyCacheDao.upsert(it.map { it.logicalId }) }
+          saveLastUpdatedTimestamp(dataStore)
+          onSendBroadcast(broadcaster, it)
+        }
+      }
+    } else {
+      defaultDownloadManager()
     }
   }
 
-  private fun syncParams(): Map<ResourceType, Map<String, String>> {
-    val configs =
-      perOrgSyncConfig(configurationRegistry, preference)
-        ?: return syncListenerManager.loadSyncParams()
-
-    if (configs.offlineFirst) return syncListenerManager.loadSyncParams()
+  private suspend fun purgeListResource() {
+    engine
+      .search<ListResource> { filter(ListResource.TITLE, { value = listResourceTitle }) }
+      .map { it.resource }
+      .firstOrNull()
+      ?.let {
+        runCatching { engine.purge(ResourceType.List, it.idPart) }.onFailure { Timber.e(it) }
+      }
+  }
 
+  private fun syncParams(): Map<ResourceType, Map<String, String>> {
+    if (
+      syncConfigOfflineFirst(
+          configurationRegistry,
+          preference,
+        )
+        .not()
+    ) {
+      return syncListenerManager.loadSyncParams()
+    }
     return when {
       hasCompletedInitialSync(preference) -> SyncParamStrategy(preference).syncParams()
       else -> syncListenerManager.loadSyncParams()
@@ -125,7 +152,11 @@ constructor(
             timestamp?.let { dataStore.saveLastUpdatedTimestamp(resourceType, timestamp) }
           }
         },
-    )
+    ) { ids ->
+      if (syncConfigOfflineFirst(configurationRegistry, preference)) {
+        runBlocking { syncStrategyCacheDao.upsert(ids) }
+      }
+    }
 
   override fun getConflictResolver(): ConflictResolver = AcceptLocalConflictResolver
Original file line number	Diff line number	Diff line change
Expand Up		@@ -27,3 +27,5 @@ data class SyncStrategyCacheEntity(
		)

		fun List<String>.toEntity() = map { SyncStrategyCacheEntity(logicalId = it) }

		fun String.toEntity() = SyncStrategyCacheEntity(logicalId = this)