diff --git a/parsely/src/main/java/com/parsely/parselyandroid/LocalStorageRepository.kt b/parsely/src/main/java/com/parsely/parselyandroid/LocalStorageRepository.kt new file mode 100644 index 00000000..1f4f903b --- /dev/null +++ b/parsely/src/main/java/com/parsely/parselyandroid/LocalStorageRepository.kt @@ -0,0 +1,82 @@ +package com.parsely.parselyandroid + +import android.content.Context +import java.io.EOFException +import java.io.FileNotFoundException +import java.io.ObjectInputStream +import java.io.ObjectOutputStream + +internal open class LocalStorageRepository(private val context: Context) { + /** + * Persist an object to storage. + * + * @param o Object to store. + */ + private fun persistObject(o: Any) { + try { + val fos = context.applicationContext.openFileOutput( + STORAGE_KEY, + Context.MODE_PRIVATE + ) + val oos = ObjectOutputStream(fos) + oos.writeObject(o) + oos.close() + } catch (ex: Exception) { + ParselyTracker.PLog("Exception thrown during queue serialization: %s", ex.toString()) + } + } + + /** + * Delete the stored queue from persistent storage. + */ + fun purgeStoredQueue() { + persistObject(ArrayList>()) + } + + /** + * Get the stored event queue from persistent storage. + * + * @return The stored queue of events. + */ + open fun getStoredQueue(): ArrayList?> { + var storedQueue: ArrayList?> = ArrayList() + try { + val fis = context.applicationContext.openFileInput(STORAGE_KEY) + val ois = ObjectInputStream(fis) + @Suppress("UNCHECKED_CAST") + storedQueue = ois.readObject() as ArrayList?> + ois.close() + } catch (ex: EOFException) { + // Nothing to do here. + } catch (ex: FileNotFoundException) { + // Nothing to do here. Means there was no saved queue. + } catch (ex: Exception) { + ParselyTracker.PLog( + "Exception thrown during queue deserialization: %s", + ex.toString() + ) + } + return storedQueue + } + + /** + * Delete an event from the stored queue. + */ + open fun expelStoredEvent() { + val storedQueue = getStoredQueue() + storedQueue.removeAt(0) + } + + /** + * Save the event queue to persistent storage. + */ + @Synchronized + open fun persistQueue(inMemoryQueue: List?>) { + ParselyTracker.PLog("Persisting event queue") + persistObject((inMemoryQueue + getStoredQueue()).distinct()) + } + + companion object { + private const val STORAGE_KEY = "parsely-events.ser" + } +} diff --git a/parsely/src/main/java/com/parsely/parselyandroid/ParselyTracker.java b/parsely/src/main/java/com/parsely/parselyandroid/ParselyTracker.java index 7784970e..609ed087 100644 --- a/parsely/src/main/java/com/parsely/parselyandroid/ParselyTracker.java +++ b/parsely/src/main/java/com/parsely/parselyandroid/ParselyTracker.java @@ -29,18 +29,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.EOFException; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; import java.io.StringWriter; import java.util.ArrayList; import java.util.Formatter; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Timer; import java.util.TimerTask; @@ -56,13 +51,10 @@ public class ParselyTracker { private static ParselyTracker instance = null; private static final int DEFAULT_FLUSH_INTERVAL_SECS = 60; private static final int DEFAULT_ENGAGEMENT_INTERVAL_MILLIS = 10500; - private static final int QUEUE_SIZE_LIMIT = 50; - private static final int STORAGE_SIZE_LIMIT = 100; - private static final String STORAGE_KEY = "parsely-events.ser"; @SuppressWarnings("StringOperationCanBeSimplified") // private static final String ROOT_URL = "http://10.0.2.2:5001/".intern(); // emulator localhost private static final String ROOT_URL = "https://p1.parsely.com/".intern(); - protected ArrayList> eventQueue; + private final ArrayList> eventQueue; private boolean isDebug; private final Context context; private final Timer timer; @@ -72,7 +64,10 @@ public class ParselyTracker { private String lastPageviewUuid = null; @NonNull private final EventsBuilder eventsBuilder; - @NonNull final HeartbeatIntervalCalculator intervalCalculator = new HeartbeatIntervalCalculator(new Clock()); + @NonNull + private final HeartbeatIntervalCalculator intervalCalculator = new HeartbeatIntervalCalculator(new Clock()); + @NonNull + private final LocalStorageRepository localStorageRepository; /** * Create a new ParselyTracker instance. @@ -80,6 +75,7 @@ public class ParselyTracker { protected ParselyTracker(String siteId, int flushInterval, Context c) { context = c.getApplicationContext(); eventsBuilder = new EventsBuilder(context, siteId); + localStorageRepository = new LocalStorageRepository(context); // get the adkey straight away on instantiation timer = new Timer(); @@ -89,7 +85,7 @@ protected ParselyTracker(String siteId, int flushInterval, Context c) { flushManager = new FlushManager(timer, flushInterval * 1000L); - if (getStoredQueue().size() > 0) { + if (localStorageRepository.getStoredQueue().size() > 0) { startFlushTimer(); } @@ -102,6 +98,10 @@ protected ParselyTracker(String siteId, int flushInterval, Context c) { ); } + List> getInMemoryQueue() { + return eventQueue; + } + /** * Singleton instance accessor. Note: This must be called after {@link #sharedInstance(String, Context)} * @@ -411,14 +411,14 @@ public void resetVideo() { * Place a data structure representing the event into the in-memory queue for later use. *

* **Note**: Events placed into this queue will be discarded if the size of the persistent queue - * store exceeds {@link #STORAGE_SIZE_LIMIT}. + * store exceeds {@link QueueManager#STORAGE_SIZE_LIMIT}. * * @param event The event Map to enqueue. */ void enqueueEvent(Map event) { // Push it onto the queue eventQueue.add(event); - new QueueManager().execute(); + new QueueManager(this, localStorageRepository).execute(); if (!flushTimerIsActive()) { startFlushTimer(); PLog("Flush flushTimer set to %ds", (flushManager.getIntervalMillis() / 1000)); @@ -474,85 +474,9 @@ private boolean isReachable() { return netInfo != null && netInfo.isConnectedOrConnecting(); } - /** - * Save the event queue to persistent storage. - */ - private synchronized void persistQueue() { - PLog("Persisting event queue"); - ArrayList> storedQueue = getStoredQueue(); - HashSet> hs = new HashSet<>(); - hs.addAll(storedQueue); - hs.addAll(eventQueue); - storedQueue.clear(); - storedQueue.addAll(hs); - persistObject(storedQueue); - } - - /** - * Get the stored event queue from persistent storage. - * - * @return The stored queue of events. - */ - @NonNull - private ArrayList> getStoredQueue() { - ArrayList> storedQueue = null; - try { - FileInputStream fis = context.getApplicationContext().openFileInput(STORAGE_KEY); - ObjectInputStream ois = new ObjectInputStream(fis); - //noinspection unchecked - storedQueue = (ArrayList>) ois.readObject(); - ois.close(); - } catch (EOFException ex) { - // Nothing to do here. - } catch (FileNotFoundException ex) { - // Nothing to do here. Means there was no saved queue. - } catch (Exception ex) { - PLog("Exception thrown during queue deserialization: %s", ex.toString()); - } - - if (storedQueue == null) { - storedQueue = new ArrayList<>(); - } - return storedQueue; - } - void purgeEventsQueue() { eventQueue.clear(); - purgeStoredQueue(); - } - - /** - * Delete the stored queue from persistent storage. - */ - private void purgeStoredQueue() { - persistObject(new ArrayList>()); - } - - /** - * Delete an event from the stored queue. - */ - private void expelStoredEvent() { - ArrayList> storedQueue = getStoredQueue(); - storedQueue.remove(0); - } - - /** - * Persist an object to storage. - * - * @param o Object to store. - */ - private void persistObject(Object o) { - try { - FileOutputStream fos = context.getApplicationContext().openFileOutput( - STORAGE_KEY, - android.content.Context.MODE_PRIVATE - ); - ObjectOutputStream oos = new ObjectOutputStream(fos); - oos.writeObject(o); - oos.close(); - } catch (Exception ex) { - PLog("Exception thrown during queue serialization: %s", ex.toString()); - } + localStorageRepository.purgeStoredQueue(); } /** @@ -621,31 +545,14 @@ public int queueSize() { * @return The number of events stored in persistent storage. */ public int storedEventsCount() { - ArrayList> ar = getStoredQueue(); + ArrayList> ar = localStorageRepository.getStoredQueue(); return ar.size(); } - private class QueueManager extends AsyncTask { - @Override - protected Void doInBackground(Void... params) { - // if event queue is too big, push to persisted storage - if (eventQueue.size() > QUEUE_SIZE_LIMIT) { - PLog("Queue size exceeded, expelling oldest event to persistent memory"); - persistQueue(); - eventQueue.remove(0); - // if persisted storage is too big, expel one - if (storedEventsCount() > STORAGE_SIZE_LIMIT) { - expelStoredEvent(); - } - } - return null; - } - } - private class FlushQueue extends AsyncTask { @Override protected synchronized Void doInBackground(Void... params) { - ArrayList> storedQueue = getStoredQueue(); + ArrayList> storedQueue = localStorageRepository.getStoredQueue(); PLog("%d events in queue, %d stored events", eventQueue.size(), storedEventsCount()); // in case both queues have been flushed and app quits, don't crash if ((eventQueue == null || eventQueue.size() == 0) && storedQueue.size() == 0) { diff --git a/parsely/src/main/java/com/parsely/parselyandroid/QueueManager.kt b/parsely/src/main/java/com/parsely/parselyandroid/QueueManager.kt new file mode 100644 index 00000000..59d5401b --- /dev/null +++ b/parsely/src/main/java/com/parsely/parselyandroid/QueueManager.kt @@ -0,0 +1,30 @@ +package com.parsely.parselyandroid + +import android.os.AsyncTask + +@Suppress("DEPRECATION") +internal class QueueManager( + private val parselyTracker: ParselyTracker, + private val localStorageRepository: LocalStorageRepository +) : AsyncTask() { + + @Deprecated("Deprecated in Java") + override fun doInBackground(vararg params: Void?): Void? { + // if event queue is too big, push to persisted storage + if (parselyTracker.inMemoryQueue.size > QUEUE_SIZE_LIMIT) { + ParselyTracker.PLog("Queue size exceeded, expelling oldest event to persistent memory") + localStorageRepository.persistQueue(parselyTracker.inMemoryQueue) + parselyTracker.inMemoryQueue.removeAt(0) + // if persisted storage is too big, expel one + if (parselyTracker.storedEventsCount() > STORAGE_SIZE_LIMIT) { + localStorageRepository.expelStoredEvent() + } + } + return null + } + + companion object { + const val QUEUE_SIZE_LIMIT = 50 + const val STORAGE_SIZE_LIMIT = 100 + } +} diff --git a/parsely/src/test/java/com/parsely/parselyandroid/LocalStorageRepositoryTest.kt b/parsely/src/test/java/com/parsely/parselyandroid/LocalStorageRepositoryTest.kt new file mode 100644 index 00000000..2879fdab --- /dev/null +++ b/parsely/src/test/java/com/parsely/parselyandroid/LocalStorageRepositoryTest.kt @@ -0,0 +1,123 @@ +package com.parsely.parselyandroid + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import java.io.File +import org.assertj.core.api.Assertions.assertThat +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@RunWith(RobolectricTestRunner::class) +class LocalStorageRepositoryTest { + + private lateinit var sut: LocalStorageRepository + private val context = ApplicationProvider.getApplicationContext() + + @Before + fun setUp() { + sut = LocalStorageRepository(context) + } + + @Test + fun `when expelling stored event, then assert that it has no effect`() { + // given + sut.persistQueue((1..100).map { mapOf("index" to it) }) + + // when + sut.expelStoredEvent() + + // then + assertThat(sut.getStoredQueue()).hasSize(100) + } + + @Test + fun `given the list of events, when persisting the list, then querying the list returns the same result`() { + // given + val eventsList = (1..10).map { mapOf("index" to it) } + + // when + sut.persistQueue(eventsList) + + // then + assertThat(sut.getStoredQueue()).hasSize(10).containsExactlyInAnyOrderElementsOf(eventsList) + } + + @Test + fun `given no locally stored list, when requesting stored queue, then return an empty list`() { + assertThat(sut.getStoredQueue()).isEmpty() + } + + @Test + fun `given stored queue with some elements, when persisting in-memory queue, then assert there'll be no duplicates and queues will be combined`() { + // given + val storedQueue = (1..5).map { mapOf("index" to it) } + val inMemoryQueue = (3..10).map { mapOf("index" to it) } + sut.persistQueue(storedQueue) + + // when + sut.persistQueue(inMemoryQueue) + + // then + val expectedQueue = (1..10).map { mapOf("index" to it) } + assertThat(sut.getStoredQueue()).hasSize(10).containsExactlyInAnyOrderElementsOf(expectedQueue) + } + + @Test + fun `given stored queue, when purging stored queue, then assert queue is purged`() { + // given + val eventsList = (1..10).map { mapOf("index" to it) } + sut.persistQueue(eventsList) + + // when + sut.purgeStoredQueue() + + // then + assertThat(sut.getStoredQueue()).isEmpty() + } + + @Test + fun `given stored file with serialized events, when querying the queue, then list has expected events`() { + // given + val file = File(context.filesDir.path + "/parsely-events.ser") + File(ClassLoader.getSystemResource("valid-java-parsely-events.ser")?.path!!).copyTo(file) + + // when + val queue = sut.getStoredQueue() + + // then + assertThat(queue).isEqualTo( + listOf( + mapOf( + "idsite" to "example.com", + "urlref" to "http://example.com/", + "data" to mapOf( + "manufacturer" to "Google", + "os" to "android", + "os_version" to "33", + "parsely_site_uuid" to "b325e2c9-498c-4331-a967-2d6049317c77", + "ts" to 1698918720863L + ), + "pvid" to "272cc2b8-5acc-4a70-80c7-20bb6eb843e4", + "action" to "pageview", + "url" to "http://example.com/article1.html" + ), + mapOf( + "idsite" to "example.com", + "urlref" to "http://example.com/", + "data" to mapOf( + "manufacturer" to "Google", + "os" to "android", + "os_version" to "33", + "parsely_site_uuid" to "b325e2c9-498c-4331-a967-2d6049317c77", + "ts" to 1698918742375L + ), + "pvid" to "e94567ec-3459-498c-bf2e-6a1b85ed5a82", + "action" to "pageview", + "url" to "http://example.com/article1.html" + ) + ) + ) + } +} diff --git a/parsely/src/test/java/com/parsely/parselyandroid/QueueManagerTest.kt b/parsely/src/test/java/com/parsely/parselyandroid/QueueManagerTest.kt new file mode 100644 index 00000000..86613295 --- /dev/null +++ b/parsely/src/test/java/com/parsely/parselyandroid/QueueManagerTest.kt @@ -0,0 +1,108 @@ +package com.parsely.parselyandroid + +import androidx.test.core.app.ApplicationProvider +import com.parsely.parselyandroid.QueueManager.Companion.QUEUE_SIZE_LIMIT +import com.parsely.parselyandroid.QueueManager.Companion.STORAGE_SIZE_LIMIT +import org.assertj.core.api.Assertions.assertThat +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.LooperMode +import org.robolectric.shadows.ShadowLooper.shadowMainLooper + +@Suppress("DEPRECATION") +@RunWith(RobolectricTestRunner::class) +@LooperMode(LooperMode.Mode.PAUSED) +internal class QueueManagerTest { + + private lateinit var sut: QueueManager + + private val tracker = FakeTracker() + private val repository = FakeLocalRepository() + + @Before + fun setUp() { + sut = QueueManager(tracker, repository) + } + + @Test + fun `given the queue is smaller than any threshold, when querying flush manager, do nothing`() { + // given + val initialInMemoryQueue = listOf(mapOf("test" to "test")) + tracker.applyFakeQueue(initialInMemoryQueue) + + // when + sut.execute().get() + shadowMainLooper().idle(); + + // then + assertThat(tracker.inMemoryQueue).isEqualTo(initialInMemoryQueue) + assertThat(repository.getStoredQueue()).isEmpty() + } + + @Test + fun `given the in-memory queue is above the in-memory limit, when querying flush manager, then save queue to local storage and remove first event`() { + // given + val initialInMemoryQueue = (1..QUEUE_SIZE_LIMIT + 1).map { mapOf("test" to it) } + tracker.applyFakeQueue(initialInMemoryQueue) + + // when + sut.execute().get() + shadowMainLooper().idle(); + + // then + assertThat(repository.getStoredQueue()).isEqualTo(initialInMemoryQueue) + assertThat(tracker.inMemoryQueue).hasSize(QUEUE_SIZE_LIMIT) + } + + @Test + fun `given the in-memory queue is above the in-memory limit and stored events queue is above stored-queue limit, when querying flush manager, then expel the last event from local storage`() { + // given + val initialInMemoryQueue = (1..QUEUE_SIZE_LIMIT + 1).map { mapOf("in memory" to it) } + tracker.applyFakeQueue(initialInMemoryQueue) + val initialStoredQueue = (1..STORAGE_SIZE_LIMIT + 1).map { mapOf("storage" to it) } + repository.persistQueue(initialStoredQueue) + + // when + sut.execute().get() + shadowMainLooper().idle(); + + // then + assertThat(repository.wasEventExpelled).isTrue + } + + inner class FakeTracker : ParselyTracker( + "siteId", 10, ApplicationProvider.getApplicationContext() + ) { + + private var fakeQueue: List> = emptyList() + + internal override fun getInMemoryQueue(): List> = fakeQueue + + fun applyFakeQueue(fakeQueue: List>) { + this.fakeQueue = fakeQueue.toList() + } + + override fun storedEventsCount(): Int { + return repository.getStoredQueue().size + } + } + + class FakeLocalRepository : + LocalStorageRepository(ApplicationProvider.getApplicationContext()) { + + private var localFileQueue = emptyList?>() + var wasEventExpelled = false + + override fun persistQueue(inMemoryQueue: List?>) { + this.localFileQueue += inMemoryQueue + } + + override fun getStoredQueue() = ArrayList(localFileQueue) + + override fun expelStoredEvent() { + wasEventExpelled = true + } + } +} diff --git a/parsely/src/test/resources/valid-java-parsely-events.ser b/parsely/src/test/resources/valid-java-parsely-events.ser new file mode 100644 index 00000000..2d514a8c Binary files /dev/null and b/parsely/src/test/resources/valid-java-parsely-events.ser differ