feat: Add max persistence age override option (#273)

UnitTests/MPBackendControllerTests.m

@@ -53,6 +53,7 @@ + (dispatch_queue_t)messageQueue;
 @property (nonatomic, strong) MPKitContainer *kitContainer;
 @property (nonatomic, strong, nullable) NSString *dataPlanId;
 @property (nonatomic, strong, nullable) NSNumber *dataPlanVersion;
+@property (nonatomic, strong, nonnull) MParticleOptions *options;
 
 @end
 
@@ -93,6 +94,7 @@ - (void)requestConfig:(void(^ _Nullable)(BOOL uploadBatch))completionHandler;
 - (MPExecStatus)checkForKitsAndUploadWithCompletionHandler:(void (^ _Nullable)(BOOL didShortCircuit))completionHandler;
 - (void)uploadBatchesWithCompletionHandler:(void(^)(BOOL success))completionHandler;
 - (NSMutableArray<NSDictionary<NSString *, id> *> *)userIdentitiesForUserId:(NSNumber *)userId;
+- (void)cleanUp:(NSTimeInterval)currentTime;
 
 @end
 
@@ -2062,4 +2064,27 @@ - (void)testUserIdentitiesForUserIdMultipleInvalidIdTypes {
     XCTAssertEqualObjects(currentUserIdentities[0], validUserId);
 }
 
+- (void)testPersistanceMaxAgeCleanup {
+    NSTimeInterval maxAge = 24 * 60 * 60; // 24 hours
+
+    MParticle *instance = [MParticle sharedInstance];
+    MParticleOptions *options = [[MParticleOptions alloc] init];
+    options.persistenceMaxAgeSeconds = @(maxAge); // 24 hours
+    instance.options = options;
+
+    MPBackendController *backendController = [[MPBackendController alloc] init];
+    MPPersistenceController *persistenceController = [[MPPersistenceController alloc] init];
+    id mockPersistenceController = OCMPartialMock(persistenceController);
+
+    NSTimeInterval currentTime = [[NSDate date] timeIntervalSince1970];
+    [[mockPersistenceController expect] deleteRecordsOlderThan:(currentTime - maxAge)];
+
+    instance.backendController = backendController;
+    instance.persistenceController = mockPersistenceController;
+
+    [instance.backendController cleanUp:currentTime];
+
+    [mockPersistenceController verifyWithDelay:1.0];
+}
+
 @end

UnitTests/MPBaseTestCase.m

@@ -12,6 +12,10 @@
 #import "MPConnectorProtocol.h"
 #import "MPConnectorFactoryProtocol.h"
 
+@interface MParticle (Tests)
+@property (nonatomic, strong) MPPersistenceController *persistenceController;
+@end
+
 @interface MPTestConnectorFactory : NSObject <MPConnectorFactoryProtocol>
 
 @property (nonatomic) NSMutableArray *mockConnectors;
@@ -32,7 +36,13 @@ @implementation MPBaseTestCase
 
 - (void)setUpWithCompletionHandler:(void (^)(NSError * _Nullable))completion {
     [super setUp];
-    [[MParticle sharedInstance] reset:^{
+    MParticle *instance = [MParticle sharedInstance];
+    if (!instance.persistenceController) {
+        // Ensure we have a persistence controller to reset the db etc
+        instance.persistenceController = [[MPPersistenceController alloc] init];
+    }
+
+    [instance reset:^{
         MPNetworkCommunication.connectorFactory = [[MPTestConnectorFactory alloc] init];
         completion(nil);
     }];

UnitTests/MPPersistenceControllerTests.mm

@@ -859,4 +859,76 @@ - (void)testShouldNotUploadMessageToMParticle {
     XCTAssertEqual(messages.count, 0);
 }
 
+- (void)testDeleteRecordsOlderThan_DontDelete {
+    NSTimeInterval oneDayAgo = [[NSDate date] timeIntervalSince1970] - 24*60*60;
+    NSTimeInterval ninteyDaysAgo = [[NSDate date] timeIntervalSince1970] - 90*24*60*60;
+
+    // Store a message
+    MParticle *instance = [MParticle sharedInstance];
+    NSLog(@"instance: %@", instance);
+    NSDictionary *messageDictionary = @{@"test": @"test"};
+    NSData *messageData = [NSJSONSerialization dataWithJSONObject:messageDictionary options:0 error:nil];
+    MPMessage *message = [[MPMessage alloc] initWithSessionId:@17 messageId:1 UUID:@"uuid" messageType:@"test" messageData:messageData timestamp:oneDayAgo uploadStatus:MPUploadStatusBatch userId:@1 dataPlanId:nil dataPlanVersion:nil];
+    [instance.persistenceController saveMessage:message];
+    XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 1);
+
+    // Store a session
+    MPSession *session = [[MPSession alloc] initWithStartTime:oneDayAgo userId:[MPPersistenceController mpId]];
+    session.endTime = oneDayAgo;
+    session.attributesDictionary = [@{@"key1":@"value1"} mutableCopy];
+    [instance.persistenceController saveSession:session];
+    XCTAssertEqual([instance.persistenceController fetchSessions].count, 1);
+
+    // Store an upload
+    NSDictionary *uploadDictionary = @{kMPOptOutKey:@NO, kMPSessionTimeoutKey:@120, kMPUploadIntervalKey:@10, kMPLifeTimeValueKey:@0, kMPMessagesKey:@[[message dictionaryRepresentation]], kMPMessageIdKey:[[NSUUID UUID] UUIDString]};
+    MPUpload *upload = [[MPUpload alloc] initWithSessionId:@(session.sessionId) uploadDictionary:uploadDictionary dataPlanId:nil dataPlanVersion:nil];
+    upload.timestamp = oneDayAgo;
+    [instance.persistenceController saveUpload:upload];
+    XCTAssertEqual([instance.persistenceController fetchUploads].count, 1);
+
+    // Cleanup persistence using default 90 day limit
+    [instance.persistenceController deleteRecordsOlderThan:ninteyDaysAgo];
+
+    // Check that the records still exist
+    XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 1);
+    XCTAssertEqual([instance.persistenceController fetchSessions].count, 1);
+    XCTAssertEqual([instance.persistenceController fetchUploads].count, 1);
+}
+
+- (void)testDeleteRecordsOlderThan_Delete {
+    NSTimeInterval sevenDaysAgo = [[NSDate date] timeIntervalSince1970] - 7*24*60*60;
+    NSTimeInterval twoDaysAgo = [[NSDate date] timeIntervalSince1970] - 2*24*60*60;
+
+    // Store a message
+    MParticle *instance = [MParticle sharedInstance];
+    NSLog(@"instance: %@", instance);
+    NSDictionary *messageDictionary = @{@"test": @"test"};
+    NSData *messageData = [NSJSONSerialization dataWithJSONObject:messageDictionary options:0 error:nil];
+    MPMessage *message = [[MPMessage alloc] initWithSessionId:@17 messageId:1 UUID:@"uuid" messageType:@"test" messageData:messageData timestamp:sevenDaysAgo uploadStatus:MPUploadStatusBatch userId:@1 dataPlanId:nil dataPlanVersion:nil];
+    [instance.persistenceController saveMessage:message];
+    XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 1);
+
+    // Store a session
+    MPSession *session = [[MPSession alloc] initWithStartTime:sevenDaysAgo userId:[MPPersistenceController mpId]];
+    session.endTime = sevenDaysAgo;
+    session.attributesDictionary = [@{@"key1":@"value1"} mutableCopy];
+    [instance.persistenceController saveSession:session];
+    XCTAssertEqual([instance.persistenceController fetchSessions].count, 1);
+
+    // Store an upload
+    NSDictionary *uploadDictionary = @{kMPOptOutKey:@NO, kMPSessionTimeoutKey:@120, kMPUploadIntervalKey:@10, kMPLifeTimeValueKey:@0, kMPMessagesKey:@[[message dictionaryRepresentation]], kMPMessageIdKey:[[NSUUID UUID] UUIDString]};
+    MPUpload *upload = [[MPUpload alloc] initWithSessionId:@(session.sessionId) uploadDictionary:uploadDictionary dataPlanId:nil dataPlanVersion:nil];
+    upload.timestamp = sevenDaysAgo;
+    [instance.persistenceController saveUpload:upload];
+    XCTAssertEqual([instance.persistenceController fetchUploads].count, 1);
+
+    // Cleanup persistence using 2 day limit
+    [instance.persistenceController deleteRecordsOlderThan:twoDaysAgo];
+
+    // Check that the records no longer exist
+    XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 0);
+    XCTAssertEqual([instance.persistenceController fetchSessions].count, 0);
+    XCTAssertEqual([instance.persistenceController fetchUploads].count, 0);
+}
+
 @end

mParticle-Apple-SDK/Include/mParticle.h

@@ -385,6 +385,19 @@ Defaults to false. Prevents the eventsHost above from overwriting the alias endp
  */
 @property (nonatomic, strong, readwrite, nullable) NSNumber *configMaxAgeSeconds;
 
+/**
+ Set a maximum threshold for stored events, batches, and sessions, in seconds.
+ 
+ By default, data is persisted for 90 days before being deleted to minimize data loss, however
+ this can lead to excessive storage usage on some users' devices. This is exacerbated if you log
+ a large number of events, or events with a lot of data (attributes, etc).
+ 
+ You can set this to any value greater than 0 seconds, so if you have storage usage concerns, set a lower
+ value such as 48 hours or 1 week. Or alternatively, if you have data loss concerns, you can set this to an even
+ longer value than the default.
+ */
+@property (nonatomic, strong, nullable) NSNumber *persistenceMaxAgeSeconds;
+
 /**
  Set an array of instances of kit (MPKitProtocol wrapped in MPSideloadedKit) objects to be "sideloaded".
  
@@ -607,6 +620,12 @@ Defaults to false. Prevents the eventsHost above from overwriting the alias endp
  */
 @property (nonatomic, readonly, nullable) NSNumber *configMaxAgeSeconds;
 
+/**
+ Maximum threshold for stored events, batches, and sessions, in seconds.
+ @see MParticleOptions
+ */
+@property (nonatomic, readonly, nullable) NSNumber *persistenceMaxAgeSeconds;
+
 #pragma mark - Initialization
 
 /**

mParticle-Apple-SDK/MPBackendController.m

@@ -1317,12 +1317,6 @@ -  (void)logCrash:(NSString *)message stackTrace:(NSString *)stackTrace plCrashR
 }
 
 - (void)logBaseEvent:(MPBaseEvent *)event completionHandler:(void (^)(MPBaseEvent *event, MPExecStatus execStatus))completionHandler {
-    if (![MPStateMachine canWriteMessagesToDB]) {
-        MPILogError(@"Not saving message for event to prevent excessive local database growth because API Key appears to be invalid based on server response");
-        completionHandler(event, MPExecStatusFail);
-        return;
-    }
-
     [MPListenerController.sharedInstance onAPICalled:_cmd parameter1:event];
 
     if (event.shouldBeginSession) {
@@ -1578,11 +1572,6 @@ - (void)startWithKey:(NSString *)apiKey secret:(NSString *)secret firstRun:(BOOL
 }
 
 - (void)saveMessage:(MPMessage *)message updateSession:(BOOL)updateSession {
-    if (![MPStateMachine canWriteMessagesToDB]) {
-        MPILogError(@"Not saving message for event to prevent excessive local database growth because API Key appears to be invalid based on server response");
-        return;
-    }
-
     NSTimeInterval lastEventTimestamp = message.timestamp ?: [[NSDate date] timeIntervalSince1970];
     if (MPStateMachine.runningInBackground) {
         self.timeOfLastEventInBackground = lastEventTimestamp;
@@ -2089,9 +2078,15 @@ - (void)endSessionIfTimedOut {
 
 - (void)cleanUp {
     NSTimeInterval currentTime = [[NSDate date] timeIntervalSince1970];
+    [self cleanUp:currentTime];
+}
+
+- (void)cleanUp:(NSTimeInterval)currentTime {
     MPPersistenceController *persistence = [MParticle sharedInstance].persistenceController;
     if (nextCleanUpTime < currentTime) {
-        [persistence deleteRecordsOlderThan:(currentTime - NINETY_DAYS)];
+        NSNumber *persistanceMaxAgeSeconds = [MParticle sharedInstance].persistenceMaxAgeSeconds;
+        NSTimeInterval maxAgeSeconds = persistanceMaxAgeSeconds == nil ? NINETY_DAYS : persistanceMaxAgeSeconds.doubleValue;
+        [persistence deleteRecordsOlderThan:(currentTime - maxAgeSeconds)];
         nextCleanUpTime = currentTime + TWENTY_FOUR_HOURS;
     }
     [persistence purgeMemory];

mParticle-Apple-SDK/Network/MPNetworkCommunication.m

@@ -403,15 +403,6 @@ - (NSNumber *)maxAgeForCache:(nonnull NSString *)cache {
     return maxAge;
 }
 
-- (void)checkResponseCodeToDisableEventLogging:(NSInteger)responseCode {
-    if (responseCode == HTTPStatusCodeBadRequest || responseCode == HTTPStatusCodeUnauthorized || responseCode == HTTPStatusCodeForbidden) {
-        [MPStateMachine setCanWriteMessagesToDB:NO];
-        MPILogError(@"API Key appears to be invalid based on server response, disabling event logging to prevent excessive local database growth");
-    } else {
-        [MPStateMachine setCanWriteMessagesToDB:YES];
-    }
-}
-
 #pragma mark Public methods
 - (NSObject<MPConnectorProtocol> *_Nonnull)makeConnector {
     if (MPNetworkCommunication.connectorFactory) {
@@ -463,9 +454,7 @@ - (void)requestConfig:(nullable NSObject<MPConnectorProtocol> *)connector withCo
 
     NSInteger responseCode = [httpResponse statusCode];
     MPILogVerbose(@"Config Response Code: %ld, Execution Time: %.2fms", (long)responseCode, ([[NSDate date] timeIntervalSince1970] - start) * 1000.0);
-
-    [self checkResponseCodeToDisableEventLogging:responseCode];
-
+
     if (responseCode == HTTPStatusCodeNotModified) {
         MPIUserDefaults *userDefaults = [MPIUserDefaults standardUserDefaults];
         [userDefaults setConfiguration:[userDefaults getConfiguration] eTag:userDefaults[kMPHTTPETagHeaderKey] requestTimestamp:[[NSDate date] timeIntervalSince1970] currentAge:ageString maxAge:maxAge];
@@ -875,9 +864,7 @@ - (void)identityApiRequestWithURL:(NSURL*)url identityRequest:(MPIdentityHTTPBas
 
 
         MPILogVerbose(@"Identity response code: %ld", (long)responseCode);
-
-        [self checkResponseCodeToDisableEventLogging:[httpResponse statusCode]];
-
+
         if (success) {
             @try {
                 NSError *serializationError = nil;

mParticle-Apple-SDK/Persistence/MPPersistenceController.mm

@@ -1557,10 +1557,6 @@ - (void)saveIntegrationAttributes:(nonnull MPIntegrationAttributes *)integration
 }
 
 - (void)saveMessage:(MPMessage *)message {
-    if (![MPStateMachine canWriteMessagesToDB]) {
-        MPILogError(@"Not saving message for event to prevent excessive local database growth because API Key appears to be invalid based on server response");
-        return;
-    }
     if (!message.shouldUploadEvent) {
         MPILogDebug(@"Not saving message for event because shouldUploadEvent was set to NO, message id: %lld, type: %@", message.messageId, message.messageType);
         return;

mParticle-Apple-SDK/Utils/MPStateMachine.h

@@ -66,8 +66,6 @@
 + (BOOL)runningInBackground;
 + (void)setRunningInBackground:(BOOL)background;
 + (BOOL)isAppExtension;
-+ (BOOL)canWriteMessagesToDB;
-+ (void)setCanWriteMessagesToDB:(BOOL)canWriteMessagesToDB;
 - (void)configureCustomModules:(nullable NSArray<NSDictionary *> *)customModuleSettings;
 - (void)configureRampPercentage:(nullable NSNumber *)rampPercentage;
 - (void)configureTriggers:(nullable NSDictionary *)triggerDictionary;

mParticle-Apple-SDK/Utils/MPStateMachine.m

@@ -33,7 +33,6 @@
 
 static MPEnvironment runningEnvironment = MPEnvironmentAutoDetect;
 static BOOL runningInBackground = NO;
-static BOOL _canWriteMessagesToDB = YES;
 
 @interface MParticle ()
 + (dispatch_queue_t)messageQueue;
@@ -348,18 +347,6 @@ + (BOOL)isAppExtension {
 #endif
 }
 
-+ (BOOL)canWriteMessagesToDB {
-    @synchronized(self) {
-        return _canWriteMessagesToDB;
-    }
-}
-
-+ (void)setCanWriteMessagesToDB:(BOOL)canWriteMessagesToDB {
-    @synchronized(self) {
-        _canWriteMessagesToDB = canWriteMessagesToDB;
-    }
-}
-
 #pragma mark Public accessors
 - (MPConsumerInfo *)consumerInfo {
     if (_consumerInfo) {

mParticle-Apple-SDK/mParticle.m

@@ -258,6 +258,14 @@ - (void)setConfigMaxAgeSeconds:(NSNumber *)configMaxAgeSeconds {
     }
 }
 
+- (void)setPersistenceMaxAgeSeconds:(NSNumber *)persistenceMaxAgeSeconds {
+    if (persistenceMaxAgeSeconds != nil && [persistenceMaxAgeSeconds doubleValue] <= 0) {
+        MPILogWarning(@"Persistence Max Age must be a positive number, disregarding value.");
+    } else {
+        _persistenceMaxAgeSeconds = persistenceMaxAgeSeconds;
+    }
+}
+
 @end
 
 @interface MPBackendController ()
@@ -491,6 +499,10 @@ - (NSNumber *)configMaxAgeSeconds {
     return self.options.configMaxAgeSeconds;
 }
 
+- (NSNumber *)persistenceMaxAgeSeconds {
+    return self.options.persistenceMaxAgeSeconds;
+}
+
 #pragma mark Initialization
 + (instancetype)sharedInstance {
     dispatch_once(&predicate, ^{