From 241e7734f6b26f617afb1a4aa51af37355f05de9 Mon Sep 17 00:00:00 2001 From: Ben Baron Date: Wed, 24 Apr 2024 08:58:27 -0500 Subject: [PATCH] feat: Add max persistence age override option (#273) --- UnitTests/MPBackendControllerTests.m | 25 +++++++ UnitTests/MPBaseTestCase.m | 12 +++- UnitTests/MPPersistenceControllerTests.mm | 72 +++++++++++++++++++ mParticle-Apple-SDK/Include/mParticle.h | 19 +++++ mParticle-Apple-SDK/MPBackendController.m | 19 ++--- .../Network/MPNetworkCommunication.m | 17 +---- .../Persistence/MPPersistenceController.mm | 4 -- mParticle-Apple-SDK/Utils/MPStateMachine.h | 2 - mParticle-Apple-SDK/Utils/MPStateMachine.m | 13 ---- mParticle-Apple-SDK/mParticle.m | 12 ++++ 10 files changed, 148 insertions(+), 47 deletions(-) diff --git a/UnitTests/MPBackendControllerTests.m b/UnitTests/MPBackendControllerTests.m index 7c234e32..02ee13be 100644 --- a/UnitTests/MPBackendControllerTests.m +++ b/UnitTests/MPBackendControllerTests.m @@ -53,6 +53,7 @@ + (dispatch_queue_t)messageQueue; @property (nonatomic, strong) MPKitContainer *kitContainer; @property (nonatomic, strong, nullable) NSString *dataPlanId; @property (nonatomic, strong, nullable) NSNumber *dataPlanVersion; +@property (nonatomic, strong, nonnull) MParticleOptions *options; @end @@ -93,6 +94,7 @@ - (void)requestConfig:(void(^ _Nullable)(BOOL uploadBatch))completionHandler; - (MPExecStatus)checkForKitsAndUploadWithCompletionHandler:(void (^ _Nullable)(BOOL didShortCircuit))completionHandler; - (void)uploadBatchesWithCompletionHandler:(void(^)(BOOL success))completionHandler; - (NSMutableArray *> *)userIdentitiesForUserId:(NSNumber *)userId; +- (void)cleanUp:(NSTimeInterval)currentTime; @end @@ -2062,4 +2064,27 @@ - (void)testUserIdentitiesForUserIdMultipleInvalidIdTypes { XCTAssertEqualObjects(currentUserIdentities[0], validUserId); } +- (void)testPersistanceMaxAgeCleanup { + NSTimeInterval maxAge = 24 * 60 * 60; // 24 hours + + MParticle *instance = [MParticle sharedInstance]; + MParticleOptions *options = [[MParticleOptions alloc] init]; + options.persistenceMaxAgeSeconds = @(maxAge); // 24 hours + instance.options = options; + + MPBackendController *backendController = [[MPBackendController alloc] init]; + MPPersistenceController *persistenceController = [[MPPersistenceController alloc] init]; + id mockPersistenceController = OCMPartialMock(persistenceController); + + NSTimeInterval currentTime = [[NSDate date] timeIntervalSince1970]; + [[mockPersistenceController expect] deleteRecordsOlderThan:(currentTime - maxAge)]; + + instance.backendController = backendController; + instance.persistenceController = mockPersistenceController; + + [instance.backendController cleanUp:currentTime]; + + [mockPersistenceController verifyWithDelay:1.0]; +} + @end diff --git a/UnitTests/MPBaseTestCase.m b/UnitTests/MPBaseTestCase.m index ff492318..e94132c3 100644 --- a/UnitTests/MPBaseTestCase.m +++ b/UnitTests/MPBaseTestCase.m @@ -12,6 +12,10 @@ #import "MPConnectorProtocol.h" #import "MPConnectorFactoryProtocol.h" +@interface MParticle (Tests) +@property (nonatomic, strong) MPPersistenceController *persistenceController; +@end + @interface MPTestConnectorFactory : NSObject @property (nonatomic) NSMutableArray *mockConnectors; @@ -32,7 +36,13 @@ @implementation MPBaseTestCase - (void)setUpWithCompletionHandler:(void (^)(NSError * _Nullable))completion { [super setUp]; - [[MParticle sharedInstance] reset:^{ + MParticle *instance = [MParticle sharedInstance]; + if (!instance.persistenceController) { + // Ensure we have a persistence controller to reset the db etc + instance.persistenceController = [[MPPersistenceController alloc] init]; + } + + [instance reset:^{ MPNetworkCommunication.connectorFactory = [[MPTestConnectorFactory alloc] init]; completion(nil); }]; diff --git a/UnitTests/MPPersistenceControllerTests.mm b/UnitTests/MPPersistenceControllerTests.mm index 4cf21162..fc5a204d 100644 --- a/UnitTests/MPPersistenceControllerTests.mm +++ b/UnitTests/MPPersistenceControllerTests.mm @@ -859,4 +859,76 @@ - (void)testShouldNotUploadMessageToMParticle { XCTAssertEqual(messages.count, 0); } +- (void)testDeleteRecordsOlderThan_DontDelete { + NSTimeInterval oneDayAgo = [[NSDate date] timeIntervalSince1970] - 24*60*60; + NSTimeInterval ninteyDaysAgo = [[NSDate date] timeIntervalSince1970] - 90*24*60*60; + + // Store a message + MParticle *instance = [MParticle sharedInstance]; + NSLog(@"instance: %@", instance); + NSDictionary *messageDictionary = @{@"test": @"test"}; + NSData *messageData = [NSJSONSerialization dataWithJSONObject:messageDictionary options:0 error:nil]; + MPMessage *message = [[MPMessage alloc] initWithSessionId:@17 messageId:1 UUID:@"uuid" messageType:@"test" messageData:messageData timestamp:oneDayAgo uploadStatus:MPUploadStatusBatch userId:@1 dataPlanId:nil dataPlanVersion:nil]; + [instance.persistenceController saveMessage:message]; + XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 1); + + // Store a session + MPSession *session = [[MPSession alloc] initWithStartTime:oneDayAgo userId:[MPPersistenceController mpId]]; + session.endTime = oneDayAgo; + session.attributesDictionary = [@{@"key1":@"value1"} mutableCopy]; + [instance.persistenceController saveSession:session]; + XCTAssertEqual([instance.persistenceController fetchSessions].count, 1); + + // Store an upload + NSDictionary *uploadDictionary = @{kMPOptOutKey:@NO, kMPSessionTimeoutKey:@120, kMPUploadIntervalKey:@10, kMPLifeTimeValueKey:@0, kMPMessagesKey:@[[message dictionaryRepresentation]], kMPMessageIdKey:[[NSUUID UUID] UUIDString]}; + MPUpload *upload = [[MPUpload alloc] initWithSessionId:@(session.sessionId) uploadDictionary:uploadDictionary dataPlanId:nil dataPlanVersion:nil]; + upload.timestamp = oneDayAgo; + [instance.persistenceController saveUpload:upload]; + XCTAssertEqual([instance.persistenceController fetchUploads].count, 1); + + // Cleanup persistence using default 90 day limit + [instance.persistenceController deleteRecordsOlderThan:ninteyDaysAgo]; + + // Check that the records still exist + XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 1); + XCTAssertEqual([instance.persistenceController fetchSessions].count, 1); + XCTAssertEqual([instance.persistenceController fetchUploads].count, 1); +} + +- (void)testDeleteRecordsOlderThan_Delete { + NSTimeInterval sevenDaysAgo = [[NSDate date] timeIntervalSince1970] - 7*24*60*60; + NSTimeInterval twoDaysAgo = [[NSDate date] timeIntervalSince1970] - 2*24*60*60; + + // Store a message + MParticle *instance = [MParticle sharedInstance]; + NSLog(@"instance: %@", instance); + NSDictionary *messageDictionary = @{@"test": @"test"}; + NSData *messageData = [NSJSONSerialization dataWithJSONObject:messageDictionary options:0 error:nil]; + MPMessage *message = [[MPMessage alloc] initWithSessionId:@17 messageId:1 UUID:@"uuid" messageType:@"test" messageData:messageData timestamp:sevenDaysAgo uploadStatus:MPUploadStatusBatch userId:@1 dataPlanId:nil dataPlanVersion:nil]; + [instance.persistenceController saveMessage:message]; + XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 1); + + // Store a session + MPSession *session = [[MPSession alloc] initWithStartTime:sevenDaysAgo userId:[MPPersistenceController mpId]]; + session.endTime = sevenDaysAgo; + session.attributesDictionary = [@{@"key1":@"value1"} mutableCopy]; + [instance.persistenceController saveSession:session]; + XCTAssertEqual([instance.persistenceController fetchSessions].count, 1); + + // Store an upload + NSDictionary *uploadDictionary = @{kMPOptOutKey:@NO, kMPSessionTimeoutKey:@120, kMPUploadIntervalKey:@10, kMPLifeTimeValueKey:@0, kMPMessagesKey:@[[message dictionaryRepresentation]], kMPMessageIdKey:[[NSUUID UUID] UUIDString]}; + MPUpload *upload = [[MPUpload alloc] initWithSessionId:@(session.sessionId) uploadDictionary:uploadDictionary dataPlanId:nil dataPlanVersion:nil]; + upload.timestamp = sevenDaysAgo; + [instance.persistenceController saveUpload:upload]; + XCTAssertEqual([instance.persistenceController fetchUploads].count, 1); + + // Cleanup persistence using 2 day limit + [instance.persistenceController deleteRecordsOlderThan:twoDaysAgo]; + + // Check that the records no longer exist + XCTAssertEqual([instance.persistenceController fetchMessagesForUploading].count, 0); + XCTAssertEqual([instance.persistenceController fetchSessions].count, 0); + XCTAssertEqual([instance.persistenceController fetchUploads].count, 0); +} + @end diff --git a/mParticle-Apple-SDK/Include/mParticle.h b/mParticle-Apple-SDK/Include/mParticle.h index 6f301bdb..065992aa 100644 --- a/mParticle-Apple-SDK/Include/mParticle.h +++ b/mParticle-Apple-SDK/Include/mParticle.h @@ -385,6 +385,19 @@ Defaults to false. Prevents the eventsHost above from overwriting the alias endp */ @property (nonatomic, strong, readwrite, nullable) NSNumber *configMaxAgeSeconds; +/** + Set a maximum threshold for stored events, batches, and sessions, in seconds. + + By default, data is persisted for 90 days before being deleted to minimize data loss, however + this can lead to excessive storage usage on some users' devices. This is exacerbated if you log + a large number of events, or events with a lot of data (attributes, etc). + + You can set this to any value greater than 0 seconds, so if you have storage usage concerns, set a lower + value such as 48 hours or 1 week. Or alternatively, if you have data loss concerns, you can set this to an even + longer value than the default. + */ +@property (nonatomic, strong, nullable) NSNumber *persistenceMaxAgeSeconds; + /** Set an array of instances of kit (MPKitProtocol wrapped in MPSideloadedKit) objects to be "sideloaded". @@ -607,6 +620,12 @@ Defaults to false. Prevents the eventsHost above from overwriting the alias endp */ @property (nonatomic, readonly, nullable) NSNumber *configMaxAgeSeconds; +/** + Maximum threshold for stored events, batches, and sessions, in seconds. + @see MParticleOptions + */ +@property (nonatomic, readonly, nullable) NSNumber *persistenceMaxAgeSeconds; + #pragma mark - Initialization /** diff --git a/mParticle-Apple-SDK/MPBackendController.m b/mParticle-Apple-SDK/MPBackendController.m index a7f94224..c331f092 100644 --- a/mParticle-Apple-SDK/MPBackendController.m +++ b/mParticle-Apple-SDK/MPBackendController.m @@ -1317,12 +1317,6 @@ - (void)logCrash:(NSString *)message stackTrace:(NSString *)stackTrace plCrashR } - (void)logBaseEvent:(MPBaseEvent *)event completionHandler:(void (^)(MPBaseEvent *event, MPExecStatus execStatus))completionHandler { - if (![MPStateMachine canWriteMessagesToDB]) { - MPILogError(@"Not saving message for event to prevent excessive local database growth because API Key appears to be invalid based on server response"); - completionHandler(event, MPExecStatusFail); - return; - } - [MPListenerController.sharedInstance onAPICalled:_cmd parameter1:event]; if (event.shouldBeginSession) { @@ -1578,11 +1572,6 @@ - (void)startWithKey:(NSString *)apiKey secret:(NSString *)secret firstRun:(BOOL } - (void)saveMessage:(MPMessage *)message updateSession:(BOOL)updateSession { - if (![MPStateMachine canWriteMessagesToDB]) { - MPILogError(@"Not saving message for event to prevent excessive local database growth because API Key appears to be invalid based on server response"); - return; - } - NSTimeInterval lastEventTimestamp = message.timestamp ?: [[NSDate date] timeIntervalSince1970]; if (MPStateMachine.runningInBackground) { self.timeOfLastEventInBackground = lastEventTimestamp; @@ -2089,9 +2078,15 @@ - (void)endSessionIfTimedOut { - (void)cleanUp { NSTimeInterval currentTime = [[NSDate date] timeIntervalSince1970]; + [self cleanUp:currentTime]; +} + +- (void)cleanUp:(NSTimeInterval)currentTime { MPPersistenceController *persistence = [MParticle sharedInstance].persistenceController; if (nextCleanUpTime < currentTime) { - [persistence deleteRecordsOlderThan:(currentTime - NINETY_DAYS)]; + NSNumber *persistanceMaxAgeSeconds = [MParticle sharedInstance].persistenceMaxAgeSeconds; + NSTimeInterval maxAgeSeconds = persistanceMaxAgeSeconds == nil ? NINETY_DAYS : persistanceMaxAgeSeconds.doubleValue; + [persistence deleteRecordsOlderThan:(currentTime - maxAgeSeconds)]; nextCleanUpTime = currentTime + TWENTY_FOUR_HOURS; } [persistence purgeMemory]; diff --git a/mParticle-Apple-SDK/Network/MPNetworkCommunication.m b/mParticle-Apple-SDK/Network/MPNetworkCommunication.m index 7267adb8..2db738dd 100644 --- a/mParticle-Apple-SDK/Network/MPNetworkCommunication.m +++ b/mParticle-Apple-SDK/Network/MPNetworkCommunication.m @@ -403,15 +403,6 @@ - (NSNumber *)maxAgeForCache:(nonnull NSString *)cache { return maxAge; } -- (void)checkResponseCodeToDisableEventLogging:(NSInteger)responseCode { - if (responseCode == HTTPStatusCodeBadRequest || responseCode == HTTPStatusCodeUnauthorized || responseCode == HTTPStatusCodeForbidden) { - [MPStateMachine setCanWriteMessagesToDB:NO]; - MPILogError(@"API Key appears to be invalid based on server response, disabling event logging to prevent excessive local database growth"); - } else { - [MPStateMachine setCanWriteMessagesToDB:YES]; - } -} - #pragma mark Public methods - (NSObject *_Nonnull)makeConnector { if (MPNetworkCommunication.connectorFactory) { @@ -463,9 +454,7 @@ - (void)requestConfig:(nullable NSObject *)connector withCo NSInteger responseCode = [httpResponse statusCode]; MPILogVerbose(@"Config Response Code: %ld, Execution Time: %.2fms", (long)responseCode, ([[NSDate date] timeIntervalSince1970] - start) * 1000.0); - - [self checkResponseCodeToDisableEventLogging:responseCode]; - + if (responseCode == HTTPStatusCodeNotModified) { MPIUserDefaults *userDefaults = [MPIUserDefaults standardUserDefaults]; [userDefaults setConfiguration:[userDefaults getConfiguration] eTag:userDefaults[kMPHTTPETagHeaderKey] requestTimestamp:[[NSDate date] timeIntervalSince1970] currentAge:ageString maxAge:maxAge]; @@ -875,9 +864,7 @@ - (void)identityApiRequestWithURL:(NSURL*)url identityRequest:(MPIdentityHTTPBas MPILogVerbose(@"Identity response code: %ld", (long)responseCode); - - [self checkResponseCodeToDisableEventLogging:[httpResponse statusCode]]; - + if (success) { @try { NSError *serializationError = nil; diff --git a/mParticle-Apple-SDK/Persistence/MPPersistenceController.mm b/mParticle-Apple-SDK/Persistence/MPPersistenceController.mm index 8ca39b9c..d5b5072f 100644 --- a/mParticle-Apple-SDK/Persistence/MPPersistenceController.mm +++ b/mParticle-Apple-SDK/Persistence/MPPersistenceController.mm @@ -1557,10 +1557,6 @@ - (void)saveIntegrationAttributes:(nonnull MPIntegrationAttributes *)integration } - (void)saveMessage:(MPMessage *)message { - if (![MPStateMachine canWriteMessagesToDB]) { - MPILogError(@"Not saving message for event to prevent excessive local database growth because API Key appears to be invalid based on server response"); - return; - } if (!message.shouldUploadEvent) { MPILogDebug(@"Not saving message for event because shouldUploadEvent was set to NO, message id: %lld, type: %@", message.messageId, message.messageType); return; diff --git a/mParticle-Apple-SDK/Utils/MPStateMachine.h b/mParticle-Apple-SDK/Utils/MPStateMachine.h index c36157ce..83adc39a 100644 --- a/mParticle-Apple-SDK/Utils/MPStateMachine.h +++ b/mParticle-Apple-SDK/Utils/MPStateMachine.h @@ -66,8 +66,6 @@ + (BOOL)runningInBackground; + (void)setRunningInBackground:(BOOL)background; + (BOOL)isAppExtension; -+ (BOOL)canWriteMessagesToDB; -+ (void)setCanWriteMessagesToDB:(BOOL)canWriteMessagesToDB; - (void)configureCustomModules:(nullable NSArray *)customModuleSettings; - (void)configureRampPercentage:(nullable NSNumber *)rampPercentage; - (void)configureTriggers:(nullable NSDictionary *)triggerDictionary; diff --git a/mParticle-Apple-SDK/Utils/MPStateMachine.m b/mParticle-Apple-SDK/Utils/MPStateMachine.m index 203dcaa0..ed9f453f 100644 --- a/mParticle-Apple-SDK/Utils/MPStateMachine.m +++ b/mParticle-Apple-SDK/Utils/MPStateMachine.m @@ -33,7 +33,6 @@ static MPEnvironment runningEnvironment = MPEnvironmentAutoDetect; static BOOL runningInBackground = NO; -static BOOL _canWriteMessagesToDB = YES; @interface MParticle () + (dispatch_queue_t)messageQueue; @@ -348,18 +347,6 @@ + (BOOL)isAppExtension { #endif } -+ (BOOL)canWriteMessagesToDB { - @synchronized(self) { - return _canWriteMessagesToDB; - } -} - -+ (void)setCanWriteMessagesToDB:(BOOL)canWriteMessagesToDB { - @synchronized(self) { - _canWriteMessagesToDB = canWriteMessagesToDB; - } -} - #pragma mark Public accessors - (MPConsumerInfo *)consumerInfo { if (_consumerInfo) { diff --git a/mParticle-Apple-SDK/mParticle.m b/mParticle-Apple-SDK/mParticle.m index 49ce84e9..38254518 100644 --- a/mParticle-Apple-SDK/mParticle.m +++ b/mParticle-Apple-SDK/mParticle.m @@ -258,6 +258,14 @@ - (void)setConfigMaxAgeSeconds:(NSNumber *)configMaxAgeSeconds { } } +- (void)setPersistenceMaxAgeSeconds:(NSNumber *)persistenceMaxAgeSeconds { + if (persistenceMaxAgeSeconds != nil && [persistenceMaxAgeSeconds doubleValue] <= 0) { + MPILogWarning(@"Persistence Max Age must be a positive number, disregarding value."); + } else { + _persistenceMaxAgeSeconds = persistenceMaxAgeSeconds; + } +} + @end @interface MPBackendController () @@ -491,6 +499,10 @@ - (NSNumber *)configMaxAgeSeconds { return self.options.configMaxAgeSeconds; } +- (NSNumber *)persistenceMaxAgeSeconds { + return self.options.persistenceMaxAgeSeconds; +} + #pragma mark Initialization + (instancetype)sharedInstance { dispatch_once(&predicate, ^{