From 5ad06b80de91105d3e41036a179f42cbc9df9b4f Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Mon, 25 Nov 2024 16:53:23 +0100 Subject: [PATCH 1/7] mask emails with random strings instead of *** --- src/libs/ExportOnyxState/common.ts | 49 ++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 46a47528f1fe..6cf398df1921 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -4,6 +4,21 @@ import type {Session} from '@src/types/onyx'; const MASKING_PATTERN = '***'; +const emailMap = new Map(); + +const getRandomLetter = () => String.fromCharCode(97 + Math.floor(Math.random() * 26)); + +const randomizeEmail = (email: string): string => { + const [localPart, domain] = email.split('@'); + const [domainName, tld] = domain.split('.'); + + const randomizePart = (part: string) => [...part].map((c) => (/[a-zA-Z0-9]/.test(c) ? getRandomLetter() : c)).join(''); + const randomLocal = randomizePart(localPart); + const randomDomain = randomizePart(domainName); + + return `${randomLocal}@${randomDomain}.${tld}`; +}; + const maskSessionDetails = (data: Record): Record => { const session = data.session as Session; const maskedData: Record = {}; @@ -38,16 +53,38 @@ const maskFragileData = (data: Record | unknown[] | null, paren return; } - const value = data[key]; + // loginList is an object that contains emails as keys, the keys should be masked as well + let propertyName = ''; + if (Str.isValidEmail(key)) { + if (emailMap.has(key)) { + // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style + propertyName = emailMap.get(key) as string; + } else { + const maskedEmail = randomizeEmail(key); + propertyName = maskedEmail; + } + } else { + propertyName = key; + } + + const value = data[propertyName]; if (typeof value === 'string' && Str.isValidEmail(value)) { - maskedData[key] = MASKING_PATTERN; - } else if (parentKey && parentKey.includes(ONYXKEYS.COLLECTION.REPORT_ACTIONS) && (key === 'text' || key === 'html')) { - maskedData[key] = MASKING_PATTERN; + let maskedEmail = ''; + if (!emailMap.has(value)) { + maskedEmail = randomizeEmail(value); + emailMap.set(value, maskedEmail); + } else { + // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style + maskedEmail = emailMap.get(value) as string; + } + maskedData[propertyName] = maskedEmail; + } else if (parentKey && parentKey.includes(ONYXKEYS.COLLECTION.REPORT_ACTIONS) && (propertyName === 'text' || propertyName === 'html')) { + maskedData[propertyName] = MASKING_PATTERN; } else if (typeof value === 'object') { - maskedData[key] = maskFragileData(value as Record, key.includes(ONYXKEYS.COLLECTION.REPORT_ACTIONS) ? key : parentKey); + maskedData[propertyName] = maskFragileData(value as Record, propertyName.includes(ONYXKEYS.COLLECTION.REPORT_ACTIONS) ? propertyName : parentKey); } else { - maskedData[key] = value; + maskedData[propertyName] = value; } }); From 770c25ed63f56ab440896ebaaa57da50c89d1e07 Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Mon, 2 Dec 2024 13:51:16 +0100 Subject: [PATCH 2/7] mask email addresses in strings --- src/libs/ExportOnyxState/common.ts | 42 +++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 6cf398df1921..57d626e8f148 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -3,11 +3,21 @@ import ONYXKEYS from '@src/ONYXKEYS'; import type {Session} from '@src/types/onyx'; const MASKING_PATTERN = '***'; +const emailRegex = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/; const emailMap = new Map(); const getRandomLetter = () => String.fromCharCode(97 + Math.floor(Math.random() * 26)); +function stringContainsEmail(text: string) { + return emailRegex.test(text); +} + +function extractEmail(text: string) { + const match = text.match(emailRegex); + return match ? match[0] : null; // Return the email if found, otherwise null +} + const randomizeEmail = (email: string): string => { const [localPart, domain] = email.split('@'); const [domainName, tld] = domain.split('.'); @@ -19,6 +29,10 @@ const randomizeEmail = (email: string): string => { return `${randomLocal}@${randomDomain}.${tld}`; }; +function replaceEmailInString(text: string, emailReplacement: string) { + return text.replace(emailRegex, emailReplacement); +} + const maskSessionDetails = (data: Record): Record => { const session = data.session as Session; const maskedData: Record = {}; @@ -43,7 +57,20 @@ const maskFragileData = (data: Record | unknown[] | null, paren } if (Array.isArray(data)) { - return data.map((item): unknown => (typeof item === 'object' ? maskFragileData(item as Record, parentKey) : item)); + return data.map((item): unknown => { + if (typeof item === 'string' && Str.isValidEmail(item)) { + let maskedEmail = ''; + if (!emailMap.has(item)) { + maskedEmail = randomizeEmail(item); + emailMap.set(item, maskedEmail); + } else { + // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style + maskedEmail = emailMap.get(item) as string; + } + return maskedEmail; + } + return typeof item === 'object' ? maskFragileData(item as Record, parentKey) : item; + }); } const maskedData: Record = {}; @@ -79,6 +106,19 @@ const maskFragileData = (data: Record | unknown[] | null, paren maskedEmail = emailMap.get(value) as string; } maskedData[propertyName] = maskedEmail; + } else if (typeof value === 'string' && stringContainsEmail(value)) { + let maskedEmailString = value; + const email = extractEmail(value) ?? ''; + + if (!emailMap.has(email)) { + const randomEmail = randomizeEmail(email); + emailMap.set(email, randomEmail); + maskedEmailString = replaceEmailInString(value, randomEmail); + } else { + // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style + maskedEmailString = replaceEmailInString(value, emailMap.get(email) as string); + } + maskedData[propertyName] = maskedEmailString; } else if (parentKey && parentKey.includes(ONYXKEYS.COLLECTION.REPORT_ACTIONS) && (propertyName === 'text' || propertyName === 'html')) { maskedData[propertyName] = MASKING_PATTERN; } else if (typeof value === 'object') { From 0c64fe6eb633b1a7a3580930475f086ecda37b79 Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Mon, 2 Dec 2024 15:35:15 +0100 Subject: [PATCH 3/7] add tests for masking --- src/libs/ExportOnyxState/common.ts | 2 +- tests/unit/ExportOnyxStateTest.ts | 47 ++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 57d626e8f148..95469c4eb793 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -143,4 +143,4 @@ const maskOnyxState = (data: Record, isMaskingFragileDataEnable return onyxState; }; -export default {maskOnyxState}; +export {maskOnyxState, emailRegex}; diff --git a/tests/unit/ExportOnyxStateTest.ts b/tests/unit/ExportOnyxStateTest.ts index 70faa061cb2a..cb4e4b846218 100644 --- a/tests/unit/ExportOnyxStateTest.ts +++ b/tests/unit/ExportOnyxStateTest.ts @@ -1,4 +1,4 @@ -import ExportOnyxState from '@libs/ExportOnyxState/common'; +import * as ExportOnyxState from '@libs/ExportOnyxState/common'; import type * as OnyxTypes from '@src/types/onyx'; type ExampleOnyxState = { @@ -40,6 +40,49 @@ describe('maskOnyxState', () => { expect(result.session.authToken).toBe('***'); expect(result.session.encryptedAuthToken).toBe('***'); - expect(result.session.email).toBe('***'); + }); + + it('should mask emails as a string value in property with a random email', () => { + const input = { + session: mockSession, + }; + + const result = ExportOnyxState.maskOnyxState(input) as ExampleOnyxState; + + expect(result.session.email).toMatch(ExportOnyxState.emailRegex); + }); + + it('should mask array of emails with random emails', () => { + const input = { + session: mockSession, + emails: ['user@example.com', 'user2@example.com'], + }; + + const result = ExportOnyxState.maskOnyxState(input, true) as Record; + + expect(result.emails.at(0)).toMatch(ExportOnyxState.emailRegex); + expect(result.emails.at(1)).toMatch(ExportOnyxState.emailRegex); + }); + + it('should mask emails in keys of objects', () => { + const input = { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'user@example.com': 'value', + session: mockSession, + }; + + const result = ExportOnyxState.maskOnyxState(input, true) as Record; + + expect(Object.keys(result).at(0)).toMatch(ExportOnyxState.emailRegex); + }); + + it('should mask emails that are part of a string', () => { + const input = { + session: mockSession, + emailString: 'user@example.com is a test string', + }; + + const result = ExportOnyxState.maskOnyxState(input, true) as Record; + expect(result.emailString).not.toContain('user@example.com'); }); }); From a9efa5a07fefdf9302cac56a1258b7f18d90ea40 Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Wed, 4 Dec 2024 09:31:13 +0100 Subject: [PATCH 4/7] update ExportOnyxState imports --- src/libs/ExportOnyxState/index.native.ts | 2 +- src/libs/ExportOnyxState/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libs/ExportOnyxState/index.native.ts b/src/libs/ExportOnyxState/index.native.ts index 2ad9af0bf54c..eb1bb879c32d 100644 --- a/src/libs/ExportOnyxState/index.native.ts +++ b/src/libs/ExportOnyxState/index.native.ts @@ -2,7 +2,7 @@ import RNFS from 'react-native-fs'; import {open} from 'react-native-quick-sqlite'; import Share from 'react-native-share'; import CONST from '@src/CONST'; -import ExportOnyxState from './common'; +import * as ExportOnyxState from './common'; const readFromOnyxDatabase = () => new Promise((resolve) => { diff --git a/src/libs/ExportOnyxState/index.ts b/src/libs/ExportOnyxState/index.ts index 66fa6744ecdc..f04ce8d5c90b 100644 --- a/src/libs/ExportOnyxState/index.ts +++ b/src/libs/ExportOnyxState/index.ts @@ -1,5 +1,5 @@ import CONST from '@src/CONST'; -import ExportOnyxState from './common'; +import * as ExportOnyxState from './common'; const readFromOnyxDatabase = () => new Promise>((resolve) => { From c7235c85a30510d56fbb4e2343fac134e455bd01 Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Fri, 6 Dec 2024 09:00:58 +0100 Subject: [PATCH 5/7] reuse logic for masking email --- src/libs/ExportOnyxState/common.ts | 33 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 95469c4eb793..770254b2056d 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -51,6 +51,18 @@ const maskSessionDetails = (data: Record): Record { + let maskedEmail = ''; + if (!emailMap.has(email)) { + maskedEmail = randomizeEmail(email); + emailMap.set(email, maskedEmail); + } else { + // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style + maskedEmail = emailMap.get(email) as string; + } + return maskedEmail; +}; + const maskFragileData = (data: Record | unknown[] | null, parentKey?: string): Record | unknown[] | null => { if (data === null) { return data; @@ -59,15 +71,7 @@ const maskFragileData = (data: Record | unknown[] | null, paren if (Array.isArray(data)) { return data.map((item): unknown => { if (typeof item === 'string' && Str.isValidEmail(item)) { - let maskedEmail = ''; - if (!emailMap.has(item)) { - maskedEmail = randomizeEmail(item); - emailMap.set(item, maskedEmail); - } else { - // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style - maskedEmail = emailMap.get(item) as string; - } - return maskedEmail; + return maskEmail(item); } return typeof item === 'object' ? maskFragileData(item as Record, parentKey) : item; }); @@ -87,7 +91,7 @@ const maskFragileData = (data: Record | unknown[] | null, paren // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style propertyName = emailMap.get(key) as string; } else { - const maskedEmail = randomizeEmail(key); + const maskedEmail = maskEmail(key); propertyName = maskedEmail; } } else { @@ -97,14 +101,7 @@ const maskFragileData = (data: Record | unknown[] | null, paren const value = data[propertyName]; if (typeof value === 'string' && Str.isValidEmail(value)) { - let maskedEmail = ''; - if (!emailMap.has(value)) { - maskedEmail = randomizeEmail(value); - emailMap.set(value, maskedEmail); - } else { - // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style - maskedEmail = emailMap.get(value) as string; - } + const maskedEmail = maskEmail(value); maskedData[propertyName] = maskedEmail; } else if (typeof value === 'string' && stringContainsEmail(value)) { let maskedEmailString = value; From 2d7fe272f2e272f765f79699ef6cae8d9b3f32d0 Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Fri, 6 Dec 2024 09:03:14 +0100 Subject: [PATCH 6/7] clean map after masking --- src/libs/ExportOnyxState/common.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 770254b2056d..8b88391dad5c 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -137,6 +137,7 @@ const maskOnyxState = (data: Record, isMaskingFragileDataEnable onyxState = maskFragileData(onyxState) as Record; } + emailMap.clear(); return onyxState; }; From 836e8e8e35a60a0e3b1aeca85f988d540afde6dd Mon Sep 17 00:00:00 2001 From: Tomasz Misiukiewicz Date: Fri, 6 Dec 2024 09:46:28 +0100 Subject: [PATCH 7/7] cleanup implementation --- src/libs/ExportOnyxState/common.ts | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/libs/ExportOnyxState/common.ts b/src/libs/ExportOnyxState/common.ts index 8b88391dad5c..9cc38aad3346 100644 --- a/src/libs/ExportOnyxState/common.ts +++ b/src/libs/ExportOnyxState/common.ts @@ -87,13 +87,7 @@ const maskFragileData = (data: Record | unknown[] | null, paren // loginList is an object that contains emails as keys, the keys should be masked as well let propertyName = ''; if (Str.isValidEmail(key)) { - if (emailMap.has(key)) { - // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style - propertyName = emailMap.get(key) as string; - } else { - const maskedEmail = maskEmail(key); - propertyName = maskedEmail; - } + propertyName = maskEmail(key); } else { propertyName = key; } @@ -101,21 +95,9 @@ const maskFragileData = (data: Record | unknown[] | null, paren const value = data[propertyName]; if (typeof value === 'string' && Str.isValidEmail(value)) { - const maskedEmail = maskEmail(value); - maskedData[propertyName] = maskedEmail; + maskedData[propertyName] = maskEmail(value); } else if (typeof value === 'string' && stringContainsEmail(value)) { - let maskedEmailString = value; - const email = extractEmail(value) ?? ''; - - if (!emailMap.has(email)) { - const randomEmail = randomizeEmail(email); - emailMap.set(email, randomEmail); - maskedEmailString = replaceEmailInString(value, randomEmail); - } else { - // eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style - maskedEmailString = replaceEmailInString(value, emailMap.get(email) as string); - } - maskedData[propertyName] = maskedEmailString; + maskedData[propertyName] = replaceEmailInString(value, maskEmail(extractEmail(value) ?? '')); } else if (parentKey && parentKey.includes(ONYXKEYS.COLLECTION.REPORT_ACTIONS) && (propertyName === 'text' || propertyName === 'html')) { maskedData[propertyName] = MASKING_PATTERN; } else if (typeof value === 'object') {