diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fa8708570..edc898ed2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -37,6 +37,7 @@ highlighting the necessary changes) - If no such branch exists, then create one from the latest released branch. - [ ] If added a foreign key constraint on `app_id_to_user_id` table, make sure to delete from this table when deleting the user as well if `deleteUserIdMappingToo` is false. +- [ ] If added a new recipe, then make sure to update the bulk import API to include the new recipe. ## Remaining TODOs for this PR diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b0589d1..88a37174a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,46 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [9.4.0] + +### Added +- Adds property `bulk_migration_parallelism` for fine-tuning the worker threads number +- Adds APIs to bulk import users + - GET `/bulk-import/users` + - POST `/bulk-import/users` + - GET `/bulk-import/users/count` + - POST `/bulk-import/users/remove` + - POST `/bulk-import/users/import` + - POST `/bulk-import/backgroundjob` + - GET `/bulk-import/backgroundjob` +- Adds `ProcessBulkImportUsers` cron job to process bulk import users +- Adds multithreaded worker support for the `ProcessBulkImportUsers` cron job for faster bulk imports +- Adds support for lazy importing users + +### Migrations + +```sql +"CREATE TABLE IF NOT EXISTS bulk_import_users ( + id CHAR(36), + app_id VARCHAR(64) NOT NULL DEFAULT 'public', + primary_user_id VARCHAR(36), + raw_data TEXT NOT NULL, + status VARCHAR(128) DEFAULT 'NEW', + error_msg TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + CONSTRAINT bulk_import_users_pkey PRIMARY KEY(app_id, id), + CONSTRAINT bulk_import_users__app_id_fkey FOREIGN KEY(app_id) REFERENCES apps(app_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS bulk_import_users_status_updated_at_index ON bulk_import_users (app_id, status, updated_at); + +CREATE INDEX IF NOT EXISTS bulk_import_users_pagination_index1 ON bulk_import_users (app_id, status, created_at DESC, + id DESC); + +CREATE INDEX IF NOT EXISTS bulk_import_users_pagination_index2 ON bulk_import_users (app_id, created_at DESC, id DESC); +``` + ## [9.3.0] ### Changes diff --git a/build.gradle b/build.gradle index 1a462ca04..17e593dbc 100644 --- a/build.gradle +++ b/build.gradle @@ -19,8 +19,7 @@ compileTestJava { options.encoding = "UTF-8" } // } //} -version = "9.3.0" - +version = "9.4.0" repositories { mavenCentral() diff --git a/config.yaml b/config.yaml index 5f6a8f80f..b25281c0a 100644 --- a/config.yaml +++ b/config.yaml @@ -170,3 +170,7 @@ core_config_version: 0 # (Optional | Default: null) string value. The encryption key used for saving OAuth client secret on the database. # oauth_client_secret_encryption_key: + +# (DIFFERENT_ACROSS_APPS | OPTIONAL | Default: number of available processor cores) int value. If specified, +# the supertokens core will use the specified number of threads to complete the migration of users. +# bulk_migration_parallelism: diff --git a/devConfig.yaml b/devConfig.yaml index 9557ada23..89ae068f5 100644 --- a/devConfig.yaml +++ b/devConfig.yaml @@ -170,3 +170,8 @@ disable_telemetry: true # (Optional | Default: null) string value. The encryption key used for saving OAuth client secret on the database. # oauth_client_secret_encryption_key: + +# (DIFFERENT_ACROSS_APPS | OPTIONAL | Default: number of available processor cores) int value. If specified, +# the supertokens core will use the specified number of threads to complete the migration of users. +# bulk_migration_parallelism: + diff --git a/src/main/java/io/supertokens/Main.java b/src/main/java/io/supertokens/Main.java index 2681a3743..f30dc5ca0 100644 --- a/src/main/java/io/supertokens/Main.java +++ b/src/main/java/io/supertokens/Main.java @@ -20,6 +20,7 @@ import io.supertokens.config.Config; import io.supertokens.config.CoreConfig; import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.cronjobs.cleanupOAuthSessionsAndChallenges.CleanupOAuthSessionsAndChallenges; import io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys; import io.supertokens.cronjobs.deleteExpiredDashboardSessions.DeleteExpiredDashboardSessions; @@ -61,6 +62,8 @@ public class Main { // this is a special variable that will be set to true by TestingProcessManager public static boolean isTesting = false; + // this flag is used in ProcessBulkImportUsersCronJobTest to skip the user validation + public static boolean isTesting_skipBulkImportUserValidationInCronJob = false; // this is a special variable that will be set to true by TestingProcessManager public static boolean makeConsolePrintSilent = false; @@ -257,6 +260,9 @@ private void init() throws IOException, StorageQueryException { // starts DeleteExpiredAccessTokenSigningKeys cronjob if the access token signing keys can change Cronjobs.addCronjob(this, DeleteExpiredAccessTokenSigningKeys.init(this, uniqueUserPoolIdsTenants)); + // initializes ProcessBulkImportUsers cronjob to process bulk import users + Cronjobs.addCronjob(this, ProcessBulkImportUsers.init(this, uniqueUserPoolIdsTenants)); + Cronjobs.addCronjob(this, CleanupOAuthSessionsAndChallenges.init(this, uniqueUserPoolIdsTenants)); // this is to ensure tenantInfos are in sync for the new cron job as well diff --git a/src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java b/src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java new file mode 100644 index 000000000..0daeedf96 --- /dev/null +++ b/src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens; + +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.useridmapping.UserIdMapping; + +public class StorageAndUserIdMappingForBulkImport extends StorageAndUserIdMapping { + + public String userIdInQuestion; + + public StorageAndUserIdMappingForBulkImport(Storage storage, + UserIdMapping userIdMapping, String userIdInQuestion) { + super(storage, userIdMapping); + this.userIdInQuestion = userIdInQuestion; + } +} diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 6032ba225..1b0da61a8 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -21,14 +21,15 @@ import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; -import io.supertokens.featureflag.EE_FEATURES; -import io.supertokens.featureflag.FeatureFlag; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.pluginInterface.*; +import io.supertokens.pluginInterface.RECIPE_ID; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.dashboard.DashboardSearchTags; import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -42,10 +43,12 @@ import io.supertokens.session.Session; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.useridmapping.UserIdType; +import io.supertokens.utils.Utils; import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; import java.util.*; +import java.util.stream.Collectors; /*This files contains functions that are common for all auth recipes*/ @@ -129,6 +132,18 @@ public static AuthRecipeUserInfo getUserById(AppIdentifier appIdentifier, Storag return StorageUtils.getAuthRecipeStorage(storage).getPrimaryUserById(appIdentifier, userId); } + public static List getUsersById(AppIdentifier appIdentifier, Storage storage, List userIds) + throws StorageQueryException { + AuthRecipeSQLStorage authStorage = StorageUtils.getAuthRecipeStorage(storage); + try { + return authStorage.startTransaction(con -> { + return authStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, userIds); + }); + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e); + } + } + public static class CreatePrimaryUserResult { public AuthRecipeUserInfo user; public boolean wasAlreadyAPrimaryUser; @@ -139,10 +154,21 @@ public CreatePrimaryUserResult(AuthRecipeUserInfo user, boolean wasAlreadyAPrima } } + public static class CreatePrimaryUserBulkResult { + public AuthRecipeUserInfo user; + public boolean wasAlreadyAPrimaryUser; + public Exception error; + + public CreatePrimaryUserBulkResult(AuthRecipeUserInfo user, boolean wasAlreadyAPrimaryUser, Exception error) { + this.user = user; + this.wasAlreadyAPrimaryUser = wasAlreadyAPrimaryUser; + this.error = error; + } + } + public static class CanLinkAccountsResult { public String recipeUserId; public String primaryUserId; - public boolean alreadyLinked; public CanLinkAccountsResult(String recipeUserId, String primaryUserId, boolean alreadyLinked) { @@ -152,6 +178,23 @@ public CanLinkAccountsResult(String recipeUserId, String primaryUserId, boolean } } + public static class CanLinkAccountsBulkResult { + public String recipeUserId; + public String primaryUserId; + public Exception error; + public AuthRecipeUserInfo authRecipeUserInfo; + public boolean alreadyLinked; + + public CanLinkAccountsBulkResult(String recipeUserId, String primaryUserId, boolean alreadyLinked, Exception error, + AuthRecipeUserInfo authRecipeUserInfo) { + this.recipeUserId = recipeUserId; + this.primaryUserId = primaryUserId; + this.alreadyLinked = alreadyLinked; + this.error = error; + this.authRecipeUserInfo = authRecipeUserInfo; + } + } + @TestOnly public static CanLinkAccountsResult canLinkAccounts(Main main, String recipeUserId, String primaryUserId) throws StorageQueryException, UnknownUserIdException, InputUserIdIsNotAPrimaryUserException, @@ -248,10 +291,77 @@ private static CanLinkAccountsResult canLinkAccountsHelper(TransactionConnection return new CanLinkAccountsResult(recipeUser.getSupertokensUserId(), primaryUser.getSupertokensUserId(), false); } + private static List canLinkMultipleAccountsHelper(TransactionConnection con, + AppIdentifier appIdentifier, + Storage storage, + Map recipeUserIdByPrimaryUserId, + List allDistinctEmailAddresses, + List phones, + Map thirdpartyUserIdToId) + throws StorageQueryException { + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + + List results = new ArrayList<>(); + + List primaryUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, + new ArrayList<>(recipeUserIdByPrimaryUserId.values())); + + List recipeUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, + new ArrayList<>(recipeUserIdByPrimaryUserId.keySet())); + + List allUsersWithExtraData = + List.of(authRecipeStorage.listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction + (appIdentifier, con, allDistinctEmailAddresses, phones, thirdpartyUserIdToId)); + + if(recipeUsers != null && primaryUsers != null) { + //collect all the really primary users into a map of userid -> authRecipeUserInfo + Map foundValidPrimaryUsers = primaryUsers.stream().filter(authRecipeUserInfo -> authRecipeUserInfo.isPrimaryUser).collect(Collectors.toMap(AuthRecipeUserInfo::getSupertokensUserId, authRecipeUserInfo -> authRecipeUserInfo)); + Map foundRecipeUsers = recipeUsers.stream().collect(Collectors.toMap(AuthRecipeUserInfo::getSupertokensUserId, authRecipeUserInfo -> authRecipeUserInfo)); + + for(Map.Entry recipeUserByPrimaryUser : recipeUserIdByPrimaryUserId.entrySet()) { + String recipeUserId = recipeUserByPrimaryUser.getKey(); + String primaryUserId = recipeUserByPrimaryUser.getValue(); + AuthRecipeUserInfo primaryUser = foundValidPrimaryUsers.get(primaryUserId); + AuthRecipeUserInfo recipeUser = foundRecipeUsers.get(recipeUserId); + if(primaryUser == null || recipeUser == null) { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, new UnknownUserIdException(), null)); + } else if(recipeUser.isPrimaryUser) { + if (recipeUser.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, true, null, null)); + } else { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, new RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException(recipeUser, "The input recipe user ID is already linked to another user ID"), null)); + } + } else { + if (recipeUser.loginMethods.length == 1) { + Set tenantIds = new HashSet<>(); + tenantIds.addAll(recipeUser.tenantIds); + tenantIds.addAll(primaryUser.tenantIds); + + try { + bulkCheckIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, + recipeUser.loginMethods[0], primaryUser, allUsersWithExtraData); + + for (LoginMethod currLoginMethod : primaryUser.loginMethods) { + bulkCheckIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, + currLoginMethod, primaryUser, allUsersWithExtraData); + } + + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, null, primaryUser)); + + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException exception) { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, exception, null)); + } + } + } + } + } + return results; + } + private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection con, AppIdentifier appIdentifier, - AuthRecipeSQLStorage authRecipeStorage, - Set tenantIds, LoginMethod currLoginMethod, - AuthRecipeUserInfo primaryUser) + AuthRecipeSQLStorage authRecipeStorage, + Set tenantIds, LoginMethod currLoginMethod, + AuthRecipeUserInfo primaryUser) throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException { // we loop through the union of both the user's tenantIds and check that the criteria for // linking accounts is not violated in any of them. We do a union and not an intersection @@ -269,9 +379,8 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection // tenants of the same storage - therefore, the storage will be the same. if (currLoginMethod.email != null) { - AuthRecipeUserInfo[] usersWithSameEmail = authRecipeStorage - .listPrimaryUsersByEmail_Transaction(appIdentifier, con, - currLoginMethod.email); + AuthRecipeUserInfo[] usersWithSameEmail = + authRecipeStorage.listPrimaryUsersByEmail_Transaction(appIdentifier, con, currLoginMethod.email); for (AuthRecipeUserInfo user : usersWithSameEmail) { if (!user.tenantIds.contains(tenantId)) { continue; @@ -285,8 +394,8 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection } if (currLoginMethod.phoneNumber != null) { - AuthRecipeUserInfo[] usersWithSamePhoneNumber = authRecipeStorage - .listPrimaryUsersByPhoneNumber_Transaction(appIdentifier, con, + AuthRecipeUserInfo[] usersWithSamePhoneNumber = + authRecipeStorage.listPrimaryUsersByPhoneNumber_Transaction(appIdentifier, con, currLoginMethod.phoneNumber); for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { if (!user.tenantIds.contains(tenantId)) { @@ -315,9 +424,88 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection userWithSameThirdParty.getSupertokensUserId(), "This user's third party login is already associated with another" + " user ID"); + + } + } + } + } + } + + private static void bulkCheckIfLoginMethodCanBeLinkedOnTenant(TransactionConnection con, AppIdentifier appIdentifier, + AuthRecipeSQLStorage authRecipeStorage, + Set tenantIds, LoginMethod currLoginMethod, + AuthRecipeUserInfo primaryUser, + List allUsersWithExtraData) + throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException { + // we loop through the union of both the user's tenantIds and check that the criteria for + // linking accounts is not violated in any of them. We do a union and not an intersection + // cause if we did an intersection, and that yields that account linking is allowed, it could + // result in one tenant having two primary users with the same email. For example: + // - tenant1 has u1 with email e, and u2 with email e, primary user (one is ep, one is tp) + // - tenant2 has u3 with email e, primary user (passwordless) + // now if we want to link u3 with u1, we have to deny it cause if we don't, it will result in + // u1 and u2 to be primary users with the same email in the same tenant. If we do an + // intersection, we will get an empty set, but if we do a union, we will get both the tenants and + // do the checks in both. + for (String tenantId : tenantIds) { + // we do not bother with getting the storage for each tenant here because + // we get the tenants from the user itself, and the user can only be shared across + // tenants of the same storage - therefore, the storage will be the same. + + if (currLoginMethod.email != null) { + List usersWithSameEmail = + allUsersWithExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod -> loginMethod.email).collect( + Collectors.toList()).contains(currLoginMethod.email)).collect(Collectors.toList()); + for (AuthRecipeUserInfo user : usersWithSameEmail) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser && !user.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's email is already associated with another user ID"); + } + } + } + + if (currLoginMethod.phoneNumber != null) { + List usersWithSamePhoneNumber = + allUsersWithExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod -> loginMethod.phoneNumber).collect( + Collectors.toList()).contains(currLoginMethod.phoneNumber)).collect(Collectors.toList()); + for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser && !user.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's phone number is already associated with another user" + + " ID"); } } + } + if (currLoginMethod.thirdParty != null) { + List extraUsersWithThirdParty = allUsersWithExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).anyMatch(loginMethod1 -> loginMethod1.thirdParty != null)).collect(Collectors.toList()); + for(AuthRecipeUserInfo extraUser : extraUsersWithThirdParty) { + if(extraUser.isPrimaryUser && extraUser.tenantIds.contains(tenantId) + && !extraUser.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + for (LoginMethod loginMethodExtra : extraUser.loginMethods) { + if (loginMethodExtra.thirdParty != null && + loginMethodExtra.thirdParty.userId.equals(currLoginMethod.thirdParty.userId) + && loginMethodExtra.thirdParty.id.equals(currLoginMethod.thirdParty.id)) { + + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + extraUser.getSupertokensUserId(), + "This user's third party login is already associated with another" + + " user ID"); + } + } + } + } } } } @@ -343,8 +531,7 @@ public static LinkAccountsResult linkAccounts(Main main, AppIdentifier appIdenti RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException, InputUserIdIsNotAPrimaryUserException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } @@ -401,6 +588,73 @@ public static LinkAccountsResult linkAccounts(Main main, AppIdentifier appIdenti } } + public static List linkMultipleAccounts(Main main, AppIdentifier appIdentifier, + Storage storage, Map recipeUserIdToPrimaryUserId, + List allDistinctEmailAddresses, List allDistinctPhones, + Map allThirdpartyUserIdsToThirdpartyIds) + throws StorageQueryException, TenantOrAppNotFoundException, FeatureNotEnabledException { + + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + throw new FeatureNotEnabledException( + "Account linking feature is not enabled for this app. Please contact support to enable it."); + } + + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + Map errorByUserId = new HashMap<>(); + try { + + List linkAccountsResults = authRecipeStorage.startTransaction(con -> { + List canLinkAccounts = canLinkMultipleAccountsHelper(con, appIdentifier, + authRecipeStorage, recipeUserIdToPrimaryUserId, allDistinctEmailAddresses, allDistinctPhones, + allThirdpartyUserIdsToThirdpartyIds); + List results = new ArrayList<>(); + Map recipeUserByPrimaryUserNeedsLinking = new HashMap<>(); + if(!canLinkAccounts.isEmpty()){ + for(CanLinkAccountsBulkResult canLinkAccountsBulkResult : canLinkAccounts) { + if (canLinkAccountsBulkResult.alreadyLinked) { + results.add(new LinkAccountsBulkResult( + canLinkAccountsBulkResult.authRecipeUserInfo, true, null)); + } else if(canLinkAccountsBulkResult.error != null) { + results.add(new LinkAccountsBulkResult( + canLinkAccountsBulkResult.authRecipeUserInfo, false, canLinkAccountsBulkResult.error)); // preparing to return the error + errorByUserId.put(canLinkAccountsBulkResult.recipeUserId, canLinkAccountsBulkResult.error); + } else { + recipeUserByPrimaryUserNeedsLinking.put(canLinkAccountsBulkResult.recipeUserId, canLinkAccountsBulkResult.primaryUserId); + } + } + // link the remaining + authRecipeStorage.linkMultipleAccounts_Transaction(appIdentifier, con, recipeUserByPrimaryUserNeedsLinking); + List linkedPrimaryUsers = getUsersById(appIdentifier, authRecipeStorage, new ArrayList<>(recipeUserByPrimaryUserNeedsLinking.values())); + + for(AuthRecipeUserInfo linkedUser : linkedPrimaryUsers){ + results.add(new LinkAccountsBulkResult(linkedUser, false, null)); + } + + authRecipeStorage.commitTransaction(con); + } + if(!errorByUserId.isEmpty()) { + throw new StorageQueryException(new BulkImportBatchInsertException("link accounts errors", errorByUserId)); + } + return results; + }); + + for(LinkAccountsBulkResult result : linkAccountsResults) { + if (!result.wasAlreadyLinked) { + io.supertokens.pluginInterface.useridmapping.UserIdMapping mappingResult = + io.supertokens.useridmapping.UserIdMapping.getUserIdMapping( + appIdentifier, authRecipeStorage, + result.user.getSupertokensUserId(), UserIdType.SUPERTOKENS); + // finally, we revoke all sessions of the recipeUser Id cause their user ID has changed. + Session.revokeAllSessionsForUser(main, appIdentifier, authRecipeStorage, + mappingResult == null ? result.user.getSupertokensUserId() : mappingResult.externalUserId, false); + } + } + return linkAccountsResults; + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e); + } + } + public static class LinkAccountsResult { public final AuthRecipeUserInfo user; public final boolean wasAlreadyLinked; @@ -411,6 +665,18 @@ public LinkAccountsResult(AuthRecipeUserInfo user, boolean wasAlreadyLinked) { } } + public static class LinkAccountsBulkResult { + public final AuthRecipeUserInfo user; + public final boolean wasAlreadyLinked; + public final Exception error; + + public LinkAccountsBulkResult(AuthRecipeUserInfo user, boolean wasAlreadyLinked, Exception error) { + this.user = user; + this.wasAlreadyLinked = wasAlreadyLinked; + this.error = error; + } + } + @TestOnly public static CreatePrimaryUserResult canCreatePrimaryUser(Main main, String recipeUserId) @@ -528,9 +794,124 @@ private static CreatePrimaryUserResult canCreatePrimaryUserHelper(TransactionCon } } + + return new CreatePrimaryUserResult(targetUser, false); } + private static List canCreatePrimaryUsersHelper(TransactionConnection con, + AppIdentifier appIdentifier, + Storage storage, + List recipeUserIds, + List allDistinctEmails, + List allPhones, + Map thirdpartyUserIdToThirdpartyId) + throws StorageQueryException, UnknownUserIdException{ + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + List targetUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, + recipeUserIds); + if (targetUsers == null || targetUsers.isEmpty()) { + throw new UnknownUserIdException(); + } + List results = new ArrayList<>(); + List allUsersWithProvidedExtraData = + List.of(authRecipeStorage. + listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction(appIdentifier, con, + allDistinctEmails, allPhones, thirdpartyUserIdToThirdpartyId)); + + for(int i = 0; i < targetUsers.size(); i++) { + AuthRecipeUserInfo targetUser = targetUsers.get(i); + if (targetUser.isPrimaryUser) { + if (targetUser.getSupertokensUserId() + .equals(recipeUserIds.get(i))) { + results.add(new CreatePrimaryUserBulkResult(targetUser, true, null)); + } else { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new RecipeUserIdAlreadyLinkedWithPrimaryUserIdException(targetUser.getSupertokensUserId(), + "This user ID is already linked to another user ID"))); + continue; + } + } + + + // this means that the user has only one login method since it's not a primary user + // nor is it linked to a primary user + assert (targetUser.loginMethods.length == 1); + LoginMethod loginMethod = targetUser.loginMethods[0]; + boolean errorFound = false; + + for (String tenantId : targetUser.tenantIds) { + if (loginMethod.email != null) { + List usersWithSameEmail = allUsersWithProvidedExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod1 -> loginMethod1.email).collect(Collectors.toList()).contains(loginMethod.email)).collect( + Collectors.toList()); + for (AuthRecipeUserInfo user : usersWithSameEmail) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser) { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's email is already associated with another user ID"))); + errorFound = true; + break; + } + } + } + + if (loginMethod.phoneNumber != null) { + List usersWithSamePhoneNumber = allUsersWithProvidedExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod1 -> loginMethod1.phoneNumber).collect(Collectors.toList()).contains(loginMethod.phoneNumber)).collect( + Collectors.toList()); + for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser) { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's phone number is already associated with another user" + + " ID"))); + errorFound = true; + break; + } + } + } + + if (loginMethod.thirdParty != null) { + List extraUsersWithThirdParty = allUsersWithProvidedExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).anyMatch(loginMethod1 -> loginMethod1.thirdParty != null)).collect(Collectors.toList()); + for(AuthRecipeUserInfo extraUser : extraUsersWithThirdParty) { + if(extraUser.isPrimaryUser && extraUser.tenantIds.contains(tenantId)) { + for (LoginMethod loginMethodExtra : extraUser.loginMethods) { + if (loginMethodExtra.thirdParty != null && + loginMethodExtra.thirdParty.userId.equals(loginMethod.thirdParty.userId) + && loginMethodExtra.thirdParty.id.equals(loginMethod.thirdParty.id)) { + + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + extraUser.getSupertokensUserId(), + "This user's third party login is already associated with another" + + " user ID"))); + errorFound = true; + break; + } + } + } + } + } + + if(!errorFound){ + results.add(new CreatePrimaryUserBulkResult(targetUser, false, null)); + } + } + } + return results; + } + + @TestOnly public static CreatePrimaryUserResult createPrimaryUser(Main main, String recipeUserId) @@ -552,8 +933,7 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } @@ -563,7 +943,7 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, return authRecipeStorage.startTransaction(con -> { try { - CreatePrimaryUserResult result = canCreatePrimaryUserHelper(con, appIdentifier, authRecipeStorage, + CreatePrimaryUserResult result = canCreatePrimaryUserHelper(con, appIdentifier, authRecipeStorage, recipeUserId); if (result.wasAlreadyAPrimaryUser) { return result; @@ -593,6 +973,71 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, } } + public static List createPrimaryUsers(Main main, + AppIdentifier appIdentifier, + Storage storage, + List recipeUserIds, + List allDistinctEmails, + List allDistinctPhones, + Map thirdpartyUserIdsToThirdpartyIds) + throws StorageQueryException, TenantOrAppNotFoundException, + FeatureNotEnabledException { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + throw new FeatureNotEnabledException( + "Account linking feature is not enabled for this app. Please contact support to enable it."); + } + + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + Map errorsByUserId = new HashMap<>(); + try { + return authRecipeStorage.startTransaction(con -> { + + try { + List results = canCreatePrimaryUsersHelper(con, appIdentifier, authRecipeStorage, + recipeUserIds, allDistinctEmails, allDistinctPhones, thirdpartyUserIdsToThirdpartyIds); + List canMakePrimaryUsers = new ArrayList<>(); + for(CreatePrimaryUserBulkResult result : results) { + if (result.wasAlreadyAPrimaryUser) { + continue; + } + if(result.error != null) { + errorsByUserId.put(result.user.getSupertokensUserId(), result.error); + continue; + } + canMakePrimaryUsers.add(result); + } + authRecipeStorage.makePrimaryUsers_Transaction(appIdentifier, con, + canMakePrimaryUsers.stream().map(canMakePrimaryUser -> canMakePrimaryUser.user.getSupertokensUserId()).collect( + Collectors.toList())); + + authRecipeStorage.commitTransaction(con); + + for(CreatePrimaryUserBulkResult result : results) { + if (result.wasAlreadyAPrimaryUser) { + continue; + } + if(result.error != null) { + errorsByUserId.put(result.user.getSupertokensUserId(), result.error); + continue; + } + result.user.isPrimaryUser = true; + } + + if(!errorsByUserId.isEmpty()) { + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("create primary users errors", errorsByUserId)); + } + + return results; + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(e); + } + }); + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e.actualException); + } + } + + public static AuthRecipeUserInfo[] getUsersByAccountInfo(TenantIdentifier tenantIdentifier, Storage storage, boolean doUnionOfAccountInfo, String email, diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java new file mode 100644 index 000000000..07dd6b912 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -0,0 +1,818 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import com.google.gson.JsonObject; +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; +import io.supertokens.config.Config; +import io.supertokens.emailpassword.EmailPassword; +import io.supertokens.emailpassword.PasswordHashing; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; +import io.supertokens.passwordless.Passwordless; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.ImportUserBase; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; +import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.passwordless.PasswordlessImportUser; +import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; +import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; +import io.supertokens.pluginInterface.totp.TOTPDevice; +import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; +import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; +import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.ThirdParty; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; +import io.supertokens.utils.Utils; +import jakarta.servlet.ServletException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +// Error codes ensure globally unique and identifiable errors in Bulk Import. +// Current range: E001 to E046. + +public class BulkImport { + + // Maximum number of users that can be added in a single /bulk-import/users POST request + public static final int MAX_USERS_TO_ADD = 10000; + // Maximum number of users to return in a single page when calling /bulk-import/users GET + public static final int GET_USERS_PAGINATION_MAX_LIMIT = 500; + // Default number of users to return when no specific limit is given in /bulk-import/users GET + public static final int GET_USERS_DEFAULT_LIMIT = 100; + // Maximum number of users that can be deleted in a single operation + public static final int DELETE_USERS_MAX_LIMIT = 500; + // Number of users to process in a single batch of ProcessBulkImportUsers Cron Job + public static final int PROCESS_USERS_BATCH_SIZE = 8000; + // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job + public static final int PROCESS_USERS_INTERVAL_SECONDS = 5*60; // 5 minutes + private static final Logger log = LoggerFactory.getLogger(BulkImport.class); + + // This map allows reusing proxy storage for all tenants in the app and closing connections after import. + private static Map userPoolToStorageMap = new HashMap<>(); + + public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) + throws StorageQueryException, TenantOrAppNotFoundException { + while (true) { + try { + StorageUtils.getBulkImportStorage(storage).addBulkImportUsers(appIdentifier, users); + break; + } catch (io.supertokens.pluginInterface.bulkimport.exceptions.DuplicateUserIdException ignored) { + // We re-generate the user id for every user and retry + for (BulkImportUser user : users) { + user.id = Utils.getUUID(); + } + } + } + } + + public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdentifier, Storage storage, + int limit, @Nullable BULK_IMPORT_USER_STATUS status, @Nullable String paginationToken) + throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException { + List users; + + BulkImportSQLStorage bulkImportStorage = StorageUtils.getBulkImportStorage(storage); + + if (paginationToken == null) { + users = bulkImportStorage + .getBulkImportUsers(appIdentifier, limit + 1, status, null, null); + } else { + BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); + users = bulkImportStorage + .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, + tokenInfo.createdAt); + } + + String nextPaginationToken = null; + int maxLoop = users.size(); + if (users.size() == limit + 1) { + maxLoop = limit; + BulkImportUser user = users.get(limit); + nextPaginationToken = new BulkImportUserPaginationToken(user.id, user.createdAt).generateToken(); + } + + List resultUsers = users.subList(0, maxLoop); + return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); + } + + public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) + throws StorageQueryException { + return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); + } + + public static long getBulkImportUsersCount(AppIdentifier appIdentifier, Storage storage, + @Nullable BULK_IMPORT_USER_STATUS status) + throws StorageQueryException { + return StorageUtils.getBulkImportStorage(storage).getBulkImportUsersCount(appIdentifier, status); + } + + public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifier appIdentifier, + BulkImportUser user) + throws StorageQueryException, InvalidConfigException, IOException, TenantOrAppNotFoundException, + DbInitException, BulkImportBatchInsertException { + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(main, firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + try { + return bulkImportProxyStorage.startTransaction(con -> { + try { + Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + + processUsersImportSteps(main, appIdentifier, bulkImportProxyStorage, List.of(user), allStoragesForApp); + + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + + AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, + primaryLM.superTokensUserId); + io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, + bulkImportProxyStorage, new AuthRecipeUserInfo[] { importedUser }); + + return importedUser; + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + throw e; + } finally { + closeAllProxyStorages(); + } + }); + } catch (StorageTransactionLogicException e) { + if(e.actualException instanceof BulkImportBatchInsertException){ + throw (BulkImportBatchInsertException) e.actualException; + } + throw new StorageQueryException(e.actualException); + } + } + + public static void processUsersImportSteps(Main main, AppIdentifier appIdentifier, + Storage bulkImportProxyStorage, List users, Storage[] allStoragesForApp) + throws StorageTransactionLogicException { + try { + processUsersLoginMethods(main, appIdentifier, bulkImportProxyStorage, users); + createPrimaryUsersAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, users); + createMultipleUserIdMapping(appIdentifier, users, allStoragesForApp); + verifyMultipleEmailForAllLoginMethods(appIdentifier, bulkImportProxyStorage, users); + createMultipleTotpDevices(main, appIdentifier, bulkImportProxyStorage, users); + createMultipleUserMetadata(appIdentifier, bulkImportProxyStorage, users); + createMultipleUserRoles(main, appIdentifier, bulkImportProxyStorage, users); + } catch ( StorageQueryException | FeatureNotEnabledException | + TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + } + + public static void processUsersLoginMethods(Main main, AppIdentifier appIdentifier, Storage storage, + List users) throws StorageTransactionLogicException { + //sort login methods together + Map> sortedLoginMethods = new HashMap<>(); + for (BulkImportUser user: users) { + for(LoginMethod loginMethod : user.loginMethods){ + if(!sortedLoginMethods.containsKey(loginMethod.recipeId)) { + sortedLoginMethods.put(loginMethod.recipeId, new ArrayList<>()); + } + sortedLoginMethods.get(loginMethod.recipeId).add(loginMethod); + } + } + + List importedUsers = new ArrayList<>(); + if (sortedLoginMethods.containsKey("emailpassword")) { + importedUsers.addAll( + processEmailPasswordLoginMethods(main, storage, sortedLoginMethods.get("emailpassword"), + appIdentifier)); + } + if (sortedLoginMethods.containsKey("thirdparty")) { + importedUsers.addAll( + processThirdpartyLoginMethods(main, storage, sortedLoginMethods.get("thirdparty"), + appIdentifier)); + } + if (sortedLoginMethods.containsKey("passwordless")) { + importedUsers.addAll(processPasswordlessLoginMethods(main, appIdentifier, storage, + sortedLoginMethods.get("passwordless"))); + } + Set actualKeys = new HashSet<>(sortedLoginMethods.keySet()); + List.of("emailpassword", "thirdparty", "passwordless").forEach(actualKeys::remove); + if(!actualKeys.isEmpty()){ + throw new StorageTransactionLogicException( + new IllegalArgumentException("E001: Unknown recipeId(s) [" + + actualKeys.stream().map(s -> s+" ") + "] for loginMethod.")); + } + + Map errorsById = new HashMap<>(); + for (Map.Entry> loginMethodEntries : sortedLoginMethods.entrySet()) { + for (LoginMethod loginMethod : loginMethodEntries.getValue()) { + try { + associateUserToTenants(main, appIdentifier, storage, loginMethod, loginMethod.tenantIds.get(0)); + } catch (StorageTransactionLogicException e){ + errorsById.put(loginMethod.superTokensUserId, e.actualException); + } + } + } + if(!errorsById.isEmpty()){ + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("tenant association errors", errorsById)); + } + } + + private static List processPasswordlessLoginMethods(Main main, AppIdentifier appIdentifier, Storage storage, + List loginMethods) + throws StorageTransactionLogicException { + try { + List usersToImport = new ArrayList<>(); + for (LoginMethod loginMethod : loginMethods) { + String userId = Utils.getUUID(); + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), loginMethod.tenantIds.get( + 0)); // the cron runs per app. The app stays the same, the tenant can change + usersToImport.add(new PasswordlessImportUser(userId, loginMethod.phoneNumber, + loginMethod.email, tenantIdentifierForLoginMethod, loginMethod.timeJoinedInMSSinceEpoch)); + loginMethod.superTokensUserId = userId; + } + + Passwordless.createPasswordlessUsers(storage, usersToImport); + + return usersToImport; + } catch (StorageQueryException | StorageTransactionLogicException e) { + if (e.getCause() instanceof BulkImportBatchInsertException) { + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof DuplicateEmailException) { + String message = "E006: A user with email " + + loginMethods.stream() + .filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get().email + " already exists in passwordless loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof DuplicatePhoneNumberException) { + String message = "E007: A user with phoneNumber " + + loginMethods.stream() + .filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get().phoneNumber + " already exists in passwordless loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } + throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E008: " + e.getMessage())); + } + } + + private static List processThirdpartyLoginMethods(Main main, Storage storage, List loginMethods, + AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + try { + List usersToImport = new ArrayList<>(); + for (LoginMethod loginMethod: loginMethods){ + String userId = Utils.getUUID(); + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), loginMethod.tenantIds.get(0)); // the cron runs per app. The app stays the same, the tenant can change + + usersToImport.add(new ThirdPartyImportUser(loginMethod.email, userId, loginMethod.thirdPartyId, + loginMethod.thirdPartyUserId, tenantIdentifierForLoginMethod, loginMethod.timeJoinedInMSSinceEpoch)); + loginMethod.superTokensUserId = userId; + } + ThirdParty.createMultipleThirdPartyUsers(storage, usersToImport); + + return usersToImport; + } catch (StorageQueryException | StorageTransactionLogicException e) { + if (e.getCause() instanceof BulkImportBatchInsertException) { + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof DuplicateThirdPartyUserException) { + LoginMethod loginMethodForError = loginMethods.stream() + .filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get(); + String message = "E005: A user with thirdPartyId " + loginMethodForError.thirdPartyId + + " and thirdPartyUserId " + loginMethodForError.thirdPartyUserId + + " already exists in thirdparty loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } + throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); + } + } + + private static List processEmailPasswordLoginMethods(Main main, Storage storage, List loginMethods, + AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + try { + + //prepare data for batch import + List usersToImport = new ArrayList<>(); + for(LoginMethod emailPasswordLoginMethod : loginMethods) { + + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), emailPasswordLoginMethod.tenantIds.get(0)); // the cron runs per app. The app stays the same, the tenant can change + + String passwordHash = emailPasswordLoginMethod.passwordHash; + if (passwordHash == null && emailPasswordLoginMethod.plainTextPassword != null) { + passwordHash = PasswordHashing.getInstance(main) + .createHashWithSalt(tenantIdentifierForLoginMethod.toAppIdentifier(), emailPasswordLoginMethod.plainTextPassword); + } + emailPasswordLoginMethod.passwordHash = passwordHash; + String userId = Utils.getUUID(); + usersToImport.add(new EmailPasswordImportUser(userId, emailPasswordLoginMethod.email, + emailPasswordLoginMethod.passwordHash, tenantIdentifierForLoginMethod, emailPasswordLoginMethod.timeJoinedInMSSinceEpoch)); + emailPasswordLoginMethod.superTokensUserId = userId; + } + + EmailPassword.createMultipleUsersWithPasswordHash(storage, usersToImport); + + return usersToImport; + } catch (StorageQueryException | StorageTransactionLogicException e) { + if(e.getCause() instanceof BulkImportBatchInsertException){ + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for(String userid : errorsByPosition.keySet()){ + Exception exception = errorsByPosition.get(userid); + if(exception instanceof DuplicateEmailException){ + String message = "E003: A user with email " + + loginMethods.stream().filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get().email + " already exists in emailpassword loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("translated", errorsByPosition)); + } + throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E002: " + e.getMessage())); + } + } + + private static void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, + String firstTenant) throws StorageTransactionLogicException { + for (String tenantId : lm.tenantIds) { + try { + if (tenantId.equals(firstTenant)) { + continue; + } + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E009: " + e.getMessage())); + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception("E010: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (AnotherPrimaryUserWithEmailAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E011: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with email " + lm.email + " already exists.")); + } catch (AnotherPrimaryUserWithPhoneNumberAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E012: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with phoneNumber " + lm.phoneNumber + " already exists.")); + } catch (AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E013: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + + lm.thirdPartyUserId + " already exists.")); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException(new Exception("E014: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with email " + lm.email + " already exists.")); + } catch (DuplicatePhoneNumberException e) { + throw new StorageTransactionLogicException(new Exception("E015: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with phoneNumber " + lm.phoneNumber + " already exists.")); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("E016: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + + lm.thirdPartyUserId + " already exists.")); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E017: " + e.getMessage())); + } + } + } + + private static void createPrimaryUsersAndLinkAccounts(Main main, + AppIdentifier appIdentifier, Storage storage, + List users) + throws StorageTransactionLogicException, StorageQueryException, FeatureNotEnabledException, + TenantOrAppNotFoundException { + List userIds = + users.stream() + .map(bulkImportUser -> getPrimaryLoginMethod(bulkImportUser).getSuperTokenOrExternalUserId()) + .collect(Collectors.toList()); + Set allEmails = new HashSet<>(); + Set allPhoneNumber = new HashSet<>(); + Map allThirdParty = new HashMap<>(); + for (BulkImportUser user : users) { + for (LoginMethod loginMethod : user.loginMethods) { + if (loginMethod.email != null) { + allEmails.add(loginMethod.email); + } + if (loginMethod.phoneNumber != null) { + allPhoneNumber.add(loginMethod.phoneNumber); + } + if (loginMethod.thirdPartyId != null && loginMethod.thirdPartyUserId != null) { + allThirdParty.put(loginMethod.thirdPartyUserId, loginMethod.thirdPartyId); + } + + } + } + + try { + AuthRecipe.createPrimaryUsers(main, appIdentifier, storage, userIds, new ArrayList<>(allEmails), + new ArrayList<>(allPhoneNumber), allThirdParty); + } catch (StorageQueryException e) { + if(e.getCause() instanceof BulkImportBatchInsertException){ + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof UnknownUserIdException) { + String message = "E020: We tried to create the primary user for the userId " + + userid + + " but it doesn't exist. This should not happen. Please contact support."; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof RecipeUserIdAlreadyLinkedWithPrimaryUserIdException) { + String message = "E021: We tried to create the primary user for the userId " + + userid + + " but it is already linked with another primary user."; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) { + String message = "E022: We tried to create the primary user for the userId " + + userid + + " but the account info is already associated with another primary user."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } + throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E018: " + e.getMessage())); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E019: " + e.getMessage())); + } + + linkAccountsForMultipleUser(main, appIdentifier, storage, users, new ArrayList<>(allEmails), + new ArrayList<>(allPhoneNumber), allThirdParty); + } + + private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIdentifier, Storage storage, + List users, + List allDistinctEmails, + List allDistinctPhones, + Map thirdpartyUserIdsToThirdpartyIds) + throws StorageTransactionLogicException { + Map recipeUserIdByPrimaryUserId = collectRecipeIdsToPrimaryIds(users); + try { + AuthRecipe.linkMultipleAccounts(main, appIdentifier, storage, recipeUserIdByPrimaryUserId, + allDistinctEmails, allDistinctPhones, thirdpartyUserIdsToThirdpartyIds); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E023: " + e.getMessage())); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E024: " + e.getMessage())); + } catch (StorageQueryException e) { + if (e.getCause() instanceof BulkImportBatchInsertException) { + Map errorByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userId : errorByPosition.keySet()) { + Exception currentException = errorByPosition.get(userId); + String recipeUID = recipeUserIdByPrimaryUserId.get(userId); + if (currentException instanceof UnknownUserIdException) { + String message = "E025: We tried to link the userId " + recipeUID + + " to the primary userId " + userId + + " but it doesn't exist."; + errorByPosition.put(userId, new Exception(message)); + } else if (currentException instanceof InputUserIdIsNotAPrimaryUserException) { + String message = "E026: We tried to link the userId " + recipeUID + + " to the primary userId " + userId + + " but it is not a primary user."; + errorByPosition.put(userId, new Exception(message)); + } else if (currentException instanceof AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) { + String message = "E027: We tried to link the userId " + userId + + " to the primary userId " + recipeUID + + " but the account info is already associated with another primary user."; + errorByPosition.put(userId, new Exception(message)); + } else if (currentException instanceof RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException) { + String message = "E028: We tried to link the userId " + recipeUID + + " to the primary userId " + userId + + " but it is already linked with another primary user."; + errorByPosition.put(userId, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("link accounts translated", errorByPosition)); + } + throw new StorageTransactionLogicException(e); + } + } + + private static Map collectRecipeIdsToPrimaryIds(List users) { + Map recipeUserIdByPrimaryUserId = new HashMap<>(); + for(BulkImportUser user: users){ + LoginMethod primaryLM = getPrimaryLoginMethod(user); + for (LoginMethod lm : user.loginMethods) { + if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { + continue; + } + recipeUserIdByPrimaryUserId.put(lm.getSuperTokenOrExternalUserId(), + primaryLM.getSuperTokenOrExternalUserId()); + } + } + return recipeUserIdByPrimaryUserId; + } + + public static void createMultipleUserIdMapping(AppIdentifier appIdentifier, + List users, Storage[] storages) throws StorageTransactionLogicException { + Map superTokensUserIdToExternalUserId = new HashMap<>(); + for(BulkImportUser user: users) { + if(user.externalUserId != null) { + LoginMethod primaryLoginMethod = getPrimaryLoginMethod(user); + superTokensUserIdToExternalUserId.put(primaryLoginMethod.superTokensUserId, user.externalUserId); + primaryLoginMethod.externalUserId = user.externalUserId; + } + } + try { + List mappingResults = UserIdMapping.createMultipleUserIdMappings( + appIdentifier, storages, + superTokensUserIdToExternalUserId, + false, true); + + } catch (StorageQueryException e) { + if(e.getCause() instanceof BulkImportBatchInsertException) { + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof ServletException) { + String message = "E030: " + e.getMessage(); + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof UserIdMappingAlreadyExistsException) { + String message = "E031: A user with externalId " + superTokensUserIdToExternalUserId.get(userid) + " already exists"; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof UnknownSuperTokensUserIdException) { + String message = "E032: We tried to create the externalUserId mapping for the superTokenUserId " + + userid + + " but it doesn't exist. This should not happen. Please contact support."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } + throw new StorageTransactionLogicException(e); + } + } + + public static void createMultipleUserMetadata(AppIdentifier appIdentifier, Storage storage, List users) + throws StorageTransactionLogicException { + + Map usersMetadata = new HashMap<>(); + for(BulkImportUser user: users) { + if (user.userMetadata != null) { + usersMetadata.put(getPrimaryLoginMethod(user).getSuperTokenOrExternalUserId(), user.userMetadata); + } + } + + try { + UserMetadata.updateMultipleUsersMetadata(appIdentifier, storage, usersMetadata); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E040: " + e.getMessage())); + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + + public static void createMultipleUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, + List users) throws StorageTransactionLogicException { + Map>> rolesToUserByTenant = new HashMap<>(); + for (BulkImportUser user : users) { + + if (user.userRoles != null) { + for (UserRole userRole : user.userRoles) { + for (String tenantId : userRole.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), + tenantId); + if(!rolesToUserByTenant.containsKey(tenantIdentifier)){ + + rolesToUserByTenant.put(tenantIdentifier, new HashMap<>()); + } + if(!rolesToUserByTenant.get(tenantIdentifier).containsKey(user.externalUserId)){ + rolesToUserByTenant.get(tenantIdentifier).put(user.externalUserId, new ArrayList<>()); + } + rolesToUserByTenant.get(tenantIdentifier).get(user.externalUserId).add(userRole.role); + } + } + } + } + try { + + UserRoles.addMultipleRolesToMultipleUsers(main, appIdentifier, storage, rolesToUserByTenant); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); + } catch (StorageTransactionLogicException e) { + if(e.actualException instanceof BulkImportBatchInsertException){ + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof UnknownRoleException) { + String message = "E034: Role does not exist! You need to pre-create the role before " + + "assigning it to the user."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("roles errors translated", errorsByPosition)); + } else { + throw new StorageTransactionLogicException(e); + } + } + + } + + public static void verifyMultipleEmailForAllLoginMethods(AppIdentifier appIdentifier, Storage storage, + List users) + throws StorageTransactionLogicException { + Map emailToUserId = new HashMap<>(); + for (BulkImportUser user : users) { + for (LoginMethod lm : user.loginMethods) { + emailToUserId.put(lm.getSuperTokenOrExternalUserId(), lm.email); + } + } + + try { + + EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils + .getEmailVerificationStorage(storage); + emailVerificationSQLStorage.startTransaction(con -> { + emailVerificationSQLStorage + .updateMultipleIsEmailVerified_Transaction(appIdentifier, con, + emailToUserId, true); + + emailVerificationSQLStorage.commitTransaction(con); + return null; + }); + + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + + public static void createMultipleTotpDevices(Main main, AppIdentifier appIdentifier, + Storage storage, List users) + throws StorageTransactionLogicException { + List devices = new ArrayList<>(); + for (BulkImportUser user : users) { + if (user.totpDevices != null) { + for(TotpDevice device : user.totpDevices){ + TOTPDevice totpDevice = new TOTPDevice(getPrimaryLoginMethod(user).getSuperTokenOrExternalUserId(), + device.deviceName, device.secretKey, device.period, device.skew, true, + System.currentTimeMillis()); + devices.add(totpDevice); + } + } + } + try { + Totp.createDevices(main, appIdentifier, storage, devices); + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(new Exception("E036: " + e.getMessage())); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E037: " + e.getMessage())); + } + } + + // Returns the primary loginMethod of the user. If no loginMethod is marked as + // primary, then the oldest loginMethod is returned. + public static BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { + BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); + for (BulkImportUser.LoginMethod lm : user.loginMethods) { + if (lm.isPrimary) { + return lm; + } + + if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { + oldestLM = lm; + } + } + return oldestLM; + } + + private static synchronized Storage getBulkImportProxyStorage(Main main, TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); + } + + private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + + try { + List allProxyStorages = new ArrayList<>(); + + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E039: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E040: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E041: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E042: " + e.getMessage())); + } + } + + private static void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); + storage.close(); + } + userPoolToStorageMap.clear(); + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java new file mode 100644 index 000000000..d2bd21634 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + package io.supertokens.bulkimport; + +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; + +public class BulkImportUserPaginationContainer { + public final List users; + public final String nextPaginationToken; + + public BulkImportUserPaginationContainer(@Nonnull List users, @Nullable String nextPaginationToken) { + this.users = users; + this.nextPaginationToken = nextPaginationToken; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java new file mode 100644 index 000000000..8a492c2ca --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.Base64; + +public class BulkImportUserPaginationToken { + public final String bulkImportUserId; + public final long createdAt; + + public BulkImportUserPaginationToken(String bulkImportUserId, long createdAt) { + this.bulkImportUserId = bulkImportUserId; + this.createdAt = createdAt; + } + + public static BulkImportUserPaginationToken extractTokenInfo(String token) throws InvalidTokenException { + try { + String decodedPaginationToken = new String(Base64.getDecoder().decode(token)); + String[] splitDecodedToken = decodedPaginationToken.split(";"); + if (splitDecodedToken.length != 2) { + throw new InvalidTokenException(); + } + String bulkImportUserId = splitDecodedToken[0]; + long createdAt = Long.parseLong(splitDecodedToken[1]); + return new BulkImportUserPaginationToken(bulkImportUserId, createdAt); + } catch (Exception e) { + throw new InvalidTokenException(); + } + } + + public String generateToken() { + return new String(Base64.getEncoder().encode((this.bulkImportUserId + ";" + this.createdAt).getBytes())); + } + + public static class InvalidTokenException extends Exception { + + private static final long serialVersionUID = 6289026174830695478L; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java new file mode 100644 index 000000000..7b70d27aa --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -0,0 +1,579 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.CoreConfig; +import io.supertokens.emailpassword.PasswordHashingUtils; +import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.utils.JsonValidatorUtils.ValueType; + +import static io.supertokens.utils.JsonValidatorUtils.parseAndValidateFieldType; +import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; + +public class BulkImportUserUtils { + private String[] allUserRoles; + private Set allExternalUserIds; + + public BulkImportUserUtils(String[] allUserRoles) { + this.allUserRoles = allUserRoles; + this.allExternalUserIds = new HashSet<>(); + } + + public BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, JsonObject userData, + String id) + throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { + List errors = new ArrayList<>(); + + String externalUserId = parseAndValidateFieldType(userData, "externalUserId", ValueType.STRING, false, + String.class, + errors, "."); + JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, + JsonObject.class, errors, "."); + List userRoles = getParsedUserRoles(main, appIdentifier, userData, errors); + List totpDevices = getParsedTotpDevices(main, appIdentifier, userData, errors); + List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); + + externalUserId = validateAndNormaliseExternalUserId(externalUserId, errors); + + validateTenantIdsForRoleAndLoginMethods(main, appIdentifier, userRoles, loginMethods, errors); + + if (!errors.isEmpty()) { + throw new InvalidBulkImportDataException(errors); + } + return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); + } + + private List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "userRoles", ValueType.ARRAY_OF_OBJECT, false, + JsonArray.class, errors, "."); + + if (jsonUserRoles == null) { + return null; + } + + List userRoles = new ArrayList<>(); + + for (JsonElement jsonUserRoleEl : jsonUserRoles) { + JsonObject jsonUserRole = jsonUserRoleEl.getAsJsonObject(); + + String role = parseAndValidateFieldType(jsonUserRole, "role", ValueType.STRING, true, String.class, errors, + " for a user role."); + JsonArray jsonTenantIds = parseAndValidateFieldType(jsonUserRole, "tenantIds", ValueType.ARRAY_OF_STRING, + true, JsonArray.class, errors, " for a user role."); + + role = validateAndNormaliseUserRole(role, errors); + List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, jsonTenantIds, errors, + " for a user role."); + + if (role != null && normalisedTenantIds != null) { + userRoles.add(new UserRole(role, normalisedTenantIds)); + } + } + return userRoles; + } + + private List getParsedTotpDevices(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totpDevices", ValueType.ARRAY_OF_OBJECT, false, + JsonArray.class, errors, "."); + + if (jsonTotpDevices == null) { + return null; + } + + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.MFA)) { + errors.add("MFA must be enabled to import totp devices."); + return null; + } + + List totpDevices = new ArrayList<>(); + for (JsonElement jsonTotpDeviceEl : jsonTotpDevices) { + JsonObject jsonTotpDevice = jsonTotpDeviceEl.getAsJsonObject(); + + String secretKey = parseAndValidateFieldType(jsonTotpDevice, "secretKey", ValueType.STRING, true, + String.class, errors, " for a totp device."); + Integer period = parseAndValidateFieldType(jsonTotpDevice, "period", ValueType.INTEGER, false, + Integer.class, errors, " for a totp device."); + Integer skew = parseAndValidateFieldType(jsonTotpDevice, "skew", ValueType.INTEGER, false, Integer.class, + errors, " for a totp device."); + String deviceName = parseAndValidateFieldType(jsonTotpDevice, "deviceName", ValueType.STRING, false, + String.class, errors, " for a totp device."); + + secretKey = validateAndNormaliseTotpSecretKey(secretKey, errors); + period = validateAndNormaliseTotpPeriod(period, errors); + skew = validateAndNormaliseTotpSkew(skew, errors); + deviceName = validateAndNormaliseTotpDeviceName(deviceName, errors); + + if (secretKey != null && period != null && skew != null) { + totpDevices.add(new TotpDevice(secretKey, period, skew, deviceName)); + } + } + return totpDevices; + } + + private List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) + throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonLoginMethods = parseAndValidateFieldType(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, + true, JsonArray.class, errors, "."); + + if (jsonLoginMethods == null) { + return new ArrayList<>(); + } + + if (jsonLoginMethods.size() == 0) { + errors.add("At least one loginMethod is required."); + return new ArrayList<>(); + } + + if (jsonLoginMethods.size() > 1) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + errors.add("Account linking must be enabled to import multiple loginMethods."); + } + } + + validateAndNormaliseIsPrimaryField(jsonLoginMethods, errors); + + List loginMethods = new ArrayList<>(); + + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + + String recipeId = parseAndValidateFieldType(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, + String.class, errors, " for a loginMethod."); + JsonArray tenantIds = parseAndValidateFieldType(jsonLoginMethodObj, "tenantIds", ValueType.ARRAY_OF_STRING, + false, JsonArray.class, errors, " for a loginMethod."); + Boolean isVerified = parseAndValidateFieldType(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, + Boolean.class, errors, " for a loginMethod."); + Boolean isPrimary = parseAndValidateFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, + Boolean.class, errors, " for a loginMethod."); + Long timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.LONG, + false, Long.class, errors, " for a loginMethod"); + + recipeId = validateAndNormaliseRecipeId(recipeId, errors); + List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, tenantIds, errors, + " for " + recipeId + " recipe."); + isPrimary = validateAndNormaliseIsPrimary(isPrimary); + isVerified = validateAndNormaliseIsVerified(isVerified); + + long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined, errors); + + if ("emailpassword".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, + String.class, errors, " for an emailpassword recipe."); + String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, + false, String.class, errors, " for an emailpassword recipe."); + String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", + ValueType.STRING, false, String.class, errors, " for an emailpassword recipe."); + String plainTextPassword = parseAndValidateFieldType(jsonLoginMethodObj, "plainTextPassword", + ValueType.STRING, false, String.class, errors, " for an emailpassword recipe."); + + if ((passwordHash == null || hashingAlgorithm == null) && plainTextPassword == null) { + errors.add("Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe."); + } + + email = validateAndNormaliseEmail(email, errors); + CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm( + hashingAlgorithm, errors); + hashingAlgorithm = normalisedHashingAlgorithm != null ? normalisedHashingAlgorithm.toString() + : hashingAlgorithm; + passwordHash = validateAndNormalisePasswordHash(main, appIdentifier, normalisedHashingAlgorithm, + passwordHash, errors); + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null, null)); + } else if ("thirdparty".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, + String.class, errors, " for a thirdparty recipe."); + String thirdPartyId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, + true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyUserId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyUserId", + ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + + email = validateAndNormaliseEmail(email, errors); + thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId, errors); + thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, null, null, null, thirdPartyId, thirdPartyUserId, null)); + } else if ("passwordless".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, + String.class, errors, " for a passwordless recipe."); + String phoneNumber = parseAndValidateFieldType(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, + false, String.class, errors, " for a passwordless recipe."); + + email = validateAndNormaliseEmail(email, errors); + phoneNumber = validateAndNormalisePhoneNumber(phoneNumber, errors); + + if (email == null && phoneNumber == null) { + errors.add("Either email or phoneNumber is required for a passwordless recipe."); + } + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, null, null, null, null, null, phoneNumber)); + } + } + return loginMethods; + } + + private String validateAndNormaliseExternalUserId(String externalUserId, List errors) { + if (externalUserId == null) { + return null; + } + + if (externalUserId.length() > 128) { + errors.add("externalUserId " + externalUserId + " is too long. Max length is 128."); + } + + if (!allExternalUserIds.add(externalUserId)) { + errors.add("externalUserId " + externalUserId + " is not unique. It is already used by another user."); + } + + // We just trim the externalUserId as per the UpdateExternalUserIdInfoAPI.java + return externalUserId.trim(); + } + + private String validateAndNormaliseUserRole(String role, List errors) { + if (role.length() > 255) { + errors.add("role " + role + " is too long. Max length is 255."); + } + + // We just trim the role as per the CreateRoleAPI.java + String normalisedRole = role.trim(); + + if (!Arrays.asList(allUserRoles).contains(normalisedRole)) { + errors.add("Role " + normalisedRole + " does not exist."); + } + + return normalisedRole; + } + + private String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + if (secretKey == null) { + return null; + } + + if (secretKey.length() > 256) { + errors.add("TOTP secretKey " + secretKey + " is too long. Max length is 256."); + } + + // We don't perform any normalisation on the secretKey in ImportTotpDeviceAPI.java + return secretKey; + } + + private Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { + // We default to 30 if period is null + if (period == null) { + return 30; + } + + if (period.intValue() < 1) { + errors.add("period should be > 0 for a totp device."); + return null; + } + return period; + } + + private Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { + // We default to 1 if skew is null + if (skew == null) { + return 1; + } + + if (skew.intValue() < 0) { + errors.add("skew should be >= 0 for a totp device."); + return null; + } + return skew; + } + + private String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { + if (deviceName == null) { + return null; + } + + if (deviceName.length() > 256) { + errors.add("TOTP deviceName " + deviceName + " is too long. Max length is 256."); + } + + // We normalise the deviceName as per the ImportTotpDeviceAPI.java + return deviceName.trim(); + } + + private void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { + // We are validating that only one loginMethod has isPrimary as true + boolean hasPrimaryLoginMethod = false; + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { + if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { + if (hasPrimaryLoginMethod) { + errors.add("No two loginMethods can have isPrimary as true."); + } + hasPrimaryLoginMethod = true; + } + } + } + } + + private String validateAndNormaliseRecipeId(String recipeId, List errors) { + if (recipeId == null) { + return null; + } + + // We don't perform any normalisation on the recipeId after reading it from request header. + // We will validate it as is. + if (!Arrays.asList("emailpassword", "thirdparty", "passwordless").contains(recipeId)) { + errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); + } + return recipeId; + } + + private List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, + JsonArray tenantIds, List errors, String errorSuffix) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantIds == null) { + return List.of(TenantIdentifier.DEFAULT_TENANT_ID); // Default to DEFAULT_TENANT_ID ("public") + } + + List normalisedTenantIds = new ArrayList<>(); + + for (JsonElement tenantIdEl : tenantIds) { + String tenantId = tenantIdEl.getAsString(); + tenantId = validateAndNormaliseTenantId(main, appIdentifier, tenantId, errors, errorSuffix); + + if (tenantId != null) { + normalisedTenantIds.add(tenantId); + } + } + return normalisedTenantIds; + } + + private String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, + List errors, String errorSuffix) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { + return tenantId; + } + + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { + errors.add("Multitenancy must be enabled before importing users to a different tenant."); + return null; + } + + // We make the tenantId lowercase while parsing from the request in WebserverAPI.java + String normalisedTenantId = tenantId.trim().toLowerCase(); + TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + Set validTenantIds = new HashSet<>(); + Arrays.stream(allTenantConfigs) + .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + + if (!validTenantIds.contains(normalisedTenantId)) { + errors.add("Invalid tenantId: " + tenantId + errorSuffix); + return null; + } + return normalisedTenantId; + } + + private Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { + // We set the default value as false + return isPrimary == null ? false : isPrimary; + } + + private Boolean validateAndNormaliseIsVerified(Boolean isVerified) { + // We set the default value as false + return isVerified == null ? false : isVerified; + } + + private long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { + // We default timeJoined to currentTime if it is null + if (timeJoined == null) { + return System.currentTimeMillis(); + } + + if (timeJoined > System.currentTimeMillis()) { + errors.add("timeJoined cannot be in future for a loginMethod."); + } + + if (timeJoined < 0) { + errors.add("timeJoined cannot be < 0 for a loginMethod."); + } + + return timeJoined.longValue(); + } + + private String validateAndNormaliseEmail(String email, List errors) { + if (email == null) { + return null; + } + + if (email.length() > 255) { + errors.add("email " + email + " is too long. Max length is 256."); + } + + // We normalise the email as per the SignUpAPI.java + return Utils.normaliseEmail(email); + } + + private CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, + List errors) { + if (hashingAlgorithm == null) { + return null; + } + + try { + // We trim the hashingAlgorithm and make it uppercase as per the ImportUserWithPasswordHashAPI.java + return CoreConfig.PASSWORD_HASHING_ALG.valueOf(hashingAlgorithm.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + errors.add( + "Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); + return null; + } + } + + private String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, + CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm, String passwordHash, List errors) + throws TenantOrAppNotFoundException { + if (hashingAlgorithm == null || passwordHash == null) { + return passwordHash; + } + + if (passwordHash.length() > 256) { + errors.add("passwordHash is too long. Max length is 256."); + } + + // We trim the passwordHash and validate it as per ImportUserWithPasswordHashAPI.java + passwordHash = passwordHash.trim(); + + try { + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, passwordHash, + hashingAlgorithm); + } catch (UnsupportedPasswordHashingFormatException e) { + errors.add(e.getMessage()); + } + + return passwordHash; + } + + private String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { + if (thirdPartyId == null) { + return null; + } + + if (thirdPartyId.length() > 28) { + errors.add("thirdPartyId " + thirdPartyId + " is too long. Max length is 28."); + } + + // We don't perform any normalisation on the thirdPartyId in SignInUpAPI.java + return thirdPartyId; + } + + private String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { + if (thirdPartyUserId == null) { + return null; + } + + if (thirdPartyUserId.length() > 256) { + errors.add("thirdPartyUserId " + thirdPartyUserId + " is too long. Max length is 256."); + } + + // We don't perform any normalisation on the thirdPartyUserId in SignInUpAPI.java + return thirdPartyUserId; + } + + private String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { + if (phoneNumber == null) { + return null; + } + + if (phoneNumber.length() > 256) { + errors.add("phoneNumber " + phoneNumber + " is too long. Max length is 256."); + } + + // We normalise the phoneNumber as per the CreateCodeAPI.java + return Utils.normalizeIfPhoneNumber(phoneNumber); + } + + private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier appIdentifier, + List userRoles, List loginMethods, List errors) + throws TenantOrAppNotFoundException { + if (loginMethods == null) { + return; + } + + // First validate that tenantIds provided for userRoles also exist in the loginMethods + if (userRoles != null) { + for (UserRole userRole : userRoles) { + for (String tenantId : userRole.tenantIds) { + if (!tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID) && loginMethods.stream() + .noneMatch(loginMethod -> loginMethod.tenantIds.contains(tenantId))) { + errors.add("TenantId " + tenantId + " for a user role does not exist in loginMethods."); + } + } + } + } + + // Now validate that all the tenants share the same storage + String commonTenantUserPoolId = null; + for (LoginMethod loginMethod : loginMethods) { + for (String tenantId : loginMethod.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Storage storage = StorageLayer.getStorage(tenantIdentifier, main); + String tenantUserPoolId = storage.getUserPoolId(); + + if (commonTenantUserPoolId == null) { + commonTenantUserPoolId = tenantUserPoolId; + } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { + errors.add("All tenants for a user must share the same database for " + loginMethod.recipeId + + " recipe."); + break; // Break to avoid adding the same error multiple times for the same loginMethod + } + } + } + } +} diff --git a/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java new file mode 100644 index 000000000..3fbcd8fbd --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport.exceptions; + +import java.util.List; + +public class InvalidBulkImportDataException extends Exception { + private static final long serialVersionUID = 1L; + public List errors; + + public InvalidBulkImportDataException(List errors) { + super("Data has missing or invalid fields. Please check the errors field for more details."); + this.errors = errors; + } + + public void addError(String error) { + this.errors.add(error); + } +} diff --git a/src/main/java/io/supertokens/config/CoreConfig.java b/src/main/java/io/supertokens/config/CoreConfig.java index e8c266fe3..46495cda0 100644 --- a/src/main/java/io/supertokens/config/CoreConfig.java +++ b/src/main/java/io/supertokens/config/CoreConfig.java @@ -344,6 +344,12 @@ public class CoreConfig { @IgnoreForAnnotationCheck private boolean isNormalizedAndValid = false; + @NotConflictingInApp + @JsonProperty + @ConfigDescription("If specified, the supertokens core will use the specified number of threads to complete the " + + "migration of users. (Default: number of available processor cores).") + private int bulk_migration_parallelism = Runtime.getRuntime().availableProcessors(); + @IgnoreForAnnotationCheck private static boolean disableOAuthValidationForTest = false; @@ -579,6 +585,10 @@ public boolean getHttpsEnabled() { return webserver_https_enabled; } + public int getBulkMigrationParallelism() { + return bulk_migration_parallelism; + } + private String getConfigFileLocation(Main main) { return new File(CLIOptions.get(main).getConfigFilePath() == null ? CLIOptions.get(main).getInstallationPath() + "config.yaml" @@ -772,6 +782,10 @@ void normalizeAndValidate(Main main, boolean includeConfigFilePath) throws Inval } } + if (bulk_migration_parallelism < 1) { + throw new InvalidConfigException("Provided bulk_migration_parallelism must be >= 1"); + } + for (String fieldId : CoreConfig.getValidFields()) { try { Field field = CoreConfig.class.getDeclaredField(fieldId); diff --git a/src/main/java/io/supertokens/cronjobs/CronTaskTest.java b/src/main/java/io/supertokens/cronjobs/CronTaskTest.java index 477d23cc5..4265c361d 100644 --- a/src/main/java/io/supertokens/cronjobs/CronTaskTest.java +++ b/src/main/java/io/supertokens/cronjobs/CronTaskTest.java @@ -28,6 +28,7 @@ public class CronTaskTest extends SingletonResource { private static final String RESOURCE_ID = "io.supertokens.cronjobs.CronTaskTest"; private Map cronTaskToInterval = new HashMap(); + private Map cronTaskToWaitTime = new HashMap(); private CronTaskTest() { @@ -51,4 +52,13 @@ public void setIntervalInSeconds(String resourceId, int interval) { public Integer getIntervalInSeconds(String resourceId) { return cronTaskToInterval.get(resourceId); } + + @TestOnly + public void setInitialWaitTimeInSeconds(String resourceId, int interval) { + cronTaskToWaitTime.put(resourceId, interval); + } + + public Integer getInitialWaitTimeInSeconds(String resourceId) { + return cronTaskToWaitTime.get(resourceId); + } } diff --git a/src/main/java/io/supertokens/cronjobs/Cronjobs.java b/src/main/java/io/supertokens/cronjobs/Cronjobs.java index 0be582b31..572aa3e2a 100644 --- a/src/main/java/io/supertokens/cronjobs/Cronjobs.java +++ b/src/main/java/io/supertokens/cronjobs/Cronjobs.java @@ -18,7 +18,6 @@ import io.supertokens.Main; import io.supertokens.ResourceDistributor; -import io.supertokens.multitenancy.MultitenancyHelper; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import org.jetbrains.annotations.TestOnly; @@ -100,6 +99,16 @@ public static void addCronjob(Main main, CronTask task) { } } + public static boolean isCronjobLoaded(Main main, CronTask task) { + if (getInstance(main) == null) { + init(main); + } + Cronjobs instance = getInstance(main); + synchronized (instance.lock) { + return instance.tasks.contains(task); + } + } + @TestOnly public List getTasks() { return this.tasks; diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java new file mode 100644 index 000000000..fe1e8f754 --- /dev/null +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2024. VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.cronjobs.bulkimport; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.config.Config; +import io.supertokens.cronjobs.CronTask; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ProcessBulkImportUsers extends CronTask { + + public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; + + private ExecutorService executorService; + + private ProcessBulkImportUsers(Main main, List> tenantsInfo) { + super("ProcessBulkImportUsers", main, tenantsInfo, true); + } + + public static ProcessBulkImportUsers init(Main main, List> tenantsInfo) { + return (ProcessBulkImportUsers) main.getResourceDistributor() + .setResource(new TenantIdentifier(null, null, null), RESOURCE_KEY, + new ProcessBulkImportUsers(main, tenantsInfo)); + } + + @Override + protected void doTaskPerApp(AppIdentifier app) + throws TenantOrAppNotFoundException, StorageQueryException { + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage bulkImportSQLStorage = (BulkImportSQLStorage) StorageLayer + .getStorage(app.getAsPublicTenantIdentifier(), main); + + //split the loaded users list into smaller chunks + int NUMBER_OF_BATCHES = Config.getConfig(app.getAsPublicTenantIdentifier(), main) + .getBulkMigrationParallelism(); + executorService = Executors.newFixedThreadPool(NUMBER_OF_BATCHES); + String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + + long newUsers = bulkImportSQLStorage.getBulkImportUsersCount(app, BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW); + long processingUsers = bulkImportSQLStorage.getBulkImportUsersCount(app, BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING); + //taking a "snapshot" here and processing in this round as many users as there are uploaded now. After this the processing will go on + //with another app and gets back here when all the apps had a chance. + long usersProcessed = 0; + + while(usersProcessed < (newUsers + processingUsers)) { + + List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, + BulkImport.PROCESS_USERS_BATCH_SIZE); + if (users == null || users.isEmpty()) { + // "No more users to process!" + break; + } + + List> loadedUsersChunks = makeChunksOf(users, NUMBER_OF_BATCHES); + + try { + List> tasks = new ArrayList<>(); + for (int i = 0; i < NUMBER_OF_BATCHES && i < loadedUsersChunks.size(); i++) { + tasks.add( + executorService.submit(new ProcessBulkUsersImportWorker(main, app, loadedUsersChunks.get(i), + bulkImportSQLStorage, bulkImportUserUtils))); + } + + for (Future task : tasks) { + while (!task.isDone()) { + Thread.sleep(1000); + } + Void result = (Void) task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up + usersProcessed += loadedUsersChunks.get(tasks.indexOf(task)).size(); + } + + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + executorService.shutdownNow(); + } + + @Override + public int getIntervalTimeSeconds() { + if (Main.isTesting) { + Integer interval = CronTaskTest.getInstance(main).getIntervalInSeconds(RESOURCE_KEY); + if (interval != null) { + return interval; + } + } + return BulkImport.PROCESS_USERS_INTERVAL_SECONDS; + } + + @Override + public int getInitialWaitTimeSeconds() { + if (Main.isTesting) { + Integer waitTime = CronTaskTest.getInstance(main).getInitialWaitTimeInSeconds(RESOURCE_KEY); + if (waitTime != null) { + return waitTime; + } + } + return 0; + } + + private List> makeChunksOf(List users, int numberOfChunks) { + List> chunks = new ArrayList<>(); + if (users != null && !users.isEmpty() && numberOfChunks > 0) { + AtomicInteger index = new AtomicInteger(0); + int chunkSize = users.size() / numberOfChunks + 1; + Stream> listStream = users.stream() + .collect(Collectors.groupingBy(x -> index.getAndIncrement() / chunkSize)) + .entrySet().stream() + .sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue); + + listStream.forEach(chunks::add); + } + return chunks; + } + +} diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java new file mode 100644 index 000000000..a19fa6723 --- /dev/null +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.cronjobs.bulkimport; + +import com.google.gson.JsonObject; +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.Config; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportTransactionRolledBackException; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.storageLayer.StorageLayer; + +import java.io.IOException; +import java.util.*; + +public class ProcessBulkUsersImportWorker implements Runnable { + + private final Map userPoolToStorageMap = new HashMap<>(); + private final Main main; + private final AppIdentifier app; + private final BulkImportSQLStorage bulkImportSQLStorage; + private final BulkImportUserUtils bulkImportUserUtils; + private final List usersToProcess; + + ProcessBulkUsersImportWorker(Main main, AppIdentifier app, List usersToProcess, BulkImportSQLStorage bulkImportSQLStorage, BulkImportUserUtils bulkImportUserUtils){ + this.main = main; + this.app = app; + this.usersToProcess = usersToProcess; + this.bulkImportSQLStorage = bulkImportSQLStorage; + this.bulkImportUserUtils = bulkImportUserUtils; + } + + @Override + public void run() { + try { + processMultipleUsers(app, usersToProcess, bulkImportUserUtils, bulkImportSQLStorage); + } catch (TenantOrAppNotFoundException | DbInitException | IOException | StorageQueryException e) { + throw new RuntimeException(e); + } + } + + private void processMultipleUsers(AppIdentifier appIdentifier, List users, + BulkImportUserUtils bulkImportUserUtils, + BulkImportSQLStorage baseTenantStorage) + throws TenantOrAppNotFoundException, StorageQueryException, IOException, + DbInitException { + BulkImportUser user = null; + try { + final Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + int userIndexPointer = 0; + List validUsers = new ArrayList<>(); + Map validationErrorsBeforeActualProcessing = new HashMap<>(); + while(userIndexPointer < users.size()) { + user = users.get(userIndexPointer); + if (Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) { + // Skip validation when the flag is enabled during testing + // Skip validation if it's a retry run. This already passed validation. A revalidation triggers + // an invalid external user id already exists validation error - which is not true! + validUsers.add(user); + } else { + // Validate the user + try { + validUsers.add(bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, + user.toJsonObject(), user.id)); + } catch (InvalidBulkImportDataException exception) { + validationErrorsBeforeActualProcessing.put(user.id, new Exception( + String.valueOf(exception.errors))); + } + } + userIndexPointer+=1; + } + + if(!validationErrorsBeforeActualProcessing.isEmpty()) { + throw new BulkImportBatchInsertException("Invalid input data", validationErrorsBeforeActualProcessing); + } + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + Map> partitionedUsers = partitionUsersByStorage(appIdentifier, validUsers); + for(SQLStorage bulkImportProxyStorage : partitionedUsers.keySet()) { + boolean shouldRetryImmediatley = true; + while (shouldRetryImmediatley) { + shouldRetryImmediatley = bulkImportProxyStorage.startTransaction(con -> { + try { + BulkImport.processUsersImportSteps(main, appIdentifier, bulkImportProxyStorage, partitionedUsers.get(bulkImportProxyStorage), + allStoragesForApp); + + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + + String[] toDelete = new String[validUsers.size()]; + for (int i = 0; i < validUsers.size(); i++) { + toDelete[i] = validUsers.get(i).id; + } + + baseTenantStorage.deleteBulkImportUsers(appIdentifier, toDelete); + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy + // storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + if (isBulkImportTransactionRolledBackIsTheRealCause(e)) { + return true; + //@see BulkImportTransactionRolledBackException for explanation + } + handleProcessUserExceptions(app, validUsers, e, baseTenantStorage); + } + return false; + }); + } + } + } catch (StorageTransactionLogicException | InvalidConfigException e) { + throw new RuntimeException(e); + } catch (BulkImportBatchInsertException insertException) { + handleProcessUserExceptions(app, users, insertException, baseTenantStorage); + } finally { + closeAllProxyStorages(); //closing it here to reuse the existing connection with all the users + } + } + + private boolean isBulkImportTransactionRolledBackIsTheRealCause(Throwable exception) { + if(exception instanceof BulkImportTransactionRolledBackException){ + return true; + } else if(exception.getCause()!=null){ + return isBulkImportTransactionRolledBackIsTheRealCause(exception.getCause()); + } + return false; + } + + private void handleProcessUserExceptions(AppIdentifier appIdentifier, List usersBatch, Exception e, + BulkImportSQLStorage baseTenantStorage) + throws StorageQueryException { + // Java doesn't allow us to reassign local variables inside a lambda expression + // so we have to use an array. + String[] errorMessage = { e.getMessage() }; + Map bulkImportUserIdToErrorMessage = new HashMap<>(); + + if (e instanceof StorageTransactionLogicException) { + StorageTransactionLogicException exception = (StorageTransactionLogicException) e; + // If the exception is due to a StorageQueryException, we want to retry the entry after sometime instead + // of marking it as FAILED. We will return early in that case. + if (exception.actualException instanceof StorageQueryException) { + Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); + return; + } + if(exception.actualException instanceof BulkImportBatchInsertException){ + handleBulkImportException(usersBatch, (BulkImportBatchInsertException) exception.actualException, bulkImportUserIdToErrorMessage); + } else { + //fail the whole batch + errorMessage[0] = exception.actualException.getMessage(); + for(BulkImportUser user : usersBatch){ + bulkImportUserIdToErrorMessage.put(user.id, errorMessage[0]); + } + } + + } else if (e instanceof InvalidBulkImportDataException) { + errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); + } else if (e instanceof InvalidConfigException) { + errorMessage[0] = e.getMessage(); + } else if (e instanceof BulkImportBatchInsertException) { + handleBulkImportException(usersBatch, (BulkImportBatchInsertException)e, bulkImportUserIdToErrorMessage); + } + + try { + baseTenantStorage.startTransaction(con -> { + baseTenantStorage.updateMultipleBulkImportUsersStatusToError_Transaction(appIdentifier, con, + bulkImportUserIdToErrorMessage); + return null; + }); + } catch (StorageTransactionLogicException e1) { + throw new StorageQueryException(e1.actualException); + } + } + + private static void handleBulkImportException(List usersBatch, BulkImportBatchInsertException exception, + Map bulkImportUserIdToErrorMessage) { + Map userIndexToError = exception.exceptionByUserId; + for(String userid : userIndexToError.keySet()){ + Optional userWithId = usersBatch.stream() + .filter(bulkImportUser -> bulkImportUser.id.equals(userid) || bulkImportUser.externalUserId.equals(userid)).findFirst(); + String id = null; + if(userWithId.isPresent()){ + id = userWithId.get().id; + } + + if(id == null) { + userWithId = usersBatch.stream() + .filter(bulkImportUser -> + bulkImportUser.loginMethods.stream() + .map(loginMethod -> loginMethod.superTokensUserId) + .anyMatch(s -> s!= null && s.equals(userid))).findFirst(); + if(userWithId.isPresent()){ + id = userWithId.get().id; + } + } + bulkImportUserIdToErrorMessage.put(id, userIndexToError.get(userid).getMessage()); + } + } + + private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); + } + + private synchronized Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + + try { + List allProxyStorages = new ArrayList<>(); + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E043: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E044: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E045: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E046: " + e.getMessage())); + } + } + + private void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); + } + userPoolToStorageMap.clear(); + } + + private Map> partitionUsersByStorage(AppIdentifier appIdentifier, List users) + throws DbInitException, TenantOrAppNotFoundException, InvalidConfigException, IOException { + Map> result = new HashMap<>(); + for(BulkImportUser user: users) { + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); + if(!result.containsKey(bulkImportProxyStorage)){ + result.put(bulkImportProxyStorage, new ArrayList<>()); + } + result.get(bulkImportProxyStorage).add(user); + } + return result; + } +} diff --git a/src/main/java/io/supertokens/emailpassword/EmailPassword.java b/src/main/java/io/supertokens/emailpassword/EmailPassword.java index 5b907c47f..61384e86a 100644 --- a/src/main/java/io/supertokens/emailpassword/EmailPassword.java +++ b/src/main/java/io/supertokens/emailpassword/EmailPassword.java @@ -32,6 +32,8 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; import io.supertokens.pluginInterface.emailpassword.PasswordResetTokenInfo; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicatePasswordResetTokenException; @@ -55,6 +57,7 @@ import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.spec.InvalidKeySpecException; +import java.util.List; public class EmailPassword { @@ -177,19 +180,57 @@ public static ImportUserResponse importUserWithPasswordHash(TenantIdentifier ten tenantIdentifier.toAppIdentifier(), main, passwordHash, hashingAlgorithm); - while (true) { - String userId = Utils.getUUID(); + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + ImportUserResponse response = null; + + try { long timeJoined = System.currentTimeMillis(); + response = createUserWithPasswordHash(tenantIdentifier, storage, email, passwordHash, timeJoined); + } catch (DuplicateEmailException e) { + AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); + AuthRecipeUserInfo userInfoToBeUpdated = null; + LoginMethod loginMethod = null; + for (AuthRecipeUserInfo currUser : allUsers) { + for (LoginMethod currLM : currUser.loginMethods) { + if (currLM.email.equals(email) && currLM.recipeId == RECIPE_ID.EMAIL_PASSWORD && currLM.tenantIds.contains(tenantIdentifier.getTenantId())) { + userInfoToBeUpdated = currUser; + loginMethod = currLM; + break; + } + } + } - EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + if (userInfoToBeUpdated != null) { + LoginMethod finalLoginMethod = loginMethod; + epStorage.startTransaction(con -> { + epStorage.updateUsersPassword_Transaction(tenantIdentifier.toAppIdentifier(), con, + finalLoginMethod.getSupertokensUserId(), passwordHash); + return null; + }); + response = new ImportUserResponse(true, userInfoToBeUpdated); + } + } + return response; + } + public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier tenantIdentifier, Storage storage, + @Nonnull String email, + @Nonnull String passwordHash, @Nullable long timeJoined) + throws StorageQueryException, DuplicateEmailException, TenantOrAppNotFoundException, + StorageTransactionLogicException { + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + while (true) { + String userId = Utils.getUUID(); try { - AuthRecipeUserInfo userInfo = epStorage.signUp(tenantIdentifier, userId, email, passwordHash, - timeJoined); + AuthRecipeUserInfo userInfo = null; + userInfo = epStorage.signUp(tenantIdentifier, userId, email, passwordHash, timeJoined); return new ImportUserResponse(false, userInfo); } catch (DuplicateUserIdException e) { // we retry with a new userId } catch (DuplicateEmailException e) { + if(epStorage instanceof BulkImportStorage){ + throw e; + } AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); AuthRecipeUserInfo userInfoToBeUpdated = null; LoginMethod loginMethod = null; @@ -217,6 +258,17 @@ public static ImportUserResponse importUserWithPasswordHash(TenantIdentifier ten } } + public static void createMultipleUsersWithPasswordHash(Storage storage, + List usersToImport) + throws StorageQueryException, TenantOrAppNotFoundException, StorageTransactionLogicException { + + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + epStorage.startTransaction(con -> { + epStorage.signUpMultipleViaBulkImport_Transaction(con, usersToImport); + return null; + }); + } + @TestOnly public static ImportUserResponse importUserWithPasswordHash(Main main, @Nonnull String email, @Nonnull String passwordHash) diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 9a978dff9..18f290816 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -26,11 +26,13 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.dashboard.DashboardSearchTags; import io.supertokens.pluginInterface.dashboard.DashboardSessionInfo; import io.supertokens.pluginInterface.dashboard.DashboardUser; import io.supertokens.pluginInterface.dashboard.exceptions.UserIdNotFoundException; import io.supertokens.pluginInterface.dashboard.sqlStorage.DashboardSQLStorage; +import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; import io.supertokens.pluginInterface.emailpassword.PasswordResetTokenInfo; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicatePasswordResetTokenException; @@ -65,12 +67,14 @@ import io.supertokens.pluginInterface.oauth.exception.OAuthClientNotFoundException; import io.supertokens.pluginInterface.passwordless.PasswordlessCode; import io.supertokens.pluginInterface.passwordless.PasswordlessDevice; +import io.supertokens.pluginInterface.passwordless.PasswordlessImportUser; import io.supertokens.pluginInterface.passwordless.exception.*; import io.supertokens.pluginInterface.passwordless.sqlStorage.PasswordlessSQLStorage; import io.supertokens.pluginInterface.session.SessionInfo; import io.supertokens.pluginInterface.session.SessionStorage; import io.supertokens.pluginInterface.session.sqlStorage.SessionSQLStorage; import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; import io.supertokens.pluginInterface.thirdparty.sqlStorage.ThirdPartySQLStorage; import io.supertokens.pluginInterface.totp.TOTPDevice; @@ -101,10 +105,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLTransactionRollbackException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; +import java.util.*; public class Start implements SessionSQLStorage, EmailPasswordSQLStorage, EmailVerificationSQLStorage, ThirdPartySQLStorage, @@ -142,6 +143,30 @@ public void constructor(String processId, boolean silent, boolean isTesting) { Start.isTesting = isTesting; } + @Override + public Storage createBulkImportProxyStorageInstance() { + throw new UnsupportedOperationException("'createBulkImportProxyStorageInstance' is not supported for in-memory db"); + + } + + @Override + public void closeConnectionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "closeConnectionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + + @Override + public void commitTransactionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "commitTransactionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + + @Override + public void rollbackTransactionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "rollbackTransactionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + @Override public STORAGE_TYPE getType() { return STORAGE_TYPE.SQL; @@ -208,7 +233,8 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev tries++; try { return startTransactionHelper(logic); - } catch (SQLException | StorageQueryException | StorageTransactionLogicException e) { + } catch (SQLException | StorageQueryException | StorageTransactionLogicException | + TenantOrAppNotFoundException e) { if ((e instanceof SQLTransactionRollbackException || (e.getMessage() != null && e.getMessage().toLowerCase().contains("deadlock"))) && tries < 3) { @@ -227,7 +253,7 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev } private T startTransactionHelper(TransactionLogic logic) - throws StorageQueryException, StorageTransactionLogicException, SQLException { + throws StorageQueryException, StorageTransactionLogicException, SQLException, TenantOrAppNotFoundException { Connection con = null; try { con = ConnectionPool.getConnection(this); @@ -628,6 +654,14 @@ public boolean isUserIdBeingUsedInNonAuthRecipe(AppIdentifier appIdentifier, Str } } + @Override + public Map> findNonAuthRecipesWhereForUserIdsUsed(AppIdentifier appIdentifier, + List userIds) + throws StorageQueryException { + throw new UnsupportedOperationException("'findNonAuthRecipesWhereForUserIdsUsed' is not supported for in-memory db"); + + } + @TestOnly @Override public void addInfoToNonAuthRecipesBasedOnUserId(TenantIdentifier tenantIdentifier, String className, String userId) @@ -716,6 +750,8 @@ public void addInfoToNonAuthRecipesBasedOnUserId(TenantIdentifier tenantIdentifi } } else if (className.equals(JWTRecipeStorage.class.getName())) { /* Since JWT recipe tables do not store userId we do not add any data to them */ + } else if (className.equals(BulkImportStorage.class.getName())){ + //ignore } else if (className.equals(OAuthStorage.class.getName())) { /* Since OAuth tables store client-related data, we don't add user-specific data here */ } else if (className.equals(ActiveUsersStorage.class.getName())) { @@ -894,6 +930,13 @@ public void deleteEmailPasswordUser_Transaction(TransactionConnection con, AppId } } + @Override + public void signUpMultipleViaBulkImport_Transaction(TransactionConnection connection, + List users) + throws StorageQueryException, StorageTransactionLogicException { + throw new UnsupportedOperationException("'signUpMultipleViaBulkImport_Transaction' is not supported for in-memory db"); + } + @Override public void deleteExpiredEmailVerificationTokens() throws StorageQueryException { try { @@ -966,6 +1009,40 @@ public void updateIsEmailVerified_Transaction(AppIdentifier appIdentifier, Trans } } + @Override + public void updateMultipleIsEmailVerified_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + Map emailToUserId, boolean isEmailVerified) + throws StorageQueryException, TenantOrAppNotFoundException { + Connection sqlCon = (Connection) con.getConnection(); + try { + EmailVerificationQueries.updateMultipleUsersIsEmailVerified_Transaction(this, sqlCon, appIdentifier, + emailToUserId, isEmailVerified); + } catch (SQLException e) { + if (e instanceof SQLiteException) { + SQLiteConfig config = Config.getConfig(this); + String serverMessage = e.getMessage(); + + if (isForeignKeyConstraintError( + serverMessage, + config.getTenantsTable(), + new String[]{"app_id"}, + new Object[]{appIdentifier.getAppId()})) { + throw new TenantOrAppNotFoundException(appIdentifier); + } + } + + boolean isPSQLPrimKeyError = e instanceof SQLiteException && isPrimaryKeyError( + e.getMessage(), + Config.getConfig(this).getEmailVerificationTable(), + new String[]{"app_id", "user_id", "email"}); + + if (!isEmailVerified || !isPSQLPrimKeyError) { + throw new StorageQueryException(e); + } + // we do not throw an error since the email is already verified + } + } + @Override public void deleteEmailVerificationUserInfo_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -1076,6 +1153,13 @@ public void updateIsEmailVerifiedToExternalUserId(AppIdentifier appIdentifier, S externalUserId); } + @Override + public void updateMultipleIsEmailVerifiedToExternalUserIds(AppIdentifier appIdentifier, + Map supertokensUserIdToExternalUserId) + throws StorageQueryException { + throw new UnsupportedOperationException("'updateMultipleIsEmailVerifiedToExternalUserIds' is not supported for in-memory db"); + } + @Override public void deleteExpiredPasswordResetTokens() throws StorageQueryException { try { @@ -1110,6 +1194,20 @@ public void deleteThirdPartyUser_Transaction(TransactionConnection con, AppIdent } } + @Override + public void importThirdPartyUsers_Transaction(TransactionConnection con, + List usersToImport) + throws StorageQueryException, StorageTransactionLogicException { + throw new UnsupportedOperationException("'importThirdPartyUsers_Transaction' is not supported for in-memory db"); + } + + @Override + public void importPasswordlessUsers_Transaction(TransactionConnection con, + List users) + throws StorageQueryException { + throw new UnsupportedOperationException("'importPasswordlessUsers_Transaction' is not supported for in-memory db"); + } + @Override public AuthRecipeUserInfo signUp( TenantIdentifier tenantIdentifier, String id, String email, @@ -1250,6 +1348,12 @@ public boolean doesUserIdExist(TenantIdentifier tenantIdentifier, String userId) } } + @Override + public List findExistingUserIds(AppIdentifier appIdentifier, List userIds) + throws StorageQueryException { + throw new UnsupportedOperationException("'findExistingUserIds' is not supported for in-memory db"); + } + @Override public AuthRecipeUserInfo getPrimaryUserById(AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -1809,6 +1913,16 @@ public JsonObject getUserMetadata_Transaction(AppIdentifier appIdentifier, Trans } } + @Override + public Map getMultipleUsersMetadatas_Transaction(AppIdentifier appIdentifier, + TransactionConnection con, + List userIds) + throws StorageQueryException { + throw new UnsupportedOperationException("'getMultipleUsersMetadatas_Transaction' is not supported for in-memory db"); + } + + + @Override public int setUserMetadata_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String userId, JsonObject metadata) @@ -1834,6 +1948,13 @@ public int setUserMetadata_Transaction(AppIdentifier appIdentifier, TransactionC } } + @Override + public void setMultipleUsersMetadatas_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + Map metadataByUserId) + throws StorageQueryException, TenantOrAppNotFoundException { + throw new UnsupportedOperationException("'setMultipleUsersMetadatas_Transaction' is not supported for in-memory db"); + } + @Override public int deleteUserMetadata_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -2078,6 +2199,13 @@ public boolean doesRoleExist_Transaction(AppIdentifier appIdentifier, Transactio } } + @Override + public List doesMultipleRoleExist_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + List roles) throws StorageQueryException { + throw new UnsupportedOperationException("'doesMultipleRoleExist_Transaction' is not supported for in-memory db"); + + } + @Override public void deleteAllRolesForUser_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -2089,6 +2217,13 @@ public void deleteAllRolesForUser_Transaction(TransactionConnection con, AppIden } } + @Override + public void addRolesToUsers_Transaction(TransactionConnection connection, + Map>> rolesToUserByTenants) + throws StorageQueryException { + throw new UnsupportedOperationException("'addRolesToUsers_Transaction' is not supported for in-memory db"); + } + @Override public void createUserIdMapping(AppIdentifier appIdentifier, String superTokensUserId, String externalUserId, @org.jetbrains.annotations.Nullable String externalUserIdInfo) @@ -2129,6 +2264,13 @@ public void createUserIdMapping(AppIdentifier appIdentifier, String superTokensU } } + @Override + public void createBulkUserIdMapping(AppIdentifier appIdentifier, + Map superTokensUserIdToExternalUserId) + throws StorageQueryException { + throw new UnsupportedOperationException("'createBulkUserIdMapping' is not supported for in-memory db"); + } + @Override public boolean deleteUserIdMapping(AppIdentifier appIdentifier, String userId, boolean isSuperTokensUserId) throws StorageQueryException { @@ -2630,6 +2772,13 @@ public TOTPDevice createDevice_Transaction(TransactionConnection con, AppIdentif } } + @Override + public void createDevices_Transaction(TransactionConnection con, AppIdentifier appIdentifier, + List devices) + throws StorageQueryException, TenantOrAppNotFoundException { + throw new UnsupportedOperationException("'createDevices_Transaction' is not supported for in-memory db"); + } + @Override public TOTPDevice getDeviceByName_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId, String deviceName) throws StorageQueryException { @@ -2838,6 +2987,18 @@ public AuthRecipeUserInfo getPrimaryUserById_Transaction(AppIdentifier appIdenti } } + @Override + public List getPrimaryUsersByIds_Transaction(AppIdentifier appIdentifier, + TransactionConnection con, List userIds) + throws StorageQueryException { + try { + Connection sqlCon = (Connection) con.getConnection(); + return GeneralQueries.getPrimaryUsersInfoForUserIds_Transaction(this, sqlCon, appIdentifier, userIds); + } catch (SQLException e) { + throw new StorageQueryException(e); + } + } + @Override public AuthRecipeUserInfo[] listPrimaryUsersByEmail_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String email) @@ -2850,6 +3011,13 @@ public AuthRecipeUserInfo[] listPrimaryUsersByEmail_Transaction(AppIdentifier ap } } + @Override + public AuthRecipeUserInfo[] listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction( + AppIdentifier appIdentifier, TransactionConnection con, List emails, List phones, + Map thirdpartyIdToThirdpartyUserId) throws StorageQueryException { + throw new UnsupportedOperationException("'listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction' is not supported for in-memory db"); + } + @Override public AuthRecipeUserInfo[] listPrimaryUsersByPhoneNumber_Transaction(AppIdentifier appIdentifier, TransactionConnection con, @@ -2905,6 +3073,19 @@ public void makePrimaryUser_Transaction(AppIdentifier appIdentifier, Transaction } } + @Override + public void makePrimaryUsers_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + List userIds) throws StorageQueryException { + try { + Connection sqlCon = (Connection) con.getConnection(); + // we do not bother returning if a row was updated here or not, cause it's happening + // in a transaction anyway. + GeneralQueries.makePrimaryUsers_Transaction(this, sqlCon, appIdentifier, userIds); + } catch (SQLException e) { + throw new StorageQueryException(e); + } + } + @Override public void linkAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String recipeUserId, String primaryUserId) throws StorageQueryException { @@ -2918,6 +3099,13 @@ public void linkAccounts_Transaction(AppIdentifier appIdentifier, TransactionCon } } + @Override + public void linkMultipleAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + Map recipeUserIdByPrimaryUserId) + throws StorageQueryException { + throw new UnsupportedOperationException("'linkMultipleAccounts_Transaction' is not supported for in-memory db"); + } + @Override public void unlinkAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String primaryUserId, String recipeUserId) @@ -2995,6 +3183,14 @@ public UserIdMapping[] getUserIdMapping_Transaction(TransactionConnection con, A } } + @Override + public List getMultipleUserIdMapping_Transaction(TransactionConnection connection, + AppIdentifier appIdentifier, List userIds, + boolean isSupertokensIds) + throws StorageQueryException { + throw new UnsupportedOperationException("'getMultipleUserIdMapping_Transaction' is not supported for in-memory db"); + } + @Override public int getUsersCountWithMoreThanOneLoginMethodOrTOTPEnabled(AppIdentifier appIdentifier) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java index 8d5ccc7d8..5df8a518d 100644 --- a/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java +++ b/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java @@ -29,6 +29,7 @@ import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; @@ -103,6 +104,36 @@ public static void updateUsersIsEmailVerified_Transaction(Start start, Connectio } } + public static void updateMultipleUsersIsEmailVerified_Transaction(Start start, Connection con, AppIdentifier appIdentifier, + Map emailToUserIds, + boolean isEmailVerified) + throws SQLException, StorageQueryException { + + if (isEmailVerified) { + String QUERY = "INSERT INTO " + getConfig(start).getEmailVerificationTable() + + "(app_id, user_id, email) VALUES(?, ?, ?)"; + PreparedStatement insertQuery = con.prepareStatement(QUERY); + for(Map.Entry emailToUser : emailToUserIds.entrySet()){ + insertQuery.setString(1, appIdentifier.getAppId()); + insertQuery.setString(2, emailToUser.getValue()); + insertQuery.setString(3, emailToUser.getKey()); + insertQuery.addBatch(); + } + insertQuery.executeBatch(); + } else { + String QUERY = "DELETE FROM " + getConfig(start).getEmailVerificationTable() + + " WHERE app_id = ? AND user_id = ? AND email = ?"; + PreparedStatement deleteQuery = con.prepareStatement(QUERY); + for (Map.Entry emailToUser : emailToUserIds.entrySet()) { + deleteQuery.setString(1, appIdentifier.getAppId()); + deleteQuery.setString(2, emailToUser.getValue()); + deleteQuery.setString(3, emailToUser.getKey()); + deleteQuery.addBatch(); + } + deleteQuery.executeBatch(); + } + } + public static void deleteAllEmailVerificationTokensForUser_Transaction(Start start, Connection con, TenantIdentifier tenantIdentifier, String userId, diff --git a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java index eb2fe4809..5e8df292f 100644 --- a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java +++ b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java @@ -35,10 +35,7 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.util.*; import java.util.stream.Collectors; @@ -960,6 +957,37 @@ public static void makePrimaryUser_Transaction(Start start, Connection sqlCon, A } } + public static void makePrimaryUsers_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, + List userIds) + throws SQLException, StorageQueryException { + + String users_update_QUERY = "UPDATE " + getConfig(start).getUsersTable() + + " SET is_linked_or_is_a_primary_user = true WHERE app_id = ? AND user_id = ?"; + String appid_to_userid_update_QUERY = "UPDATE " + getConfig(start).getAppIdToUserIdTable() + + " SET is_linked_or_is_a_primary_user = true WHERE app_id = ? AND user_id = ?"; + + PreparedStatement usersUpdateStatement = sqlCon.prepareStatement(users_update_QUERY); + PreparedStatement appIdToUserIdUpdateStatement = sqlCon.prepareStatement(appid_to_userid_update_QUERY); + int counter = 0; + for(String userId: userIds){ + usersUpdateStatement.setString(1, appIdentifier.getAppId()); + usersUpdateStatement.setString(2, userId); + usersUpdateStatement.addBatch(); + + appIdToUserIdUpdateStatement.setString(1, appIdentifier.getAppId()); + appIdToUserIdUpdateStatement.setString(2, userId); + appIdToUserIdUpdateStatement.addBatch(); + + counter++; + if(counter % 100 == 0) { + usersUpdateStatement.executeBatch(); + appIdToUserIdUpdateStatement.executeBatch(); + } + } + usersUpdateStatement.executeBatch(); + appIdToUserIdUpdateStatement.executeBatch(); + } + public static void linkAccounts_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, String recipeUserId, String primaryUserId) throws SQLException, StorageQueryException { @@ -990,6 +1018,54 @@ public static void linkAccounts_Transaction(Start start, Connection sqlCon, AppI } } + public static void linkMultipleAccounts_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, + Map recipeUserIdToPrimaryUserId) + throws SQLException, StorageQueryException { + + if(recipeUserIdToPrimaryUserId == null || recipeUserIdToPrimaryUserId.isEmpty()){ + return; + } + + String update_users_QUERY = "UPDATE " + getConfig(start).getUsersTable() + + " SET is_linked_or_is_a_primary_user = true, primary_or_recipe_user_id = ? WHERE app_id = ? AND " + + "user_id = ?"; + + String update_appid_to_userid_QUERY = "UPDATE " + getConfig(start).getAppIdToUserIdTable() + + " SET is_linked_or_is_a_primary_user = true, primary_or_recipe_user_id = ? WHERE app_id = ? AND " + + "user_id = ?"; + + PreparedStatement updateUsers = sqlCon.prepareStatement(update_users_QUERY); + PreparedStatement updateAppIdToUserId = sqlCon.prepareStatement(update_appid_to_userid_QUERY); + + int counter = 0; + for(Map.Entry linkEntry : recipeUserIdToPrimaryUserId.entrySet()) { + String primaryUserId = linkEntry.getValue(); + String recipeUserId = linkEntry.getKey(); + + updateUsers.setString(1, primaryUserId); + updateUsers.setString(2, appIdentifier.getAppId()); + updateUsers.setString(3, recipeUserId); + updateUsers.addBatch(); + + updateAppIdToUserId.setString(1, primaryUserId); + updateAppIdToUserId.setString(2, appIdentifier.getAppId()); + updateAppIdToUserId.setString(3, recipeUserId); + updateAppIdToUserId.addBatch(); + + counter++; + if (counter % 100 == 0) { + updateUsers.executeBatch(); + updateAppIdToUserId.executeBatch(); + } + } + + updateUsers.executeBatch(); + updateAppIdToUserId.executeBatch(); + + updateTimeJoinedForPrimaryUsers_Transaction(start, sqlCon, appIdentifier, + new ArrayList<>(recipeUserIdToPrimaryUserId.values())); + } + public static void unlinkAccounts_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, String primaryUserId, String recipeUserId) throws SQLException, StorageQueryException { @@ -1221,6 +1297,17 @@ public static AuthRecipeUserInfo getPrimaryUserInfoForUserId_Transaction(Start s return result.get(0); } + public static List getPrimaryUsersInfoForUserIds_Transaction(Start start, Connection con, + AppIdentifier appIdentifier, List ids) + throws SQLException, StorageQueryException { + + List result = getPrimaryUserInfoForUserIds_Transaction(start, con, appIdentifier, ids); + if (result.isEmpty()) { + return null; + } + return result; + } + private static List getPrimaryUserInfoForUserIds(Start start, AppIdentifier appIdentifier, List userIds) @@ -1670,6 +1757,25 @@ public static void updateTimeJoinedForPrimaryUser_Transaction(Start start, Conne }); } + public static void updateTimeJoinedForPrimaryUsers_Transaction(Start start, Connection sqlCon, + AppIdentifier appIdentifier, List primaryUserIds) + throws SQLException, StorageQueryException { + String QUERY = "UPDATE " + getConfig(start).getUsersTable() + + " SET primary_or_recipe_user_time_joined = (SELECT MIN(time_joined) FROM " + + getConfig(start).getUsersTable() + " WHERE app_id = ? AND primary_or_recipe_user_id = ?) WHERE " + + " app_id = ? AND primary_or_recipe_user_id = ?"; + PreparedStatement updateStatement = sqlCon.prepareStatement(QUERY); + for(String primaryUserId : primaryUserIds) { + updateStatement.setString(1, appIdentifier.getAppId()); + updateStatement.setString(2, primaryUserId); + updateStatement.setString(3, appIdentifier.getAppId()); + updateStatement.setString(4, primaryUserId); + updateStatement.addBatch(); + } + + updateStatement.executeBatch(); + } + private static class AllAuthRecipeUsersResultHolder { String userId; String tenantId; diff --git a/src/main/java/io/supertokens/passwordless/Passwordless.java b/src/main/java/io/supertokens/passwordless/Passwordless.java index 9b5753c4c..2ff4e28ef 100644 --- a/src/main/java/io/supertokens/passwordless/Passwordless.java +++ b/src/main/java/io/supertokens/passwordless/Passwordless.java @@ -21,7 +21,6 @@ import io.supertokens.config.Config; import io.supertokens.emailpassword.exceptions.EmailChangeNotAllowedException; import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.MultitenancyHelper; import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.passwordless.exceptions.*; import io.supertokens.pluginInterface.RECIPE_ID; @@ -42,6 +41,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.passwordless.PasswordlessCode; import io.supertokens.pluginInterface.passwordless.PasswordlessDevice; +import io.supertokens.pluginInterface.passwordless.PasswordlessImportUser; import io.supertokens.pluginInterface.passwordless.exception.*; import io.supertokens.pluginInterface.passwordless.sqlStorage.PasswordlessSQLStorage; import io.supertokens.storageLayer.StorageLayer; @@ -457,53 +457,37 @@ public static ConsumeCodeResponse consumeCode(TenantIdentifier tenantIdentifier, } if (user == null) { - while (true) { - try { - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); - user = passwordlessStorage.createUser(tenantIdentifier, userId, consumedDevice.email, - consumedDevice.phoneNumber, timeJoined); + long timeJoined = System.currentTimeMillis(); + user = createPasswordlessUser(tenantIdentifier, storage, consumedDevice.email, + consumedDevice.phoneNumber, timeJoined); - // Set email as verified, if using email - if (setEmailVerified && consumedDevice.email != null) { + // Set email as verified, if using email + if (setEmailVerified && consumedDevice.email != null) { + try { + AuthRecipeUserInfo finalUser = user; + EmailVerificationSQLStorage evStorage = + StorageUtils.getEmailVerificationStorage(storage); + evStorage.startTransaction(con -> { try { - AuthRecipeUserInfo finalUser = user; - EmailVerificationSQLStorage evStorage = - StorageUtils.getEmailVerificationStorage(storage); - evStorage.startTransaction(con -> { - try { - evStorage.updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + evStorage.updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, finalUser.getSupertokensUserId(), consumedDevice.email, true); - evStorage.commitTransaction(con); + evStorage.commitTransaction(con); - return null; - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } - }); - user.loginMethods[0].setVerified(); // newly created user has only one loginMethod - } catch (StorageTransactionLogicException e) { - if (e.actualException instanceof TenantOrAppNotFoundException) { - throw (TenantOrAppNotFoundException) e.actualException; - } - throw new StorageQueryException(e); + return null; + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); } + }); + user.loginMethods[0].setVerified(); // newly created user has only one loginMethod + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof TenantOrAppNotFoundException) { + throw (TenantOrAppNotFoundException) e.actualException; } - - return new ConsumeCodeResponse(true, user, consumedDevice.email, consumedDevice.phoneNumber, - consumedDevice); - } catch (DuplicateEmailException | DuplicatePhoneNumberException e) { - // Getting these would mean that between getting the user and trying creating it: - // 1. the user managed to do a full create+consume flow - // 2. the users email or phoneNumber was updated to the new one (including device cleanup) - // These should be almost impossibly rare, so it's safe to just ask the user to restart. - // Also, both would make the current login fail if done before the transaction - // by cleaning up the device/code this consume would've used. - throw new RestartFlowException(); - } catch (DuplicateUserIdException e) { - // We can retry.. + throw new StorageQueryException(e); } } + + return new ConsumeCodeResponse(true, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); } else { if (setEmailVerified && consumedDevice.email != null) { // Set email verification @@ -543,6 +527,42 @@ public static ConsumeCodeResponse consumeCode(TenantIdentifier tenantIdentifier, return new ConsumeCodeResponse(false, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); } + public static AuthRecipeUserInfo createPasswordlessUser(TenantIdentifier tenantIdentifier, Storage storage, + String email, String phoneNumber, long timeJoined) + throws TenantOrAppNotFoundException, StorageQueryException, RestartFlowException { + PasswordlessSQLStorage passwordlessStorage = StorageUtils.getPasswordlessStorage(storage); + + while (true) { + try { + String userId = Utils.getUUID(); + return passwordlessStorage.createUser(tenantIdentifier, userId, email, phoneNumber, timeJoined); + } catch (DuplicateEmailException | DuplicatePhoneNumberException e) { + // Getting these would mean that between getting the user and trying creating it: + // 1. the user managed to do a full create+consume flow + // 2. the users email or phoneNumber was updated to the new one (including device cleanup) + // These should be almost impossibly rare, so it's safe to just ask the user to restart. + // Also, both would make the current login fail if done before the transaction + // by cleaning up the device/code this consume would've used. + throw new RestartFlowException(); + } catch (DuplicateUserIdException e) { + // We can retry.. + } + } + } + + public static void createPasswordlessUsers(Storage storage, + List importUsers) + throws TenantOrAppNotFoundException, StorageQueryException, + StorageTransactionLogicException { + PasswordlessSQLStorage passwordlessStorage = StorageUtils.getPasswordlessStorage(storage); + + passwordlessStorage.startTransaction(con -> { + passwordlessStorage.importPasswordlessUsers_Transaction(con, importUsers); + passwordlessStorage.commitTransaction(con); + return null; + }); + } + @TestOnly public static void removeCode(Main main, String codeId) throws StorageQueryException, StorageTransactionLogicException { diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index 4e03fadd4..c054b7feb 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -31,7 +31,10 @@ import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.multitenancy.*; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.MultitenancyStorage; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.useridmapping.UserIdMapping; import io.supertokens.useridmapping.UserIdType; @@ -55,8 +58,15 @@ public Storage getUnderlyingStorage() { return storage; } - public static Storage getNewStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, - boolean doNotLog) throws InvalidConfigException { + public static Storage getNewStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog) throws InvalidConfigException { + return getNewInstance(main, config, tenantIdentifier, doNotLog, false); + } + + public static Storage getNewBulkImportProxyStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog) throws InvalidConfigException { + return getNewInstance(main, config, tenantIdentifier, doNotLog, true); + } + + private static Storage getNewInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog, boolean isBulkImportProxy) throws InvalidConfigException { Storage result; if (StorageLayer.ucl == null) { result = new Start(main); @@ -76,8 +86,15 @@ public static Storage getNewStorageInstance(Main main, JsonObject config, Tenant } if (storageLayer != null && !main.isForceInMemoryDB() && (storageLayer.canBeUsed(config) || CLIOptions.get(main).isForceNoInMemoryDB())) { - result = storageLayer; + if (isBulkImportProxy) { + result = storageLayer.createBulkImportProxyStorageInstance(); + } else { + result = storageLayer; + } } else { + if (isBulkImportProxy) { + throw new QuitProgramException("Creating a bulk import proxy storage instance with in-memory DB is not supported."); + } result = new Start(main); } } @@ -558,4 +575,44 @@ public static StorageAndUserIdMapping findStorageAndUserIdMappingForUser( throw new IllegalStateException("should never come here"); } } + + public static List findStorageAndUserIdMappingForBulkUserImport( + AppIdentifier appIdentifier, Storage[] storages, List userIds, + UserIdType userIdType) throws StorageQueryException { + + if (storages.length == 0) { + throw new IllegalStateException("No storages were provided!"); + } + + if (storages[0].getType() != STORAGE_TYPE.SQL) { + // for non sql plugin, there will be only one storage as multitenancy is not supported + assert storages.length == 1; + List results = new ArrayList<>(); + for(String userId : userIds) { + results.add(new StorageAndUserIdMapping(storages[0], new UserIdMapping(userId, null, null))); + } + return results; + } + List allMappingsFromAllStorages = new ArrayList<>(); + if (userIdType != UserIdType.ANY) { + for (Storage storage : storages) { + List existingIdsInStorage = ((AuthRecipeStorage)storage).findExistingUserIds(appIdentifier, userIds); + List mappingsFromThisStorage = io.supertokens.useridmapping.UserIdMapping.getMultipleUserIdMapping( + appIdentifier, storage, + userIds, userIdType); + + for(String existingId : existingIdsInStorage) { + UserIdMapping mappingForId = mappingsFromThisStorage.stream() + .filter(userIdMapping -> (userIdType == UserIdType.SUPERTOKENS && userIdMapping.superTokensUserId.equals(existingId)) + || (userIdType == UserIdType.EXTERNAL && userIdMapping.externalUserId.equals(existingId)) ) + .findFirst().orElse(null); + allMappingsFromAllStorages.add(new StorageAndUserIdMappingForBulkImport(storage, mappingForId, existingId)); + } + } + } else { + throw new IllegalStateException("UserIdType.ANY is not supported for this method"); + } + return allMappingsFromAllStorages; + } + } diff --git a/src/main/java/io/supertokens/thirdparty/ThirdParty.java b/src/main/java/io/supertokens/thirdparty/ThirdParty.java index e29b3e44c..a7f18bb54 100644 --- a/src/main/java/io/supertokens/thirdparty/ThirdParty.java +++ b/src/main/java/io/supertokens/thirdparty/ThirdParty.java @@ -34,6 +34,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateUserIdException; import io.supertokens.pluginInterface.thirdparty.sqlStorage.ThirdPartySQLStorage; @@ -209,22 +210,12 @@ private static SignInUpResponse signInUpHelper(TenantIdentifier tenantIdentifier while (true) { // loop for sign in + sign up - while (true) { - // loop for sign up - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); + long timeJoined = System.currentTimeMillis(); - try { - AuthRecipeUserInfo createdUser = tpStorage.signUp(tenantIdentifier, userId, email, - new LoginMethod.ThirdParty(thirdPartyId, thirdPartyUserId), timeJoined); - - return new SignInUpResponse(true, createdUser); - } catch (DuplicateUserIdException e) { - // we try again.. - } catch (DuplicateThirdPartyUserException e) { - // we try to sign in - break; - } + try { + return createThirdPartyUser( tenantIdentifier, storage, thirdPartyId, thirdPartyUserId, email, timeJoined); + } catch (DuplicateThirdPartyUserException e) { + // The user already exists, we will try to update the email if needed below } // we try to get user and update their email @@ -346,6 +337,37 @@ private static SignInUpResponse signInUpHelper(TenantIdentifier tenantIdentifier } } + public static SignInUpResponse createThirdPartyUser(TenantIdentifier tenantIdentifier, Storage storage, + String thirdPartyId, String thirdPartyUserId, String email, long timeJoined) + throws StorageQueryException, TenantOrAppNotFoundException, DuplicateThirdPartyUserException { + ThirdPartySQLStorage tpStorage = StorageUtils.getThirdPartyStorage(storage); + + while (true) { + // loop for sign up + String userId = Utils.getUUID(); + + try { + AuthRecipeUserInfo createdUser = tpStorage.signUp(tenantIdentifier, userId, email, + new LoginMethod.ThirdParty(thirdPartyId, thirdPartyUserId), timeJoined); + return new SignInUpResponse(true, createdUser); + } catch (DuplicateUserIdException e) { + // we try again.. + } + } + } + + public static void createMultipleThirdPartyUsers(Storage storage, + List usersToImport) + throws StorageQueryException, StorageTransactionLogicException, TenantOrAppNotFoundException { + + ThirdPartySQLStorage tpStorage = StorageUtils.getThirdPartyStorage(storage); + tpStorage.startTransaction(con -> { + tpStorage.importThirdPartyUsers_Transaction(con, usersToImport); + tpStorage.commitTransaction(con); + return null; + }); + } + @Deprecated public static AuthRecipeUserInfo getUser(AppIdentifier appIdentifier, Storage storage, String userId) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/totp/Totp.java b/src/main/java/io/supertokens/totp/Totp.java index d2afce084..c8b705668 100644 --- a/src/main/java/io/supertokens/totp/Totp.java +++ b/src/main/java/io/supertokens/totp/Totp.java @@ -5,10 +5,10 @@ import io.supertokens.config.Config; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.mfa.Mfa; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; @@ -33,6 +33,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; +import java.util.List; public class Totp { private static String generateSecret() throws NoSuchAlgorithmException { @@ -143,6 +144,26 @@ public static TOTPDevice createDevice(Main main, AppIdentifier appIdentifier, St } } + public static void createDevices(Main main, AppIdentifier appIdentifier, Storage storage, List devices) + throws StorageQueryException, FeatureNotEnabledException, + StorageTransactionLogicException { + + try { + Mfa.checkForMFAFeature(appIdentifier, main); + + TOTPSQLStorage totpStorage = StorageUtils.getTOTPStorage(storage); + + totpStorage.startTransaction(con -> { + totpStorage.createDevices_Transaction(con, appIdentifier, devices); + totpStorage.commitTransaction(con); + return null; + }); + + } catch (TenantOrAppNotFoundException e ) { + throw new StorageTransactionLogicException(e); + } + } + public static TOTPDevice registerDevice(AppIdentifier appIdentifier, Storage storage, Main main, String userId, String deviceName, int skew, int period) throws StorageQueryException, DeviceAlreadyExistsException, NoSuchAlgorithmException, diff --git a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java index 5f81c81ee..453ec6e54 100644 --- a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java +++ b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java @@ -18,10 +18,12 @@ import io.supertokens.Main; import io.supertokens.StorageAndUserIdMapping; +import io.supertokens.StorageAndUserIdMappingForBulkImport; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; import io.supertokens.pluginInterface.emailverification.EmailVerificationStorage; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -45,9 +47,31 @@ import javax.annotation.Nullable; import java.util.*; +import java.util.stream.Collectors; public class UserIdMapping { + public static class UserIdBulkMappingResult { + public String supertokensUserId; + public String externalUserId; + public Exception error; + + public UserIdBulkMappingResult(String supertokensUserId, String externalUserId, Exception error) { + this.supertokensUserId = supertokensUserId; + this.error = error; + this.externalUserId = externalUserId; + } + + @Override + public String toString() { + return "UserIdBulkMappingResult{" + + "supertokensUserId='" + supertokensUserId + '\'' + + ", externalUserId='" + externalUserId + '\'' + + ", error=" + error + + '}'; + } + } + @TestOnly public static void createUserIdMapping(AppIdentifier appIdentifier, Storage[] storages, String superTokensUserId, String externalUserId, @@ -158,11 +182,211 @@ public static void createUserIdMapping(AppIdentifier appIdentifier, Storage[] st } } + StorageUtils.getUserIdMappingStorage(userStorage) .createUserIdMapping(appIdentifier, superTokensUserId, externalUserId, externalUserIdInfo); } + //support method for the primary intention of bulk importing users. + public static List createMultipleUserIdMappings(AppIdentifier appIdentifier, Storage[] storages, + Map superTokensUserIdToExternalUserId, boolean force, + boolean makeExceptionForEmailVerification) + throws StorageQueryException { + + // We first need to check if the external user id exists across all app storages because we do not want + // 2 users from different user pool but same app to point to same external user id. + // We may still end up having that situation due to race conditions, as we are not taking any app level lock, + // but we are okay with it as of now, by returning prioritized mapping based on which the tenant the request + // came from. + // This issue - https://github.com/supertokens/supertokens-core/issues/610 - must be resolved when the + // race condition is fixed. + + List mappingResults = new ArrayList<>(); + + // with external id + List mappingAndStorageWithExternal = + StorageLayer.findStorageAndUserIdMappingForBulkUserImport( + appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.values()), UserIdType.EXTERNAL); + + // with supertokens id + List mappingAndStorageWithSupertokens = + StorageLayer.findStorageAndUserIdMappingForBulkUserImport( + appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.keySet()), UserIdType.SUPERTOKENS); + + //with external id treated as supertokens id - should not happen + List mappingAndStoragesAsInvalid = StorageLayer.findStorageAndUserIdMappingForBulkUserImport( + appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.values()), UserIdType.SUPERTOKENS); + + Map> userIdsUsedInNonAuthRecipes = + storages[0].findNonAuthRecipesWhereForUserIdsUsed(appIdentifier, new ArrayList<>(superTokensUserIdToExternalUserId.keySet())); + + //for collecting which users needs to be updated + Map supertokensToExternalUserIdsToUpdateEmailVerified = new HashMap<>(); + List noErrorFound = new ArrayList<>(); + + for(Map.Entry supertokensIdToExternalId : superTokensUserIdToExternalUserId.entrySet()) { + String supertokensId = supertokensIdToExternalId.getKey(); + String externalId = supertokensIdToExternalId.getValue(); + StorageAndUserIdMapping mappingByExternal = findStorageAndUserIdMappingForUser(externalId, + mappingAndStorageWithExternal, false); + if (mappingByExternal != null && mappingByExternal.userIdMapping != null) { + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, + new UserIdMappingAlreadyExistsException( + supertokensId.equals(mappingByExternal.userIdMapping.superTokensUserId), + externalId.equals(mappingByExternal.userIdMapping.externalUserId)))); + continue; + } + StorageAndUserIdMapping mappingBySupertokens = findStorageAndUserIdMappingForUser(supertokensId, + mappingAndStorageWithSupertokens, true); + if (mappingBySupertokens == null) { + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, + new UnknownSuperTokensUserIdException())); + continue; + } + Storage userStorage = mappingBySupertokens.storage; + + // if a userIdMapping is created with force, then we skip the following checks + if (!force) { + // We do not allow for a UserIdMapping to be created when the externalUserId is a SuperTokens userId. + // There could be a case where User_1 has a userId mapping and a new SuperTokens User, User_2 is created + // whose userId is equal to the User_1's externalUserId. + // Theoretically this could happen but the likelihood of generating a non-unique UUID is low enough that we + // ignore it. + + { + if (findStorageAndUserIdMappingForUser(externalId, mappingAndStoragesAsInvalid, true) != null) { + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, + new ServletException(new WebserverAPI.BadRequestException( + "Cannot create a userId mapping where the externalId is also a SuperTokens userID")))); + continue; + } + } + + List storageClasses; + if (userIdsUsedInNonAuthRecipes.containsKey(supertokensId)) { + storageClasses = userIdsUsedInNonAuthRecipes.get(supertokensId); + } else { + storageClasses = new ArrayList<>(); + } + + if (makeExceptionForEmailVerification) { + // check that none of the non-auth recipes are using the superTokensUserId + + if (storageClasses.size() == 1 && + storageClasses.get(0).equals(EmailVerificationStorage.class.getName())) { + // if the userId is used in email verification, then we do an exception and update the + // isEmailVerified + // to the externalUserId. We do this because we automatically set the isEmailVerified to true for + // passwordless + // and third party sign in up when the user info from provider says the email is verified and If + // we don't make + // an exception, then the creation of userIdMapping for the user will be blocked. And, to + // overcome that the + // email will have to be unverified first, then the userIdMapping should be created and then the + // email must be + // verified again on the externalUserId, which is not a good user experience. + supertokensToExternalUserIdsToUpdateEmailVerified.put(supertokensId, externalId); + + } else if (!storageClasses.isEmpty()) { + createBulkIdMappingErrorForNonAuthRecipeUsage(storageClasses, mappingResults, supertokensId, + externalId); + continue; + } + } else { + //if we are not making any exceptions, then having the id used is an error! + if (!storageClasses.isEmpty()) { + createBulkIdMappingErrorForNonAuthRecipeUsage(storageClasses, mappingResults, supertokensId, + externalId); + continue; + } + } + + noErrorFound.add(mappingBySupertokens); + } + } + //userstorage - group users by storage + Map> partitionedMappings = partitionUsersByStorage(noErrorFound); + for(Storage storage : partitionedMappings.keySet()){ + + List mappingsForCurrentStorage = partitionedMappings.get(storage); + Map mappingInCurrentStorageThatNeedsToBeDone = new HashMap<>(); + Map supertokensIdToExternalIdInCurrentStorageForEmailUpdate = new HashMap<>(); + + for(StorageAndUserIdMapping storageAndUserIdMapping: mappingsForCurrentStorage) { + String userIdInQuestion = ((StorageAndUserIdMappingForBulkImport)storageAndUserIdMapping).userIdInQuestion; + + if(supertokensToExternalUserIdsToUpdateEmailVerified.keySet().contains(userIdInQuestion)){ + supertokensIdToExternalIdInCurrentStorageForEmailUpdate.put(userIdInQuestion, + superTokensUserIdToExternalUserId.get(userIdInQuestion)); + } + mappingInCurrentStorageThatNeedsToBeDone.put(userIdInQuestion, superTokensUserIdToExternalUserId.get(userIdInQuestion)); + } + + StorageUtils.getUserIdMappingStorage(storage).createBulkUserIdMapping(appIdentifier, mappingInCurrentStorageThatNeedsToBeDone); + + EmailVerificationStorage emailVerificationStorage = StorageUtils.getEmailVerificationStorage(storage); + emailVerificationStorage.updateMultipleIsEmailVerifiedToExternalUserIds(appIdentifier, supertokensIdToExternalIdInCurrentStorageForEmailUpdate); + + for(String supertokensIdForResult : mappingInCurrentStorageThatNeedsToBeDone.keySet()) { + mappingResults.add(new UserIdBulkMappingResult(supertokensIdForResult, mappingInCurrentStorageThatNeedsToBeDone.get(supertokensIdForResult), null)); + } + } + + Map errors = new HashMap<>(); + for(UserIdBulkMappingResult result : mappingResults){ + if(result.error != null) { + errors.put(result.supertokensUserId, result.error); + } + } + if(!errors.isEmpty()) { + throw new StorageQueryException(new BulkImportBatchInsertException("useridmapping errors", errors)); + } + return mappingResults; + } + + private static void createBulkIdMappingErrorForNonAuthRecipeUsage(List storageClasses, + List mappingResults, + String supertokensId, String externalId) { + String recipeName = storageClasses.get(0); + String[] parts = recipeName.split("[.]"); + recipeName = parts[parts.length - 1]; + recipeName = recipeName.replace("Storage", ""); + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, new ServletException(new WebserverAPI.BadRequestException( + "UserId is already in use in " + recipeName + " recipe")))); + } + + private static Map> partitionUsersByStorage(List storageAndMappings){ + Map> results = new HashMap<>(); + for(StorageAndUserIdMapping storageAndUserIdMapping : storageAndMappings) { + if(!results.containsKey(storageAndUserIdMapping.storage)){ + results.put(storageAndUserIdMapping.storage, new ArrayList<>()); + } + results.get(storageAndUserIdMapping.storage).add(storageAndUserIdMapping); + } + return results; + } + + private static StorageAndUserIdMapping findStorageAndUserIdMappingForUser(String userId, List findIn, boolean supertokensId) { + List mappings = findIn.stream().filter(storageAndUserIdMapping -> { + if(storageAndUserIdMapping instanceof StorageAndUserIdMappingForBulkImport && ((StorageAndUserIdMappingForBulkImport) storageAndUserIdMapping).userIdInQuestion != null) { + return ((StorageAndUserIdMappingForBulkImport)storageAndUserIdMapping).userIdInQuestion.equals(userId); + } else if(storageAndUserIdMapping.userIdMapping != null) { + if(supertokensId) { + return userId.equals(storageAndUserIdMapping.userIdMapping.superTokensUserId); + } else { + return userId.equals(storageAndUserIdMapping.userIdMapping.externalUserId); + } + } + return false; + }).collect(Collectors.toList()); // theoretically it shouldn't happen that there are more than one element in the list + if(mappings.size() > 1 && !(mappings.get(0) instanceof StorageAndUserIdMappingForBulkImport)) { + throw new IllegalStateException("more than one mapping exists for Id."); + } + return mappings.isEmpty() ? null : mappings.get(0); + } + + @TestOnly public static void createUserIdMapping(Main main, String superTokensUserId, String externalUserId, @@ -208,6 +432,26 @@ public static io.supertokens.pluginInterface.useridmapping.UserIdMapping getUser } } + public static List getMultipleUserIdMapping( + AppIdentifier appIdentifier, Storage storage, List userIds, + UserIdType userIdType) + throws StorageQueryException { + UserIdMappingSQLStorage uidMappingStorage = + (UserIdMappingSQLStorage) storage; + + try { + return uidMappingStorage.startTransaction(con -> { + return uidMappingStorage.getMultipleUserIdMapping_Transaction(con, appIdentifier, userIds, userIdType == UserIdType.SUPERTOKENS); + }); + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof StorageQueryException) { + throw (StorageQueryException) e.actualException; + } else { + throw new IllegalStateException(e.actualException); + } + } + } + public static io.supertokens.pluginInterface.useridmapping.UserIdMapping getUserIdMapping( TransactionConnection con, AppIdentifier appIdentifier, Storage storage, String userId, diff --git a/src/main/java/io/supertokens/usermetadata/UserMetadata.java b/src/main/java/io/supertokens/usermetadata/UserMetadata.java index 938f6f749..545e5eaa1 100644 --- a/src/main/java/io/supertokens/usermetadata/UserMetadata.java +++ b/src/main/java/io/supertokens/usermetadata/UserMetadata.java @@ -30,6 +30,8 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Map; public class UserMetadata { @@ -76,6 +78,42 @@ public static JsonObject updateUserMetadata(AppIdentifier appIdentifier, Storage } } + public static void updateMultipleUsersMetadata(AppIdentifier appIdentifier, Storage storage, + @Nonnull Map metadataToUpdateByUserId) + throws StorageQueryException, StorageTransactionLogicException, TenantOrAppNotFoundException { + UserMetadataSQLStorage umdStorage = StorageUtils.getUserMetadataStorage(storage); + + try { + umdStorage.startTransaction(con -> { + Map originalMetadatas = umdStorage.getMultipleUsersMetadatas_Transaction(appIdentifier, con, + new ArrayList<>(metadataToUpdateByUserId.keySet())); + + // updating only the already existing ones. The others don't need update + for(Map.Entry metadataByUserId : originalMetadatas.entrySet()){ + JsonObject originalMetadata = metadataByUserId.getValue(); + String userId = metadataByUserId.getKey(); + JsonObject updatedMetadata = originalMetadata == null ? new JsonObject() : originalMetadata; + MetadataUtils.shallowMergeMetadataUpdate(updatedMetadata, metadataToUpdateByUserId.get(userId)); + metadataToUpdateByUserId.put(userId, updatedMetadata); + } + + try { + umdStorage.setMultipleUsersMetadatas_Transaction(appIdentifier, con, metadataToUpdateByUserId); + umdStorage.commitTransaction(con); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + + return null; + }); + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof TenantOrAppNotFoundException) { + throw (TenantOrAppNotFoundException) e.actualException; + } + throw e; + } + } + @TestOnly public static JsonObject getUserMetadata(Main main, @Nonnull String userId) throws StorageQueryException { Storage storage = StorageLayer.getStorage(main); diff --git a/src/main/java/io/supertokens/userroles/UserRoles.java b/src/main/java/io/supertokens/userroles/UserRoles.java index 21b132aa1..8d88b637a 100644 --- a/src/main/java/io/supertokens/userroles/UserRoles.java +++ b/src/main/java/io/supertokens/userroles/UserRoles.java @@ -19,6 +19,7 @@ import io.supertokens.Main; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; @@ -31,7 +32,7 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; -import java.util.Arrays; +import java.util.*; public class UserRoles { // add a role to a user and return true, if the role is already mapped to the user return false, but if @@ -57,6 +58,55 @@ public static boolean addRoleToUser(Main main, TenantIdentifier tenantIdentifier } } + public static void addMultipleRolesToMultipleUsers(Main main, AppIdentifier appIdentifier, Storage storage, + Map>> rolesToUserByTenant) + throws StorageTransactionLogicException, TenantOrAppNotFoundException { + + // Roles are stored in public tenant storage and role to user mapping is stored in the tenant's storage + // We do this because it's not straight forward to replicate roles to all storages of an app + Storage appStorage = StorageLayer.getStorage( + appIdentifier.getAsPublicTenantIdentifier(), main); + + try { + UserRolesSQLStorage userRolesStorage = StorageUtils.getUserRolesStorage(storage); + UserRolesSQLStorage publicRoleStorage = StorageUtils.getUserRolesStorage(appStorage); + Map errorsByUser = new HashMap<>(); + publicRoleStorage.startTransaction(con -> { + Set rolesToSearchFor = new HashSet<>(); + for (TenantIdentifier tenantIdentifier : rolesToUserByTenant.keySet()) { + for(String userId : rolesToUserByTenant.get(tenantIdentifier).keySet()){ + rolesToSearchFor.addAll(rolesToUserByTenant.get(tenantIdentifier).get(userId)); + } + } + List rolesFound = ((UserRolesSQLStorage) appStorage).doesMultipleRoleExist_Transaction( + appIdentifier, con, + new ArrayList<>(rolesToSearchFor)); + + for (Map> rolesToUsers : rolesToUserByTenant.values()) { + for (String userId : rolesToUsers.keySet()) { + List rolesOfUser = rolesToUsers.get(userId); + if (!new HashSet<>(rolesFound).containsAll(rolesOfUser)) { //wrapping in hashset for performance reasons + errorsByUser.put(userId, new UnknownRoleException()); + } + } + } + if (!errorsByUser.isEmpty()) { + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("Roles errors", errorsByUser)); + } + return null; + }); + userRolesStorage.startTransaction(con -> { + userRolesStorage.addRolesToUsers_Transaction(con, rolesToUserByTenant); + userRolesStorage.commitTransaction(con); + return null; + }); + + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + @TestOnly public static boolean addRoleToUser(Main main, String userId, String role) throws StorageQueryException, UnknownRoleException { diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java new file mode 100644 index 000000000..89a8ea932 --- /dev/null +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.utils; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class JsonValidatorUtils { + @SuppressWarnings("unchecked") + public static T parseAndValidateFieldType(JsonObject jsonObject, String key, ValueType expectedType, + boolean isRequired, Class targetType, List errors, String errorSuffix) { + if (jsonObject.has(key)) { + if (validateJsonFieldType(jsonObject, key, expectedType)) { + T value; + switch (expectedType) { + case STRING: + value = (T) jsonObject.get(key).getAsString(); + break; + case INTEGER: + Integer intValue = jsonObject.get(key).getAsNumber().intValue(); + value = (T) intValue; + break; + case LONG: + Long longValue = jsonObject.get(key).getAsNumber().longValue(); + value = (T) longValue; + break; + case BOOLEAN: + Boolean boolValue = jsonObject.get(key).getAsBoolean(); + value = (T) boolValue; + break; + case OBJECT: + value = (T) jsonObject.get(key).getAsJsonObject(); + break; + case ARRAY_OF_OBJECT, ARRAY_OF_STRING: + value = (T) jsonObject.get(key).getAsJsonArray(); + break; + default: + value = null; + break; + } + if (value != null) { + return targetType.cast(value); + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else if (isRequired) { + errors.add(key + " is required" + errorSuffix); + } + return null; + } + + public enum ValueType { + STRING, + INTEGER, + LONG, + BOOLEAN, + OBJECT, + ARRAY_OF_STRING, + ARRAY_OF_OBJECT + } + + private static String getTypeForErrorMessage(ValueType type) { + return switch (type) { + case STRING -> "string"; + case INTEGER -> "integer"; + case LONG -> "integer"; // choosing integer over long because it is user facing + case BOOLEAN -> "boolean"; + case OBJECT -> "object"; + case ARRAY_OF_STRING -> "array of string"; + case ARRAY_OF_OBJECT -> "array of object"; + }; + } + + public static boolean validateJsonFieldType(JsonObject jsonObject, String key, ValueType expectedType) { + if (jsonObject.has(key)) { + return switch (expectedType) { + case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() + && !jsonObject.get(key).getAsString().isBlank(); + case INTEGER, LONG -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); + case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); + case OBJECT -> jsonObject.get(key).isJsonObject(); + case ARRAY_OF_OBJECT, ARRAY_OF_STRING -> jsonObject.get(key).isJsonArray() + && validateArrayElements(jsonObject.getAsJsonArray(key), expectedType); + default -> false; + }; + } + return false; + } + + public static boolean validateArrayElements(JsonArray array, ValueType expectedType) { + List elements = new ArrayList<>(); + array.forEach(elements::add); + + return switch (expectedType) { + case ARRAY_OF_OBJECT -> elements.stream().allMatch(JsonElement::isJsonObject); + case ARRAY_OF_STRING -> + elements.stream().allMatch(el -> el.isJsonPrimitive() && el.getAsJsonPrimitive().isString() + && !el.getAsString().isBlank()); + default -> false; + }; + } +} diff --git a/src/main/java/io/supertokens/utils/Utils.java b/src/main/java/io/supertokens/utils/Utils.java index 9775bb0bb..d02ebce5b 100644 --- a/src/main/java/io/supertokens/utils/Utils.java +++ b/src/main/java/io/supertokens/utils/Utils.java @@ -20,11 +20,13 @@ import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.i18n.phonenumbers.PhoneNumberUtil; import com.google.i18n.phonenumbers.NumberParseException; +import com.google.i18n.phonenumbers.PhoneNumberUtil; import com.google.i18n.phonenumbers.Phonenumber; import io.supertokens.Main; import io.supertokens.config.Config; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; import io.supertokens.jwt.exceptions.UnsupportedJWTSigningAlgorithmException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; @@ -51,8 +53,8 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.RSAPublicKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Arrays; import java.util.Base64; import java.util.Base64.Decoder; import java.util.Base64.Encoder; @@ -432,6 +434,12 @@ public static JsonElement toJsonTreeWithNulls(Object src) { return new GsonBuilder().serializeNulls().create().toJsonTree(src); } + + public static boolean isAccountLinkingEnabled(Main main, AppIdentifier appIdentifier) throws StorageQueryException, TenantOrAppNotFoundException { + return Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .anyMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA); + } + public static boolean containsUrl(String urlToCheckIfContains, String whatItContains, boolean careForProtocol) throws MalformedURLException { URL urlToCheck = new URL(urlToCheckIfContains); @@ -467,4 +475,5 @@ public static String snakeCaseToCamelCase(String toCamelCase) { } return toCamelCase; } + } diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 9a2940898..f6517c8d8 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -26,6 +26,10 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; +import io.supertokens.webserver.api.bulkimport.BulkImportAPI; +import io.supertokens.webserver.api.bulkimport.CountBulkImportUsersAPI; +import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; +import io.supertokens.webserver.api.bulkimport.ImportUserAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -280,6 +284,11 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); addAPI(new GetTenantCoreConfigForDashboardAPI(main)); + addAPI(new BulkImportAPI(main)); + addAPI(new DeleteBulkImportUserAPI(main)); + addAPI(new ImportUserAPI(main)); + addAPI(new CountBulkImportUsersAPI(main)); + addAPI(new OAuthAuthAPI(main)); addAPI(new OAuthTokenAPI(main)); addAPI(new CreateUpdateOrGetOAuthClientAPI(main)); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java new file mode 100644 index 000000000..50856db03 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.bulkimport.BulkImportUserPaginationToken; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class BulkImportAPI extends WebserverAPI { + public BulkImportAPI(Main main) { + super(main, "bulkimport"); + } + + @Override + public String getPath() { + return "/bulk-import/users"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); + String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); + Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); + + if (limit != null) { + if (limit > BulkImport.GET_USERS_PAGINATION_MAX_LIMIT) { + throw new ServletException( + new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_MAX_LIMIT)); + } else if (limit < 1) { + throw new ServletException(new BadRequestException("limit must a positive integer with min value 1")); + } + } else { + limit = BulkImport.GET_USERS_DEFAULT_LIMIT; + } + + BULK_IMPORT_USER_STATUS status = null; + if (statusString != null) { + try { + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException(new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + BulkImportUserPaginationContainer users = BulkImport.getUsers(appIdentifier, storage, limit, status, paginationToken); + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + + JsonArray usersJson = new JsonArray(); + for (BulkImportUser user : users.users) { + usersJson.add(user.toJsonObject()); + } + result.add("users", usersJson); + + if (users.nextPaginationToken != null) { + result.addProperty("nextPaginationToken", users.nextPaginationToken); + } + super.sendJsonResponse(200, result, resp); + } catch (BulkImportUserPaginationToken.InvalidTokenException e) { + Logging.debug(main, null, Utils.exceptionStacktraceToString(e)); + throw new ServletException(new BadRequestException("invalid pagination token")); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); + + if (users.size() == 0) { + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + return; + } + + if (users.size() > BulkImport.MAX_USERS_TO_ADD) { + JsonObject errorResponseJson = new JsonObject(); + String errorMsg = users.size() <= 0 ? "You need to add at least one user." + : "You can only add " + BulkImport.MAX_USERS_TO_ADD + " users at a time."; + errorResponseJson.addProperty("error", errorMsg); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + String[] allUserRoles = null; + + try { + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + + JsonArray errorsJson = new JsonArray(); + List usersToAdd = new ArrayList<>(); + + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + for (int i = 0; i < users.size(); i++) { + try { + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID()); + usersToAdd.add(user); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonObject errorObj = new JsonObject(); + + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + errorObj.addProperty("index", i); + errorObj.add("errors", errors); + errorsJson.add(errorObj); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new ServletException(e); + } + } + + if (errorsJson.size() > 0) { + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.addProperty("error", + "Data has missing or invalid fields. Please check the users field for more details."); + errorResponseJson.add("users", errorsJson); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + try { + BulkImport.addUsers(appIdentifier, storage, usersToAdd); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new ServletException(e); + } + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + } +} diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java new file mode 100644 index 000000000..4536209b6 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import com.google.gson.JsonObject; +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; + +public class CountBulkImportUsersAPI extends WebserverAPI { + public CountBulkImportUsersAPI(Main main) { + super(main, "bulkimport"); + } + + @Override + public String getPath() { + return "/bulk-import/users/count"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); + + BULK_IMPORT_USER_STATUS status = null; + if (statusString != null) { + try { + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException( + new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, status); + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.addProperty("count", count); + super.sendJsonResponse(200, result, resp); + + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java new file mode 100644 index 000000000..343d04cc0 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.util.List; + +public class DeleteBulkImportUserAPI extends WebserverAPI { + public DeleteBulkImportUserAPI(Main main) { + super(main, "bulkimport"); + } + + @Override + public String getPath() { + return "/bulk-import/users/remove"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); + + if (arr.size() == 0) { + JsonObject result = new JsonObject(); + result.add("deletedIds", new JsonArray()); + result.add("invalidIds", new JsonArray()); + super.sendJsonResponse(200, result, resp); + return; + } + + if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " + + BulkImport.DELETE_USERS_MAX_LIMIT + " elements")); + } + + String[] userIds = new String[arr.size()]; + + for (int i = 0; i < userIds.length; i++) { + String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); + if (userId.isEmpty()) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); + } + userIds[i] = userId; + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); + + JsonArray deletedIdsJson = new JsonArray(); + JsonArray invalidIds = new JsonArray(); + + for (String userId : userIds) { + if (deletedIds.contains(userId)) { + deletedIdsJson.add(new JsonPrimitive(userId)); + } else { + invalidIds.add(new JsonPrimitive(userId)); + } + } + + JsonObject result = new JsonObject(); + result.add("deletedIds", deletedIdsJson); + result.add("invalidIds", invalidIds); + + super.sendJsonResponse(200, result, resp); + + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java new file mode 100644 index 000000000..646e21915 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; + +public class ImportUserAPI extends WebserverAPI { + public ImportUserAPI(Main main) { + super(main, "bulkimport"); + } + + @Override + public String getPath() { + return "/bulk-import/import"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(req); + + AppIdentifier appIdentifier = null; + Storage storage = null; + String[] allUserRoles = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } + + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + + try { + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, jsonUser, + Utils.getUUID()); + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, user); + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.add("user", importedUser.toJson()); + super.sendJsonResponse(200, result, resp); + } catch (BulkImportBatchInsertException e) { + JsonArray errors = new JsonArray(); + errors.addAll( + e.exceptionByUserId.values().stream().map(exc -> exc.getMessage()).map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll) + ); + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (StorageQueryException storageQueryException){ + JsonArray errors = new JsonArray(); + errors.add(new JsonPrimitive(storageQueryException.getMessage())); + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/CronjobTest.java b/src/test/java/io/supertokens/test/CronjobTest.java index dc3a557bd..a29cdf8fe 100644 --- a/src/test/java/io/supertokens/test/CronjobTest.java +++ b/src/test/java/io/supertokens/test/CronjobTest.java @@ -32,7 +32,6 @@ import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.multitenancy.*; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.storageLayer.StorageLayer; import org.junit.AfterClass; import org.junit.Before; @@ -1049,6 +1048,7 @@ public void testThatThereAreTasksOfAllCronTaskClassesAndHaveCorrectIntervals() t intervals.put("io.supertokens.cronjobs.telemetry.Telemetry", 86400); intervals.put("io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys", 86400); + intervals.put("io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers", 60); intervals.put("io.supertokens.cronjobs.cleanupOAuthSessionsAndChallenges.CleanupOAuthSessionsAndChallenges", 86400); @@ -1065,6 +1065,7 @@ public void testThatThereAreTasksOfAllCronTaskClassesAndHaveCorrectIntervals() t delays.put("io.supertokens.cronjobs.telemetry.Telemetry", 0); delays.put("io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys", 0); + delays.put("io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers", 0); delays.put("io.supertokens.cronjobs.cleanupOAuthSessionsAndChallenges.CleanupOAuthSessionsAndChallenges", 0); @@ -1079,4 +1080,42 @@ public void testThatThereAreTasksOfAllCronTaskClassesAndHaveCorrectIntervals() t process.kill(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + + @Test + public void testThatIsCronJobLoadedReturnsTheGoodValues() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + boolean isLoaded = Cronjobs.isCronjobLoaded(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + + assertFalse(isLoaded); + + Cronjobs.addCronjob(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + isLoaded = Cronjobs.isCronjobLoaded(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + + assertTrue(isLoaded); + + Cronjobs.removeCronjob(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + isLoaded = Cronjobs.isCronjobLoaded(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + + assertFalse(isLoaded); + + //removing twice doesn't do anything funky + Cronjobs.removeCronjob(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + isLoaded = Cronjobs.isCronjobLoaded(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + + assertFalse(isLoaded); + + //adding twice doesn't do anything funky + Cronjobs.addCronjob(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + isLoaded = Cronjobs.isCronjobLoaded(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + + assertTrue(isLoaded); + Cronjobs.addCronjob(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + isLoaded = Cronjobs.isCronjobLoaded(process.getProcess(), CounterCronJob.getInstance(process.getProcess())); + + assertTrue(isLoaded); + } } diff --git a/src/test/java/io/supertokens/test/Utils.java b/src/test/java/io/supertokens/test/Utils.java index 0657dcad4..fd8a83fc8 100644 --- a/src/test/java/io/supertokens/test/Utils.java +++ b/src/test/java/io/supertokens/test/Utils.java @@ -83,6 +83,7 @@ public static String getCdiVersionStringLatestForTests() { public static void reset() { Main.isTesting = true; + Main.isTesting_skipBulkImportUserValidationInCronJob = false; PluginInterfaceTesting.isTesting = true; Main.makeConsolePrintSilent = true; String installDir = "../"; diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java new file mode 100644 index 000000000..6b4717046 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -0,0 +1,830 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.multitenancy.*; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.test.httpRequest.HttpResponseException; +import io.supertokens.userroles.UserRoles; +import org.jetbrains.annotations.NotNull; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.UUID; + +import static org.junit.Assert.*; + +public class BulkImportFlowTest { + + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void testWithALotOfUsers() throws Exception { + Main main = startCronProcess("14"); + + int NUMBER_OF_USERS_TO_UPLOAD = 100000; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // upload a bunch of users through the API + { + for (int i = 0; i < (NUMBER_OF_USERS_TO_UPLOAD / 10000); i++) { + JsonObject request = generateUsersJson(10000, i * 10000); // API allows 10k users upload at once + JsonObject response = uploadBulkImportUsersJson(main, request); + assertEquals("OK", response.get("status").getAsString()); + } + + } + + long processingStarted = System.currentTimeMillis(); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + { + long count = NUMBER_OF_USERS_TO_UPLOAD; + while(true) { + try { + JsonObject response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + count = response.get("count").getAsLong(); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + int failedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + count = newUsersNumber + processingUsersNumber; + + if (count == 0) { + break; + } + } catch (Exception e) { + if(e instanceof SocketTimeoutException) { + //ignore + } else { + throw e; + } + } + Thread.sleep(5000); + } + } + + long processingFinished = System.currentTimeMillis(); + System.out.println("Processed " + NUMBER_OF_USERS_TO_UPLOAD + " users in " + (processingFinished - processingStarted) / 1000 + + " seconds ( or " + (processingFinished - processingStarted) / 60000 + " minutes)"); + + // after processing finished, make sure every user got processed correctly + { + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore + failedImportedUsersNumber); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); + } + + } + + @Test + public void testBatchWithOneUser() throws Exception { + Main main = startCronProcess("14"); + + int NUMBER_OF_USERS_TO_UPLOAD = 1; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + int failedUsersNumber = 0; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + count = response.get("count").getAsLong(); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + failedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); // 5 seconds + } + + //print failed users + JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD , usersInCore + failedImportedUsersNumber); + assertEquals(0, failedImportedUsersNumber); + + + } + + @Test + public void testBatchWithDuplicate() throws Exception { + Main main = startCronProcess("14"); + + int NUMBER_OF_USERS_TO_UPLOAD = 2; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + usersJson.get("users").getAsJsonArray().add(generateUsersJson(1, 0).get("users").getAsJsonArray().get(0).getAsJsonObject()); + usersJson.get("users").getAsJsonArray().add(generateUsersJson(1, 1).get("users").getAsJsonArray().get(0).getAsJsonObject()); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + int failedUsersNumber = 0; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + count = response.get("count").getAsLong(); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + failedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); // 5 seconds + } + + //print failed users + JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD + 2, usersInCore + failedImportedUsersNumber); + assertEquals(2, failedImportedUsersNumber); + + + for(JsonElement userJson : failedUsersLs.get("users").getAsJsonArray()) { + String errorMessage = userJson.getAsJsonObject().get("errorMessage").getAsString(); + assertTrue(errorMessage.startsWith("E003:")); + } + + } + + @Test + public void testBatchWithDuplicateUserIdMappingWithInputValidation() throws Exception { + Main main = startCronProcess("14"); + + int NUMBER_OF_USERS_TO_UPLOAD = 20; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + //set the first and last users' externalId to the same value + usersJson.get("users").getAsJsonArray().get(0).getAsJsonObject().addProperty("externalUserId", + "some-text-external-id"); + usersJson.get("users").getAsJsonArray().get(19).getAsJsonObject().addProperty("externalUserId", + "some-text-external-id"); + + try { + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + } catch (HttpResponseException expected) { + assertEquals(400, expected.statusCode); + assertEquals("Http error. Status Code: 400. Message: {\"error\":\"Data has missing or invalid fields. Please check the users field for more details.\",\"users\":[{\"index\":19,\"errors\":[\"externalUserId some-text-external-id is not unique. It is already used by another user.\"]}]}", + expected.getMessage()); + } + } + + @Test + public void testBatchWithInvalidInput() throws Exception { + Main main = startCronProcess("14"); + + int NUMBER_OF_USERS_TO_UPLOAD = 2; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + usersJson.get("users").getAsJsonArray().get(0).getAsJsonObject().addProperty("externalUserId", + Boolean.FALSE); // invalid, should be string + try { + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + } catch (HttpResponseException exception) { + assertEquals(400, exception.statusCode); + assertEquals("Http error. Status Code: 400. Message: {\"error\":\"Data has missing or invalid " + + "fields. Please check the users field for more details.\",\"users\":[{\"index\":0,\"errors\":" + + "[\"externalUserId should be of type string.\"]}]}", exception.getMessage()); + } + } + + @Test + public void testBatchWithMissingRole() throws Exception { + Main main = startCronProcess("14"); + + int NUMBER_OF_USERS_TO_UPLOAD = 2; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Creating only one user role before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + try { + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + } catch (HttpResponseException exception) { + assertEquals(400, exception.statusCode); + assertEquals(400, exception.statusCode); + assertEquals("Http error. Status Code: 400. Message: {\"error\":\"Data has missing or " + + "invalid fields. Please check the users field for more details.\",\"users\":[{\"index\":0,\"errors\"" + + ":[\"Role role2 does not exist.\"]},{\"index\":1,\"errors\":[\"Role role2 does not exist.\"]}]}", + exception.getMessage()); + } + } + + @Test + public void testBatchWithOnlyOneWithDuplicate() throws Exception { + Main main = startCronProcess("8", 10); + + int NUMBER_OF_USERS_TO_UPLOAD = 9; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + //create tenant t1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + usersJson.get("users").getAsJsonArray().add(generateUsersJson(1, 0).get("users").getAsJsonArray().get(0).getAsJsonObject()); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + int failedUsersNumber = 0; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); + } + + //print failed users + JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD + 1, usersInCore + failedImportedUsersNumber); + assertEquals(1, failedImportedUsersNumber); + + + for(JsonElement userJson : failedUsersLs.get("users").getAsJsonArray()) { + String errorMessage = userJson.getAsJsonObject().get("errorMessage").getAsString(); + assertTrue(errorMessage.startsWith("E003:")); + } + + } + + @Test + public void testBatchWithOneThreadWorks() throws Exception { + Main main = startCronProcess("1"); + + int NUMBER_OF_USERS_TO_UPLOAD = 5; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); // 5 seconds + } + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); + assertEquals(0, failedImportedUsersNumber); + } + + @Test + public void testFirstLazyImportAfterBulkImport() throws Exception { + Main main = startCronProcess("14", 10); + + + int NUMBER_OF_USERS_TO_UPLOAD = 100; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + // lazy import most of the users + int successfully_lazy_imported = 0; + for (int i = 0; i < allUsersJson.get("users").getAsJsonArray().size() / 10 * 9; i++) { + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(i).getAsJsonObject(); + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + assertEquals("OK", lazyImportResponse.get("status").getAsString()); + assertNotNull(lazyImportResponse.get("user")); + successfully_lazy_imported++; + } + + // bulk import all of the users + { + JsonObject bulkUploadResponse = uploadBulkImportUsersJson(main, allUsersJson); + assertEquals("OK", bulkUploadResponse.get("status").getAsString()); + } + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + { + long count = NUMBER_OF_USERS_TO_UPLOAD; + while(count != 0) { + JsonObject response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + + Thread.sleep(60000); // one minute + } + } + + + // expect: lazy imported users are already there, duplicate.. errors + // expect: not lazy imported users are imported successfully + { + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + assertEquals(successfully_lazy_imported, failedImportedUsersNumber); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); // lazy + bulk = all users + } + + JsonObject failedUsers = loadBulkImportUsersWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + JsonArray faileds = failedUsers.getAsJsonArray("users"); + for (JsonElement failedUser : faileds) { + String errorMessage = failedUser.getAsJsonObject().get("errorMessage").getAsString(); + assertTrue(errorMessage.startsWith("E003:") || errorMessage.startsWith("E005:") + || errorMessage.startsWith("E006:") || errorMessage.startsWith("E007:")); // duplicate email, phone, etc errors + } + + } + + @Test + public void testLazyImport() throws Exception { + Main main = startCronProcess("1"); + + int NUMBER_OF_USERS_TO_UPLOAD = 100; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + // lazy import most of the users + int successfully_lazy_imported = 0; + for (int i = 0; i < allUsersJson.get("users").getAsJsonArray().size(); i++) { + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(i).getAsJsonObject(); + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + assertEquals("OK", lazyImportResponse.get("status").getAsString()); + assertNotNull(lazyImportResponse.get("user")); + successfully_lazy_imported++; + } + + // expect: lazy imported users are already there, duplicate.. errors + // expect: not lazy imported users are imported successfully + { + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, successfully_lazy_imported ); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); + } + } + + @Test + public void testLazyImportUnknownRecipeLoginMethod() throws Exception { + Main main = startCronProcess("1"); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(1, 0); + allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject().get("loginMethods") + .getAsJsonArray().get(0).getAsJsonObject().addProperty("recipeId", "not-existing-recipe"); + + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); + try { + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + } catch (HttpResponseException expected) { + assertEquals(400, expected.statusCode); + assertNotNull(expected.getMessage()); + assertEquals("Http error. Status Code: 400. Message: {\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}", + expected.getMessage()); + } + } + + @Test + public void testLazyImportDuplicatesFail() throws Exception { + Main main = startCronProcess("1"); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(1, 0); + + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + assertEquals("OK", lazyImportResponse.get("status").getAsString()); + assertNotNull(lazyImportResponse.get("user")); + + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(1, usersInCore); + + JsonObject userToImportLazyAgain = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); + try { + JsonObject lazyImportResponseTwo = lazyImportUser(main, userToImportLazyAgain); + } catch (HttpResponseException expected) { + assertEquals(400, expected.statusCode); + } + } + + private static JsonObject lazyImportUser(Main main, JsonObject user) + throws HttpResponseException, IOException { + return HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + user, 100000, 100000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadBulkImportUsersCountWithStatus(Main main, BulkImportStorage.BULK_IMPORT_USER_STATUS status) + throws HttpResponseException, IOException { + Map params = new HashMap<>(); + if(status!= null) { + params.put("status", status.name()); + } + return HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 10000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadBulkImportUsersWithStatus(Main main, BulkImportStorage.BULK_IMPORT_USER_STATUS status) + throws HttpResponseException, IOException { + Map params = new HashMap<>(); + if(status!= null) { + params.put("status", status.name()); + } + return HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 10000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadUsersCount(Main main) throws HttpResponseException, IOException { + Map params = new HashMap<>(); + + return HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/users/count", + params, 10000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject generateUsersJson(int numberOfUsers, int startIndex) { + JsonObject userJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + + JsonArray usersArray = new JsonArray(); + for (int i = 0; i < numberOfUsers; i++) { + JsonObject user = new JsonObject(); + + user.addProperty("externalUserId", UUID.randomUUID().toString()); + user.add("userMetadata", parser.parse("{\"key1\":"+ UUID.randomUUID().toString() + ",\"key2\":{\"key3\":\"value3\"}}")); + user.add("userRoles", parser.parse( + "[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); + + //JsonArray tenanatIds = parser.parse("[\"public\", \"t1\"]").getAsJsonArray(); + JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); + String email = " johndoe+" + (i + startIndex) + "@gmail.com "; + + Random random = new Random(); + + JsonArray loginMethodsArray = new JsonArray(); + //if(random.nextInt(2) == 0){ + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + //} + if(random.nextInt(2) == 0){ + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + } + if(random.nextInt(2) == 0){ + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds, "+910000" + (startIndex + i))); + } + if(loginMethodsArray.size() == 0) { + int methodNumber = random.nextInt(3); + switch (methodNumber) { + case 0: + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + break; + case 1: + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + break; + case 2: + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds, "+911000" + (startIndex + i))); + break; + } + } + user.add("loginMethods", loginMethodsArray); + + usersArray.add(user); + } + + userJsonObject.add("users", usersArray); + return userJsonObject; + } + + private static JsonObject createEmailLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "emailpassword"); + loginMethod.addProperty("passwordHash", + "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("hashingAlgorithm", "argon2"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", true); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createThirdPartyLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("recipeId", "thirdparty"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("thirdPartyId", "google"); + loginMethod.addProperty("thirdPartyUserId", String.valueOf(email.hashCode())); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createPasswordlessLoginMethod(String email, JsonArray tenantIds, String phoneNumber) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "passwordless"); + loginMethod.addProperty("phoneNumber", phoneNumber); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private void setFeatureFlags(Main main, EE_FEATURES[] features) { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); + } + + + private static JsonObject uploadBulkImportUsersJson(Main main, JsonObject request) throws IOException, HttpResponseException { + return HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + @NotNull + private Main startCronProcess(String parallelism) throws IOException, InterruptedException, TenantOrAppNotFoundException { + return startCronProcess(parallelism, 5*60); + } + + + @NotNull + private Main startCronProcess(String parallelism, int intervalInSeconds) throws IOException, InterruptedException, TenantOrAppNotFoundException { + String[] args = { "../" }; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", parallelism); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + Main main = process.getProcess(); + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); + CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, intervalInSeconds); + + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + Cronjobs.addCronjob(main, (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY)); + return main; + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java new file mode 100644 index 000000000..cd399e61c --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -0,0 +1,552 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.userroles.UserRoles; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class BulkImportTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldAddUsersInBulkImportUsersTable() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + List users = generateBulkImportUser(10); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + BulkImport.addUsers(new AppIdentifier(null, null), storage, users); + + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, + BULK_IMPORT_USER_STATUS.NEW, null, null); + + // Verify that all users are present in addedUsers + for (BulkImportUser user : users) { + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.id.equals(addedUser.id)) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertEquals(user.toRawDataForDbStorage(), matchingUser.toRawDataForDbStorage()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + List users = generateBulkImportUser(10); + + // We are setting the id of the second user to be the same as the first user to ensure a duplicate id is present + users.get(1).id = users.get(0).id; + + List initialIds = users.stream().map(user -> user.id).collect(Collectors.toList()); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + BulkImport.addUsers(appIdentifier, storage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, + null, null); + + // Verify that the other properties are same but ids changed + for (BulkImportUser user : users) { + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.toRawDataForDbStorage().equals(addedUser.toRawDataForDbStorage())) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertFalse(initialIds.contains(matchingUser.id)); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void testGetUsersStatusFilter() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.NEW, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to PROCESSING + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.PROCESSING, null); + } + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to FAILED + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.FAILED, null); + } + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.FAILED, null, null); + assertEquals(10, addedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void randomPaginationTest() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + + // We are setting a high initial wait time to ensure the cron job doesn't run while we are running the tests + CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, + 1000000); + + process.startProcess(); + + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + + int numberOfUsers = 500; + // Insert users in batches + { + int batchSize = 100; + for (int i = 0; i < numberOfUsers; i += batchSize) { + List users = generateBulkImportUser(batchSize); + BulkImport.addUsers(new AppIdentifier(null, null), storage, users); + // Adding a delay between each batch to ensure the createdAt different + Thread.sleep(1000); + } + } + + // Get all inserted users + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, + null); + assertEquals(numberOfUsers, addedUsers.size()); + + // We are sorting the users based on createdAt and id like we do in the storage layer + List sortedUsers = addedUsers.stream() + .sorted((user1, user2) -> { + int compareResult = Long.compare(user2.createdAt, user1.createdAt); + if (compareResult == 0) { + return user2.id.compareTo(user1.id); + } + return compareResult; + }) + .collect(Collectors.toList()); + + int[] limits = new int[] { 10, 14, 20, 23, 50, 100, 110, 150, 200, 510 }; + + for (int limit : limits) { + int indexIntoUsers = 0; + String paginationToken = null; + do { + BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, + limit, null, paginationToken); + + for (BulkImportUser actualUser : users.users) { + BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); + + assertEquals(expectedUser.id, actualUser.id); + assertEquals(expectedUser.status, actualUser.status); + assertEquals(expectedUser.toRawDataForDbStorage(), actualUser.toRawDataForDbStorage()); + indexIntoUsers++; + } + + paginationToken = users.nextPaginationToken; + } while (paginationToken != null); + + assert (indexIntoUsers == sortedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void testGetBulkImportUsersCount() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.NEW); + assertEquals(10, count); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to PROCESSING + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.PROCESSING, null); + } + storage.commitTransaction(con); + return null; + }); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.PROCESSING); + assertEquals(10, count); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to FAILED + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.FAILED, null); + } + storage.commitTransaction(con); + return null; + }); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.FAILED); + assertEquals(10, count); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportTheUserInTheSameTenant() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(1); + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, users.get(0)); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(0), importedUser); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportTheUserInMultipleTenantsWithDifferentStorages() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + AuthRecipeUserInfo importedUser1 = BulkImport.importUser(main, appIdentifier, bulkImportUserT1); + AuthRecipeUserInfo importedUser2 = BulkImport.importUser(main, appIdentifier, bulkImportUserT2); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, + bulkImportUserT1, + importedUser1); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, + bulkImportUserT2, + importedUser2); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportUsersConcurrently() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(10); + + // Concurrently import users + ExecutorService executor = Executors.newFixedThreadPool(10); + List> futures = new ArrayList<>(); + + for (BulkImportUser user : users) { + Future future = executor.submit(() -> { + return BulkImport.importUser(main, appIdentifier, user); + }); + futures.add(future); + } + + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + + for (int i = 0; i < users.size(); i++) { + AuthRecipeUserInfo importedUser = futures.get(i).get(); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(i), + importedUser); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportWithPlainTextPassword() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(1); + BulkImportUser bulkImportUser = users.get(0); + + // Set passwordHash to null and plainTextPassword to a value to ensure we do a plainTextPassword import + for (LoginMethod lm : bulkImportUser.loginMethods) { + if (lm.recipeId == "emailpassword") { + lm.passwordHash = null; + lm.hashingAlgorithm = null; + lm.plainTextPassword = "testPass@123"; + } + } + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, bulkImportUser); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), bulkImportUser, importedUser); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java new file mode 100644 index 000000000..3fac0ca38 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.supertokens.Main; +import io.supertokens.emailpassword.PasswordHashing; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.*; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.totp.TOTPDevice; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.totp.Totp; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class BulkImportTestUtils { + + public static List generateBulkImportUser(int numberOfUsers) { + return generateBulkImportUser(numberOfUsers, List.of("public", "t1"), 0); + } + + public static List generateBulkImportUser(int numberOfUsers, List tenants, int startIndex) { + return generateBulkImportUserWithRoles(numberOfUsers, tenants, startIndex, List.of("role1", "role2")); + } + + public static List generateBulkImportUserWithRoles(int numberOfUsers, List tenants, int startIndex, List roles) { + List users = new ArrayList<>(); + JsonParser parser = new JsonParser(); + + for (int i = startIndex; i < numberOfUsers + startIndex; i++) { + String email = "user" + i + "@example.com"; + String id = io.supertokens.utils.Utils.getUUID(); + String externalId = io.supertokens.utils.Utils.getUUID(); + + JsonObject userMetadata = parser.parse("{\"key1\":\""+id+"\",\"key2\":{\"key3\":\"value3\"}}") + .getAsJsonObject(); + + List userRoles = new ArrayList<>(); + for(String roleName : roles) { + userRoles.add(new UserRole(roleName, tenants)); + } + + List totpDevices = new ArrayList<>(); + totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); + + List loginMethods = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", + "BCRYPT", null, null, null, null)); + loginMethods + .add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, null, + "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add( + new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, + null, null, null)); + users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); + } + return users; + } + + public static void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + // User pool 1 - (null, null, null), (null, null, t1) + // User pool 2 - (null, null, t2) + + { // tenant 1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + } + { // tenant 2 + JsonObject config = new JsonObject(); + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config)); + } + } + + public static void assertBulkImportUserAndAuthRecipeUserAreEqual(Main main, AppIdentifier appIdentifier, + TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, + AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm1 : authRecipeUser.loginMethods) { + for (LoginMethod lm2 : bulkImportUser.loginMethods) { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(main, lm1, lm2); + } + } + } + assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, + UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); + + String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + + private static void assertLoginMethodEquals(Main main, io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) + throws TenantOrAppNotFoundException { + assertEquals(lm1.email, lm2.email); + assertEquals(lm1.verified, lm2.isVerified); + assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); + + switch (lm2.recipeId) { + case "emailpassword": + // If lm2.passwordHash is null then the user was imported using plainTextPassword + // We check if the plainTextPassword matches the stored passwordHash + if (lm2.passwordHash == null) { + assertTrue(PasswordHashing.getInstance(main).verifyPasswordWithHash(lm2.plainTextPassword, + lm1.passwordHash)); + } else { + assertEquals(lm1.passwordHash, lm2.passwordHash); + } + break; + case "thirdparty": + assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); + assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); + break; + case "passwordless": + assertEquals(lm1.phoneNumber, lm2.phoneNumber); + break; + default: + break; + } + } + + private static void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { + assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); + for (int i = 0; i < createdTotpDevices.length; i++) { + assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); + assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); + assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); + assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java new file mode 100644 index 000000000..831653c4c --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -0,0 +1,666 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +package io.supertokens.test.bulkimport; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.UserPaginationContainer; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.TestingProcessManager.TestingProcess; +import io.supertokens.test.Utils; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.userroles.UserRoles; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import java.util.ArrayList; +import java.util.List; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUserWithRoles; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class ProcessBulkImportUsersCronJobTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 1; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + BulkImportUser bulkImportUser = users.get(0); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storage, container.users); + + TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, publicTenant, storage, + bulkImportUser, + container.users[0]); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInNotSoLargeNumbersInTheSameTenant() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "8"); + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 15; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 1000, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInLargeNumbersInTheSameTenant() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 1000; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(2 * 60000); // minute + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 1000, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStoragesOnMultipleThreads() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "3"); + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + BulkImport.addUsers(appIdentifier, storage, usersT1); + BulkImport.addUsers(appIdentifier, storage, usersT2); + + Thread.sleep(12000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 100, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 100, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, + bulkImportUserT1, + containerT1.users[0]); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, + bulkImportUserT2, + containerT2.users[0]); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStoragesOnOneThreads() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "1"); + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + BulkImport.addUsers(appIdentifier, storage, usersT1); + BulkImport.addUsers(appIdentifier, storage, usersT2); + + Thread.sleep(12000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 100, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 100, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, + bulkImportUserT1, + containerT1.users[0]); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, + bulkImportUserT2, + containerT2.users[0]); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInLargeNumberInMultipleTenantsWithDifferentStorages() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(500, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(500, List.of(t2.getTenantId()), 500); + + List allUsers = new ArrayList<>(); + allUsers.addAll(usersT1); + allUsers.addAll(usersT2); + + BulkImport.addUsers(appIdentifier, storage, allUsers); + + Thread.sleep(2 * 60000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 500, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 500, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInLargeNumberInMultipleTenantsWithDifferentStoragesOnOneThread() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "1"); + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(50, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(50, List.of(t2.getTenantId()), 50); + + List allUsers = new ArrayList<>(); + allUsers.addAll(usersT1); + allUsers.addAll(usersT2); + + BulkImport.addUsers(appIdentifier, storage, allUsers); + + Thread.sleep(2 * 60000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 500, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 500, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldDeleteEverythingFromTheDBIfAnythingFails() throws Exception { + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // note the missing role creation here! + + List users = generateBulkImportUser(1); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(12000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals("E034: Role does not exist! You need to pre-create the role before assigning it to the user.", + usersAfterProcessing.get(0).errorMessage); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(0, container.users.length); + } + + + @Test + public void shouldDeleteEverythingFromTheDBIfAnythingFailsOnMultipleThreads() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "8"); + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // note the missing role creation here! + + List users = generateBulkImportUser(100); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(60000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(100, usersAfterProcessing.size()); + + for(BulkImportUser userAfterProcessing: usersAfterProcessing){ + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, userAfterProcessing.status); // should process every user and every one of them should fail because of the missing role + assertEquals("E034: Role does not exist! You need to pre-create the role before assigning it to the user.", + userAfterProcessing.errorMessage); + } + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(0, container.users.length); + } + + @Test + public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "8"); + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Create one user role before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + } + + List users = generateBulkImportUserWithRoles(99, List.of("public", "t1"), 0, List.of("role1")); + users.addAll(generateBulkImportUserWithRoles(1, List.of("public", "t1"), 99, List.of("notExistingRole"))); + + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(60000); // one minute + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + + int numberOfFailed = 0; + for(int i = 0; i < usersAfterProcessing.size(); i++){ + if(usersAfterProcessing.get(i).status == BULK_IMPORT_USER_STATUS.FAILED) { + assertEquals( + "E034: Role does not exist! You need to pre-create the role before assigning it to the user.", + usersAfterProcessing.get(i).errorMessage); + numberOfFailed++; + } + } + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(99, container.users.length); + assertEquals(1, numberOfFailed); + } + + + @Test + public void shouldThrowTenantDoesNotExistError() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + List users = generateBulkImportUser(1); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals( + "[Invalid tenantId: t1 for a user role., Invalid tenantId: t1 for a user role., Invalid tenantId: t1 for emailpassword recipe., Invalid tenantId: t1 for thirdparty recipe., Invalid tenantId: t1 for passwordless recipe.]", + usersAfterProcessing.get(0).errorMessage); + } + + @Test + public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + BulkImportTestUtils.createTenants(main); + + List users = generateBulkImportUser(1, List.of("t1", "t2"), 0); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(12000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals( + "[All tenants for a user must share the same database for emailpassword recipe., All tenants for a user must share the same database for thirdparty recipe., All tenants for a user must share the same database for passwordless recipe.]", + usersAfterProcessing.get(0).errorMessage); + } + + private TestingProcess startCronProcess() throws InterruptedException, TenantOrAppNotFoundException { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + + Main main = process.getProcess(); + + FeatureFlagTestContent.getInstance(main) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); + CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1); + + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + Cronjobs.addCronjob(main, (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY)); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL) { + return null; + } + + return process; + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java new file mode 100644 index 000000000..ff7d12f02 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -0,0 +1,687 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.userroles.UserRoles; + +public class AddBulkImportUsersTest { + private String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: users field is not present + testBadRequest(main, new JsonObject(), "Field name 'users' is invalid in JSON input"); + + // CASE 2: users field type in incorrect + testBadRequest(main, new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), + "Field name 'users' is invalid in JSON input"); + + // CASE 3: users array length is greater than 10000 + testBadRequest(main, generateUsersJson(10001).getAsJsonObject(), + "{\"error\":\"You can only add 10000 users at a time.\"}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: loginMethods field is not present + testBadRequest(main, new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); + + // CASE 2: loginMethods field type in incorrect + testBadRequest(main, + new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); + + // CASE 3: loginMethods array is empty + testBadRequest(main, + new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: MFA must be enabled to import totp devices + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); + + // CASE 2: secretKey is required in totpDevices + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + } + + // CASE 1: tenantIds is required for a user role + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); + + // CASE 2: Role doesn't exist + JsonObject requestBody2 = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: Field type is invalid + JsonObject requestBody = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); + + // CASE 2: recipeId is invalid + JsonObject requestBody2 = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: email, passwordHash and hashingAlgorithm are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe.\"]}]}"); + + // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\",\"Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe.\"]}]}"); + + // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt + JsonObject requestBody3 = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody3, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: email, thirdPartyId and thirdPartyUserId are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); + + // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: email and phoneNumber are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + + // CASE 2: email and phoneNumber field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisabled() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking must be enabled to import multiple loginMethods.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // CASE 1: Multitenancy is not enabled + JsonObject requestBody = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); + + // CASE 2: Invalid tenantId + setFeatureFlags(main, + new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); + + JsonObject requestBody2 = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); + + // CASE 3: Two or more tenants do not share the same storage + + BulkImportTestUtils.createTenants(main); + + JsonObject requestBody3 = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody3, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same database for thirdparty recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfTwoLoginMethodsHaveIsPrimaryTrue() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + setFeatureFlags(main, + new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); + + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}") + .getAsJsonObject(); + + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + JsonObject request = generateUsersJson(10000); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldNormaliseFields() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + JsonObject request = generateUsersJson(1); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + JsonObject getResponse = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + new HashMap<>(), 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", getResponse.get("status").getAsString()); + JsonArray bulkImportUsers = getResponse.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + + // Test if default values were set in totpDevices + JsonArray totpDevices = bulkImportUserJson.getAsJsonArray("totpDevices"); + for (int i = 0; i < totpDevices.size(); i++) { + JsonObject totpDevice = totpDevices.get(i).getAsJsonObject(); + assertEquals(30, totpDevice.get("period").getAsInt()); + assertEquals(1, totpDevice.get("skew").getAsInt()); + } + + JsonArray loginMethods = bulkImportUserJson.getAsJsonArray("loginMethods"); + for (int i = 0; i < loginMethods.size(); i++) { + JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); + if (loginMethod.has("email")) { + assertEquals("johndoe+0@gmail.com", loginMethod.get("email").getAsString()); + } + if (loginMethod.has("phoneNumber")) { + assertEquals("+919999999999", loginMethod.get("phoneNumber").getAsString()); + } + if (loginMethod.has("hashingAlgorithm")) { + assertEquals("ARGON2", loginMethod.get("hashingAlgorithm").getAsString()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldFailIfANewFieldWasAddedToBulkImportUser() throws Exception { + List bulkImportUsers = generateBulkImportUser(1); + BulkImportUser user = bulkImportUsers.get(0); + + checkFields(user, "BulkImportUser", + Arrays.asList("id", "externalUserId", "userMetadata", "userRoles", "totpDevices", + "loginMethods", "status", "primaryUserId", "errorMessage", "createdAt", + "updatedAt")); + + checkLoginMethodFields(user.loginMethods.get(0), "LoginMethod", + Arrays.asList("tenantIds", "isVerified", "isPrimary", "timeJoinedInMSSinceEpoch", + "recipeId", "email", "passwordHash", "plainTextPassword", "hashingAlgorithm", + "phoneNumber", "thirdPartyId", "thirdPartyUserId", "externalUserId", "superTokensUserId")); + + checkTotpDeviceFields(user.totpDevices.get(0), "TotpDevice", + Arrays.asList("secretKey", "period", "skew", "deviceName")); + + checkUserRoleFields(user.userRoles.get(0), "UserRole", + Arrays.asList("role", "tenantIds")); + } + + private void checkFields(Object object, String objectType, List expectedFields) { + Field[] actualFields = object.getClass().getDeclaredFields(); + List actualFieldNames = Arrays.stream(actualFields) + .map(Field::getName) + .collect(Collectors.toList()); + + List extraFields = actualFieldNames.stream() + .filter(fieldName -> !expectedFields.contains(fieldName)) + .collect(Collectors.toList()); + + if (!extraFields.isEmpty()) { + fail("The following extra field(s) are present in " + objectType + ": " + String.join(", ", extraFields)); + } + } + + private void checkLoginMethodFields(BulkImportUser.LoginMethod loginMethod, String objectType, + List expectedFields) { + checkFields(loginMethod, objectType, expectedFields); + } + + private void checkTotpDeviceFields(BulkImportUser.TotpDevice totpDevice, String objectType, + List expectedFields) { + checkFields(totpDevice, objectType, expectedFields); + } + + private void checkUserRoleFields(BulkImportUser.UserRole userRole, String objectType, List expectedFields) { + checkFields(userRole, objectType, expectedFields); + } + + private String getResponseMessageFromError(String response) { + return response.substring(response.indexOf("Message: ") + "Message: ".length()); + } + + private void testBadRequest(Main main, JsonObject requestBody, String expectedErrorMessage) throws Exception { + try { + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + requestBody, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, expectedErrorMessage); + } + } + + public static JsonObject generateUsersJson(int numberOfUsers) { + JsonObject userJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + + JsonArray usersArray = new JsonArray(); + for (int i = 0; i < numberOfUsers; i++) { + JsonObject user = new JsonObject(); + + user.addProperty("externalUserId", UUID.randomUUID().toString()); + user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); + user.add("userRoles", parser.parse( + "[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); + + JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); + String email = " johndoe+" + i + "@gmail.com "; + + JsonArray loginMethodsArray = new JsonArray(); + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds)); + user.add("loginMethods", loginMethodsArray); + + usersArray.add(user); + } + + userJsonObject.add("users", usersArray); + return userJsonObject; + } + + private static JsonObject createEmailLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "emailpassword"); + loginMethod.addProperty("passwordHash", + "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("hashingAlgorithm", "argon2"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", true); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createThirdPartyLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("recipeId", "thirdparty"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("thirdPartyId", "google"); + loginMethod.addProperty("thirdPartyUserId", "112618388912586834161"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createPasswordlessLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "passwordless"); + loginMethod.addProperty("phoneNumber", "+91-9999999999"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private void setFeatureFlags(Main main, EE_FEATURES[] features) { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java new file mode 100644 index 000000000..6d1e14a61 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + + +public class CountBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", + e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "NEW"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "PROCESSING"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "FAILED"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java new file mode 100644 index 000000000..28906ba95 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class DeleteBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", + e.getMessage()); + } + } + { + try { + // Create a string array of 500 uuids + JsonObject request = new JsonObject(); + JsonArray ids = new JsonArray(); + for (int i = 0; i < 501; i++) { + ids.add(new JsonPrimitive(io.supertokens.utils.Utils.getUUID())); + } + request.add("ids", ids); + + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", + e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Call the API with empty array + { + JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals(0, response.get("deletedIds").getAsJsonArray().size()); + assertEquals(0, response.get("invalidIds").getAsJsonArray().size()); + } + + { + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Insert users + List users = generateBulkImportUser(5); + BulkImport.addUsers(appIdentifier, storage, users); + + String invalidId = io.supertokens.utils.Utils.getUUID(); + JsonObject request = new JsonObject(); + JsonArray validIds = new JsonArray(); + for (BulkImportUser user : users) { + validIds.add(new JsonPrimitive(user.id)); + } + validIds.add(new JsonPrimitive(invalidId)); + + request.add("ids", validIds); + + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + response.get("deletedIds").getAsJsonArray().forEach(id -> { + assertTrue(validIds.contains(id)); + }); + + assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + } + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java new file mode 100644 index 000000000..8db075610 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +public class GetBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "0"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: limit must a positive integer with min value 1", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "501"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Max limit allowed is 500", e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("paginationToken", "invalid_token"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: invalid pagination token", e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create a bulk import user to test the GET API + String rawData = "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}"; + { + JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); + JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assert res.get("status").getAsString().equals("OK"); + } + + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + JsonArray bulkImportUsers = response.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + bulkImportUserJson.get("status").getAsString().equals("NEW"); + BulkImportUser.forTesting_fromJson(bulkImportUserJson).toRawDataForDbStorage().equals(rawData); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java new file mode 100644 index 000000000..5a1b1db85 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.userroles.UserRoles; + +public class ImportUserTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + try { + List users = BulkImportTestUtils.generateBulkImportUser(1); + JsonObject request = users.get(0).toJsonObject(); + + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: {\"errors\":[\"Role role1 does not exist.\",\"Invalid tenantId: t1 for a user role.\",\"Role role2 does not exist.\",\"Invalid tenantId: t1 for a user role.\",\"Invalid tenantId: t1 for emailpassword recipe.\",\"Invalid tenantId: t1 for thirdparty recipe.\",\"Invalid tenantId: t1 for passwordless recipe.\"]}", + e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + List users = BulkImportTestUtils.generateBulkImportUser(1); + JsonObject request = users.get(0).toJsonObject(); + + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertNotNull(response.get("user")); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java b/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java index d239f9c9d..cca62b70c 100644 --- a/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java +++ b/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.jwt.JWTRecipeStorage; import io.supertokens.pluginInterface.multitenancy.*; import io.supertokens.pluginInterface.nonAuthRecipe.NonAuthRecipeStorage; @@ -81,7 +82,8 @@ public void testDeletingAppDeleteNonAuthRecipeData() throws Exception { List classesToSkip = List.of( JWTRecipeStorage.class.getName(), ActiveUsersStorage.class.getName(), - OAuthStorage.class.getName() + OAuthStorage.class.getName(), + BulkImportStorage.class.getName() ); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); @@ -189,7 +191,8 @@ public void testDisassociationOfUserDeletesNonAuthRecipeData() throws Exception List classesToSkip = List.of( JWTRecipeStorage.class.getName(), ActiveUsersStorage.class.getName(), - OAuthStorage.class.getName() + OAuthStorage.class.getName(), + BulkImportStorage.class.getName() ); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); diff --git a/src/test/java/io/supertokens/test/multitenant/TestAppData.java b/src/test/java/io/supertokens/test/multitenant/TestAppData.java index d51ed6ff0..3295511ea 100644 --- a/src/test/java/io/supertokens/test/multitenant/TestAppData.java +++ b/src/test/java/io/supertokens/test/multitenant/TestAppData.java @@ -21,6 +21,7 @@ import io.supertokens.ActiveUsers; import io.supertokens.Main; import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; import io.supertokens.dashboard.Dashboard; import io.supertokens.emailpassword.EmailPassword; import io.supertokens.emailverification.EmailVerification; @@ -40,6 +41,7 @@ import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; import io.supertokens.thirdparty.ThirdParty; import io.supertokens.totp.Totp; import io.supertokens.useridmapping.UserIdMapping; @@ -177,6 +179,8 @@ null, null, new JsonObject() UserIdMapping.createUserIdMapping(process.getProcess(), app.toAppIdentifier(), appStorage, plUser.user.getSupertokensUserId(), "externalid", null, false); + BulkImport.addUsers(app.toAppIdentifier(), appStorage, BulkImportTestUtils.generateBulkImportUser(1)); + OAuth.addOrUpdateClient(process.getProcess(), app.toAppIdentifier(), appStorage, "test", "secret123", false, false); OAuth.createLogoutRequestAndReturnRedirectUri(process.getProcess(), app.toAppIdentifier(), appStorage, "test", "http://localhost", "sessionHandle", "state"); ((OAuthStorage) appStorage).addOAuthM2MTokenForStats(app.toAppIdentifier(), "test", 1000, 2000); diff --git a/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java b/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java index 3bb6857a5..a495687da 100644 --- a/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java +++ b/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java @@ -31,6 +31,7 @@ import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.jwt.JWTRecipeStorage; @@ -200,11 +201,13 @@ public void testUserDisassociationForNotAuthRecipes() throws Exception { if (name.equals(UserMetadataStorage.class.getName()) || name.equals(JWTRecipeStorage.class.getName()) || name.equals(ActiveUsersStorage.class.getName()) + || name.equals(BulkImportStorage.class.getName()) || name.equals(OAuthStorage.class.getName()) ) { // user metadata is app specific and does not have any tenant specific data // JWT storage does not have any user specific data // Active users storage does not have tenant specific data + // BulkImportStorage continue; } diff --git a/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java b/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java index 38419fb3c..1af8acb90 100644 --- a/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java +++ b/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java @@ -25,6 +25,7 @@ import io.supertokens.pluginInterface.ActiveUsersStorage; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.jwt.JWTRecipeStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; @@ -804,7 +805,8 @@ public void checkThatCreateUserIdMappingHasAllNonAuthRecipeChecks() throws Excep List nonAuthRecipesWhichDontNeedUserIdMapping = List.of( JWTRecipeStorage.class.getName(), ActiveUsersStorage.class.getName(), - OAuthStorage.class.getName() + OAuthStorage.class.getName(), + BulkImportStorage.class.getName() ); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); @@ -888,7 +890,8 @@ public void checkThatDeleteUserIdMappingHasAllNonAuthRecipeChecks() throws Excep List nonAuthRecipesWhichDontNeedUserIdMapping = List.of( JWTRecipeStorage.class.getName(), ActiveUsersStorage.class.getName(), - OAuthStorage.class.getName() + OAuthStorage.class.getName(), + BulkImportStorage.class.getName() ); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); Set> classes = reflections.getSubTypesOf(NonAuthRecipeStorage.class);