From 0ea0211fca87c453d78e3fe7f509bff88403d592 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 20 Mar 2024 12:58:56 +0530 Subject: [PATCH 01/41] feat: Add BulkImport APIs and cron --- src/main/java/io/supertokens/Main.java | 4 + .../io/supertokens/bulkimport/BulkImport.java | 90 +++ .../BulkImportUserPaginationContainer.java | 34 + .../BulkImportUserPaginationToken.java | 53 ++ .../bulkimport/BulkImportUserUtils.java | 551 ++++++++++++++ .../InvalidBulkImportDataException.java | 33 + .../bulkimport/ProcessBulkImportUsers.java | 504 +++++++++++++ .../emailpassword/EmailPassword.java | 68 +- .../java/io/supertokens/inmemorydb/Start.java | 5 + .../passwordless/Passwordless.java | 88 ++- .../storageLayer/StorageLayer.java | 14 +- .../io/supertokens/thirdparty/ThirdParty.java | 39 +- .../supertokens/utils/JsonValidatorUtils.java | 123 ++++ .../io/supertokens/webserver/Webserver.java | 3 + .../api/bulkimport/BulkImportAPI.java | 250 +++++++ .../test/bulkimport/BulkImportTest.java | 265 +++++++ .../test/bulkimport/BulkImportTestUtils.java | 59 ++ .../ProcessBulkImportUsersCronJobTest.java | 297 ++++++++ .../apis/AddBulkImportUsersTest.java | 692 ++++++++++++++++++ .../apis/DeleteBulkImportUsersTest.java | 173 +++++ .../apis/GetBulkImportUsersTest.java | 160 ++++ 21 files changed, 3420 insertions(+), 85 deletions(-) create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImport.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java create mode 100644 src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java create mode 100644 src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java create mode 100644 src/main/java/io/supertokens/utils/JsonValidatorUtils.java create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/Main.java b/src/main/java/io/supertokens/Main.java index 2998efb7b..7375e6e08 100644 --- a/src/main/java/io/supertokens/Main.java +++ b/src/main/java/io/supertokens/Main.java @@ -20,6 +20,7 @@ import io.supertokens.config.Config; import io.supertokens.config.CoreConfig; import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys; import io.supertokens.cronjobs.deleteExpiredDashboardSessions.DeleteExpiredDashboardSessions; import io.supertokens.cronjobs.deleteExpiredEmailVerificationTokens.DeleteExpiredEmailVerificationTokens; @@ -254,6 +255,9 @@ private void init() throws IOException, StorageQueryException { // starts DeleteExpiredAccessTokenSigningKeys cronjob if the access token signing keys can change Cronjobs.addCronjob(this, DeleteExpiredAccessTokenSigningKeys.init(this, uniqueUserPoolIdsTenants)); + // starts ProcessBulkImportUsers cronjob to process bulk import users + Cronjobs.addCronjob(this, ProcessBulkImportUsers.init(this, uniqueUserPoolIdsTenants)); + // this is to ensure tenantInfos are in sync for the new cron job as well MultitenancyHelper.getInstance(this).refreshCronjobs(); diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java new file mode 100644 index 000000000..8bfabc7e1 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; + + +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class BulkImport { + + public static final int MAX_USERS_TO_ADD = 10000; + public static final int GET_USERS_PAGINATION_LIMIT = 500; + public static final int GET_USERS_DEFAULT_LIMIT = 100; + public static final int DELETE_USERS_LIMIT = 500; + public static final int PROCESS_USERS_BATCH_SIZE = 1000; + public static final int PROCESS_USERS_INTERVAL = 60; + + public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) + throws StorageQueryException, TenantOrAppNotFoundException { + while (true) { + try { + StorageUtils.getBulkImportStorage(storage).addBulkImportUsers(appIdentifier, users); + break; + } catch (io.supertokens.pluginInterface.bulkimport.exceptions.DuplicateUserIdException ignored) { + // We re-generate the user id for every user and retry + for (BulkImportUser user : users) { + user.id = Utils.getUUID(); + } + } + } + } + + public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdentifier, Storage storage, + @Nonnull Integer limit, @Nullable BULK_IMPORT_USER_STATUS status, @Nullable String paginationToken) + throws StorageQueryException, BulkImportUserPaginationToken.InvalidTokenException { + List users; + + BulkImportSQLStorage bulkImportStorage = StorageUtils.getBulkImportStorage(storage); + + if (paginationToken == null) { + users = bulkImportStorage + .getBulkImportUsers(appIdentifier, limit + 1, status, null, null); + } else { + BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); + users = bulkImportStorage + .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, tokenInfo.createdAt); + } + + String nextPaginationToken = null; + int maxLoop = users.size(); + if (users.size() == limit + 1) { + maxLoop = limit; + BulkImportUser user = users.get(limit); + nextPaginationToken = new BulkImportUserPaginationToken(user.id, user.createdAt).generateToken(); + } + + List resultUsers = users.subList(0, maxLoop); + return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); + } + + public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) throws StorageQueryException { + return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java new file mode 100644 index 000000000..d2bd21634 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationContainer.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + package io.supertokens.bulkimport; + +import java.util.List; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; + +public class BulkImportUserPaginationContainer { + public final List users; + public final String nextPaginationToken; + + public BulkImportUserPaginationContainer(@Nonnull List users, @Nullable String nextPaginationToken) { + this.users = users; + this.nextPaginationToken = nextPaginationToken; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java new file mode 100644 index 000000000..8a492c2ca --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserPaginationToken.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.Base64; + +public class BulkImportUserPaginationToken { + public final String bulkImportUserId; + public final long createdAt; + + public BulkImportUserPaginationToken(String bulkImportUserId, long createdAt) { + this.bulkImportUserId = bulkImportUserId; + this.createdAt = createdAt; + } + + public static BulkImportUserPaginationToken extractTokenInfo(String token) throws InvalidTokenException { + try { + String decodedPaginationToken = new String(Base64.getDecoder().decode(token)); + String[] splitDecodedToken = decodedPaginationToken.split(";"); + if (splitDecodedToken.length != 2) { + throw new InvalidTokenException(); + } + String bulkImportUserId = splitDecodedToken[0]; + long createdAt = Long.parseLong(splitDecodedToken[1]); + return new BulkImportUserPaginationToken(bulkImportUserId, createdAt); + } catch (Exception e) { + throw new InvalidTokenException(); + } + } + + public String generateToken() { + return new String(Base64.getEncoder().encode((this.bulkImportUserId + ";" + this.createdAt).getBytes())); + } + + public static class InvalidTokenException extends Exception { + + private static final long serialVersionUID = 6289026174830695478L; + } +} diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java new file mode 100644 index 000000000..834ed0e4a --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.CoreConfig; +import io.supertokens.emailpassword.PasswordHashingUtils; +import io.supertokens.emailpassword.exceptions.UnsupportedPasswordHashingFormatException; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.utils.JsonValidatorUtils.ValueType; + +import static io.supertokens.utils.JsonValidatorUtils.parseAndValidateFieldType; +import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; + +public class BulkImportUserUtils { + public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, + JsonObject userData, String id, String[] allUserRoles, Set allExternalUserIds) + throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { + List errors = new ArrayList<>(); + + String externalUserId = parseAndValidateFieldType(userData, "externalUserId", ValueType.STRING, false, + String.class, + errors, "."); + JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, + JsonObject.class, errors, "."); + List userRoles = getParsedUserRoles(main, appIdentifier, userData, allUserRoles, errors); + List totpDevices = getParsedTotpDevices(userData, errors); + List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); + + externalUserId = validateAndNormaliseExternalUserId(externalUserId, allExternalUserIds, errors); + + validateTenantIdsForRoleAndLoginMethods(main, appIdentifier, userRoles, loginMethods, errors); + + if (!errors.isEmpty()) { + throw new InvalidBulkImportDataException(errors); + } + return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); + } + + private static List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, + String[] allUserRoles, List errors) throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "userRoles", ValueType.ARRAY_OF_OBJECT, false, + JsonArray.class, errors, "."); + + if (jsonUserRoles == null) { + return null; + } + + List userRoles = new ArrayList<>(); + + for (JsonElement jsonUserRoleEl : jsonUserRoles) { + JsonObject jsonUserRole = jsonUserRoleEl.getAsJsonObject(); + + String role = parseAndValidateFieldType(jsonUserRole, "role", ValueType.STRING, true, String.class, errors, + " for a user role."); + JsonArray jsonTenantIds = parseAndValidateFieldType(jsonUserRole, "tenantIds", ValueType.ARRAY_OF_STRING, + true, JsonArray.class, errors, " for a user role."); + + role = validateAndNormaliseUserRole(role, allUserRoles, errors); + List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, jsonTenantIds, errors, + " for a user role."); + + if (role != null && normalisedTenantIds != null) { + userRoles.add(new UserRole(role, normalisedTenantIds)); + } + } + return userRoles; + } + + private static List getParsedTotpDevices(JsonObject userData, List errors) { + JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totpDevices", ValueType.ARRAY_OF_OBJECT, false, + JsonArray.class, errors, "."); + + if (jsonTotpDevices == null) { + return null; + } + + List totpDevices = new ArrayList<>(); + for (JsonElement jsonTotpDeviceEl : jsonTotpDevices) { + JsonObject jsonTotpDevice = jsonTotpDeviceEl.getAsJsonObject(); + + String secretKey = parseAndValidateFieldType(jsonTotpDevice, "secretKey", ValueType.STRING, true, + String.class, errors, " for a totp device."); + Integer period = parseAndValidateFieldType(jsonTotpDevice, "period", ValueType.INTEGER, false, + Integer.class, errors, " for a totp device."); + Integer skew = parseAndValidateFieldType(jsonTotpDevice, "skew", ValueType.INTEGER, false, Integer.class, + errors, " for a totp device."); + String deviceName = parseAndValidateFieldType(jsonTotpDevice, "deviceName", ValueType.STRING, false, + String.class, errors, " for a totp device."); + + secretKey = validateAndNormaliseTotpSecretKey(secretKey, errors); + period = validateAndNormaliseTotpPeriod(period, errors); + skew = validateAndNormaliseTotpSkew(skew, errors); + deviceName = validateAndNormaliseTotpDeviceName(deviceName, errors); + + if (secretKey != null && period != null && skew != null) { + totpDevices.add(new TotpDevice(secretKey, period, skew, deviceName)); + } + } + return totpDevices; + } + + private static List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) + throws StorageQueryException, TenantOrAppNotFoundException { + JsonArray jsonLoginMethods = parseAndValidateFieldType(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, + true, JsonArray.class, errors, "."); + + if (jsonLoginMethods == null) { + return new ArrayList<>(); + } + + if (jsonLoginMethods.size() == 0) { + errors.add("At least one loginMethod is required."); + return new ArrayList<>(); + } + + validateAndNormaliseIsPrimaryField(jsonLoginMethods, errors); + + List loginMethods = new ArrayList<>(); + + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + + String recipeId = parseAndValidateFieldType(jsonLoginMethodObj, "recipeId", ValueType.STRING, true, + String.class, errors, " for a loginMethod."); + JsonArray tenantIds = parseAndValidateFieldType(jsonLoginMethodObj, "tenantIds", ValueType.ARRAY_OF_STRING, + false, JsonArray.class, errors, " for a loginMethod."); + Boolean isVerified = parseAndValidateFieldType(jsonLoginMethodObj, "isVerified", ValueType.BOOLEAN, false, + Boolean.class, errors, " for a loginMethod."); + Boolean isPrimary = parseAndValidateFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN, false, + Boolean.class, errors, " for a loginMethod."); + Long timeJoined = parseAndValidateFieldType(jsonLoginMethodObj, "timeJoinedInMSSinceEpoch", ValueType.LONG, + false, Long.class, errors, " for a loginMethod"); + + recipeId = validateAndNormaliseRecipeId(recipeId, errors); + List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, tenantIds, errors, + " for " + recipeId + " recipe."); + isPrimary = validateAndNormaliseIsPrimary(isPrimary); + isVerified = validateAndNormaliseIsVerified(isVerified); + + long timeJoinedInMSSinceEpoch = validateAndNormaliseTimeJoined(timeJoined, errors); + + if ("emailpassword".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, + String.class, errors, " for an emailpassword recipe."); + String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, + true, String.class, errors, " for an emailpassword recipe."); + String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", + ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + + email = validateAndNormaliseEmail(email, errors); + CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm( + hashingAlgorithm, errors); + hashingAlgorithm = normalisedHashingAlgorithm != null ? normalisedHashingAlgorithm.toString() + : hashingAlgorithm; + passwordHash = validateAndNormalisePasswordHash(main, appIdentifier, normalisedHashingAlgorithm, + passwordHash, errors); + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null)); + } else if ("thirdparty".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, + String.class, errors, " for a thirdparty recipe."); + String thirdPartyId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyId", ValueType.STRING, + true, String.class, errors, " for a thirdparty recipe."); + String thirdPartyUserId = parseAndValidateFieldType(jsonLoginMethodObj, "thirdPartyUserId", + ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); + + email = validateAndNormaliseEmail(email, errors); + thirdPartyId = validateAndNormaliseThirdPartyId(thirdPartyId, errors); + thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, null, null, thirdPartyId, thirdPartyUserId, null)); + } else if ("passwordless".equals(recipeId)) { + String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, + String.class, errors, " for a passwordless recipe."); + String phoneNumber = parseAndValidateFieldType(jsonLoginMethodObj, "phoneNumber", ValueType.STRING, + false, String.class, errors, " for a passwordless recipe."); + + email = validateAndNormaliseEmail(email, errors); + phoneNumber = validateAndNormalisePhoneNumber(phoneNumber, errors); + + if (email == null && phoneNumber == null) { + errors.add("Either email or phoneNumber is required for a passwordless recipe."); + } + + loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, + timeJoinedInMSSinceEpoch, email, null, null, null, null, phoneNumber)); + } + } + return loginMethods; + } + + private static String validateAndNormaliseExternalUserId(String externalUserId, Set allExternalUserIds, + List errors) { + if (externalUserId == null) { + return null; + } + + if (externalUserId.length() > 255) { + errors.add("externalUserId " + externalUserId + " is too long. Max length is 128."); + } + + if (!allExternalUserIds.add(externalUserId)) { + errors.add("externalUserId " + externalUserId + " is not unique. It is already used by another user."); + } + + // We just trim the externalUserId as per the UpdateExternalUserIdInfoAPI.java + return externalUserId.trim(); + } + + private static String validateAndNormaliseUserRole(String role, String[] allUserRoles, List errors) { + if (role.length() > 255) { + errors.add("role " + role + " is too long. Max length is 255."); + } + + // We just trim the role as per the CreateRoleAPI.java + String normalisedRole = role.trim(); + + if (!Arrays.asList(allUserRoles).contains(normalisedRole)) { + errors.add("Role " + normalisedRole + " does not exist."); + } + + return normalisedRole; + } + + private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + if (secretKey == null) { + return null; + } + + if (secretKey.length() > 256) { + errors.add("TOTP secretKey " + secretKey + " is too long. Max length is 256."); + } + + // We don't perform any normalisation on the secretKey in ImportTotpDeviceAPI.java + return secretKey; + } + + private static Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { + // We default to 30 if period is null + if (period == null) { + return 30; + } + + if (period.intValue() < 1) { + errors.add("period should be > 0 for a totp device."); + return null; + } + return period; + } + + private static Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { + // We default to 1 if skew is null + if (skew == null) { + return 1; + } + + if (skew.intValue() < 0) { + errors.add("skew should be >= 0 for a totp device."); + return null; + } + return skew; + } + + private static String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { + if (deviceName == null) { + return null; + } + + if (deviceName.length() > 256) { + errors.add("TOTP deviceName " + deviceName + " is too long. Max length is 256."); + } + + // We normalise the deviceName as per the ImportTotpDeviceAPI.java + return deviceName.trim(); + } + + private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { + // We are validating that only one loginMethod has isPrimary as true + boolean hasPrimaryLoginMethod = false; + for (JsonElement jsonLoginMethod : jsonLoginMethods) { + JsonObject jsonLoginMethodObj = jsonLoginMethod.getAsJsonObject(); + if (validateJsonFieldType(jsonLoginMethodObj, "isPrimary", ValueType.BOOLEAN)) { + if (jsonLoginMethodObj.get("isPrimary").getAsBoolean()) { + if (hasPrimaryLoginMethod) { + errors.add("No two loginMethods can have isPrimary as true."); + } + hasPrimaryLoginMethod = true; + } + } + } + } + + private static String validateAndNormaliseRecipeId(String recipeId, List errors) { + if (recipeId == null) { + return null; + } + + // We don't perform any normalisation on the recipeId after reading it from request header. + // We will validate it as is. + if (!Arrays.asList("emailpassword", "thirdparty", "passwordless").contains(recipeId)) { + errors.add("Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!"); + } + return recipeId; + } + + private static List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, + JsonArray tenantIds, List errors, String errorSuffix) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantIds == null) { + return List.of(TenantIdentifier.DEFAULT_TENANT_ID); // Default to DEFAULT_TENANT_ID ("public") + } + + List normalisedTenantIds = new ArrayList<>(); + + for (JsonElement tenantIdEl : tenantIds) { + String tenantId = tenantIdEl.getAsString(); + tenantId = validateAndNormaliseTenantId(main, appIdentifier, tenantId, errors, errorSuffix); + + if (tenantId != null) { + normalisedTenantIds.add(tenantId); + } + } + return normalisedTenantIds; + } + + private static String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, + List errors, String errorSuffix) + throws StorageQueryException, TenantOrAppNotFoundException { + if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { + return tenantId; + } + + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.MULTI_TENANCY)) { + errors.add("Multitenancy must be enabled before importing users to a different tenant."); + return null; + } + + // We make the tenantId lowercase while parsing from the request in WebserverAPI.java + String normalisedTenantId = tenantId.trim().toLowerCase(); + TenantConfig[] allTenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + Set validTenantIds = new HashSet<>(); + Arrays.stream(allTenantConfigs) + .forEach(tenantConfig -> validTenantIds.add(tenantConfig.tenantIdentifier.getTenantId())); + + if (!validTenantIds.contains(normalisedTenantId)) { + errors.add("Invalid tenantId: " + tenantId + errorSuffix); + return null; + } + return normalisedTenantId; + } + + private static Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { + // We set the default value as false + return isPrimary == null ? false : isPrimary; + } + + private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { + // We set the default value as false + return isVerified == null ? false : isVerified; + } + + private static long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { + // We default timeJoined to currentTime if it is null + if (timeJoined == null) { + return System.currentTimeMillis(); + } + + if (timeJoined > System.currentTimeMillis()) { + errors.add("timeJoined cannot be in future for a loginMethod."); + } + + if (timeJoined < 0) { + errors.add("timeJoined cannot be < 0 for a loginMethod."); + } + + return timeJoined.longValue(); + } + + private static String validateAndNormaliseEmail(String email, List errors) { + if (email == null) { + return null; + } + + if (email.length() > 255) { + errors.add("email " + email + " is too long. Max length is 256."); + } + + // We normalise the email as per the SignUpAPI.java + return Utils.normaliseEmail(email); + } + + private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, + List errors) { + if (hashingAlgorithm == null) { + return null; + } + + try { + // We trim the hashingAlgorithm and make it uppercase as per the ImportUserWithPasswordHashAPI.java + return CoreConfig.PASSWORD_HASHING_ALG.valueOf(hashingAlgorithm.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + errors.add( + "Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!"); + return null; + } + } + + private static String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, + CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm, String passwordHash, List errors) + throws TenantOrAppNotFoundException { + if (hashingAlgorithm == null || passwordHash == null) { + return passwordHash; + } + + if (passwordHash.length() > 256) { + errors.add("passwordHash is too long. Max length is 256."); + } + + // We trim the passwordHash and validate it as per ImportUserWithPasswordHashAPI.java + passwordHash = passwordHash.trim(); + + try { + PasswordHashingUtils.assertSuperTokensSupportInputPasswordHashFormat(appIdentifier, main, passwordHash, + hashingAlgorithm); + } catch (UnsupportedPasswordHashingFormatException e) { + errors.add(e.getMessage()); + } + + return passwordHash; + } + + private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { + if (thirdPartyId == null) { + return null; + } + + if (thirdPartyId.length() > 28) { + errors.add("thirdPartyId " + thirdPartyId + " is too long. Max length is 28."); + } + + // We don't perform any normalisation on the thirdPartyId in SignInUpAPI.java + return thirdPartyId; + } + + private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { + if (thirdPartyUserId == null) { + return null; + } + + if (thirdPartyUserId.length() > 256) { + errors.add("thirdPartyUserId " + thirdPartyUserId + " is too long. Max length is 256."); + } + + // We don't perform any normalisation on the thirdPartyUserId in SignInUpAPI.java + return thirdPartyUserId; + } + + private static String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { + if (phoneNumber == null) { + return null; + } + + if (phoneNumber.length() > 256) { + errors.add("phoneNumber " + phoneNumber + " is too long. Max length is 256."); + } + + // We normalise the phoneNumber as per the CreateCodeAPI.java + return Utils.normalizeIfPhoneNumber(phoneNumber); + } + + private static void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier appIdentifier, + List userRoles, List loginMethods, List errors) + throws TenantOrAppNotFoundException { + if (loginMethods == null) { + return; + } + + // First validate that tenantIds provided for userRoles also exist in the loginMethods + if (userRoles != null) { + for (UserRole userRole : userRoles) { + for (String tenantId : userRole.tenantIds) { + if (!tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID) && loginMethods.stream() + .noneMatch(loginMethod -> loginMethod.tenantIds.contains(tenantId))) { + errors.add("TenantId " + tenantId + " for a user role does not exist in loginMethods."); + } + } + } + } + + // Now validate that all the tenants share the same storage + String commonTenantUserPoolId = null; + for (LoginMethod loginMethod : loginMethods) { + for (String tenantId : loginMethod.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Storage storage = StorageLayer.getStorage(tenantIdentifier, main); + String tenantUserPoolId = storage.getUserPoolId(); + + if (commonTenantUserPoolId == null) { + commonTenantUserPoolId = tenantUserPoolId; + } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { + errors.add("All tenants for a user must share the same storage."); + } + } + } + } +} diff --git a/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java new file mode 100644 index 000000000..3fbcd8fbd --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/exceptions/InvalidBulkImportDataException.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport.exceptions; + +import java.util.List; + +public class InvalidBulkImportDataException extends Exception { + private static final long serialVersionUID = 1L; + public List errors; + + public InvalidBulkImportDataException(List errors) { + super("Data has missing or invalid fields. Please check the errors field for more details."); + this.errors = errors; + } + + public void addError(String error) { + this.errors.add(error); + } +} diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java new file mode 100644 index 000000000..730fd43b4 --- /dev/null +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -0,0 +1,504 @@ +/* + * Copyright (c) 2024. VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.cronjobs.bulkimport; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.config.Config; +import io.supertokens.cronjobs.CronTask; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.emailpassword.EmailPassword; +import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; +import io.supertokens.passwordless.Passwordless; +import io.supertokens.passwordless.exceptions.RestartFlowException; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; +import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; +import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; +import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; +import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; +import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.ThirdParty; +import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; +import jakarta.servlet.ServletException; + +public class ProcessBulkImportUsers extends CronTask { + + public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; + private Map userPoolToStorageMap = new HashMap<>(); + + private ProcessBulkImportUsers(Main main, List> tenantsInfo) { + super("ProcessBulkImportUsers", main, tenantsInfo, true); + } + + public static ProcessBulkImportUsers init(Main main, List> tenantsInfo) { + return (ProcessBulkImportUsers) main.getResourceDistributor() + .setResource(new TenantIdentifier(null, null, null), RESOURCE_KEY, + new ProcessBulkImportUsers(main, tenantsInfo)); + } + + @Override + protected void doTaskPerApp(AppIdentifier app) + throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, + DbInitException { + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportSQLStorage bulkImportSQLStorage = (BulkImportSQLStorage) StorageLayer + .getStorage(app.getAsPublicTenantIdentifier(), main); + + AppIdentifier appIdentifier = new AppIdentifier(app.getConnectionUriDomain(), app.getAppId()); + + List users = bulkImportSQLStorage.getBulkImportUsersForProcessing(appIdentifier, + BulkImport.PROCESS_USERS_BATCH_SIZE); + + for (BulkImportUser user : users) { + processUser(appIdentifier, user); + } + + closeAllProxyStorages(); + } + + @Override + public int getIntervalTimeSeconds() { + if (Main.isTesting) { + Integer interval = CronTaskTest.getInstance(main).getIntervalInSeconds(RESOURCE_KEY); + if (interval != null) { + return interval; + } + } + return BulkImport.PROCESS_USERS_INTERVAL; + } + + @Override + public int getInitialWaitTimeSeconds() { + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + if (Main.isTesting) { + return 5; + } + return 0; + } + + private Storage getProxyStorage(TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + Config.getBaseConfigAsJsonObject(main), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(true); + return bulkImportProxyStorage; + } + + public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException { + List allProxyStorages = new ArrayList<>(); + + Map resources = main + .getResourceDistributor() + .getAllResourcesWithResourceKey(RESOURCE_KEY); + for (ResourceDistributor.KeyClass key : resources.keySet()) { + if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { + allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); + } + } + return allProxyStorages.toArray(new Storage[0]); + } + + private void closeAllProxyStorages() { + for (Storage storage : userPoolToStorageMap.values()) { + storage.close(); + } + } + + private void processUser(AppIdentifier appIdentifier, BulkImportUser user) + throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, + DbInitException { + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + try { + bulkImportProxyStorage.startTransaction(con -> { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); + } + + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserIdMapping(main, appIdentifier, user, primaryLM); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + + ((BulkImportSQLStorage) bulkImportProxyStorage).deleteBulkImportUser_Transaction(appIdentifier, con, + user.id); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + try { + Connection connection = (Connection) con.getConnection(); + connection.commit(); + connection.setAutoCommit(true); + } catch (SQLException e) { + throw new StorageTransactionLogicException(e); + } + + return null; + }); + } catch (StorageTransactionLogicException e) { + handleProcessUserExceptions(appIdentifier, user, (BulkImportSQLStorage) bulkImportProxyStorage, e); + } + } + + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, + BulkImportSQLStorage bulkImportSQLStorage, Exception e) + throws StorageQueryException { + + // Java doesn't allow us to reassign local variables inside a lambda expression + // so we have to use an array. + String[] errorMessage = { e.getMessage() }; + + if (e instanceof StorageTransactionLogicException) { + StorageTransactionLogicException exception = (StorageTransactionLogicException) e; + errorMessage[0] = exception.actualException.getMessage(); + } + + String[] userId = { user.id }; + + try { + bulkImportSQLStorage.startTransaction(con -> { + bulkImportSQLStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, + BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + try { + Connection connection = (Connection) con.getConnection(); + connection.commit(); + connection.setAutoCommit(true); + } catch (SQLException ex) { + throw new StorageTransactionLogicException(ex); + } + return null; + }); + } catch (StorageTransactionLogicException e1) { + throw new StorageQueryException(e1.actualException); + } + } + + private void processUserLoginMethod(AppIdentifier appIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + String firstTenant = lm.tenantIds.get(0); + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), firstTenant); + + if (lm.recipeId.equals("emailpassword")) { + processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("thirdparty")) { + processThirdPartyLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("passwordless")) { + processPasswordlessLoginMethod(tenantIdentifier, storage, lm); + } else { + throw new StorageTransactionLogicException( + new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); + } + + associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); + } + + private void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + try { + ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, + lm.passwordHash, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException( + new Exception("A user with email " + lm.email + " already exists")); + } + } + + private void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) + throws StorageTransactionLogicException { + try { + SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( + tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, + lm.timeJoinedInMSSinceEpoch); + + lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId + + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); + } + } + + private void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) + throws StorageTransactionLogicException { + try { + AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, + lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensOrExternalUserId = userInfo.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { + throw new StorageTransactionLogicException(e); + } + } + + private void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, + String firstTenant) throws StorageTransactionLogicException { + for (String tenantId : lm.tenantIds) { + try { + if (tenantId.equals(firstTenant)) { + continue; + } + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.superTokensOrExternalUserId); + } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException + | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException + | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException + | AnotherPrimaryUserWithEmailAlreadyExistsException + | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + private void createPrimaryUserAndLinkAccounts(Main main, + AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) + throws StorageTransactionLogicException { + if (user.loginMethods.size() == 1) { + return; + } + + try { + AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId); + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "We tried to create the primary user for the userId " + primaryLM.superTokensOrExternalUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException + | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + + for (LoginMethod lm : user.loginMethods) { + try { + if (lm.superTokensOrExternalUserId.equals(primaryLM.superTokensOrExternalUserId)) { + continue; + } + + AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.superTokensOrExternalUserId, + primaryLM.superTokensOrExternalUserId); + + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId + + " to the primary userId " + primaryLM.superTokensOrExternalUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (InputUserIdIsNotAPrimaryUserException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId + + " to the primary userId " + primaryLM.superTokensOrExternalUserId + + " but it is not a primary user. This should not happen. Please contact support.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException + | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + } + } + + private void createUserIdMapping(Main main, AppIdentifier appIdentifier, + BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.externalUserId != null) { + try { + UserIdMapping.createUserIdMapping( + appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), + primaryLM.superTokensOrExternalUserId, user.externalUserId, + null, false, true); + + primaryLM.superTokensOrExternalUserId = user.externalUserId; + } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException | InvalidConfigException + | IOException | DbInitException e) { + throw new StorageTransactionLogicException(e); + } catch (UserIdMappingAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A user with externalId " + user.externalUserId + " already exists")); + } catch (UnknownSuperTokensUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to create the externalUserId mapping for the superTokenUserId " + + primaryLM.superTokensOrExternalUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } + } + } + + private void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, + LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.userMetadata != null) { + try { + UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + user.userMetadata); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + private void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, + BulkImportUser user) throws StorageTransactionLogicException { + if (user.userRoles != null) { + for (UserRole userRole : user.userRoles) { + try { + for (String tenantId : userRole.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), + tenantId); + + UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); + } + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownRoleException e) { + throw new StorageTransactionLogicException(new Exception("Role " + userRole.role + + " does not exist! You need pre-create the role before assigning it to the user.")); + } + } + } + } + + private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, Storage storage, + List loginMethods) throws StorageTransactionLogicException { + + for (LoginMethod lm : loginMethods) { + try { + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), lm.tenantIds.get(0)); + + EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils + .getEmailVerificationStorage(storage); + emailVerificationSQLStorage + .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + lm.superTokensOrExternalUserId, lm.email, true); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, + List totpDevices, LoginMethod primaryLM) throws StorageTransactionLogicException { + for (TotpDevice totpDevice : totpDevices) { + try { + Totp.createDevice(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, + true, System.currentTimeMillis()); + } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(e); + } catch (DeviceAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + } + } + } + + // Returns the primary loginMethod of the user. If no loginMethod is marked as + // primary, then the oldest loginMethod is returned. + private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { + BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); + for (BulkImportUser.LoginMethod lm : user.loginMethods) { + if (lm.isPrimary) { + return lm; + } + + if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { + oldestLM = lm; + } + } + return oldestLM; + } +} diff --git a/src/main/java/io/supertokens/emailpassword/EmailPassword.java b/src/main/java/io/supertokens/emailpassword/EmailPassword.java index 9a92592f3..d35baac2b 100644 --- a/src/main/java/io/supertokens/emailpassword/EmailPassword.java +++ b/src/main/java/io/supertokens/emailpassword/EmailPassword.java @@ -183,41 +183,51 @@ public static ImportUserResponse importUserWithPasswordHash(TenantIdentifier ten tenantIdentifier.toAppIdentifier(), main, passwordHash, hashingAlgorithm); - while (true) { - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + ImportUserResponse response = null; - EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + try { + long timeJoined = System.currentTimeMillis(); + response = createUserWithPasswordHash(tenantIdentifier, storage, email, passwordHash, timeJoined); + } catch (DuplicateEmailException e) { + AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); + AuthRecipeUserInfo userInfoToBeUpdated = null; + LoginMethod loginMethod = null; + for (AuthRecipeUserInfo currUser : allUsers) { + for (LoginMethod currLM : currUser.loginMethods) { + if (currLM.email.equals(email) && currLM.recipeId == RECIPE_ID.EMAIL_PASSWORD && currLM.tenantIds.contains(tenantIdentifier.getTenantId())) { + userInfoToBeUpdated = currUser; + loginMethod = currLM; + break; + } + } + } + if (userInfoToBeUpdated != null) { + LoginMethod finalLoginMethod = loginMethod; + epStorage.startTransaction(con -> { + epStorage.updateUsersPassword_Transaction(tenantIdentifier.toAppIdentifier(), con, + finalLoginMethod.getSupertokensUserId(), passwordHash); + return null; + }); + response = new ImportUserResponse(true, userInfoToBeUpdated); + } + } + return response; + } + public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier tenantIdentifier, Storage storage, + @Nonnull String email, + @Nonnull String passwordHash, @Nullable long timeJoined) + throws StorageQueryException, DuplicateEmailException, TenantOrAppNotFoundException { + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + while (true) { + String userId = Utils.getUUID(); try { - AuthRecipeUserInfo userInfo = epStorage.signUp(tenantIdentifier, userId, email, passwordHash, - timeJoined); + AuthRecipeUserInfo userInfo = null; + userInfo = epStorage.signUp(tenantIdentifier, userId, email, passwordHash, timeJoined); return new ImportUserResponse(false, userInfo); } catch (DuplicateUserIdException e) { // we retry with a new userId - } catch (DuplicateEmailException e) { - AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); - AuthRecipeUserInfo userInfoToBeUpdated = null; - LoginMethod loginMethod = null; - for (AuthRecipeUserInfo currUser : allUsers) { - for (LoginMethod currLM : currUser.loginMethods) { - if (currLM.email.equals(email) && currLM.recipeId == RECIPE_ID.EMAIL_PASSWORD && currLM.tenantIds.contains(tenantIdentifier.getTenantId())) { - userInfoToBeUpdated = currUser; - loginMethod = currLM; - break; - } - } - } - - if (userInfoToBeUpdated != null) { - LoginMethod finalLoginMethod = loginMethod; - epStorage.startTransaction(con -> { - epStorage.updateUsersPassword_Transaction(tenantIdentifier.toAppIdentifier(), con, - finalLoginMethod.getSupertokensUserId(), passwordHash); - return null; - }); - return new ImportUserResponse(true, userInfoToBeUpdated); - } } } } diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index b23434718..d4edd411b 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -135,6 +135,11 @@ public void constructor(String processId, boolean silent, boolean isTesting) { Start.isTesting = isTesting; } + @Override + public Storage createBulkImportProxyStorageInstance() { + return this; + } + @Override public STORAGE_TYPE getType() { return STORAGE_TYPE.SQL; diff --git a/src/main/java/io/supertokens/passwordless/Passwordless.java b/src/main/java/io/supertokens/passwordless/Passwordless.java index ba76d7f45..c0ad15f1e 100644 --- a/src/main/java/io/supertokens/passwordless/Passwordless.java +++ b/src/main/java/io/supertokens/passwordless/Passwordless.java @@ -456,52 +456,37 @@ public static ConsumeCodeResponse consumeCode(TenantIdentifier tenantIdentifier, } if (user == null) { - while (true) { - try { - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); - user = passwordlessStorage.createUser(tenantIdentifier, userId, consumedDevice.email, - consumedDevice.phoneNumber, timeJoined); + long timeJoined = System.currentTimeMillis(); + user = createPasswordlessUser(tenantIdentifier, storage, consumedDevice.email, + consumedDevice.phoneNumber, timeJoined); - // Set email as verified, if using email - if (setEmailVerified && consumedDevice.email != null) { + // Set email as verified, if using email + if (setEmailVerified && consumedDevice.email != null) { + try { + AuthRecipeUserInfo finalUser = user; + EmailVerificationSQLStorage evStorage = + StorageUtils.getEmailVerificationStorage(storage); + evStorage.startTransaction(con -> { try { - AuthRecipeUserInfo finalUser = user; - EmailVerificationSQLStorage evStorage = - StorageUtils.getEmailVerificationStorage(storage); - evStorage.startTransaction(con -> { - try { - evStorage.updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - finalUser.getSupertokensUserId(), consumedDevice.email, true); - evStorage.commitTransaction(con); - - return null; - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } - }); - user.loginMethods[0].setVerified(); // newly created user has only one loginMethod - } catch (StorageTransactionLogicException e) { - if (e.actualException instanceof TenantOrAppNotFoundException) { - throw (TenantOrAppNotFoundException) e.actualException; - } - throw new StorageQueryException(e); + evStorage.updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + finalUser.getSupertokensUserId(), consumedDevice.email, true); + evStorage.commitTransaction(con); + + return null; + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); } + }); + user.loginMethods[0].setVerified(); // newly created user has only one loginMethod + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof TenantOrAppNotFoundException) { + throw (TenantOrAppNotFoundException) e.actualException; } - - return new ConsumeCodeResponse(true, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); - } catch (DuplicateEmailException | DuplicatePhoneNumberException e) { - // Getting these would mean that between getting the user and trying creating it: - // 1. the user managed to do a full create+consume flow - // 2. the users email or phoneNumber was updated to the new one (including device cleanup) - // These should be almost impossibly rare, so it's safe to just ask the user to restart. - // Also, both would make the current login fail if done before the transaction - // by cleaning up the device/code this consume would've used. - throw new RestartFlowException(); - } catch (DuplicateUserIdException e) { - // We can retry.. + throw new StorageQueryException(e); } } + + return new ConsumeCodeResponse(true, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); } else { if (setEmailVerified && consumedDevice.email != null) { // Set email verification @@ -541,6 +526,29 @@ public static ConsumeCodeResponse consumeCode(TenantIdentifier tenantIdentifier, return new ConsumeCodeResponse(false, user, consumedDevice.email, consumedDevice.phoneNumber, consumedDevice); } + public static AuthRecipeUserInfo createPasswordlessUser(TenantIdentifier tenantIdentifier, Storage storage, + String email, String phoneNumber, long timeJoined) + throws TenantOrAppNotFoundException, StorageQueryException, RestartFlowException { + PasswordlessSQLStorage passwordlessStorage = StorageUtils.getPasswordlessStorage(storage); + + while (true) { + try { + String userId = Utils.getUUID(); + return passwordlessStorage.createUser(tenantIdentifier, userId, email, phoneNumber, timeJoined); + } catch (DuplicateEmailException | DuplicatePhoneNumberException e) { + // Getting these would mean that between getting the user and trying creating it: + // 1. the user managed to do a full create+consume flow + // 2. the users email or phoneNumber was updated to the new one (including device cleanup) + // These should be almost impossibly rare, so it's safe to just ask the user to restart. + // Also, both would make the current login fail if done before the transaction + // by cleaning up the device/code this consume would've used. + throw new RestartFlowException(); + } catch (DuplicateUserIdException e) { + // We can retry.. + } + } + } + @TestOnly public static void removeCode(Main main, String codeId) throws StorageQueryException, StorageTransactionLogicException { diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index 5145e7c1c..711767703 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -56,6 +56,14 @@ public Storage getUnderlyingStorage() { } public static Storage getNewStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog) throws InvalidConfigException { + return getNewInstance(main, config, tenantIdentifier, doNotLog, false); + } + + public static Storage getNewBulkImportProxyStorageInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog) throws InvalidConfigException { + return getNewInstance(main, config, tenantIdentifier, doNotLog, true); + } + + private static Storage getNewInstance(Main main, JsonObject config, TenantIdentifier tenantIdentifier, boolean doNotLog, boolean isBulkImportProxy) throws InvalidConfigException { Storage result; if (StorageLayer.ucl == null) { result = new Start(main); @@ -75,7 +83,11 @@ public static Storage getNewStorageInstance(Main main, JsonObject config, Tenant } if (storageLayer != null && !main.isForceInMemoryDB() && (storageLayer.canBeUsed(config) || CLIOptions.get(main).isForceNoInMemoryDB())) { - result = storageLayer; + if (isBulkImportProxy) { + result = storageLayer.createBulkImportProxyStorageInstance(); + } else { + result = storageLayer; + } } else { result = new Start(main); } diff --git a/src/main/java/io/supertokens/thirdparty/ThirdParty.java b/src/main/java/io/supertokens/thirdparty/ThirdParty.java index d49f0a93c..3628aab9d 100644 --- a/src/main/java/io/supertokens/thirdparty/ThirdParty.java +++ b/src/main/java/io/supertokens/thirdparty/ThirdParty.java @@ -206,22 +206,12 @@ private static SignInUpResponse signInUpHelper(TenantIdentifier tenantIdentifier while (true) { // loop for sign in + sign up - while (true) { - // loop for sign up - String userId = Utils.getUUID(); - long timeJoined = System.currentTimeMillis(); + long timeJoined = System.currentTimeMillis(); - try { - AuthRecipeUserInfo createdUser = tpStorage.signUp(tenantIdentifier, userId, email, - new LoginMethod.ThirdParty(thirdPartyId, thirdPartyUserId), timeJoined); - - return new SignInUpResponse(true, createdUser); - } catch (DuplicateUserIdException e) { - // we try again.. - } catch (DuplicateThirdPartyUserException e) { - // we try to sign in - break; - } + try { + return createThirdPartyUser( tenantIdentifier, storage, thirdPartyId, thirdPartyUserId, email, timeJoined); + } catch (DuplicateThirdPartyUserException e) { + // The user already exists, we will try to update the email if needed below } // we try to get user and update their email @@ -341,6 +331,25 @@ private static SignInUpResponse signInUpHelper(TenantIdentifier tenantIdentifier } } + public static SignInUpResponse createThirdPartyUser(TenantIdentifier tenantIdentifier, Storage storage, + String thirdPartyId, String thirdPartyUserId, String email, long timeJoined) + throws StorageQueryException, TenantOrAppNotFoundException, DuplicateThirdPartyUserException { + ThirdPartySQLStorage tpStorage = StorageUtils.getThirdPartyStorage(storage); + + while (true) { + // loop for sign up + String userId = Utils.getUUID(); + + try { + AuthRecipeUserInfo createdUser = tpStorage.signUp(tenantIdentifier, userId, email, + new LoginMethod.ThirdParty(thirdPartyId, thirdPartyUserId), timeJoined); + return new SignInUpResponse(true, createdUser); + } catch (DuplicateUserIdException e) { + // we try again.. + } + } + } + @Deprecated public static AuthRecipeUserInfo getUser(AppIdentifier appIdentifier, Storage storage, String userId) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/utils/JsonValidatorUtils.java b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java new file mode 100644 index 000000000..89a8ea932 --- /dev/null +++ b/src/main/java/io/supertokens/utils/JsonValidatorUtils.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.utils; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +public class JsonValidatorUtils { + @SuppressWarnings("unchecked") + public static T parseAndValidateFieldType(JsonObject jsonObject, String key, ValueType expectedType, + boolean isRequired, Class targetType, List errors, String errorSuffix) { + if (jsonObject.has(key)) { + if (validateJsonFieldType(jsonObject, key, expectedType)) { + T value; + switch (expectedType) { + case STRING: + value = (T) jsonObject.get(key).getAsString(); + break; + case INTEGER: + Integer intValue = jsonObject.get(key).getAsNumber().intValue(); + value = (T) intValue; + break; + case LONG: + Long longValue = jsonObject.get(key).getAsNumber().longValue(); + value = (T) longValue; + break; + case BOOLEAN: + Boolean boolValue = jsonObject.get(key).getAsBoolean(); + value = (T) boolValue; + break; + case OBJECT: + value = (T) jsonObject.get(key).getAsJsonObject(); + break; + case ARRAY_OF_OBJECT, ARRAY_OF_STRING: + value = (T) jsonObject.get(key).getAsJsonArray(); + break; + default: + value = null; + break; + } + if (value != null) { + return targetType.cast(value); + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else { + errors.add(key + " should be of type " + getTypeForErrorMessage(expectedType) + errorSuffix); + } + } else if (isRequired) { + errors.add(key + " is required" + errorSuffix); + } + return null; + } + + public enum ValueType { + STRING, + INTEGER, + LONG, + BOOLEAN, + OBJECT, + ARRAY_OF_STRING, + ARRAY_OF_OBJECT + } + + private static String getTypeForErrorMessage(ValueType type) { + return switch (type) { + case STRING -> "string"; + case INTEGER -> "integer"; + case LONG -> "integer"; // choosing integer over long because it is user facing + case BOOLEAN -> "boolean"; + case OBJECT -> "object"; + case ARRAY_OF_STRING -> "array of string"; + case ARRAY_OF_OBJECT -> "array of object"; + }; + } + + public static boolean validateJsonFieldType(JsonObject jsonObject, String key, ValueType expectedType) { + if (jsonObject.has(key)) { + return switch (expectedType) { + case STRING -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isString() + && !jsonObject.get(key).getAsString().isBlank(); + case INTEGER, LONG -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isNumber(); + case BOOLEAN -> jsonObject.get(key).isJsonPrimitive() && jsonObject.getAsJsonPrimitive(key).isBoolean(); + case OBJECT -> jsonObject.get(key).isJsonObject(); + case ARRAY_OF_OBJECT, ARRAY_OF_STRING -> jsonObject.get(key).isJsonArray() + && validateArrayElements(jsonObject.getAsJsonArray(key), expectedType); + default -> false; + }; + } + return false; + } + + public static boolean validateArrayElements(JsonArray array, ValueType expectedType) { + List elements = new ArrayList<>(); + array.forEach(elements::add); + + return switch (expectedType) { + case ARRAY_OF_OBJECT -> elements.stream().allMatch(JsonElement::isJsonObject); + case ARRAY_OF_STRING -> + elements.stream().allMatch(el -> el.isJsonPrimitive() && el.getAsJsonPrimitive().isString() + && !el.getAsString().isBlank()); + default -> false; + }; + } +} diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 700fb4ba1..96716c746 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -26,6 +26,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; +import io.supertokens.webserver.api.bulkimport.BulkImportAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -260,6 +261,8 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); + addAPI(new BulkImportAPI(main)); + StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java new file mode 100644 index 000000000..0fbf8055f --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.bulkimport.BulkImportUserPaginationToken; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class BulkImportAPI extends WebserverAPI { + public BulkImportAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); + String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); + Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); + + if (limit != null) { + if (limit > BulkImport.GET_USERS_PAGINATION_LIMIT) { + throw new ServletException( + new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_LIMIT)); + } else if (limit < 1) { + throw new ServletException(new BadRequestException("limit must a positive integer with min value 1")); + } + } else { + limit = BulkImport.GET_USERS_DEFAULT_LIMIT; + } + + BULK_IMPORT_USER_STATUS status = null; + if (statusString != null) { + try { + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException(new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + BulkImportUserPaginationContainer users = BulkImport.getUsers(appIdentifier, storage, limit, status, paginationToken); + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + + JsonArray usersJson = new JsonArray(); + for (BulkImportUser user : users.users) { + usersJson.add(user.toJsonObject()); + } + result.add("users", usersJson); + + if (users.nextPaginationToken != null) { + result.addProperty("nextPaginationToken", users.nextPaginationToken); + } + super.sendJsonResponse(200, result, resp); + } catch (BulkImportUserPaginationToken.InvalidTokenException e) { + Logging.debug(main, null, Utils.exceptionStacktraceToString(e)); + throw new ServletException(new BadRequestException("invalid pagination token")); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); + + if (users.size() <= 0 || users.size() > BulkImport.MAX_USERS_TO_ADD) { + JsonObject errorResponseJson = new JsonObject(); + String errorMsg = users.size() <= 0 ? "You need to add at least one user." + : "You can only add " + BulkImport.MAX_USERS_TO_ADD + " users at a time."; + errorResponseJson.addProperty("error", errorMsg); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + String[] allUserRoles = null; + + try { + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (StorageQueryException e) { + throw new ServletException(e); + } + + JsonArray errorsJson = new JsonArray(); + Set allExternalUserIds = new HashSet<>(); + List usersToAdd = new ArrayList<>(); + + for (int i = 0; i < users.size(); i++) { + try { + BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID(), allUserRoles, allExternalUserIds); + usersToAdd.add(user); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonObject errorObj = new JsonObject(); + + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + errorObj.addProperty("index", i); + errorObj.add("errors", errors); + errorsJson.add(errorObj); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new ServletException(e); + } + } + + if (errorsJson.size() > 0) { + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.addProperty("error", + "Data has missing or invalid fields. Please check the users field for more details."); + errorResponseJson.add("users", errorsJson); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } + + try { + BulkImport.addUsers(appIdentifier, storage, usersToAdd); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new ServletException(e); + } + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + } + + @Override + protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); + + if (arr.size() == 0) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); + } + + if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " + + BulkImport.DELETE_USERS_LIMIT + " elements")); + } + + String[] userIds = new String[arr.size()]; + + for (int i = 0; i < userIds.length; i++) { + String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); + if (userId.isEmpty()) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); + } + userIds[i] = userId; + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); + + JsonArray deletedIdsJson = new JsonArray(); + JsonArray invalidIds = new JsonArray(); + + for (String userId : userIds) { + if (deletedIds.contains(userId)) { + deletedIdsJson.add(new JsonPrimitive(userId)); + } else { + invalidIds.add(new JsonPrimitive(userId)); + } + } + + JsonObject result = new JsonObject(); + result.add("deletedIds", deletedIdsJson); + result.add("invalidIds", invalidIds); + + super.sendJsonResponse(200, result, resp); + + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java new file mode 100644 index 000000000..ca0be8534 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class BulkImportTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldAddUsersInBulkImportUsersTable() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + List users = generateBulkImportUser(10); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + BulkImport.addUsers(new AppIdentifier(null, null), storage, users); + + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BULK_IMPORT_USER_STATUS.NEW, null, null); + + // Verify that all users are present in addedUsers + for (BulkImportUser user : users) { + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.id.equals(addedUser.id)) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertEquals(user.toRawDataForDbStorage(), matchingUser.toRawDataForDbStorage()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + List users = generateBulkImportUser(10); + + // We are setting the id of the second user to be the same as the first user to ensure a duplicate id is present + users.get(1).id = users.get(0).id; + + List initialIds = users.stream().map(user -> user.id).collect(Collectors.toList()); + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + BulkImport.addUsers(appIdentifier, storage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + + // Verify that the other properties are same but ids changed + for (BulkImportUser user : users) { + BulkImportUser matchingUser = addedUsers.stream() + .filter(addedUser -> user.toRawDataForDbStorage().equals(addedUser.toRawDataForDbStorage())) + .findFirst() + .orElse(null); + + assertNotNull(matchingUser); + assertEquals(BULK_IMPORT_USER_STATUS.NEW, matchingUser.status); + assertFalse(initialIds.contains(matchingUser.id)); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void testGetUsersStatusFilter() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to PROCESSING + String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); + + storage.startTransaction(con -> { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.PROCESSING, null); + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + assertEquals(10, addedUsers.size()); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to FAILED + String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); + + storage.startTransaction(con -> { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.FAILED, null); + storage.commitTransaction(con); + return null; + }); + + List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.FAILED, null, null); + assertEquals(10, addedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void randomPaginationTest() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + + int numberOfUsers = 500; + // Insert users in batches + { + int batchSize = 100; + for (int i = 0; i < numberOfUsers; i += batchSize) { + List users = generateBulkImportUser(batchSize); + BulkImport.addUsers(new AppIdentifier(null, null), storage, users); + // Adding a delay between each batch to ensure the createdAt different + Thread.sleep(1000); + } + } + + // Get all inserted users + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); + assertEquals(numberOfUsers, addedUsers.size()); + + // We are sorting the users based on createdAt and id like we do in the storage layer + List sortedUsers = addedUsers.stream() + .sorted((user1, user2) -> { + int compareResult = Long.compare(user2.createdAt, user1.createdAt); + if (compareResult == 0) { + return user2.id.compareTo(user1.id); + } + return compareResult; + }) + .collect(Collectors.toList()); + + int[] limits = new int[]{10, 14, 20, 23, 50, 100, 110, 150, 200, 510}; + + for (int limit : limits) { + int indexIntoUsers = 0; + String paginationToken = null; + do { + BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, limit, null, paginationToken); + + for (BulkImportUser actualUser : users.users) { + BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); + + assertEquals(expectedUser.id, actualUser.id); + assertEquals(expectedUser.status, actualUser.status); + assertEquals(expectedUser.toRawDataForDbStorage(), actualUser.toRawDataForDbStorage()); + indexIntoUsers++; + } + + paginationToken = users.nextPaginationToken; + } while (paginationToken != null); + + assert (indexIntoUsers == sortedUsers.size()); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java new file mode 100644 index 000000000..6b822d610 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + + +package io.supertokens.test.bulkimport; + +import java.util.ArrayList; +import java.util.List; + +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; + +public class BulkImportTestUtils { + public static List generateBulkImportUser(int numberOfUsers) { + List users = new ArrayList<>(); + JsonParser parser = new JsonParser(); + + for (int i = 0; i < numberOfUsers; i++) { + String email = "user" + i + "@example.com"; + String id = io.supertokens.utils.Utils.getUUID(); + String externalId = io.supertokens.utils.Utils.getUUID(); + + JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); + + List userRoles = new ArrayList<>(); + userRoles.add(new UserRole("role1", List.of("public"))); + userRoles.add(new UserRole("role2", List.of("public"))); + + List totpDevices = new ArrayList<>(); + totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); + + List loginMethods = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + loginMethods.add(new LoginMethod(List.of("public", "t1"), "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod(List.of("public", "t1"), "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add(new LoginMethod(List.of("public", "t1"), "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); + users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); + } + return users; + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java new file mode 100644 index 000000000..78bff7e32 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -0,0 +1,297 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + */ + +package io.supertokens.test.bulkimport; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.UserPaginationContainer; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.authRecipe.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; +import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.totp.TOTPDevice; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.TestingProcessManager.TestingProcess; +import io.supertokens.test.Utils; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.List; + +public class ProcessBulkImportUsersCronJobTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldProcessBulkImportUsers() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 1; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + BulkImportUser bulkImportUser = users.get(0); + + // Thread.sleep(600000); + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + System.out.println("Users after processing: " + usersAfterProcessing.size()); + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storage, container.users); + + for (AuthRecipeUserInfo user : container.users) { + for (LoginMethod lm1 : user.loginMethods) { + bulkImportUser.loginMethods.forEach(lm2 -> { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(lm1, lm2); + } + }); + } + + JsonObject createdUserMetadata = UserMetadata.getUserMetadata(main, user.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, createdUserMetadata); + + String[] createdUserRoles = UserRoles.getRolesForUser(main, user.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + assertEquals(bulkImportUser.externalUserId, user.getSupertokensOrExternalUserId()); + + + TOTPDevice[] createdTotpDevices = Totp.getDevices(main, user.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List users = generateBulkImportUser(1); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals("Role role1 does not exist! You need pre-create the role before assigning it to the user.", + usersAfterProcessing.get(0).errorMessage); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(0, container.users.length); + } + + @Test + public void shouldThrowTenantDoesNotExistError() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List users = generateBulkImportUser(1); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals( + "Tenant with the following connectionURIDomain, appId and tenantId combination not found: (, public, t1)", + usersAfterProcessing.get(0).errorMessage); + } + + private TestingProcess startCronProcess() throws InterruptedException { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + + Main main = process.getProcess(); + + FeatureFlagTestContent.getInstance(main) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 100000); + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL) { + return null; + } + + return process; + } + + private void assertLoginMethodEquals(LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { + assertEquals(lm1.email, lm2.email); + assertEquals(lm1.verified, lm2.isVerified); + assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); + + switch (lm2.recipeId) { + case "emailpassword": + assertEquals(lm1.passwordHash, lm2.passwordHash); + break; + case "thirdparty": + assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); + assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); + break; + case "passwordless": + assertEquals(lm1.phoneNumber, lm2.phoneNumber); + break; + default: + break; + } + } + + private void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { + assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); + for (int i = 0; i < createdTotpDevices.length; i++) { + assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); + assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); + assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); + assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); + } + } + + private void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + { // tenant 1 (t1 in the same storage as public tenant) + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + } + { // tenant 2 (t2 in the different storage than public tenant) + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + JsonObject config = new JsonObject(); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config)); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java new file mode 100644 index 000000000..3303ebca7 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -0,0 +1,692 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.HashMap; +import java.util.UUID; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; +import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.userroles.UserRoles; + +public class AddBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + public String getResponseMessageFromError(String response) { + return response.substring(response.indexOf("Message: ") + "Message: ".length()); + } + + @Test + public void shouldThrow400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + } + + String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + + // users is required in the json body + { + // CASE 1: users field is not present + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "Field name 'users' is invalid in JSON input"); + } + // CASE 2: users field type in incorrect + try { + JsonObject request = new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "Field name 'users' is invalid in JSON input"); + } + } + // loginMethod array is required in the user object + { + // CASE 1: loginMethods field is not present + try { + JsonObject request = new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); + } + // CASE 2: loginMethods field type in incorrect + try { + JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); + } + // CASE 3: loginMethods array is empty + try { + JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); + } + } + // Invalid field type of non required fields outside loginMethod + { + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); + } + // Non-unique externalUserIds + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); + } + // secretKey is required in totpDevices + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); + } + // Invalid role (tenantIds is required) + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); + } + // Invalid role (role doesn't exist) + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); + } + } + // Invalid field type of non required fields inside loginMethod + { + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); + } + } + // Invalid recipeId + { + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); + } + } + // Invalid field type in emailpassword recipe + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + } + // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + } + // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); + } + } + // Invalid field type in thirdparty recipe + { + // CASE 1: email, thirdPartyId and thirdPartyUserId are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); + } + // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); + } + } + // Invalid field type in passwordless recipe + { + // CASE 1: email and phoneNumber are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + } + // CASE 2: email and phoneNumber field type is incorrect + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + } + } + // Validate tenantId + { + // CASE 1: Invalid tenantId when multitenancy is not enabled + try { + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); + } + // CASE 2: Invalid tenantId when multitenancy is enabled + try { + FeatureFlagTestContent.getInstance(process.getProcess()) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); + + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); + } + // CASE 3. Two more tenants do not share the same storage + try { + FeatureFlagTestContent.getInstance(process.getProcess()) + .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); + + createTenants(process.getProcess()); + + JsonObject request = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); + } + } + // No two loginMethods can have isPrimary as true + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); + } + } + // Can't import less than 1 user at a time + { + try { + JsonObject request = generateUsersJson(0); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"You need to add at least one user.\"}"); + } + } + // Can't import more than 10000 users at a time + { + try { + JsonObject request = generateUsersJson(10001); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, "{\"error\":\"You can only add 10000 users at a time.\"}"); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + } + + JsonObject request = generateUsersJson(10000); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldNormaliseFields() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + } + + JsonObject request = generateUsersJson(1); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + + JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + new HashMap<>(), 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", getResponse.get("status").getAsString()); + JsonArray bulkImportUsers = getResponse.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + + // Test if default values were set in totpDevices + JsonArray totpDevices = bulkImportUserJson.getAsJsonArray("totpDevices"); + for (int i = 0; i < totpDevices.size(); i++) { + JsonObject totpDevice = totpDevices.get(i).getAsJsonObject(); + assertEquals(30, totpDevice.get("period").getAsInt()); + assertEquals(1, totpDevice.get("skew").getAsInt()); + } + + JsonArray loginMethods = bulkImportUserJson.getAsJsonArray("loginMethods"); + for (int i = 0; i < loginMethods.size(); i++) { + JsonObject loginMethod = loginMethods.get(i).getAsJsonObject(); + if (loginMethod.has("email")) { + assertEquals("johndoe+0@gmail.com", loginMethod.get("email").getAsString()); + } + if (loginMethod.has("phoneNumber")) { + assertEquals("+919999999999", loginMethod.get("phoneNumber").getAsString()); + } + if (loginMethod.has("hashingAlgorithm")) { + assertEquals("ARGON2", loginMethod.get("hashingAlgorithm").getAsString()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + public static JsonObject generateUsersJson(int numberOfUsers) { + JsonObject userJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + + JsonArray usersArray = new JsonArray(); + for (int i = 0; i < numberOfUsers; i++) { + JsonObject user = new JsonObject(); + + user.addProperty("externalUserId", UUID.randomUUID().toString()); + user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); + user.add("userRoles", parser.parse("[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); + + JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); + String email = " johndoe+" + i + "@gmail.com "; + + JsonArray loginMethodsArray = new JsonArray(); + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds)); + user.add("loginMethods", loginMethodsArray); + + usersArray.add(user); + } + + userJsonObject.add("users", usersArray); + return userJsonObject; + } + + private static JsonObject createEmailLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "emailpassword"); + loginMethod.addProperty("passwordHash", "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("hashingAlgorithm", "argon2"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", true); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createThirdPartyLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("recipeId", "thirdparty"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("thirdPartyId", "google"); + loginMethod.addProperty("thirdPartyUserId", "112618388912586834161"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createPasswordlessLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "passwordless"); + loginMethod.addProperty("phoneNumber", "+91-9999999999"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + // User pool 1 - (null, null, null), (null, null, t1) + // User pool 2 - (null, null, t2) + + { // tenant 1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject() + ) + ); + } + { // tenant 2 + JsonObject config = new JsonObject(); + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config + ) + ); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java new file mode 100644 index 000000000..db2fe1707 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; + +import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; + +public class DeleteBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + { + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", e.getMessage()); + } + } + { + try { + JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", e.getMessage()); + } + } + { + try { + JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain an empty string", e.getMessage()); + } + } + { + try { + // Create a string array of 500 uuids + JsonObject request = new JsonObject(); + JsonArray ids = new JsonArray(); + for (int i = 0; i < 501; i++) { + ids.add(new JsonPrimitive(io.supertokens.utils.Utils.getUUID())); + } + request.add("ids", ids); + + HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Insert users + List users = generateBulkImportUser(5); + BulkImport.addUsers(appIdentifier, storage, users); + + String invalidId = io.supertokens.utils.Utils.getUUID(); + JsonObject request = new JsonObject(); + JsonArray validIds = new JsonArray(); + for (BulkImportUser user : users) { + validIds.add(new JsonPrimitive(user.id)); + } + validIds.add(new JsonPrimitive(invalidId)); + + request.add("ids", validIds); + + JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000000, 1000000, null, Utils.getCdiVersionStringLatestForTests(), null); + + response.get("deletedIds").getAsJsonArray().forEach(id -> { + assertTrue(validIds.contains(id)); + }); + + assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java new file mode 100644 index 000000000..181bcd336 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + +public class GetBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "0"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: limit must a positive integer with min value 1", + e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("limit", "501"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Max limit allowed is 500", e.getMessage()); + } + + try { + Map params = new HashMap<>(); + params.put("paginationToken", "invalid_token"); + HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: invalid pagination token", e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // Create a bulk import user to test the GET API + String rawData = "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}"; + { + JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); + JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assert res.get("status").getAsString().equals("OK"); + } + + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + assertEquals("OK", response.get("status").getAsString()); + JsonArray bulkImportUsers = response.get("users").getAsJsonArray(); + assertEquals(1, bulkImportUsers.size()); + JsonObject bulkImportUserJson = bulkImportUsers.get(0).getAsJsonObject(); + bulkImportUserJson.get("status").getAsString().equals("NEW"); + BulkImportUser.forTesting_fromJson(bulkImportUserJson).toRawDataForDbStorage().equals(rawData); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } +} From f86211ab7a9d9d6eaaf0490c583a598f863ab6d2 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 20 Mar 2024 13:03:13 +0530 Subject: [PATCH 02/41] chore: update pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 94c0f5211..aa8544d05 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -35,6 +35,8 @@ highlighting the necessary changes) latest branch (`git branch --all`) whose `X.Y` is greater than the latest released tag. - If no such branch exists, then create one from the latest released branch. - [ ] If added a foreign key constraint on `app_id_to_user_id` table, make sure to delete from this table when deleting the user as well if `deleteUserIdMappingToo` is false. +- [ ] If added a new recipe, then make sure to update the bulk import API to include the new recipe. + ## Remaining TODOs for this PR - [ ] Item1 From 05dc4a198b36fd0e669f0c0ae362b8b15d51c681 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 20 Mar 2024 14:15:38 +0530 Subject: [PATCH 03/41] fix: Use the correct tenant config to create the proxy storage --- .../bulkimport/ProcessBulkImportUsers.java | 48 +++++++++++++------ .../ProcessBulkImportUsersCronJobTest.java | 1 - 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 730fd43b4..aa6ff8825 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; +import com.google.gson.JsonObject; + import io.supertokens.Main; import io.supertokens.ResourceDistributor; import io.supertokens.authRecipe.AuthRecipe; @@ -62,6 +64,7 @@ import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; @@ -114,7 +117,7 @@ protected void doTaskPerApp(AppIdentifier app) BulkImport.PROCESS_USERS_BATCH_SIZE); for (BulkImportUser user : users) { - processUser(appIdentifier, user); + processUser(appIdentifier, user, bulkImportSQLStorage); } closeAllProxyStorages(); @@ -147,12 +150,23 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) return userPoolToStorageMap.get(userPoolId); } - SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, - Config.getBaseConfigAsJsonObject(main), tenantIdentifier, true); + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); - userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(true); - return bulkImportProxyStorage; + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(true); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); } public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) @@ -161,9 +175,10 @@ public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifie Map resources = main .getResourceDistributor() - .getAllResourcesWithResourceKey(RESOURCE_KEY); + .getAllResourcesWithResourceKey(StorageLayer.RESOURCE_KEY); for (ResourceDistributor.KeyClass key : resources.keySet()) { if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { + System.out.println("Adding storage for tenant: " + key.getTenantIdentifier().getTenantId()); allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); } } @@ -176,7 +191,7 @@ private void closeAllProxyStorages() { } } - private void processUser(AppIdentifier appIdentifier, BulkImportUser user) + private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportSQLStorage baseTenantStorage) throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, DbInitException { // Since all the tenants of a user must share the storage, we will just use the @@ -202,8 +217,12 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user) createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); - ((BulkImportSQLStorage) bulkImportProxyStorage).deleteBulkImportUser_Transaction(appIdentifier, con, - user.id); + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage + baseTenantStorage.startTransaction(con2 -> { + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, + user.id); + return null; + }); // We need to commit the transaction manually because we have overridden that in the proxy storage try { @@ -217,12 +236,11 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user) return null; }); } catch (StorageTransactionLogicException e) { - handleProcessUserExceptions(appIdentifier, user, (BulkImportSQLStorage) bulkImportProxyStorage, e); + handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } - private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, - BulkImportSQLStorage bulkImportSQLStorage, Exception e) + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { // Java doesn't allow us to reassign local variables inside a lambda expression @@ -237,8 +255,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport String[] userId = { user.id }; try { - bulkImportSQLStorage.startTransaction(con -> { - bulkImportSQLStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, + baseTenantStorage.startTransaction(con -> { + baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); // We need to commit the transaction manually because we have overridden that in the proxy storage diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 78bff7e32..00c73bfb6 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -110,7 +110,6 @@ public void shouldProcessBulkImportUsers() throws Exception { BulkImportUser bulkImportUser = users.get(0); - // Thread.sleep(600000); Thread.sleep(6000); List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, From 5b4658af1f1479344817ff7ff2687bf8b8b238fb Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 21 Mar 2024 16:17:37 +0530 Subject: [PATCH 04/41] fix: PR changes --- .../bulkimport/BulkImportUserUtils.java | 83 +++++++++++------- .../io/supertokens/cronjobs/CronTaskTest.java | 10 +++ .../bulkimport/ProcessBulkImportUsers.java | 84 ++++++++----------- .../java/io/supertokens/inmemorydb/Start.java | 22 ++++- .../api/bulkimport/BulkImportAPI.java | 7 +- .../test/bulkimport/BulkImportTest.java | 22 +++-- .../ProcessBulkImportUsersCronJobTest.java | 2 + .../apis/AddBulkImportUsersTest.java | 56 +++++++++++-- 8 files changed, 192 insertions(+), 94 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 834ed0e4a..f884680d4 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -52,8 +52,16 @@ import static io.supertokens.utils.JsonValidatorUtils.validateJsonFieldType; public class BulkImportUserUtils { - public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, - JsonObject userData, String id, String[] allUserRoles, Set allExternalUserIds) + private String[] allUserRoles; + private Set allExternalUserIds; + + public BulkImportUserUtils(String[] allUserRoles) { + this.allUserRoles = allUserRoles; + this.allExternalUserIds = new HashSet<>(); + } + + public BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifier appIdentifier, JsonObject userData, + String id) throws InvalidBulkImportDataException, StorageQueryException, TenantOrAppNotFoundException { List errors = new ArrayList<>(); @@ -62,11 +70,11 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi errors, "."); JsonObject userMetadata = parseAndValidateFieldType(userData, "userMetadata", ValueType.OBJECT, false, JsonObject.class, errors, "."); - List userRoles = getParsedUserRoles(main, appIdentifier, userData, allUserRoles, errors); - List totpDevices = getParsedTotpDevices(userData, errors); + List userRoles = getParsedUserRoles(main, appIdentifier, userData, errors); + List totpDevices = getParsedTotpDevices(main, appIdentifier, userData, errors); List loginMethods = getParsedLoginMethods(main, appIdentifier, userData, errors); - externalUserId = validateAndNormaliseExternalUserId(externalUserId, allExternalUserIds, errors); + externalUserId = validateAndNormaliseExternalUserId(externalUserId, errors); validateTenantIdsForRoleAndLoginMethods(main, appIdentifier, userRoles, loginMethods, errors); @@ -76,8 +84,8 @@ public static BulkImportUser createBulkImportUserFromJSON(Main main, AppIdentifi return new BulkImportUser(id, externalUserId, userMetadata, userRoles, totpDevices, loginMethods); } - private static List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, - String[] allUserRoles, List errors) throws StorageQueryException, TenantOrAppNotFoundException { + private List getParsedUserRoles(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) throws StorageQueryException, TenantOrAppNotFoundException { JsonArray jsonUserRoles = parseAndValidateFieldType(userData, "userRoles", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); @@ -95,7 +103,7 @@ private static List getParsedUserRoles(Main main, AppIdentifier appIde JsonArray jsonTenantIds = parseAndValidateFieldType(jsonUserRole, "tenantIds", ValueType.ARRAY_OF_STRING, true, JsonArray.class, errors, " for a user role."); - role = validateAndNormaliseUserRole(role, allUserRoles, errors); + role = validateAndNormaliseUserRole(role, errors); List normalisedTenantIds = validateAndNormaliseTenantIds(main, appIdentifier, jsonTenantIds, errors, " for a user role."); @@ -106,7 +114,8 @@ private static List getParsedUserRoles(Main main, AppIdentifier appIde return userRoles; } - private static List getParsedTotpDevices(JsonObject userData, List errors) { + private List getParsedTotpDevices(Main main, AppIdentifier appIdentifier, JsonObject userData, + List errors) throws StorageQueryException, TenantOrAppNotFoundException { JsonArray jsonTotpDevices = parseAndValidateFieldType(userData, "totpDevices", ValueType.ARRAY_OF_OBJECT, false, JsonArray.class, errors, "."); @@ -114,6 +123,12 @@ private static List getParsedTotpDevices(JsonObject userData, List t == EE_FEATURES.MFA)) { + errors.add("MFA must be enabled to import totp devices."); + return null; + } + List totpDevices = new ArrayList<>(); for (JsonElement jsonTotpDeviceEl : jsonTotpDevices) { JsonObject jsonTotpDevice = jsonTotpDeviceEl.getAsJsonObject(); @@ -139,7 +154,7 @@ private static List getParsedTotpDevices(JsonObject userData, List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, + private List getParsedLoginMethods(Main main, AppIdentifier appIdentifier, JsonObject userData, List errors) throws StorageQueryException, TenantOrAppNotFoundException { JsonArray jsonLoginMethods = parseAndValidateFieldType(userData, "loginMethods", ValueType.ARRAY_OF_OBJECT, @@ -154,6 +169,13 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return new ArrayList<>(); } + if (jsonLoginMethods.size() > 1) { + if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + errors.add("Account linking or MFA must be enabled to import multiple loginMethods."); + } + } + validateAndNormaliseIsPrimaryField(jsonLoginMethods, errors); List loginMethods = new ArrayList<>(); @@ -232,8 +254,7 @@ private static List getParsedLoginMethods(Main main, AppIdentifier return loginMethods; } - private static String validateAndNormaliseExternalUserId(String externalUserId, Set allExternalUserIds, - List errors) { + private String validateAndNormaliseExternalUserId(String externalUserId, List errors) { if (externalUserId == null) { return null; } @@ -250,7 +271,7 @@ private static String validateAndNormaliseExternalUserId(String externalUserId, return externalUserId.trim(); } - private static String validateAndNormaliseUserRole(String role, String[] allUserRoles, List errors) { + private String validateAndNormaliseUserRole(String role, List errors) { if (role.length() > 255) { errors.add("role " + role + " is too long. Max length is 255."); } @@ -265,7 +286,7 @@ private static String validateAndNormaliseUserRole(String role, String[] allUser return normalisedRole; } - private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + private String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { if (secretKey == null) { return null; } @@ -278,7 +299,7 @@ private static String validateAndNormaliseTotpSecretKey(String secretKey, List errors) { + private Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { // We default to 30 if period is null if (period == null) { return 30; @@ -291,7 +312,7 @@ private static Integer validateAndNormaliseTotpPeriod(Integer period, List errors) { + private Integer validateAndNormaliseTotpSkew(Integer skew, List errors) { // We default to 1 if skew is null if (skew == null) { return 1; @@ -304,7 +325,7 @@ private static Integer validateAndNormaliseTotpSkew(Integer skew, List e return skew; } - private static String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { + private String validateAndNormaliseTotpDeviceName(String deviceName, List errors) { if (deviceName == null) { return null; } @@ -317,7 +338,7 @@ private static String validateAndNormaliseTotpDeviceName(String deviceName, List return deviceName.trim(); } - private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { + private void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethods, List errors) { // We are validating that only one loginMethod has isPrimary as true boolean hasPrimaryLoginMethod = false; for (JsonElement jsonLoginMethod : jsonLoginMethods) { @@ -333,7 +354,7 @@ private static void validateAndNormaliseIsPrimaryField(JsonArray jsonLoginMethod } } - private static String validateAndNormaliseRecipeId(String recipeId, List errors) { + private String validateAndNormaliseRecipeId(String recipeId, List errors) { if (recipeId == null) { return null; } @@ -346,7 +367,7 @@ private static String validateAndNormaliseRecipeId(String recipeId, List return recipeId; } - private static List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, + private List validateAndNormaliseTenantIds(Main main, AppIdentifier appIdentifier, JsonArray tenantIds, List errors, String errorSuffix) throws StorageQueryException, TenantOrAppNotFoundException { if (tenantIds == null) { @@ -366,7 +387,7 @@ private static List validateAndNormaliseTenantIds(Main main, AppIdentifi return normalisedTenantIds; } - private static String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, + private String validateAndNormaliseTenantId(Main main, AppIdentifier appIdentifier, String tenantId, List errors, String errorSuffix) throws StorageQueryException, TenantOrAppNotFoundException { if (tenantId == null || tenantId.equals(TenantIdentifier.DEFAULT_TENANT_ID)) { @@ -393,17 +414,17 @@ private static String validateAndNormaliseTenantId(Main main, AppIdentifier appI return normalisedTenantId; } - private static Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { + private Boolean validateAndNormaliseIsPrimary(Boolean isPrimary) { // We set the default value as false return isPrimary == null ? false : isPrimary; } - private static Boolean validateAndNormaliseIsVerified(Boolean isVerified) { + private Boolean validateAndNormaliseIsVerified(Boolean isVerified) { // We set the default value as false return isVerified == null ? false : isVerified; } - private static long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { + private long validateAndNormaliseTimeJoined(Long timeJoined, List errors) { // We default timeJoined to currentTime if it is null if (timeJoined == null) { return System.currentTimeMillis(); @@ -420,7 +441,7 @@ private static long validateAndNormaliseTimeJoined(Long timeJoined, List return timeJoined.longValue(); } - private static String validateAndNormaliseEmail(String email, List errors) { + private String validateAndNormaliseEmail(String email, List errors) { if (email == null) { return null; } @@ -433,7 +454,7 @@ private static String validateAndNormaliseEmail(String email, List error return Utils.normaliseEmail(email); } - private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, + private CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgorithm(String hashingAlgorithm, List errors) { if (hashingAlgorithm == null) { return null; @@ -449,7 +470,7 @@ private static CoreConfig.PASSWORD_HASHING_ALG validateAndNormaliseHashingAlgori } } - private static String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, + private String validateAndNormalisePasswordHash(Main main, AppIdentifier appIdentifier, CoreConfig.PASSWORD_HASHING_ALG hashingAlgorithm, String passwordHash, List errors) throws TenantOrAppNotFoundException { if (hashingAlgorithm == null || passwordHash == null) { @@ -473,7 +494,7 @@ private static String validateAndNormalisePasswordHash(Main main, AppIdentifier return passwordHash; } - private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { + private String validateAndNormaliseThirdPartyId(String thirdPartyId, List errors) { if (thirdPartyId == null) { return null; } @@ -486,7 +507,7 @@ private static String validateAndNormaliseThirdPartyId(String thirdPartyId, List return thirdPartyId; } - private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { + private String validateAndNormaliseThirdPartyUserId(String thirdPartyUserId, List errors) { if (thirdPartyUserId == null) { return null; } @@ -499,7 +520,7 @@ private static String validateAndNormaliseThirdPartyUserId(String thirdPartyUser return thirdPartyUserId; } - private static String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { + private String validateAndNormalisePhoneNumber(String phoneNumber, List errors) { if (phoneNumber == null) { return null; } @@ -512,7 +533,7 @@ private static String validateAndNormalisePhoneNumber(String phoneNumber, List userRoles, List loginMethods, List errors) throws TenantOrAppNotFoundException { if (loginMethods == null) { diff --git a/src/main/java/io/supertokens/cronjobs/CronTaskTest.java b/src/main/java/io/supertokens/cronjobs/CronTaskTest.java index 477d23cc5..4265c361d 100644 --- a/src/main/java/io/supertokens/cronjobs/CronTaskTest.java +++ b/src/main/java/io/supertokens/cronjobs/CronTaskTest.java @@ -28,6 +28,7 @@ public class CronTaskTest extends SingletonResource { private static final String RESOURCE_ID = "io.supertokens.cronjobs.CronTaskTest"; private Map cronTaskToInterval = new HashMap(); + private Map cronTaskToWaitTime = new HashMap(); private CronTaskTest() { @@ -51,4 +52,13 @@ public void setIntervalInSeconds(String resourceId, int interval) { public Integer getIntervalInSeconds(String resourceId) { return cronTaskToInterval.get(resourceId); } + + @TestOnly + public void setInitialWaitTimeInSeconds(String resourceId, int interval) { + cronTaskToWaitTime.put(resourceId, interval); + } + + public Integer getInitialWaitTimeInSeconds(String resourceId) { + return cronTaskToWaitTime.get(resourceId); + } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index aa6ff8825..0be95228a 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -17,8 +17,6 @@ package io.supertokens.cronjobs.bulkimport; import java.io.IOException; -import java.sql.Connection; -import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -87,7 +85,7 @@ public class ProcessBulkImportUsers extends CronTask { public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; - private Map userPoolToStorageMap = new HashMap<>(); + private Map userPoolToStorageMap = new HashMap<>(); private ProcessBulkImportUsers(Main main, List> tenantsInfo) { super("ProcessBulkImportUsers", main, tenantsInfo, true); @@ -113,7 +111,7 @@ protected void doTaskPerApp(AppIdentifier app) AppIdentifier appIdentifier = new AppIdentifier(app.getConnectionUriDomain(), app.getAppId()); - List users = bulkImportSQLStorage.getBulkImportUsersForProcessing(appIdentifier, + List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(appIdentifier, BulkImport.PROCESS_USERS_BATCH_SIZE); for (BulkImportUser user : users) { @@ -136,9 +134,11 @@ public int getIntervalTimeSeconds() { @Override public int getInitialWaitTimeSeconds() { - // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer if (Main.isTesting) { - return 5; + Integer waitTime = CronTaskTest.getInstance(main).getInitialWaitTimeInSeconds(RESOURCE_KEY); + if (waitTime != null) { + return waitTime; + } } return 0; } @@ -178,17 +178,18 @@ public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifie .getAllResourcesWithResourceKey(StorageLayer.RESOURCE_KEY); for (ResourceDistributor.KeyClass key : resources.keySet()) { if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { - System.out.println("Adding storage for tenant: " + key.getTenantIdentifier().getTenantId()); allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); } } return allProxyStorages.toArray(new Storage[0]); } - private void closeAllProxyStorages() { - for (Storage storage : userPoolToStorageMap.values()) { + private void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); storage.close(); } + userPoolToStorageMap.clear(); } private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportSQLStorage baseTenantStorage) @@ -206,41 +207,41 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI try { bulkImportProxyStorage.startTransaction(con -> { - for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); - } + try { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); + } - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserIdMapping(main, appIdentifier, user, primaryLM); - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); - - // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage - baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, - user.id); + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserIdMapping(main, appIdentifier, user, primaryLM); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage + baseTenantStorage.startTransaction(con2 -> { + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, + user.id); + return null; + }); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return null; - }); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - try { - Connection connection = (Connection) con.getConnection(); - connection.commit(); - connection.setAutoCommit(true); - } catch (SQLException e) { - throw new StorageTransactionLogicException(e); + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + throw e; } - - return null; }); } catch (StorageTransactionLogicException e) { handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } - private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, BulkImportSQLStorage baseTenantStorage) + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, + BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { // Java doesn't allow us to reassign local variables inside a lambda expression @@ -252,21 +253,10 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport errorMessage[0] = exception.actualException.getMessage(); } - String[] userId = { user.id }; - try { baseTenantStorage.startTransaction(con -> { - baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userId, + baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - try { - Connection connection = (Connection) con.getConnection(); - connection.commit(); - connection.setAutoCommit(true); - } catch (SQLException ex) { - throw new StorageTransactionLogicException(ex); - } return null; }); } catch (StorageTransactionLogicException e1) { diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index d4edd411b..d5cdd89b2 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -137,7 +137,27 @@ public void constructor(String processId, boolean silent, boolean isTesting) { @Override public Storage createBulkImportProxyStorageInstance() { - return this; + // throw not implemented error + throw new UnsupportedOperationException("Unimplemented method 'createBulkImportProxyStorageInstance'"); + + } + + @Override + public void closeConnectionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "closeConnectionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + + @Override + public void commitTransactionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "commitTransactionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); + } + + @Override + public void rollbackTransactionForBulkImportProxyStorage() throws StorageQueryException { + throw new UnsupportedOperationException( + "rollbackTransactionForBulkImportProxyStorage should only be called from BulkImportProxyStorage"); } @Override diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 0fbf8055f..abb8fe65d 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -59,6 +59,7 @@ public String getPath() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); @@ -118,6 +119,7 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); @@ -148,12 +150,12 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S } JsonArray errorsJson = new JsonArray(); - Set allExternalUserIds = new HashSet<>(); List usersToAdd = new ArrayList<>(); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); for (int i = 0; i < users.size(); i++) { try { - BulkImportUser user = BulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID(), allUserRoles, allExternalUserIds); + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, users.get(i).getAsJsonObject(), Utils.getUUID()); usersToAdd.add(user); } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { JsonObject errorObj = new JsonObject(); @@ -191,6 +193,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S @Override protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index ca0be8534..d7847e36a 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -32,6 +32,8 @@ import io.supertokens.ProcessState; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserPaginationContainer; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; @@ -161,10 +163,10 @@ public void testGetUsersStatusFilter() throws Exception { BulkImport.addUsers(appIdentifier, storage, users); // Update the users status to PROCESSING - String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); - storage.startTransaction(con -> { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.PROCESSING, null); + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.PROCESSING, null); + } storage.commitTransaction(con); return null; }); @@ -179,10 +181,10 @@ public void testGetUsersStatusFilter() throws Exception { BulkImport.addUsers(appIdentifier, storage, users); // Update the users status to FAILED - String[] userIds = users.stream().map(user -> user.id).toArray(String[]::new); - storage.startTransaction(con -> { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, userIds, BULK_IMPORT_USER_STATUS.FAILED, null); + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.FAILED, null); + } storage.commitTransaction(con); return null; }); @@ -199,7 +201,13 @@ public void testGetUsersStatusFilter() throws Exception { public void randomPaginationTest() throws Exception { String[] args = {"../"}; - TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); + + // We are setting a high initial wait time to ensure the cron job doesn't run while we are running the tests + CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1000000); + + process.startProcess(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 00c73bfb6..b71372d5e 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -215,6 +215,8 @@ private TestingProcess startCronProcess() throws InterruptedException { .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 100000); process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 3303ebca7..d38c18de0 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -194,8 +194,24 @@ public void shouldThrow400Error() throws Exception { assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); } + // MFA must be enabled to import totpDevices + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); + } // secretKey is required in totpDevices try { + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); JsonObject request = new JsonParser() .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") .getAsJsonObject(); @@ -387,6 +403,28 @@ public void shouldThrow400Error() throws Exception { "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); } } + // Disabling all feature flags to be able to get the desired error messages + { + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{}); + } + // More than two loginMethods when either of account linking or MFA is not enabled + { + // CASE 1: email, passwordHash and hashingAlgorithm are not present + try { + JsonObject request = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}").getAsJsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, + "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking or MFA must be enabled to import multiple loginMethods.\"]}]}"); + } + } + // Validate tenantId { // CASE 1: Invalid tenantId when multitenancy is not enabled @@ -405,11 +443,12 @@ public void shouldThrow400Error() throws Exception { assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); } + // Now enabling Account linking and Multitenancy for further tests + { + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); + } // CASE 2: Invalid tenantId when multitenancy is enabled try { - FeatureFlagTestContent.getInstance(process.getProcess()) - .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); - JsonObject request = new JsonParser().parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); @@ -425,9 +464,6 @@ public void shouldThrow400Error() throws Exception { } // CASE 3. Two more tenants do not share the same storage try { - FeatureFlagTestContent.getInstance(process.getProcess()) - .setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[]{EE_FEATURES.MULTI_TENANCY}); - createTenants(process.getProcess()); JsonObject request = new JsonParser().parse( @@ -505,6 +541,8 @@ public void shouldReturn200Response() throws Exception { return; } + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); @@ -532,6 +570,8 @@ public void shouldNormaliseFields() throws Exception { return; } + setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); @@ -689,4 +729,8 @@ null, null, new JsonObject() ); } } + + private void setFeatureFlags(Main main, EE_FEATURES[] features) { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); + } } From d721cd5ce63d2426ce619cd7fa75536032c2d3fa Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Fri, 29 Mar 2024 11:49:11 +0530 Subject: [PATCH 05/41] fix: PR changes --- .../bulkimport/ProcessBulkImportUsers.java | 116 ++++++++++++++---- 1 file changed, 92 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 0be95228a..918a9cefa 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -48,6 +48,7 @@ import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; @@ -205,7 +206,23 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI LoginMethod primaryLM = getPrimaryLoginMethod(user); + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); try { + // If primaryUserId is not null, it means we may have already processed this user but failed to delete the entry + // If the primaryUserId exists in the database, we'll delete the corresponding entry from the bulkImportUser table and proceed to skip this user. + if (user.primaryUserId != null) { + AuthRecipeUserInfo processedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, + user.primaryUserId); + + if (processedUser != null && isProcessedUserFromSameBulkImportUserEntry(processedUser, user)) { + baseTenantStorage.startTransaction(con2 -> { + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); + return null; + }); + return; + } + } + bulkImportProxyStorage.startTransaction(con -> { try { for (LoginMethod lm : user.loginMethods) { @@ -219,15 +236,24 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. + // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this + // case. + baseTenantStorage.updateBulkImportUserPrimaryUserId(appIdentifier, user.id, + primaryLM.superTokensUserId); + + // We need to commit the transaction manually because we have overridden that in the proxy storage + // If this fails, the primaryUserId will be updated in the bulkImportUser but it wouldn’t actually exist. + // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user won't exist, and we'll simply re-process it. + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage + // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist in the database. + // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user will exist, and we'll simply delete the entry. baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, - user.id); + baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); return null; }); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return null; } catch (StorageTransactionLogicException e) { // We need to rollback the transaction manually because we have overridden that in the proxy storage @@ -291,7 +317,7 @@ private void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, lm.passwordHash, lm.timeJoinedInMSSinceEpoch); - lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); } catch (DuplicateEmailException e) { @@ -307,7 +333,7 @@ private void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Sto tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, lm.timeJoinedInMSSinceEpoch); - lm.superTokensOrExternalUserId = userInfo.user.getSupertokensUserId(); + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); } catch (DuplicateThirdPartyUserException e) { @@ -322,7 +348,7 @@ private void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, S AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); - lm.superTokensOrExternalUserId = userInfo.getSupertokensUserId(); + lm.superTokensUserId = userInfo.getSupertokensUserId(); } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { throw new StorageTransactionLogicException(e); } @@ -338,7 +364,7 @@ private void associateUserToTenants(Main main, AppIdentifier appIdentifier, Stor TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), tenantId); - Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.superTokensOrExternalUserId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException @@ -357,12 +383,12 @@ private void createPrimaryUserAndLinkAccounts(Main main, } try { - AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId); + AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException(new Exception( - "We tried to create the primary user for the userId " + primaryLM.superTokensOrExternalUserId + "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() + " but it doesn't exist. This should not happen. Please contact support.")); } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { @@ -372,24 +398,24 @@ private void createPrimaryUserAndLinkAccounts(Main main, for (LoginMethod lm : user.loginMethods) { try { - if (lm.superTokensOrExternalUserId.equals(primaryLM.superTokensOrExternalUserId)) { + if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { continue; } - AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.superTokensOrExternalUserId, - primaryLM.superTokensOrExternalUserId); + AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), + primaryLM.getSuperTokenOrExternalUserId()); } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId - + " to the primary userId " + primaryLM.superTokensOrExternalUserId + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + " but it doesn't exist. This should not happen. Please contact support.")); } catch (InputUserIdIsNotAPrimaryUserException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.superTokensOrExternalUserId - + " to the primary userId " + primaryLM.superTokensOrExternalUserId + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + " but it is not a primary user. This should not happen. Please contact support.")); } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { @@ -405,10 +431,10 @@ private void createUserIdMapping(Main main, AppIdentifier appIdentifier, try { UserIdMapping.createUserIdMapping( appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), - primaryLM.superTokensOrExternalUserId, user.externalUserId, + primaryLM.superTokensUserId, user.externalUserId, null, false, true); - primaryLM.superTokensOrExternalUserId = user.externalUserId; + primaryLM.externalUserId = user.externalUserId; } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException e) { throw new StorageTransactionLogicException(e); @@ -418,7 +444,7 @@ appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), } catch (UnknownSuperTokensUserIdException e) { throw new StorageTransactionLogicException( new Exception("We tried to create the externalUserId mapping for the superTokenUserId " - + primaryLM.superTokensOrExternalUserId + + primaryLM.superTokensUserId + " but it doesn't exist. This should not happen. Please contact support.")); } } @@ -428,7 +454,7 @@ private void createUserMetadata(AppIdentifier appIdentifier, Storage storage, Bu LoginMethod primaryLM) throws StorageTransactionLogicException { if (user.userMetadata != null) { try { - UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), user.userMetadata); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); @@ -471,7 +497,7 @@ private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Transact .getEmailVerificationStorage(storage); emailVerificationSQLStorage .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - lm.superTokensOrExternalUserId, lm.email, true); + lm.getSuperTokenOrExternalUserId(), lm.email, true); } catch (TenantOrAppNotFoundException | StorageQueryException e) { throw new StorageTransactionLogicException(e); } @@ -482,7 +508,7 @@ private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage s List totpDevices, LoginMethod primaryLM) throws StorageTransactionLogicException { for (TotpDevice totpDevice : totpDevices) { try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.superTokensOrExternalUserId, + Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, true, System.currentTimeMillis()); } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { @@ -509,4 +535,46 @@ private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { } return oldestLM; } + + private boolean isProcessedUserFromSameBulkImportUserEntry( + AuthRecipeUserInfo processedUser, BulkImportUser bulkImportUser) { + if (bulkImportUser == null || processedUser == null || bulkImportUser.loginMethods == null || + processedUser.loginMethods == null) { + return false; + } + + for (LoginMethod lm1 : bulkImportUser.loginMethods) { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : processedUser.loginMethods) { + if (lm2.recipeId.toString().equals(lm1.recipeId)) { + if (lm1.email != null && !lm1.email.equals(lm2.email)) { + return false; + } + + switch (lm1.recipeId) { + case "emailpassword": + if (lm1.passwordHash != null && !lm1.passwordHash.equals(lm2.passwordHash)) { + return false; + } + break; + case "thirdparty": + if ((lm1.thirdPartyId != null && !lm1.thirdPartyId.equals(lm2.thirdParty.id)) + || (lm1.thirdPartyUserId != null + && !lm1.thirdPartyUserId.equals(lm2.thirdParty.userId))) { + return false; + } + break; + case "passwordless": + if (lm1.phoneNumber != null && !lm1.phoneNumber.equals(lm2.phoneNumber)) { + return false; + } + break; + default: + return false; + } + } + } + } + + return true; + } } From 769cbb648b06c6aa005a2dc99008a9a6250b75b6 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 2 Apr 2024 14:39:32 +0530 Subject: [PATCH 06/41] fix: PR changes --- .../io/supertokens/authRecipe/AuthRecipe.java | 8 +- .../bulkimport/BulkImportUserUtils.java | 5 +- .../bulkimport/ProcessBulkImportUsers.java | 6 +- src/main/java/io/supertokens/utils/Utils.java | 8 + .../test/bulkimport/BulkImportTestUtils.java | 17 +- .../ProcessBulkImportUsersCronJobTest.java | 100 ++- .../apis/AddBulkImportUsersTest.java | 809 ++++++++---------- 7 files changed, 479 insertions(+), 474 deletions(-) diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 271b93d52..8b4ffa91c 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -42,6 +42,8 @@ import io.supertokens.session.Session; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.useridmapping.UserIdType; +import io.supertokens.utils.Utils; + import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; @@ -327,8 +329,7 @@ public static LinkAccountsResult linkAccounts(Main main, AppIdentifier appIdenti RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException, InputUserIdIsNotAPrimaryUserException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } @@ -532,8 +533,7 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index f884680d4..7e81c0125 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -170,9 +170,8 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden } if (jsonLoginMethods.size() > 1) { - if (Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) - .noneMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA)) { - errors.add("Account linking or MFA must be enabled to import multiple loginMethods."); + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + errors.add("Account linking must be enabled to import multiple loginMethods."); } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 918a9cefa..1d80ab6fa 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -110,13 +110,11 @@ protected void doTaskPerApp(AppIdentifier app) BulkImportSQLStorage bulkImportSQLStorage = (BulkImportSQLStorage) StorageLayer .getStorage(app.getAsPublicTenantIdentifier(), main); - AppIdentifier appIdentifier = new AppIdentifier(app.getConnectionUriDomain(), app.getAppId()); - - List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(appIdentifier, + List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, BulkImport.PROCESS_USERS_BATCH_SIZE); for (BulkImportUser user : users) { - processUser(appIdentifier, user, bulkImportSQLStorage); + processUser(app, user, bulkImportSQLStorage); } closeAllProxyStorages(); diff --git a/src/main/java/io/supertokens/utils/Utils.java b/src/main/java/io/supertokens/utils/Utils.java index ecd3a0479..eb84cff5c 100644 --- a/src/main/java/io/supertokens/utils/Utils.java +++ b/src/main/java/io/supertokens/utils/Utils.java @@ -25,6 +25,8 @@ import com.google.i18n.phonenumbers.Phonenumber; import io.supertokens.Main; import io.supertokens.config.Config; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlag; import io.supertokens.jwt.exceptions.UnsupportedJWTSigningAlgorithmException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; @@ -50,6 +52,7 @@ import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Arrays; import java.util.Base64; import java.util.Base64.Decoder; import java.util.Base64.Encoder; @@ -427,4 +430,9 @@ public static JsonObject addLegacySigningKeyInfos(AppIdentifier appIdentifier, M public static JsonElement toJsonTreeWithNulls(Object src) { return new GsonBuilder().serializeNulls().create().toJsonTree(src); } + + public static boolean isAccountLinkingEnabled(Main main, AppIdentifier appIdentifier) throws StorageQueryException, TenantOrAppNotFoundException { + return Arrays.stream(FeatureFlag.getInstance(main, appIdentifier).getEnabledFeatures()) + .anyMatch(t -> t == EE_FEATURES.ACCOUNT_LINKING || t == EE_FEATURES.MFA); + } } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 6b822d610..cfee7fa64 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -29,11 +29,16 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; public class BulkImportTestUtils { + public static List generateBulkImportUser(int numberOfUsers) { + return generateBulkImportUser(numberOfUsers, List.of("public", "t1"), 0); + } + + public static List generateBulkImportUser(int numberOfUsers, List tenants, int startIndex) { List users = new ArrayList<>(); JsonParser parser = new JsonParser(); - for (int i = 0; i < numberOfUsers; i++) { + for (int i = startIndex; i < numberOfUsers + startIndex; i++) { String email = "user" + i + "@example.com"; String id = io.supertokens.utils.Utils.getUUID(); String externalId = io.supertokens.utils.Utils.getUUID(); @@ -41,17 +46,17 @@ public static List generateBulkImportUser(int numberOfUsers) { JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); List userRoles = new ArrayList<>(); - userRoles.add(new UserRole("role1", List.of("public"))); - userRoles.add(new UserRole("role2", List.of("public"))); + userRoles.add(new UserRole("role1", tenants)); + userRoles.add(new UserRole("role2", tenants)); List totpDevices = new ArrayList<>(); totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); List loginMethods = new ArrayList<>(); long currentTimeMillis = System.currentTimeMillis(); - loginMethods.add(new LoginMethod(List.of("public", "t1"), "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); - loginMethods.add(new LoginMethod(List.of("public", "t1"), "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); - loginMethods.add(new LoginMethod(List.of("public", "t1"), "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); + loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index b71372d5e..8e29c106a 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -29,8 +29,9 @@ import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; @@ -89,7 +90,7 @@ public void beforeEach() { } @Test - public void shouldProcessBulkImportUsers() throws Exception { + public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -115,7 +116,6 @@ public void shouldProcessBulkImportUsers() throws Exception { List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, null, null); - System.out.println("Users after processing: " + usersAfterProcessing.size()); assertEquals(0, usersAfterProcessing.size()); UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); @@ -123,29 +123,65 @@ public void shouldProcessBulkImportUsers() throws Exception { UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storage, container.users); - for (AuthRecipeUserInfo user : container.users) { - for (LoginMethod lm1 : user.loginMethods) { - bulkImportUser.loginMethods.forEach(lm2 -> { - if (lm2.recipeId.equals(lm1.recipeId.toString())) { - assertLoginMethodEquals(lm1, lm2); - } - }); - } + TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); - JsonObject createdUserMetadata = UserMetadata.getUserMetadata(main, user.getSupertokensOrExternalUserId()); - assertEquals(bulkImportUser.userMetadata, createdUserMetadata); + assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, bulkImportUser, + container.users[0]); - String[] createdUserRoles = UserRoles.getRolesForUser(main, user.getSupertokensOrExternalUserId()); - String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); - assertArrayEquals(bulkImportUserRoles, createdUserRoles); - - assertEquals(bulkImportUser.externalUserId, user.getSupertokensOrExternalUserId()); + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + @Test + public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); - TOTPDevice[] createdTotpDevices = Totp.getDevices(main, user.getSupertokensOrExternalUserId()); - assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } + createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + BulkImport.addUsers(appIdentifier, storage, List.of(bulkImportUserT1, bulkImportUserT2)); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 100, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 100, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, bulkImportUserT1, + containerT1.users[0]); + assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, bulkImportUserT2, + containerT2.users[0]); + process.kill(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @@ -228,6 +264,30 @@ private TestingProcess startCronProcess() throws InterruptedException { return process; } + private void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, + TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, + AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { + for (LoginMethod lm1 : authRecipeUser.loginMethods) { + bulkImportUser.loginMethods.forEach(lm2 -> { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(lm1, lm2); + } + }); + } + assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, + UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); + + String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + private void assertLoginMethodEquals(LoginMethod lm1, io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { assertEquals(lm1.email, lm2.email); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index d38c18de0..774d841f3 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -60,6 +60,8 @@ import io.supertokens.userroles.UserRoles; public class AddBulkImportUsersTest { + private String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + @Rule public TestRule watchman = Utils.getOnFailure(); @@ -73,459 +75,376 @@ public void beforeEach() { Utils.reset(); } - public String getResponseMessageFromError(String response) { - return response.substring(response.indexOf("Message: ") + "Message: ".length()); + @Test + public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // CASE 1: users field is not present + testBadRequest(process.getProcess(), new JsonObject(), "Field name 'users' is invalid in JSON input"); + + // CASE 2: users field type in incorrect + testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), + "Field name 'users' is invalid in JSON input"); + + // CASE 3: users array is empty + testBadRequest(process.getProcess(), generateUsersJson(0).getAsJsonObject(), + "{\"error\":\"You need to add at least one user.\"}"); + + // CASE 4: users array length is greater than 10000 + testBadRequest(process.getProcess(), generateUsersJson(10001).getAsJsonObject(), + "{\"error\":\"You can only add 10000 users at a time.\"}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @Test - public void shouldThrow400Error() throws Exception { - String[] args = { "../" }; + public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); - TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; + } + + // CASE 1: loginMethods field is not present + testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); + + // CASE 2: loginMethods field type in incorrect + testBadRequest(process.getProcess(), + new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); + + // CASE 3: loginMethods array is empty + testBadRequest(process.getProcess(), + new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(), + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); - + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { return; } - // Create user roles - { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, + "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") + .getAsJsonObject(); - String genericErrMsg = "Data has missing or invalid fields. Please check the users field for more details."; + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); - // users is required in the json body - { - // CASE 1: users field is not present - try { - JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "Field name 'users' is invalid in JSON input"); - } - // CASE 2: users field type in incorrect - try { - JsonObject request = new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "Field name 'users' is invalid in JSON input"); - } + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // loginMethod array is required in the user object - { - // CASE 1: loginMethods field is not present - try { - JsonObject request = new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); - } - // CASE 2: loginMethods field type in incorrect - try { - JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); - } - // CASE 3: loginMethods array is empty - try { - JsonObject request = new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); - } + + // CASE 1: MFA must be enabled to import totp devices + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); + + // CASE 2: secretKey is required in totpDevices + setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type of non required fields outside loginMethod + + // Create user roles { - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); - } - // Non-unique externalUserIds - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); - } - // MFA must be enabled to import totpDevices - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); - } - // secretKey is required in totpDevices - try { - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); - } - // Invalid role (tenantIds is required) - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); - } - // Invalid role (role doesn't exist) - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); - } + UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); } - // Invalid field type of non required fields inside loginMethod - { - try { - JsonObject request = new JsonParser().parse( - "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); - } + + // CASE 1: tenantIds is required for a user role + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); + + // CASE 2: Role doesn't exist + JsonObject requestBody2 = new JsonParser() + .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid recipeId - { - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); - } + + // CASE 1: Field type is invalid + JsonObject requestBody = new JsonParser() + .parse( + "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); + + // CASE 2: recipeId is invalid + JsonObject requestBody2 = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type in emailpassword recipe - { - // CASE 1: email, passwordHash and hashingAlgorithm are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); - } - // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect - try { - JsonObject request = new JsonParser().parse( + + // CASE 1: email, passwordHash and hashingAlgorithm are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + + // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); - } - // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt - try { - JsonObject request = new JsonParser().parse( + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + + // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt + JsonObject requestBody3 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); - } + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type in thirdparty recipe - { - // CASE 1: email, thirdPartyId and thirdPartyUserId are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); - } - // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect - try { - JsonObject request = new JsonParser().parse( + + // CASE 1: email, thirdPartyId and thirdPartyUserId are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); + + // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); - } + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Invalid field type in passwordless recipe - { - // CASE 1: email and phoneNumber are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); - } - // CASE 2: email and phoneNumber field type is incorrect - try { - JsonObject request = new JsonParser().parse( + + // CASE 1: email and phoneNumber are not present + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + + // CASE 2: email and phoneNumber field type is incorrect + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); - } - } - // Disabling all feature flags to be able to get the desired error messages - { - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{}); + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisabled() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // More than two loginMethods when either of account linking or MFA is not enabled - { - // CASE 1: email, passwordHash and hashingAlgorithm are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking or MFA must be enabled to import multiple loginMethods.\"]}]}"); - } + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking must be enabled to import multiple loginMethods.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } - // Validate tenantId - { - // CASE 1: Invalid tenantId when multitenancy is not enabled - try { - JsonObject request = new JsonParser().parse( + // CASE 1: Multitenancy is not enabled + JsonObject requestBody = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); - } - // Now enabling Account linking and Multitenancy for further tests - { - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY}); - } - // CASE 2: Invalid tenantId when multitenancy is enabled - try { - JsonObject request = new JsonParser().parse( + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); + + // CASE 2: Invalid tenantId + setFeatureFlags(process.getProcess(), + new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); + + JsonObject requestBody2 = new JsonParser() + .parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); - } - // CASE 3. Two more tenants do not share the same storage - try { - createTenants(process.getProcess()); - - JsonObject request = new JsonParser().parse( - "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") - .getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); - } - } - // No two loginMethods can have isPrimary as true - { - // CASE 1: email, passwordHash and hashingAlgorithm are not present - try { - JsonObject request = new JsonParser() - .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, - "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); - } - } - // Can't import less than 1 user at a time - { - try { - JsonObject request = generateUsersJson(0); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"You need to add at least one user.\"}"); - } - } - // Can't import more than 10000 users at a time - { - try { - JsonObject request = generateUsersJson(10001); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - String responseString = getResponseMessageFromError(e.getMessage()); - assertEquals(400, e.statusCode); - assertEquals(responseString, "{\"error\":\"You can only add 10000 users at a time.\"}"); - } + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); + + // CASE 3: Two or more tenants do not share the same storage + + createTenants(process.getProcess()); + + JsonObject requestBody3 = new JsonParser().parse( + "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldThrow400IfTwoLoginMethodsHaveIsPrimaryTrue() throws Exception { + TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + + if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + return; } + setFeatureFlags(process.getProcess(), + new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); + + JsonObject requestBody = new JsonParser() + .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}") + .getAsJsonObject(); + + testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); + process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } @@ -541,7 +460,7 @@ public void shouldReturn200Response() throws Exception { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { @@ -551,8 +470,8 @@ public void shouldReturn200Response() throws Exception { JsonObject request = generateUsersJson(10000); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); process.kill(); @@ -570,7 +489,7 @@ public void shouldNormaliseFields() throws Exception { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[]{EE_FEATURES.MFA}); + setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { @@ -580,8 +499,8 @@ public void shouldNormaliseFields() throws Exception { JsonObject request = generateUsersJson(1); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + "http://localhost:3567/bulk-import/users", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", @@ -620,6 +539,24 @@ public void shouldNormaliseFields() throws Exception { Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + private String getResponseMessageFromError(String response) { + return response.substring(response.indexOf("Message: ") + "Message: ".length()); + } + + private void testBadRequest(Main main, JsonObject requestBody, String expectedErrorMessage) throws Exception { + try { + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + requestBody, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + String responseString = getResponseMessageFromError(e.getMessage()); + assertEquals(400, e.statusCode); + assertEquals(responseString, expectedErrorMessage); + } + } + public static JsonObject generateUsersJson(int numberOfUsers) { JsonObject userJsonObject = new JsonObject(); JsonParser parser = new JsonParser(); @@ -630,7 +567,8 @@ public static JsonObject generateUsersJson(int numberOfUsers) { user.addProperty("externalUserId", UUID.randomUUID().toString()); user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); - user.add("userRoles", parser.parse("[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("userRoles", parser.parse( + "[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); @@ -654,11 +592,12 @@ private static JsonObject createEmailLoginMethod(String email, JsonArray tenantI loginMethod.add("tenantIds", tenantIds); loginMethod.addProperty("email", email); loginMethod.addProperty("recipeId", "emailpassword"); - loginMethod.addProperty("passwordHash", "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("passwordHash", + "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); loginMethod.addProperty("hashingAlgorithm", "argon2"); loginMethod.addProperty("isVerified", true); loginMethod.addProperty("isPrimary", true); - loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); return loginMethod; } @@ -705,9 +644,7 @@ private void createTenants(Main main) new EmailPasswordConfig(true), new ThirdPartyConfig(true, null), new PasswordlessConfig(true), - null, null, new JsonObject() - ) - ); + null, null, new JsonObject())); } { // tenant 2 JsonObject config = new JsonObject(); @@ -724,9 +661,7 @@ null, null, new JsonObject() new EmailPasswordConfig(true), new ThirdPartyConfig(true, null), new PasswordlessConfig(true), - null, null, config - ) - ); + null, null, config)); } } From f443a8ca52824ca8a9b8d5852fd91021e7e7f701 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 2 Apr 2024 18:58:41 +0530 Subject: [PATCH 07/41] fix: PR changes --- .../cronjobs/bulkimport/ProcessBulkImportUsers.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 1d80ab6fa..4f6104ef4 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -116,8 +116,6 @@ protected void doTaskPerApp(AppIdentifier app) for (BulkImportUser user : users) { processUser(app, user, bulkImportSQLStorage); } - - closeAllProxyStorages(); } @Override @@ -257,6 +255,8 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI // We need to rollback the transaction manually because we have overridden that in the proxy storage bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); throw e; + } finally { + closeAllProxyStorages(); } }); } catch (StorageTransactionLogicException e) { From 73b75a8bbe9b31d633b46526e436b7da85c05185 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 4 Apr 2024 19:24:20 +0530 Subject: [PATCH 08/41] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 2 +- .../bulkimport/ProcessBulkImportUsers.java | 2 +- .../apis/AddBulkImportUsersTest.java | 57 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 8bfabc7e1..e552834b7 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -39,7 +39,7 @@ public class BulkImport { public static final int GET_USERS_DEFAULT_LIMIT = 100; public static final int DELETE_USERS_LIMIT = 500; public static final int PROCESS_USERS_BATCH_SIZE = 1000; - public static final int PROCESS_USERS_INTERVAL = 60; + public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) throws StorageQueryException, TenantOrAppNotFoundException { diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4f6104ef4..4ce47d81c 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -126,7 +126,7 @@ public int getIntervalTimeSeconds() { return interval; } } - return BulkImport.PROCESS_USERS_INTERVAL; + return BulkImport.PROCESS_USERS_INTERVAL_SECONDS; } @Override diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 774d841f3..e199928c2 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -16,13 +16,18 @@ package io.supertokens.test.bulkimport.apis; +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; import org.junit.AfterClass; import org.junit.Assert; @@ -44,6 +49,7 @@ import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; @@ -539,6 +545,57 @@ public void shouldNormaliseFields() throws Exception { Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldFailIfANewFieldWasAddedToBulkImportUser() throws Exception { + List bulkImportUsers = generateBulkImportUser(1); + BulkImportUser user = bulkImportUsers.get(0); + + checkFields(user, "BulkImportUser", + Arrays.asList("id", "externalUserId", "userMetadata", "userRoles", "totpDevices", + "loginMethods", "status", "primaryUserId", "errorMessage", "createdAt", + "updatedAt")); + + checkLoginMethodFields(user.loginMethods.get(0), "LoginMethod", + Arrays.asList("tenantIds", "isVerified", "isPrimary", "timeJoinedInMSSinceEpoch", + "recipeId", "email", "passwordHash", "hashingAlgorithm", + "phoneNumber", "thirdPartyId", "thirdPartyUserId", "externalUserId", "superTokensUserId")); + + checkTotpDeviceFields(user.totpDevices.get(0), "TotpDevice", + Arrays.asList("secretKey", "period", "skew", "deviceName")); + + checkUserRoleFields(user.userRoles.get(0), "UserRole", + Arrays.asList("role", "tenantIds")); + } + + private void checkFields(Object object, String objectType, List expectedFields) { + Field[] actualFields = object.getClass().getDeclaredFields(); + List actualFieldNames = Arrays.stream(actualFields) + .map(Field::getName) + .collect(Collectors.toList()); + + List extraFields = actualFieldNames.stream() + .filter(fieldName -> !expectedFields.contains(fieldName)) + .collect(Collectors.toList()); + + if (!extraFields.isEmpty()) { + fail("The following extra field(s) are present in " + objectType + ": " + String.join(", ", extraFields)); + } + } + + private void checkLoginMethodFields(BulkImportUser.LoginMethod loginMethod, String objectType, + List expectedFields) { + checkFields(loginMethod, objectType, expectedFields); + } + + private void checkTotpDeviceFields(BulkImportUser.TotpDevice totpDevice, String objectType, + List expectedFields) { + checkFields(totpDevice, objectType, expectedFields); + } + + private void checkUserRoleFields(BulkImportUser.UserRole userRole, String objectType, List expectedFields) { + checkFields(userRole, objectType, expectedFields); + } + private String getResponseMessageFromError(String response) { return response.substring(response.indexOf("Message: ") + "Message: ".length()); } From 6d59616db147bb5f155abf80d1268b3e6fd67c5c Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 8 Apr 2024 12:29:43 +0530 Subject: [PATCH 09/41] fix: PR changes --- src/main/java/io/supertokens/Main.java | 2 + .../bulkimport/BulkImportUserUtils.java | 3 +- .../bulkimport/ProcessBulkImportUsers.java | 38 ++++++++++++----- .../api/bulkimport/BulkImportAPI.java | 2 - src/test/java/io/supertokens/test/Utils.java | 1 + .../ProcessBulkImportUsersCronJobTest.java | 42 ++++++++++++++++++- .../apis/AddBulkImportUsersTest.java | 2 +- 7 files changed, 74 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/supertokens/Main.java b/src/main/java/io/supertokens/Main.java index 7375e6e08..efbc3712d 100644 --- a/src/main/java/io/supertokens/Main.java +++ b/src/main/java/io/supertokens/Main.java @@ -65,6 +65,8 @@ public class Main { // this is a special variable that will be set to true by TestingProcessManager public static boolean isTesting = false; + // this flag is used in ProcessBulkImportUsersCronJobTest to skip the user validation + public static boolean isTesting_skipBulkImportUserValidationInCronJob = false; // this is a special variable that will be set to true by TestingProcessManager public static boolean makeConsolePrintSilent = false; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 7e81c0125..127d30ded 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -563,7 +563,8 @@ private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier ap if (commonTenantUserPoolId == null) { commonTenantUserPoolId = tenantUserPoolId; } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { - errors.add("All tenants for a user must share the same storage."); + errors.add("All tenants for a user must share the same storage for " + loginMethod.recipeId + + " recipe."); } } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4ce47d81c..a28b55c4c 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -32,6 +32,8 @@ import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; import io.supertokens.config.Config; import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; @@ -113,8 +115,11 @@ protected void doTaskPerApp(AppIdentifier app) List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, BulkImport.PROCESS_USERS_BATCH_SIZE); + String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + for (BulkImportUser user : users) { - processUser(app, user, bulkImportSQLStorage); + processUser(app, user, bulkImportUserUtils, bulkImportSQLStorage); } } @@ -189,21 +194,30 @@ private void closeAllProxyStorages() throws StorageQueryException { userPoolToStorageMap.clear(); } - private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportSQLStorage baseTenantStorage) + private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportUserUtils bulkImportUserUtils, + BulkImportSQLStorage baseTenantStorage) throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, DbInitException { - // Since all the tenants of a user must share the storage, we will just use the - // storage of the first tenantId of the first loginMethod - TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + try { + if (Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) { + // Skip validation when the flag is enabled during testing + } else { + // Validate the user + bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id); + } - SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod - LoginMethod primaryLM = getPrimaryLoginMethod(user); + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); - try { + SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); // If primaryUserId is not null, it means we may have already processed this user but failed to delete the entry // If the primaryUserId exists in the database, we'll delete the corresponding entry from the bulkImportUser table and proceed to skip this user. if (user.primaryUserId != null) { @@ -259,7 +273,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI closeAllProxyStorages(); } }); - } catch (StorageTransactionLogicException e) { + } catch (StorageTransactionLogicException | InvalidBulkImportDataException e) { handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } @@ -275,6 +289,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport if (e instanceof StorageTransactionLogicException) { StorageTransactionLogicException exception = (StorageTransactionLogicException) e; errorMessage[0] = exception.actualException.getMessage(); + } else if (e instanceof InvalidBulkImportDataException) { + errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); } try { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index abb8fe65d..c3c196beb 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -18,9 +18,7 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; import com.google.gson.JsonArray; import com.google.gson.JsonObject; diff --git a/src/test/java/io/supertokens/test/Utils.java b/src/test/java/io/supertokens/test/Utils.java index 7a133782c..2c86519bb 100644 --- a/src/test/java/io/supertokens/test/Utils.java +++ b/src/test/java/io/supertokens/test/Utils.java @@ -80,6 +80,7 @@ public static String getCdiVersionStringLatestForTests() { public static void reset() { Main.isTesting = true; + Main.isTesting_skipBulkImportUserValidationInCronJob = false; PluginInterfaceTesting.isTesting = true; Main.makeConsolePrintSilent = true; String installDir = "../"; diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 8e29c106a..26b3814cc 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -191,6 +191,9 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { // Creating a non-existing user role will result in an error. // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -225,6 +228,12 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + List users = generateBulkImportUser(1); BulkImport.addUsers(appIdentifier, storage, users); @@ -236,7 +245,38 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); assertEquals( - "Tenant with the following connectionURIDomain, appId and tenantId combination not found: (, public, t1)", + "[Invalid tenantId: t1 for a user role., Invalid tenantId: t1 for a user role., Invalid tenantId: t1 for emailpassword recipe., Invalid tenantId: t1 for thirdparty recipe., Invalid tenantId: t1 for passwordless recipe.]", + usersAfterProcessing.get(0).errorMessage); + } + + @Test + public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + createTenants(main); + + + List users = generateBulkImportUser(1, List.of("t1", "t2"), 0); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); + assertEquals( + "[All tenants for a user must share the same storage for emailpassword recipe., All tenants for a user must share the same storage for thirdparty recipe., All tenants for a user must share the same storage for passwordless recipe.]", usersAfterProcessing.get(0).errorMessage); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index e199928c2..baf205057 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -426,7 +426,7 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { .getAsJsonObject(); testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage for thirdparty recipe.\"]}]}"); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From b6c63f4ad4f68f4e33ec1b788c108c1b4bf507ea Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 9 Apr 2024 12:49:16 +0530 Subject: [PATCH 10/41] fix: Update version --- CHANGELOG.md | 8 ++++++++ build.gradle | 2 +- coreDriverInterfaceSupported.json | 3 ++- src/main/java/io/supertokens/utils/SemVer.java | 1 + 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaf903f10..699f96a71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [9.1.0] - 2024-04-10 + +### Added + +- Adds APIs to bulk import users +- Adds `ProcessBulkImportUsers` cron job to process bulk import users + + ## [9.0.0] - 2024-03-13 ### Added diff --git a/build.gradle b/build.gradle index 6080bc20d..08687c8db 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ compileTestJava { options.encoding = "UTF-8" } // } //} -version = "9.0.0" +version = "9.1.0" repositories { diff --git a/coreDriverInterfaceSupported.json b/coreDriverInterfaceSupported.json index 00fa393ac..c27931c8e 100644 --- a/coreDriverInterfaceSupported.json +++ b/coreDriverInterfaceSupported.json @@ -18,6 +18,7 @@ "2.21", "3.0", "4.0", - "5.0" + "5.0", + "5.1" ] } diff --git a/src/main/java/io/supertokens/utils/SemVer.java b/src/main/java/io/supertokens/utils/SemVer.java index e02de95fb..73939d94b 100644 --- a/src/main/java/io/supertokens/utils/SemVer.java +++ b/src/main/java/io/supertokens/utils/SemVer.java @@ -35,6 +35,7 @@ public class SemVer implements Comparable { public static final SemVer v3_0 = new SemVer("3.0"); public static final SemVer v4_0 = new SemVer("4.0"); public static final SemVer v5_0 = new SemVer("5.0"); + public static final SemVer v5_1 = new SemVer("5.1"); final private String version; From 173e7fc07a9c934af7eaaac932724e14f9bd798c Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 10 Apr 2024 19:48:37 +0530 Subject: [PATCH 11/41] fix: PR changes --- .../cronjobs/bulkimport/ProcessBulkImportUsers.java | 5 +++-- .../supertokens/test/bulkimport/BulkImportTest.java | 12 ++++++------ .../ProcessBulkImportUsersCronJobTest.java | 10 +++++----- .../bulkimport/apis/DeleteBulkImportUsersTest.java | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index a28b55c4c..671e38884 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -146,7 +146,7 @@ public int getInitialWaitTimeSeconds() { } private Storage getProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -165,6 +165,7 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); bulkImportProxyStorage.initStorage(true); + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } } @@ -172,7 +173,7 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) } public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) - throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException { + throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException, StorageQueryException { List allProxyStorages = new ArrayList<>(); Map resources = main diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index d7847e36a..f430f443f 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -76,7 +76,7 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); BulkImport.addUsers(new AppIdentifier(null, null), storage, users); - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that all users are present in addedUsers for (BulkImportUser user : users) { @@ -116,7 +116,7 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { AppIdentifier appIdentifier = new AppIdentifier(null, null); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that the other properties are same but ids changed for (BulkImportUser user : users) { @@ -153,7 +153,7 @@ public void testGetUsersStatusFilter() throws Exception { List users = generateBulkImportUser(10); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.NEW, null, null); assertEquals(10, addedUsers.size()); } @@ -171,7 +171,7 @@ public void testGetUsersStatusFilter() throws Exception { return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); assertEquals(10, addedUsers.size()); } @@ -189,7 +189,7 @@ public void testGetUsersStatusFilter() throws Exception { return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, null, BULK_IMPORT_USER_STATUS.FAILED, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.FAILED, null, null); assertEquals(10, addedUsers.size()); } @@ -229,7 +229,7 @@ public void randomPaginationTest() throws Exception { } // Get all inserted users - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), null, null, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, null); assertEquals(numberOfUsers, addedUsers.size()); // We are sorting the users based on createdAt and id like we do in the storage layer diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 26b3814cc..6e763009c 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -113,7 +113,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(0, usersAfterProcessing.size()); @@ -161,7 +161,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(0, usersAfterProcessing.size()); @@ -207,7 +207,7 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(1, usersAfterProcessing.size()); @@ -239,7 +239,7 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(1, usersAfterProcessing.size()); @@ -270,7 +270,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { Thread.sleep(6000); - List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, null, null, + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); assertEquals(1, usersAfterProcessing.size()); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index db2fe1707..7dad2ac1b 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -158,7 +158,7 @@ public void shouldReturn200Response() throws Exception { JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", "http://localhost:3567/bulk-import/users", - request, 1000000, 1000000, null, Utils.getCdiVersionStringLatestForTests(), null); + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); response.get("deletedIds").getAsJsonArray().forEach(id -> { assertTrue(validIds.contains(id)); From 4f6ab132a9d22592f523daf516ed55a95a8877e4 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 18 Apr 2024 16:48:30 +0530 Subject: [PATCH 12/41] fix: PR changes --- .../java/io/supertokens/bulkimport/BulkImportUserUtils.java | 2 +- .../test/bulkimport/ProcessBulkImportUsersCronJobTest.java | 2 +- .../test/bulkimport/apis/AddBulkImportUsersTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 127d30ded..1d090b7fe 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -563,7 +563,7 @@ private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier ap if (commonTenantUserPoolId == null) { commonTenantUserPoolId = tenantUserPoolId; } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { - errors.add("All tenants for a user must share the same storage for " + loginMethod.recipeId + errors.add("All tenants for a user must share the same database for " + loginMethod.recipeId + " recipe."); } } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 6e763009c..da6b9050f 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -276,7 +276,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); assertEquals( - "[All tenants for a user must share the same storage for emailpassword recipe., All tenants for a user must share the same storage for thirdparty recipe., All tenants for a user must share the same storage for passwordless recipe.]", + "[All tenants for a user must share the same database for emailpassword recipe., All tenants for a user must share the same database for thirdparty recipe., All tenants for a user must share the same database for passwordless recipe.]", usersAfterProcessing.get(0).errorMessage); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index baf205057..90fad26cb 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -426,7 +426,7 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { .getAsJsonObject(); testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same storage for thirdparty recipe.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same database for thirdparty recipe.\"]}]}"); process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); From fcdfb54dea5a5d13fdb75c22945f8acb0d8d1175 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 29 Apr 2024 13:08:04 +0530 Subject: [PATCH 13/41] fix: Rename DeleteBulkImportUser API path --- .../bulkimport/ProcessBulkImportUsers.java | 4 +- .../io/supertokens/webserver/Webserver.java | 2 + .../api/bulkimport/BulkImportAPI.java | 60 ---------- .../bulkimport/DeleteBulkImportUserAPI.java | 108 ++++++++++++++++++ .../apis/DeleteBulkImportUsersTest.java | 20 ++-- 5 files changed, 123 insertions(+), 71 deletions(-) create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 843813241..4584b9360 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -19,8 +19,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import com.google.gson.JsonObject; @@ -164,7 +166,7 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) normalisedConfigs.get(key), tenantIdentifier, true); userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(true, new ArrayList<>(List.of(tenantIdentifier))); + bulkImportProxyStorage.initStorage(true, new ArrayList<>()); bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 96716c746..c36f9aec2 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.BulkImportAPI; +import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -262,6 +263,7 @@ private void setupRoutes() { addAPI(new RequestStatsAPI(main)); addAPI(new BulkImportAPI(main)); + addAPI(new DeleteBulkImportUserAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index c3c196beb..3fc53e263 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -188,64 +188,4 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S result.addProperty("status", "OK"); super.sendJsonResponse(200, result, resp); } - - @Override - protected void doDelete(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - // API is app specific - JsonObject input = InputParser.parseJsonObjectOrThrowError(req); - JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); - - if (arr.size() == 0) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); - } - - if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " - + BulkImport.DELETE_USERS_LIMIT + " elements")); - } - - String[] userIds = new String[arr.size()]; - - for (int i = 0; i < userIds.length; i++) { - String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); - if (userId.isEmpty()) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); - } - userIds[i] = userId; - } - - AppIdentifier appIdentifier = null; - Storage storage = null; - - try { - appIdentifier = getAppIdentifier(req); - storage = enforcePublicTenantAndGetPublicTenantStorage(req); - } catch (TenantOrAppNotFoundException | BadPermissionException e) { - throw new ServletException(e); - } - - try { - List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); - - JsonArray deletedIdsJson = new JsonArray(); - JsonArray invalidIds = new JsonArray(); - - for (String userId : userIds) { - if (deletedIds.contains(userId)) { - deletedIdsJson.add(new JsonPrimitive(userId)); - } else { - invalidIds.add(new JsonPrimitive(userId)); - } - } - - JsonObject result = new JsonObject(); - result.add("deletedIds", deletedIdsJson); - result.add("invalidIds", invalidIds); - - super.sendJsonResponse(200, result, resp); - - } catch (StorageQueryException e) { - throw new ServletException(e); - } - } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java new file mode 100644 index 000000000..b78f12c11 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; +import java.util.List; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class DeleteBulkImportUserAPI extends WebserverAPI { + public DeleteBulkImportUserAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users/remove"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); + + if (arr.size() == 0) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); + } + + if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " + + BulkImport.DELETE_USERS_LIMIT + " elements")); + } + + String[] userIds = new String[arr.size()]; + + for (int i = 0; i < userIds.length; i++) { + String userId = InputParser.parseStringFromElementOrThrowError(arr.get(i), "ids", false); + if (userId.isEmpty()) { + throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain an empty string")); + } + userIds[i] = userId; + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + } catch (TenantOrAppNotFoundException | BadPermissionException e) { + throw new ServletException(e); + } + + try { + List deletedIds = BulkImport.deleteUsers(appIdentifier, storage, userIds); + + JsonArray deletedIdsJson = new JsonArray(); + JsonArray invalidIds = new JsonArray(); + + for (String userId : userIds) { + if (deletedIds.contains(userId)) { + deletedIdsJson.add(new JsonPrimitive(userId)); + } else { + invalidIds.add(new JsonPrimitive(userId)); + } + } + + JsonObject result = new JsonObject(); + result.add("deletedIds", deletedIdsJson); + result.add("invalidIds", invalidIds); + + super.sendJsonResponse(200, result, resp); + + } catch (StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index 7dad2ac1b..50042e7d6 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -75,8 +75,8 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -86,8 +86,8 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -97,8 +97,8 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -115,8 +115,8 @@ public void shouldReturn400Error() throws Exception { } request.add("ids", ids); - HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); @@ -156,8 +156,8 @@ public void shouldReturn200Response() throws Exception { request.add("ids", validIds); - JsonObject response = HttpRequestForTesting.sendJsonDELETERequest(process.getProcess(), "", - "http://localhost:3567/bulk-import/users", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); response.get("deletedIds").getAsJsonArray().forEach(id -> { From ddfaa1c39775e63393f36b86f7d187ec7c1f3811 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 29 Apr 2024 13:38:56 +0530 Subject: [PATCH 14/41] fix: disable bulk import for in-memory db --- .../bulkimport/ProcessBulkImportUsers.java | 2 +- .../java/io/supertokens/inmemorydb/Start.java | 3 +- .../api/bulkimport/BulkImportAPI.java | 11 ++ .../bulkimport/DeleteBulkImportUserAPI.java | 6 + .../test/bulkimport/BulkImportTest.java | 13 +- .../ProcessBulkImportUsersCronJobTest.java | 20 +++ .../apis/AddBulkImportUsersTest.java | 127 ++++++++++-------- .../apis/DeleteBulkImportUsersTest.java | 17 ++- .../apis/GetBulkImportUsersTest.java | 19 +-- 9 files changed, 140 insertions(+), 78 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4584b9360..daf1160e8 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -107,7 +107,7 @@ protected void doTaskPerApp(AppIdentifier app) throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, DbInitException { - if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 77b611a10..ae9b57ba9 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -137,8 +137,7 @@ public void constructor(String processId, boolean silent, boolean isTesting) { @Override public Storage createBulkImportProxyStorageInstance() { - // throw not implemented error - throw new UnsupportedOperationException("Unimplemented method 'createBulkImportProxyStorageInstance'"); + throw new UnsupportedOperationException("'createBulkImportProxyStorageInstance' is not supported for in-memory db"); } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 3fc53e263..496fe3ba2 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -38,6 +38,7 @@ import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; import io.supertokens.utils.Utils; import io.supertokens.webserver.InputParser; import io.supertokens.webserver.WebserverAPI; @@ -58,6 +59,11 @@ public String getPath() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); String paginationToken = InputParser.getQueryParamOrThrowError(req, "paginationToken", true); Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); @@ -118,6 +124,11 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index b78f12c11..a9032cc18 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -30,6 +30,7 @@ import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; import io.supertokens.webserver.InputParser; import io.supertokens.webserver.WebserverAPI; import jakarta.servlet.ServletException; @@ -49,6 +50,11 @@ public String getPath() { @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index f430f443f..5eaf38842 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -29,6 +29,7 @@ import org.junit.Test; import org.junit.rules.TestRule; +import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserPaginationContainer; @@ -66,8 +67,9 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -100,8 +102,9 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -140,8 +143,9 @@ public void testGetUsersStatusFilter() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -209,8 +213,9 @@ public void randomPaginationTest() throws Exception { process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index da6b9050f..a8e4cbc52 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -94,6 +94,10 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); @@ -137,6 +141,10 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + // Create user roles before inserting bulk users { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); @@ -197,6 +205,10 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); @@ -225,6 +237,10 @@ public void shouldThrowTenantDoesNotExistError() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -254,6 +270,10 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { TestingProcess process = startCronProcess(); Main main = process.getProcess(); + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 90fad26cb..2ff85b4a4 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -85,24 +85,25 @@ public void beforeEach() { public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } // CASE 1: users field is not present - testBadRequest(process.getProcess(), new JsonObject(), "Field name 'users' is invalid in JSON input"); + testBadRequest(main, new JsonObject(), "Field name 'users' is invalid in JSON input"); // CASE 2: users field type in incorrect - testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), + testBadRequest(main, new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), "Field name 'users' is invalid in JSON input"); // CASE 3: users array is empty - testBadRequest(process.getProcess(), generateUsersJson(0).getAsJsonObject(), + testBadRequest(main, generateUsersJson(0).getAsJsonObject(), "{\"error\":\"You need to add at least one user.\"}"); // CASE 4: users array length is greater than 10000 - testBadRequest(process.getProcess(), generateUsersJson(10001).getAsJsonObject(), + testBadRequest(main, generateUsersJson(10001).getAsJsonObject(), "{\"error\":\"You can only add 10000 users at a time.\"}"); process.kill(); @@ -113,24 +114,25 @@ public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } // CASE 1: loginMethods field is not present - testBadRequest(process.getProcess(), new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), + testBadRequest(main, new JsonParser().parse("{\"users\":[{}]}").getAsJsonObject(), "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]}]}"); // CASE 2: loginMethods field type in incorrect - testBadRequest(process.getProcess(), + testBadRequest(main, new JsonParser().parse("{\"users\":[{\"loginMethods\": \"string\"}]}").getAsJsonObject(), "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods should be of type array of object.\"]}]}"); // CASE 3: loginMethods array is empty - testBadRequest(process.getProcess(), + testBadRequest(main, new JsonParser().parse("{\"users\":[{\"loginMethods\": []}]}").getAsJsonObject(), "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"At least one loginMethod is required.\"]}]}"); @@ -143,8 +145,9 @@ public void shouldThrow400IfLoginMethodsAreMissingInUserObject() throws Exceptio public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -152,7 +155,7 @@ public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception .parse("{\"users\":[{\"externalUserId\":[],\"userMetaData\":[],\"userRoles\":{},\"totpDevices\":{}}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"externalUserId should be of type string.\",\"userRoles should be of type array of object.\",\"totpDevices should be of type array of object.\",\"loginMethods is required.\"]}]}"); @@ -164,15 +167,16 @@ public void shouldThrow400IfNonRequiredFieldsHaveInvalidType() throws Exception public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } JsonObject requestBody = new JsonParser() .parse("{\"users\":[{\"externalUserId\":\"id1\"}, {\"externalUserId\":\"id1\"}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"loginMethods is required.\"]},{\"index\":1,\"errors\":[\"loginMethods is required.\",\"externalUserId id1 is not unique. It is already used by another user.\"]}]}"); process.kill(); @@ -183,8 +187,9 @@ public void shouldThrow400IfNonUniqueExternalIdsArePassed() throws Exception { public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -193,12 +198,12 @@ public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception .parse("{\"users\":[{\"totpDevices\":[{\"secret\": \"secret\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"MFA must be enabled to import totp devices.\",\"loginMethods is required.\"]}]}"); // CASE 2: secretKey is required in totpDevices - setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"secretKey is required for a totp device.\",\"loginMethods is required.\"]}]}"); process.kill(); @@ -209,14 +214,15 @@ public void shouldThrow400IfTotpDevicesAreNotPassedCorrectly() throws Exception public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } // Create user roles { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); } // CASE 1: tenantIds is required for a user role @@ -224,7 +230,7 @@ public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role1\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"tenantIds is required for a user role.\",\"loginMethods is required.\"]}]}"); // CASE 2: Role doesn't exist @@ -232,7 +238,7 @@ public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { .parse("{\"users\":[{\"userRoles\":[{\"role\":\"role5\", \"tenantIds\": [\"public\"]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Role role5 does not exist.\",\"loginMethods is required.\"]}]}"); process.kill(); @@ -243,8 +249,9 @@ public void shouldThrow400IfUserRolesAreNotPassedCorrectly() throws Exception { public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -254,7 +261,7 @@ public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception "{\"users\":[{\"loginMethods\":[{\"recipeId\":[],\"tenantIds\":{},\"isPrimary\":[],\"isVerified\":[],\"timeJoinedInMSSinceEpoch\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"recipeId should be of type string for a loginMethod.\",\"tenantIds should be of type array of string for a loginMethod.\",\"isVerified should be of type boolean for a loginMethod.\",\"isPrimary should be of type boolean for a loginMethod.\",\"timeJoinedInMSSinceEpoch should be of type integer for a loginMethod\"]}]}"); // CASE 2: recipeId is invalid @@ -262,7 +269,7 @@ public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"invalid_recipe_id\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}]}"); process.kill(); @@ -273,8 +280,9 @@ public void shouldThrow400IfLoginMethodsHaveInvalidFieldType() throws Exception public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -283,7 +291,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect @@ -292,7 +300,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":[],\"passwordHash\":[],\"hashingAlgorithm\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt @@ -301,7 +309,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"invalid_algorithm\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody3, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid hashingAlgorithm for emailpassword recipe. Pass one of bcrypt, argon2 or, firebase_scrypt!\"]}]}"); process.kill(); @@ -312,8 +320,9 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -322,7 +331,7 @@ public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Except .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for a thirdparty recipe.\",\"thirdPartyId is required for a thirdparty recipe.\",\"thirdPartyUserId is required for a thirdparty recipe.\"]}]}"); // CASE 2: email, passwordHash and thirdPartyUserId field type is incorrect @@ -331,7 +340,7 @@ public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Except "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"thirdparty\",\"email\":[],\"thirdPartyId\":[],\"thirdPartyUserId\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a thirdparty recipe.\",\"thirdPartyId should be of type string for a thirdparty recipe.\",\"thirdPartyUserId should be of type string for a thirdparty recipe.\"]}]}"); process.kill(); @@ -342,8 +351,9 @@ public void shouldThrow400IfThirdPartyRecipeHasInvalidFieldTypes() throws Except public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -352,7 +362,7 @@ public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exce .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); // CASE 2: email and phoneNumber field type is incorrect @@ -361,7 +371,7 @@ public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exce "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":[],\"phoneNumber\":[]}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for a passwordless recipe.\",\"phoneNumber should be of type string for a passwordless recipe.\",\"Either email or phoneNumber is required for a passwordless recipe.\"]}]}"); process.kill(); @@ -372,15 +382,16 @@ public void shouldThrow400IfPasswordlessRecipeHasInvalidFieldTypes() throws Exce public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisabled() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } JsonObject requestBody = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Account linking must be enabled to import multiple loginMethods.\"]}]}"); process.kill(); @@ -391,8 +402,9 @@ public void shouldThrow400IfAUserHasMultipleLoginMethodsAndAccountLinkingIsDisab public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -402,11 +414,11 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Multitenancy must be enabled before importing users to a different tenant.\"]}]}"); // CASE 2: Invalid tenantId - setFeatureFlags(process.getProcess(), + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); JsonObject requestBody2 = new JsonParser() @@ -414,18 +426,18 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"invalid\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody2, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"Invalid tenantId: invalid for passwordless recipe.\"]}]}"); // CASE 3: Two or more tenants do not share the same storage - createTenants(process.getProcess()); + createTenants(main); JsonObject requestBody3 = new JsonParser().parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody3, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody3, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"All tenants for a user must share the same database for thirdparty recipe.\"]}]}"); process.kill(); @@ -436,19 +448,20 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { public void shouldThrow400IfTwoLoginMethodsHaveIsPrimaryTrue() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(new String[] { "../" }); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - setFeatureFlags(process.getProcess(), + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY }); JsonObject requestBody = new JsonParser() .parse("{\"users\":[{\"loginMethods\":[{\"recipeId\":\"emailpassword\",\"email\":\"johndoe@gmail.com\",\"passwordHash\":\"$2a\",\"hashingAlgorithm\":\"bcrypt\",\"isPrimary\":true},{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\",\"isPrimary\":true}]}]}") .getAsJsonObject(); - testBadRequest(process.getProcess(), requestBody, "{\"error\":\"" + genericErrMsg + testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg + "\",\"users\":[{\"index\":0,\"errors\":[\"No two loginMethods can have isPrimary as true.\"]}]}"); process.kill(); @@ -461,21 +474,22 @@ public void shouldReturn200Response() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } JsonObject request = generateUsersJson(10000); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users", request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); @@ -490,26 +504,27 @@ public void shouldNormaliseFields() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - setFeatureFlags(process.getProcess(), new EE_FEATURES[] { EE_FEATURES.MFA }); + setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.MFA }); // Create user roles before inserting bulk users { - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role1", null); - UserRoles.createNewRoleOrModifyItsPermissions(process.getProcess(), "role2", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } JsonObject request = generateUsersJson(1); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); - JsonObject getResponse = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + JsonObject getResponse = HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", new HashMap<>(), 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index 50042e7d6..ebd9b3624 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -34,6 +34,7 @@ import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; +import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.bulkimport.BulkImport; import io.supertokens.pluginInterface.STORAGE_TYPE; @@ -67,15 +68,16 @@ public void shouldReturn400Error() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } { try { JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -86,7 +88,7 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -97,7 +99,7 @@ public void shouldReturn400Error() throws Exception { { try { JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -115,7 +117,7 @@ public void shouldReturn400Error() throws Exception { } request.add("ids", ids); - HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { @@ -134,8 +136,9 @@ public void shouldReturn200Response() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -156,7 +159,7 @@ public void shouldReturn200Response() throws Exception { request.add("ids", validIds); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java index 181bcd336..8db075610 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/GetBulkImportUsersTest.java @@ -34,6 +34,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; @@ -62,15 +63,16 @@ public void shouldReturn400Error() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } try { Map params = new HashMap<>(); params.put("status", "INVALID_STATUS"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -84,7 +86,7 @@ public void shouldReturn400Error() throws Exception { try { Map params = new HashMap<>(); params.put("limit", "0"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -97,7 +99,7 @@ public void shouldReturn400Error() throws Exception { try { Map params = new HashMap<>(); params.put("limit", "501"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -109,7 +111,7 @@ public void shouldReturn400Error() throws Exception { try { Map params = new HashMap<>(); params.put("paginationToken", "invalid_token"); - HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); fail("The API should have thrown an error"); @@ -128,8 +130,9 @@ public void shouldReturn200Response() throws Exception { TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); - if (StorageLayer.getStorage(process.getProcess()).getType() != STORAGE_TYPE.SQL) { + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } @@ -137,14 +140,14 @@ public void shouldReturn200Response() throws Exception { String rawData = "{\"users\":[{\"loginMethods\":[{\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}]}]}"; { JsonObject request = new JsonParser().parse(rawData).getAsJsonObject(); - JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(process.getProcess(), "", + JsonObject res = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assert res.get("status").getAsString().equals("OK"); } Map params = new HashMap<>(); - JsonObject response = HttpRequestForTesting.sendGETRequest(process.getProcess(), "", + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); assertEquals("OK", response.get("status").getAsString()); From c7ae1e486eddf358ac7580161ddbab5721d38c87 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 29 Apr 2024 17:57:16 +0530 Subject: [PATCH 15/41] fix: a bug with createTotpDevices --- .../bulkimport/ProcessBulkImportUsers.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index daf1160e8..44066ce04 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -245,7 +245,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserIdMapping(main, appIdentifier, user, primaryLM); verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user.totpDevices, primaryLM); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); @@ -522,17 +522,19 @@ private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Transact } private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, - List totpDevices, LoginMethod primaryLM) throws StorageTransactionLogicException { - for (TotpDevice totpDevice : totpDevices) { - try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, - true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { - throw new StorageTransactionLogicException(e); - } catch (DeviceAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.totpDevices != null) { + for (TotpDevice totpDevice : user.totpDevices) { + try { + Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), + totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, + true, System.currentTimeMillis()); + } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(e); + } catch (DeviceAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + } } } } From d24bebfb0c7482139446d934f25d06d01c5c89b0 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 8 May 2024 10:57:14 +0530 Subject: [PATCH 16/41] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 13 +++- .../bulkimport/BulkImportUserUtils.java | 1 + .../bulkimport/ProcessBulkImportUsers.java | 74 ++++++++++++------- .../storageLayer/StorageLayer.java | 3 + .../api/bulkimport/BulkImportAPI.java | 4 +- .../bulkimport/DeleteBulkImportUserAPI.java | 4 +- 6 files changed, 65 insertions(+), 34 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index e552834b7..918b1e903 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -29,16 +29,21 @@ import java.util.List; -import javax.annotation.Nonnull; import javax.annotation.Nullable; public class BulkImport { + // Maximum number of users that can be added in a single /bulk-import/users POST request public static final int MAX_USERS_TO_ADD = 10000; - public static final int GET_USERS_PAGINATION_LIMIT = 500; + // Maximum number of users to return in a single page when calling /bulk-import/users GET + public static final int GET_USERS_PAGINATION_MAX_LIMIT = 500; + // Default number of users to return when no specific limit is given in /bulk-import/users GET public static final int GET_USERS_DEFAULT_LIMIT = 100; - public static final int DELETE_USERS_LIMIT = 500; + // Maximum number of users that can be deleted in a single operation + public static final int DELETE_USERS_MAX_LIMIT = 500; + // Number of users to process in a single batch of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_BATCH_SIZE = 1000; + // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) @@ -57,7 +62,7 @@ public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 1d090b7fe..4c0bc73ba 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -565,6 +565,7 @@ private void validateTenantIdsForRoleAndLoginMethods(Main main, AppIdentifier ap } else if (!commonTenantUserPoolId.equals(tenantUserPoolId)) { errors.add("All tenants for a user must share the same database for " + loginMethod.recipeId + " recipe."); + break; // Break to avoid adding the same error multiple times for the same loginMethod } } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 44066ce04..847bbaf01 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -147,7 +147,7 @@ public int getInitialWaitTimeSeconds() { return 0; } - private Storage getProxyStorage(TenantIdentifier tenantIdentifier) + private synchronized Storage getProxyStorage(TenantIdentifier tenantIdentifier) throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { @@ -166,7 +166,13 @@ private Storage getProxyStorage(TenantIdentifier tenantIdentifier) normalisedConfigs.get(key), tenantIdentifier, true); userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(true, new ArrayList<>()); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. + // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries + // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. + + // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, + // causing other queries to stall indefinitely. bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } @@ -178,13 +184,9 @@ public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifie throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException, StorageQueryException { List allProxyStorages = new ArrayList<>(); - Map resources = main - .getResourceDistributor() - .getAllResourcesWithResourceKey(StorageLayer.RESOURCE_KEY); - for (ResourceDistributor.KeyClass key : resources.keySet()) { - if (key.getTenantIdentifier().toAppIdentifier().equals(appIdentifier)) { - allProxyStorages.add(getProxyStorage(key.getTenantIdentifier())); - } + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); } @@ -221,17 +223,39 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI LoginMethod primaryLM = getPrimaryLoginMethod(user); AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); - // If primaryUserId is not null, it means we may have already processed this user but failed to delete the entry - // If the primaryUserId exists in the database, we'll delete the corresponding entry from the bulkImportUser table and proceed to skip this user. + + /* + * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. + * This is necessary because the bulk_import_users entries are always in the public tenant storage, + * but the actual user data could be in a different storage. + * + * If transactions are committed individually, in this order: + * 1. Commit the transaction that imports the user. + * 2. Commit the transaction that deletes the corresponding bulk import entry. + * + * There's a risk where the first commit succeeds, but the second fails. This creates a situation where + * the bulk import entry is re-processed, even though the user has already been imported into the database. + * + * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. + * The processing logic now follows these steps: + * + * 1. Import the user and get the `primaryUserId` (transaction uncommitted). + * 2. Update the `primaryUserId` in the corresponding bulk import entry. + * 3. Commit the import transaction from step 1. + * 4. Delete the bulk import entry. + * + * If step 2 or any earlier step fails, nothing is committed, preventing partial state. + * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the next run. + * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. + * + * The following code implements this logic. + */ if (user.primaryUserId != null) { - AuthRecipeUserInfo processedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, + AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, user.primaryUserId); - if (processedUser != null && isProcessedUserFromSameBulkImportUserEntry(processedUser, user)) { - baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); - return null; - }); + if (importedUser != null && isProcessedUserFromSameBulkImportUserEntry(importedUser, user)) { + baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[] { user.id }); return; } } @@ -263,10 +287,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist in the database. // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user will exist, and we'll simply delete the entry. - baseTenantStorage.startTransaction(con2 -> { - baseTenantStorage.deleteBulkImportUser_Transaction(appIdentifier, con2, user.id); - return null; - }); + baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[] { user.id }); return null; } catch (StorageTransactionLogicException e) { // We need to rollback the transaction manually because we have overridden that in the proxy storage @@ -555,15 +576,16 @@ private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { return oldestLM; } + // Checks if the importedUser was processed from the same bulkImportUser entry. private boolean isProcessedUserFromSameBulkImportUserEntry( - AuthRecipeUserInfo processedUser, BulkImportUser bulkImportUser) { - if (bulkImportUser == null || processedUser == null || bulkImportUser.loginMethods == null || - processedUser.loginMethods == null) { + AuthRecipeUserInfo importedUser, BulkImportUser bulkImportEntry) { + if (bulkImportEntry == null || importedUser == null || bulkImportEntry.loginMethods == null || + importedUser.loginMethods == null) { return false; } - for (LoginMethod lm1 : bulkImportUser.loginMethods) { - for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : processedUser.loginMethods) { + for (LoginMethod lm1 : bulkImportEntry.loginMethods) { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : importedUser.loginMethods) { if (lm2.recipeId.toString().equals(lm1.recipeId)) { if (lm1.email != null && !lm1.email.equals(lm2.email)) { return false; diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index 2d09299d0..928bc9c08 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -89,6 +89,9 @@ private static Storage getNewInstance(Main main, JsonObject config, TenantIdenti result = storageLayer; } } else { + if (isBulkImportProxy) { + throw new QuitProgramException("Creating a bulk import proxy storage instance with in-memory DB is not supported."); + } result = new Start(main); } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index 496fe3ba2..c30f63f82 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -69,9 +69,9 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se Integer limit = InputParser.getIntQueryParamOrThrowError(req, "limit", true); if (limit != null) { - if (limit > BulkImport.GET_USERS_PAGINATION_LIMIT) { + if (limit > BulkImport.GET_USERS_PAGINATION_MAX_LIMIT) { throw new ServletException( - new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_LIMIT)); + new BadRequestException("Max limit allowed is " + BulkImport.GET_USERS_PAGINATION_MAX_LIMIT)); } else if (limit < 1) { throw new ServletException(new BadRequestException("limit must a positive integer with min value 1")); } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index a9032cc18..5562fafca 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -62,9 +62,9 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); } - if (arr.size() > BulkImport.DELETE_USERS_LIMIT) { + if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot contain more than " - + BulkImport.DELETE_USERS_LIMIT + " elements")); + + BulkImport.DELETE_USERS_MAX_LIMIT + " elements")); } String[] userIds = new String[arr.size()]; From 09c1b85d338f679382bb1afeda84d8df02be6adf Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 23 May 2024 12:38:14 +0530 Subject: [PATCH 17/41] feat: Add an api to import user in sync --- .../io/supertokens/bulkimport/BulkImport.java | 417 +++++++++++++++++- .../bulkimport/ProcessBulkImportUsers.java | 326 ++------------ .../io/supertokens/webserver/Webserver.java | 2 + .../api/bulkimport/ImportUserAPI.java | 103 +++++ .../test/bulkimport/BulkImportTest.java | 193 +++++++- .../test/bulkimport/BulkImportTestUtils.java | 137 +++++- .../ProcessBulkImportUsersCronJobTest.java | 133 +----- .../apis/AddBulkImportUsersTest.java | 56 +-- .../test/bulkimport/apis/ImportUserTest.java | 151 +++++++ 9 files changed, 1016 insertions(+), 502 deletions(-) create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 918b1e903..ef6d5d18d 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -18,19 +18,69 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; +import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.authRecipe.AuthRecipe; +import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; +import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; +import io.supertokens.config.Config; +import io.supertokens.emailpassword.EmailPassword; +import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; +import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; +import io.supertokens.passwordless.Passwordless; +import io.supertokens.passwordless.exceptions.RestartFlowException; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; +import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; +import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; +import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; +import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.ThirdParty; +import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; +import io.supertokens.totp.Totp; +import io.supertokens.useridmapping.UserIdMapping; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; import io.supertokens.utils.Utils; +import jakarta.servlet.ServletException; - +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.annotation.Nullable; +import com.google.gson.JsonObject; + public class BulkImport { // Maximum number of users that can be added in a single /bulk-import/users POST request @@ -46,6 +96,9 @@ public class BulkImport { // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; + // This map allows reusing proxy storage for all tenants in the app and closing connections after import. + private static Map userPoolToStorageMap = new HashMap<>(); + public static void addUsers(AppIdentifier appIdentifier, Storage storage, List users) throws StorageQueryException, TenantOrAppNotFoundException { while (true) { @@ -74,7 +127,8 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdenti } else { BulkImportUserPaginationToken tokenInfo = BulkImportUserPaginationToken.extractTokenInfo(paginationToken); users = bulkImportStorage - .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, tokenInfo.createdAt); + .getBulkImportUsers(appIdentifier, limit + 1, status, tokenInfo.bulkImportUserId, + tokenInfo.createdAt); } String nextPaginationToken = null; @@ -89,7 +143,364 @@ public static BulkImportUserPaginationContainer getUsers(AppIdentifier appIdenti return new BulkImportUserPaginationContainer(resultUsers, nextPaginationToken); } - public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) throws StorageQueryException { + public static List deleteUsers(AppIdentifier appIdentifier, Storage storage, String[] userIds) + throws StorageQueryException { return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); } + + public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifier appIdentifier, + BulkImportUser user) + throws StorageQueryException, InvalidConfigException, IOException, TenantOrAppNotFoundException, + DbInitException { + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(main, firstTenantIdentifier); + + LoginMethod primaryLM = getPrimaryLoginMethod(user); + + try { + return bulkImportProxyStorage.startTransaction(con -> { + try { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); + } + + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + + Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + + AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, primaryLM.superTokensUserId); + io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, bulkImportProxyStorage, new AuthRecipeUserInfo[]{importedUser}); + + return importedUser; + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + throw e; + } finally { + closeAllProxyStorages(); + } + }); + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e.actualException); + } + } + + public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + String firstTenant = lm.tenantIds.get(0); + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), firstTenant); + + if (lm.recipeId.equals("emailpassword")) { + processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("thirdparty")) { + processThirdPartyLoginMethod(tenantIdentifier, storage, lm); + } else if (lm.recipeId.equals("passwordless")) { + processPasswordlessLoginMethod(tenantIdentifier, storage, lm); + } else { + throw new StorageTransactionLogicException( + new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); + } + + associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); + } + + private static void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + LoginMethod lm) throws StorageTransactionLogicException { + try { + ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, + lm.passwordHash, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException( + new Exception("A user with email " + lm.email + " already exists")); + } + } + + private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) + throws StorageTransactionLogicException { + try { + SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( + tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, + lm.timeJoinedInMSSinceEpoch); + + lm.superTokensUserId = userInfo.user.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId + + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); + } + } + + private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + LoginMethod lm) + throws StorageTransactionLogicException { + try { + AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, + lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); + + lm.superTokensUserId = userInfo.getSupertokensUserId(); + } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { + throw new StorageTransactionLogicException(e); + } + } + + private static void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, + String firstTenant) throws StorageTransactionLogicException { + for (String tenantId : lm.tenantIds) { + try { + if (tenantId.equals(firstTenant)) { + continue; + } + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), tenantId); + Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); + } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException + | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException + | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException + | AnotherPrimaryUserWithEmailAlreadyExistsException + | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + public static void createPrimaryUserAndLinkAccounts(Main main, + AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) + throws StorageTransactionLogicException { + if (user.loginMethods.size() == 1) { + return; + } + + try { + AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException + | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + + for (LoginMethod lm : user.loginMethods) { + try { + if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { + continue; + } + + AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), + primaryLM.getSuperTokenOrExternalUserId()); + + } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (InputUserIdIsNotAPrimaryUserException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it is not a primary user. This should not happen. Please contact support.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException + | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException( + new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } + } + } + + public static void createUserIdMapping(AppIdentifier appIdentifier, + BulkImportUser user, LoginMethod primaryLM, Storage[] storages) throws StorageTransactionLogicException { + if (user.externalUserId != null) { + try { + UserIdMapping.createUserIdMapping( + appIdentifier, storages, + primaryLM.superTokensUserId, user.externalUserId, + null, false, true); + + primaryLM.externalUserId = user.externalUserId; + } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } catch (UserIdMappingAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A user with externalId " + user.externalUserId + " already exists")); + } catch (UnknownSuperTokensUserIdException e) { + throw new StorageTransactionLogicException( + new Exception("We tried to create the externalUserId mapping for the superTokenUserId " + + primaryLM.superTokensUserId + + " but it doesn't exist. This should not happen. Please contact support.")); + } + } + } + + public static void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, + LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.userMetadata != null) { + try { + UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), + user.userMetadata); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + public static void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, + BulkImportUser user) throws StorageTransactionLogicException { + if (user.userRoles != null) { + for (UserRole userRole : user.userRoles) { + try { + for (String tenantId : userRole.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), + tenantId); + + UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); + } + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (UnknownRoleException e) { + throw new StorageTransactionLogicException(new Exception("Role " + userRole.role + + " does not exist! You need pre-create the role before assigning it to the user.")); + } + } + } + } + + public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, + Storage storage, + List loginMethods) throws StorageTransactionLogicException { + + for (LoginMethod lm : loginMethods) { + try { + + TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), lm.tenantIds.get(0)); + + EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils + .getEmailVerificationStorage(storage); + emailVerificationSQLStorage + .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, + lm.getSuperTokenOrExternalUserId(), lm.email, true); + } catch (TenantOrAppNotFoundException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + } + + public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, + BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { + if (user.totpDevices != null) { + for (TotpDevice totpDevice : user.totpDevices) { + try { + Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), + totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, + true, System.currentTimeMillis()); + } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(e); + } catch (DeviceAlreadyExistsException e) { + throw new StorageTransactionLogicException( + new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + } + } + } + } + + // Returns the primary loginMethod of the user. If no loginMethod is marked as + // primary, then the oldest loginMethod is returned. + public static BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { + BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); + for (BulkImportUser.LoginMethod lm : user.loginMethods) { + if (lm.isPrimary) { + return lm; + } + + if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { + oldestLM = lm; + } + } + return oldestLM; + } + + private static synchronized Storage getBulkImportProxyStorage(Main main, TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, + StorageQueryException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. + // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries + // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. + + // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, + // causing other queries to stall indefinitely. + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); + } + + private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + + try { + List allProxyStorages = new ArrayList<>(); + + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + + private static void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); + storage.close(); + } + userPoolToStorageMap.clear(); + } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 847bbaf01..7e1ef2fdf 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -19,35 +19,20 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import com.google.gson.JsonObject; import io.supertokens.Main; import io.supertokens.ResourceDistributor; -import io.supertokens.authRecipe.AuthRecipe; -import io.supertokens.authRecipe.exception.AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException; -import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; -import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; -import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserUtils; import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; import io.supertokens.config.Config; import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; -import io.supertokens.emailpassword.EmailPassword; -import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; -import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; -import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; -import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; -import io.supertokens.passwordless.Passwordless; -import io.supertokens.passwordless.exceptions.RestartFlowException; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; @@ -56,12 +41,7 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; -import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; -import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -70,22 +50,8 @@ import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; import io.supertokens.pluginInterface.sqlStorage.SQLStorage; -import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; -import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; -import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; -import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; -import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; -import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; import io.supertokens.storageLayer.StorageLayer; -import io.supertokens.thirdparty.ThirdParty; -import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; -import io.supertokens.totp.Totp; -import io.supertokens.useridmapping.UserIdMapping; -import io.supertokens.usermetadata.UserMetadata; -import io.supertokens.userroles.UserRoles; -import jakarta.servlet.ServletException; public class ProcessBulkImportUsers extends CronTask { @@ -147,7 +113,7 @@ public int getInitialWaitTimeSeconds() { return 0; } - private synchronized Storage getProxyStorage(TenantIdentifier tenantIdentifier) + private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { @@ -180,15 +146,20 @@ private synchronized Storage getProxyStorage(TenantIdentifier tenantIdentifier) throw new TenantOrAppNotFoundException(tenantIdentifier); } - public Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) - throws TenantOrAppNotFoundException, InvalidConfigException, IOException, DbInitException, StorageQueryException { - List allProxyStorages = new ArrayList<>(); + private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { - TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); - for (TenantConfig tenantConfig : tenantConfigs) { - allProxyStorages.add(getProxyStorage(tenantConfig.tenantIdentifier)); + try { + List allProxyStorages = new ArrayList<>(); + + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + throw new StorageTransactionLogicException(e); } - return allProxyStorages.toArray(new Storage[0]); } private void closeAllProxyStorages() throws StorageQueryException { @@ -218,11 +189,11 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); - SQLStorage bulkImportProxyStorage = (SQLStorage) getProxyStorage(firstTenantIdentifier); + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); - LoginMethod primaryLM = getPrimaryLoginMethod(user); + LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getProxyStorage(firstTenantIdentifier); + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); /* * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. @@ -263,15 +234,18 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI bulkImportProxyStorage.startTransaction(con -> { try { for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(appIdentifier, bulkImportProxyStorage, lm); + BulkImport.processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); } - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserIdMapping(main, appIdentifier, user, primaryLM); - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + + Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + BulkImport.createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + + BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + BulkImport.createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + BulkImport.createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + BulkImport.createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this @@ -328,254 +302,6 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport } } - private void processUserLoginMethod(AppIdentifier appIdentifier, Storage storage, - LoginMethod lm) throws StorageTransactionLogicException { - String firstTenant = lm.tenantIds.get(0); - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), firstTenant); - - if (lm.recipeId.equals("emailpassword")) { - processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); - } else if (lm.recipeId.equals("thirdparty")) { - processThirdPartyLoginMethod(tenantIdentifier, storage, lm); - } else if (lm.recipeId.equals("passwordless")) { - processPasswordlessLoginMethod(tenantIdentifier, storage, lm); - } else { - throw new StorageTransactionLogicException( - new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); - } - - associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); - } - - private void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, - LoginMethod lm) throws StorageTransactionLogicException { - try { - ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, - lm.passwordHash, lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } catch (DuplicateEmailException e) { - throw new StorageTransactionLogicException( - new Exception("A user with email " + lm.email + " already exists")); - } - } - - private void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) - throws StorageTransactionLogicException { - try { - SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( - tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, - lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } catch (DuplicateThirdPartyUserException e) { - throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId - + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); - } - } - - private void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) - throws StorageTransactionLogicException { - try { - AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, - lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { - throw new StorageTransactionLogicException(e); - } - } - - private void associateUserToTenants(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm, - String firstTenant) throws StorageTransactionLogicException { - for (String tenantId : lm.tenantIds) { - try { - if (tenantId.equals(firstTenant)) { - continue; - } - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), tenantId); - Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException - | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException - | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException - | AnotherPrimaryUserWithEmailAlreadyExistsException - | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { - throw new StorageTransactionLogicException(e); - } - } - } - - private void createPrimaryUserAndLinkAccounts(Main main, - AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) - throws StorageTransactionLogicException { - if (user.loginMethods.size() == 1) { - return; - } - - try { - AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException - | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); - } - - for (LoginMethod lm : user.loginMethods) { - try { - if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { - continue; - } - - AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), - primaryLM.getSuperTokenOrExternalUserId()); - - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownUserIdException e) { - throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (InputUserIdIsNotAPrimaryUserException e) { - throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it is not a primary user. This should not happen. Please contact support.")); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException - | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); - } - } - } - - private void createUserIdMapping(Main main, AppIdentifier appIdentifier, - BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.externalUserId != null) { - try { - UserIdMapping.createUserIdMapping( - appIdentifier, getAllProxyStoragesForApp(main, appIdentifier), - primaryLM.superTokensUserId, user.externalUserId, - null, false, true); - - primaryLM.externalUserId = user.externalUserId; - } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException | InvalidConfigException - | IOException | DbInitException e) { - throw new StorageTransactionLogicException(e); - } catch (UserIdMappingAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception("A user with externalId " + user.externalUserId + " already exists")); - } catch (UnknownSuperTokensUserIdException e) { - throw new StorageTransactionLogicException( - new Exception("We tried to create the externalUserId mapping for the superTokenUserId " - + primaryLM.superTokensUserId - + " but it doesn't exist. This should not happen. Please contact support.")); - } - } - } - - private void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, - LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.userMetadata != null) { - try { - UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - user.userMetadata); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } - } - } - - private void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, - BulkImportUser user) throws StorageTransactionLogicException { - if (user.userRoles != null) { - for (UserRole userRole : user.userRoles) { - try { - for (String tenantId : userRole.tenantIds) { - TenantIdentifier tenantIdentifier = new TenantIdentifier( - appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), - tenantId); - - UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); - } - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownRoleException e) { - throw new StorageTransactionLogicException(new Exception("Role " + userRole.role - + " does not exist! You need pre-create the role before assigning it to the user.")); - } - } - } - } - - private void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, Storage storage, - List loginMethods) throws StorageTransactionLogicException { - - for (LoginMethod lm : loginMethods) { - try { - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), lm.tenantIds.get(0)); - - EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils - .getEmailVerificationStorage(storage); - emailVerificationSQLStorage - .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - lm.getSuperTokenOrExternalUserId(), lm.email, true); - } catch (TenantOrAppNotFoundException | StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } - } - } - - private void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, - BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.totpDevices != null) { - for (TotpDevice totpDevice : user.totpDevices) { - try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, - true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { - throw new StorageTransactionLogicException(e); - } catch (DeviceAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); - } - } - } - } - - // Returns the primary loginMethod of the user. If no loginMethod is marked as - // primary, then the oldest loginMethod is returned. - private BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { - BulkImportUser.LoginMethod oldestLM = user.loginMethods.get(0); - for (BulkImportUser.LoginMethod lm : user.loginMethods) { - if (lm.isPrimary) { - return lm; - } - - if (lm.timeJoinedInMSSinceEpoch < oldestLM.timeJoinedInMSSinceEpoch) { - oldestLM = lm; - } - } - return oldestLM; - } - // Checks if the importedUser was processed from the same bulkImportUser entry. private boolean isProcessedUserFromSameBulkImportUserEntry( AuthRecipeUserInfo importedUser, BulkImportUser bulkImportEntry) { diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index c36f9aec2..23e953ef9 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -28,6 +28,7 @@ import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.BulkImportAPI; import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; +import io.supertokens.webserver.api.bulkimport.ImportUserAPI; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -264,6 +265,7 @@ private void setupRoutes() { addAPI(new BulkImportAPI(main)); addAPI(new DeleteBulkImportUserAPI(main)); + addAPI(new ImportUserAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java new file mode 100644 index 000000000..ae29fdd63 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.utils.Utils; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class ImportUserAPI extends WebserverAPI { + public ImportUserAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/import"; + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(input, "user", false); + + AppIdentifier appIdentifier = null; + Storage storage = null; + String[] allUserRoles = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } + + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + + try { + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, jsonUser, + Utils.getUUID()); + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, user); + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.add("user", importedUser.toJson()); + super.sendJsonResponse(200, result, resp); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (StorageQueryException | TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index 5eaf38842..a673b7682 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -20,7 +20,12 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.junit.AfterClass; @@ -35,15 +40,21 @@ import io.supertokens.bulkimport.BulkImportUserPaginationContainer; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; +import io.supertokens.userroles.UserRoles; import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; @@ -63,7 +74,7 @@ public void beforeEach() { @Test public void shouldAddUsersInBulkImportUsersTable() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -78,7 +89,8 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); BulkImport.addUsers(new AppIdentifier(null, null), storage, users); - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 100, + BULK_IMPORT_USER_STATUS.NEW, null, null); // Verify that all users are present in addedUsers for (BulkImportUser user : users) { @@ -98,7 +110,7 @@ public void shouldAddUsersInBulkImportUsersTable() throws Exception { @Test public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -119,7 +131,8 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { AppIdentifier appIdentifier = new AppIdentifier(null, null); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 1000, BULK_IMPORT_USER_STATUS.NEW, + null, null); // Verify that the other properties are same but ids changed for (BulkImportUser user : users) { @@ -139,7 +152,7 @@ public void shouldCreatedNewIdsIfDuplicateIdIsFound() throws Exception { @Test public void testGetUsersStatusFilter() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -148,7 +161,7 @@ public void testGetUsersStatusFilter() throws Exception { if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; } - + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -157,7 +170,8 @@ public void testGetUsersStatusFilter() throws Exception { List users = generateBulkImportUser(10); BulkImport.addUsers(appIdentifier, storage, users); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.NEW, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.NEW, null, null); assertEquals(10, addedUsers.size()); } @@ -169,13 +183,15 @@ public void testGetUsersStatusFilter() throws Exception { // Update the users status to PROCESSING storage.startTransaction(con -> { for (BulkImportUser user : users) { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.PROCESSING, null); + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.PROCESSING, null); } storage.commitTransaction(con); return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.PROCESSING, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.PROCESSING, null, null); assertEquals(10, addedUsers.size()); } @@ -187,13 +203,15 @@ public void testGetUsersStatusFilter() throws Exception { // Update the users status to FAILED storage.startTransaction(con -> { for (BulkImportUser user : users) { - storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, BULK_IMPORT_USER_STATUS.FAILED, null); + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.FAILED, null); } storage.commitTransaction(con); return null; }); - List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, BULK_IMPORT_USER_STATUS.FAILED, null, null); + List addedUsers = storage.getBulkImportUsers(appIdentifier, 100, + BULK_IMPORT_USER_STATUS.FAILED, null, null); assertEquals(10, addedUsers.size()); } @@ -203,12 +221,13 @@ public void testGetUsersStatusFilter() throws Exception { @Test public void randomPaginationTest() throws Exception { - String[] args = {"../"}; + String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); // We are setting a high initial wait time to ensure the cron job doesn't run while we are running the tests - CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1000000); + CronTaskTest.getInstance(process.getProcess()).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, + 1000000); process.startProcess(); @@ -234,7 +253,8 @@ public void randomPaginationTest() throws Exception { } // Get all inserted users - List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, null); + List addedUsers = storage.getBulkImportUsers(new AppIdentifier(null, null), 1000, null, null, + null); assertEquals(numberOfUsers, addedUsers.size()); // We are sorting the users based on createdAt and id like we do in the storage layer @@ -248,13 +268,14 @@ public void randomPaginationTest() throws Exception { }) .collect(Collectors.toList()); - int[] limits = new int[]{10, 14, 20, 23, 50, 100, 110, 150, 200, 510}; + int[] limits = new int[] { 10, 14, 20, 23, 50, 100, 110, 150, 200, 510 }; for (int limit : limits) { int indexIntoUsers = 0; String paginationToken = null; do { - BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, limit, null, paginationToken); + BulkImportUserPaginationContainer users = BulkImport.getUsers(new AppIdentifier(null, null), storage, + limit, null, paginationToken); for (BulkImportUser actualUser : users.users) { BulkImportUser expectedUser = sortedUsers.get(indexIntoUsers); @@ -275,4 +296,144 @@ public void randomPaginationTest() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldImportTheUserInTheSameTenant() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(1); + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, users.get(0)); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(0), importedUser); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportTheUserInMultipleTenantsWithDifferentStorages() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(1, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(1, List.of(t2.getTenantId()), 1); + + BulkImportUser bulkImportUserT1 = usersT1.get(0); + BulkImportUser bulkImportUserT2 = usersT2.get(0); + + AuthRecipeUserInfo importedUser1 = BulkImport.importUser(main, appIdentifier, bulkImportUserT1); + AuthRecipeUserInfo importedUser2 = BulkImport.importUser(main, appIdentifier, bulkImportUserT2); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + bulkImportUserT1, + importedUser1); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + bulkImportUserT2, + importedUser2); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldImportUsersConcurrently() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(10); + + // Concurrently import users + ExecutorService executor = Executors.newFixedThreadPool(10); + List> futures = new ArrayList<>(); + + for (BulkImportUser user : users) { + Future future = executor.submit(() -> { + return BulkImport.importUser(main, appIdentifier, user); + }); + futures.add(future); + } + + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + + for (int i = 0; i < users.size(); i++) { + AuthRecipeUserInfo importedUser = futures.get(i).get(); + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(i), + importedUser); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index cfee7fa64..4faea47ab 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -14,19 +14,45 @@ * under the License. */ - package io.supertokens.test.bulkimport; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import io.supertokens.Main; +import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; +import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.totp.TOTPDevice; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.thirdparty.InvalidProviderConfigException; +import io.supertokens.totp.Totp; +import io.supertokens.usermetadata.UserMetadata; +import io.supertokens.userroles.UserRoles; public class BulkImportTestUtils { @@ -43,7 +69,8 @@ public static List generateBulkImportUser(int numberOfUsers, Lis String id = io.supertokens.utils.Utils.getUUID(); String externalId = io.supertokens.utils.Utils.getUUID(); - JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}").getAsJsonObject(); + JsonObject userMetadata = parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}") + .getAsJsonObject(); List userRoles = new ArrayList<>(); userRoles.add(new UserRole("role1", tenants)); @@ -53,12 +80,110 @@ public static List generateBulkImportUser(int numberOfUsers, Lis totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); List loginMethods = new ArrayList<>(); - long currentTimeMillis = System.currentTimeMillis(); - loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", "BCRYPT", null, null, null)); - loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, "thirdPartyId" + i, "thirdPartyUserId" + i, null)); - loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, null, null)); + long currentTimeMillis = System.currentTimeMillis(); + loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", + "BCRYPT", null, null, null)); + loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, + "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, + null, null, null)); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; } + + public static void createTenants(Main main) + throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, + FeatureNotEnabledException, IOException, InvalidConfigException, + CannotModifyBaseConfigException, BadPermissionException { + // User pool 1 - (null, null, null), (null, null, t1) + // User pool 2 - (null, null, t2) + + { // tenant 1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + } + { // tenant 2 + JsonObject config = new JsonObject(); + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); + + StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) + .modifyConfigToAddANewUserPoolForTesting(config, 1); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, config)); + } + } + + public static void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, + TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, + AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm1 : authRecipeUser.loginMethods) { + bulkImportUser.loginMethods.forEach(lm2 -> { + if (lm2.recipeId.equals(lm1.recipeId.toString())) { + assertLoginMethodEquals(lm1, lm2); + } + }); + } + assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); + assertEquals(bulkImportUser.userMetadata, + UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); + + String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); + assertArrayEquals(bulkImportUserRoles, createdUserRoles); + + TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, + authRecipeUser.getSupertokensOrExternalUserId()); + assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); + } + + private static void assertLoginMethodEquals(io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { + assertEquals(lm1.email, lm2.email); + assertEquals(lm1.verified, lm2.isVerified); + assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); + + switch (lm2.recipeId) { + case "emailpassword": + assertEquals(lm1.passwordHash, lm2.passwordHash); + break; + case "thirdparty": + assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); + assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); + break; + case "passwordless": + assertEquals(lm1.phoneNumber, lm2.phoneNumber); + break; + default: + break; + } + } + + private static void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { + assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); + for (int i = 0; i < createdTotpDevices.length; i++) { + assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); + assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); + assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); + assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); + } + } } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index a8e4cbc52..a17f842e7 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -26,36 +26,18 @@ import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; -import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; -import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; -import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; -import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.exceptions.InvalidConfigException; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; -import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; -import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; -import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; -import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.pluginInterface.totp.TOTPDevice; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.TestingProcessManager.TestingProcess; import io.supertokens.test.Utils; -import io.supertokens.thirdparty.InvalidProviderConfigException; -import io.supertokens.totp.Totp; import io.supertokens.useridmapping.UserIdMapping; -import io.supertokens.usermetadata.UserMetadata; import io.supertokens.userroles.UserRoles; import org.junit.AfterClass; @@ -64,15 +46,10 @@ import org.junit.Test; import org.junit.rules.TestRule; -import com.google.gson.JsonObject; - import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import java.io.IOException; import java.util.List; public class ProcessBulkImportUsersCronJobTest { @@ -104,7 +81,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - createTenants(main); + BulkImportTestUtils.createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -129,7 +106,8 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); - assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, bulkImportUser, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, + bulkImportUser, container.users[0]); process.kill(); @@ -151,7 +129,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - createTenants(main); + BulkImportTestUtils.createTenants(main); TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); @@ -185,9 +163,11 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); - assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, bulkImportUserT1, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + bulkImportUserT1, containerT1.users[0]); - assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, bulkImportUserT2, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + bulkImportUserT2, containerT2.users[0]); process.kill(); @@ -209,7 +189,7 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { return; } - createTenants(main); + BulkImportTestUtils.createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); @@ -282,8 +262,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - createTenants(main); - + BulkImportTestUtils.createTenants(main); List users = generateBulkImportUser(1, List.of("t1", "t2"), 0); BulkImport.addUsers(appIdentifier, storage, users); @@ -323,96 +302,4 @@ private TestingProcess startCronProcess() throws InterruptedException { return process; } - - private void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, - TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, - AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { - for (LoginMethod lm1 : authRecipeUser.loginMethods) { - bulkImportUser.loginMethods.forEach(lm2 -> { - if (lm2.recipeId.equals(lm1.recipeId.toString())) { - assertLoginMethodEquals(lm1, lm2); - } - }); - } - assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); - assertEquals(bulkImportUser.userMetadata, - UserMetadata.getUserMetadata(appIdentifier, storage, authRecipeUser.getSupertokensOrExternalUserId())); - - String[] createdUserRoles = UserRoles.getRolesForUser(tenantIdentifier, storage, - authRecipeUser.getSupertokensOrExternalUserId()); - String[] bulkImportUserRoles = bulkImportUser.userRoles.stream().map(r -> r.role).toArray(String[]::new); - assertArrayEquals(bulkImportUserRoles, createdUserRoles); - - TOTPDevice[] createdTotpDevices = Totp.getDevices(appIdentifier, storage, - authRecipeUser.getSupertokensOrExternalUserId()); - assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); - } - - private void assertLoginMethodEquals(LoginMethod lm1, - io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { - assertEquals(lm1.email, lm2.email); - assertEquals(lm1.verified, lm2.isVerified); - assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); - - switch (lm2.recipeId) { - case "emailpassword": - assertEquals(lm1.passwordHash, lm2.passwordHash); - break; - case "thirdparty": - assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); - assertEquals(lm1.thirdParty.userId, lm2.thirdPartyUserId); - break; - case "passwordless": - assertEquals(lm1.phoneNumber, lm2.phoneNumber); - break; - default: - break; - } - } - - private void assertTotpDevicesEquals(TOTPDevice[] createdTotpDevices, TotpDevice[] bulkImportTotpDevices) { - assertEquals(createdTotpDevices.length, bulkImportTotpDevices.length); - for (int i = 0; i < createdTotpDevices.length; i++) { - assertEquals(createdTotpDevices[i].deviceName, bulkImportTotpDevices[i].deviceName); - assertEquals(createdTotpDevices[i].period, bulkImportTotpDevices[i].period); - assertEquals(createdTotpDevices[i].secretKey, bulkImportTotpDevices[i].secretKey); - assertEquals(createdTotpDevices[i].skew, bulkImportTotpDevices[i].skew); - } - } - - private void createTenants(Main main) - throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, - FeatureNotEnabledException, IOException, InvalidConfigException, - CannotModifyBaseConfigException, BadPermissionException { - { // tenant 1 (t1 in the same storage as public tenant) - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); - - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, new JsonObject())); - } - { // tenant 2 (t2 in the different storage than public tenant) - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); - - JsonObject config = new JsonObject(); - - StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) - .modifyConfigToAddANewUserPoolForTesting(config, 1); - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, config)); - } - } } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 2ff85b4a4..c4ac061d8 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -21,7 +21,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.fail; -import java.io.IOException; import java.lang.reflect.Field; import java.util.Arrays; import java.util.HashMap; @@ -44,25 +43,13 @@ import io.supertokens.ProcessState; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; -import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; -import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.multitenancy.exception.CannotModifyBaseConfigException; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.exceptions.InvalidConfigException; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.multitenancy.EmailPasswordConfig; -import io.supertokens.pluginInterface.multitenancy.PasswordlessConfig; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; -import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; -import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; -import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; import io.supertokens.test.httpRequest.HttpRequestForTesting; -import io.supertokens.thirdparty.InvalidProviderConfigException; import io.supertokens.userroles.UserRoles; public class AddBulkImportUsersTest { @@ -431,7 +418,7 @@ public void shouldThrow400IfInvalidTenantIdIsPassed() throws Exception { // CASE 3: Two or more tenants do not share the same storage - createTenants(main); + BulkImportTestUtils.createTenants(main); JsonObject requestBody3 = new JsonParser().parse( "{\"users\":[{\"loginMethods\":[{\"tenantIds\":[\"public\"],\"recipeId\":\"passwordless\",\"email\":\"johndoe@gmail.com\"}, {\"tenantIds\":[\"t2\"],\"recipeId\":\"thirdparty\", \"email\":\"johndoe@gmail.com\", \"thirdPartyId\":\"id\", \"thirdPartyUserId\":\"id\"}]}]}") @@ -698,45 +685,6 @@ private static JsonObject createPasswordlessLoginMethod(String email, JsonArray return loginMethod; } - private void createTenants(Main main) - throws StorageQueryException, TenantOrAppNotFoundException, InvalidProviderConfigException, - FeatureNotEnabledException, IOException, InvalidConfigException, - CannotModifyBaseConfigException, BadPermissionException { - // User pool 1 - (null, null, null), (null, null, t1) - // User pool 2 - (null, null, t2) - - { // tenant 1 - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); - - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, new JsonObject())); - } - { // tenant 2 - JsonObject config = new JsonObject(); - TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); - - StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) - .modifyConfigToAddANewUserPoolForTesting(config, 1); - - Multitenancy.addNewOrUpdateAppOrTenant( - main, - new TenantIdentifier(null, null, null), - new TenantConfig( - tenantIdentifier, - new EmailPasswordConfig(true), - new ThirdPartyConfig(true, null), - new PasswordlessConfig(true), - null, null, config)); - } - } - private void setFeatureFlags(Main main, EE_FEATURES[] features) { FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java new file mode 100644 index 000000000..e1d5206a3 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.util.List; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.userroles.UserRoles; + +public class ImportUserTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + try { + JsonObject request = new JsonObject(); + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals("Http error. Status Code: 400. Message: Field name 'user' is invalid in JSON input", + e.getMessage()); + } + } + + { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + try { + JsonObject request = new JsonObject(); + List users = BulkImportTestUtils.generateBulkImportUser(1); + request.add("user", users.get(0).toJsonObject()); + + HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: {\"errors\":[\"Role role1 does not exist.\",\"Invalid tenantId: t1 for a user role.\",\"Role role2 does not exist.\",\"Invalid tenantId: t1 for a user role.\",\"Invalid tenantId: t1 for emailpassword recipe.\",\"Invalid tenantId: t1 for thirdparty recipe.\",\"Invalid tenantId: t1 for passwordless recipe.\"]}", + e.getMessage()); + } + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + JsonObject request = new JsonObject(); + List users = BulkImportTestUtils.generateBulkImportUser(1); + request.add("user", users.get(0).toJsonObject()); + + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertNotNull(response.get("user")); + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} From 119ca1dd582e968d86605c412ce957b757c51c2a Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Thu, 23 May 2024 17:35:24 +0530 Subject: [PATCH 18/41] feat: Add an api to get count of bulk import users --- .../io/supertokens/bulkimport/BulkImport.java | 23 ++- .../bulkimport/ProcessBulkImportUsers.java | 76 ++++----- .../io/supertokens/webserver/Webserver.java | 2 + .../bulkimport/CountBulkImportUsersAPI.java | 86 ++++++++++ .../test/bulkimport/BulkImportTest.java | 66 ++++++++ .../apis/CountBulkImportUsersTest.java | 148 ++++++++++++++++++ 6 files changed, 361 insertions(+), 40 deletions(-) create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index ef6d5d18d..8fa6c9e38 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -148,6 +148,12 @@ public static List deleteUsers(AppIdentifier appIdentifier, Storage stor return StorageUtils.getBulkImportStorage(storage).deleteBulkImportUsers(appIdentifier, userIds); } + public static long getBulkImportUsersCount(AppIdentifier appIdentifier, Storage storage, + @Nullable BULK_IMPORT_USER_STATUS status) + throws StorageQueryException { + return StorageUtils.getBulkImportStorage(storage).getBulkImportUsersCount(appIdentifier, status); + } + public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifier appIdentifier, BulkImportUser user) throws StorageQueryException, InvalidConfigException, IOException, TenantOrAppNotFoundException, @@ -180,8 +186,10 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); - AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, primaryLM.superTokensUserId); - io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, bulkImportProxyStorage, new AuthRecipeUserInfo[]{importedUser}); + AuthRecipeUserInfo importedUser = AuthRecipe.getUserById(appIdentifier, bulkImportProxyStorage, + primaryLM.superTokensUserId); + io.supertokens.useridmapping.UserIdMapping.populateExternalUserIdForUsers(appIdentifier, + bulkImportProxyStorage, new AuthRecipeUserInfo[] { importedUser }); return importedUser; } catch (StorageTransactionLogicException e) { @@ -257,7 +265,11 @@ private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdenti lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); lm.superTokensUserId = userInfo.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException | RestartFlowException e) { + } catch (RestartFlowException e) { + String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists." + : "A user with phoneNumber " + lm.phoneNumber + " already exists."; + throw new StorageTransactionLogicException(new Exception(errorMessage)); + } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); } } @@ -485,13 +497,14 @@ private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appI try { List allProxyStorages = new ArrayList<>(); - + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); for (TenantConfig tenantConfig : tenantConfigs) { allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException + | StorageQueryException e) { throw new StorageTransactionLogicException(e); } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 7e1ef2fdf..1bdf4ebb0 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -70,8 +70,7 @@ public static ProcessBulkImportUsers init(Main main, List @Override protected void doTaskPerApp(AppIdentifier app) - throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, - DbInitException { + throws TenantOrAppNotFoundException, StorageQueryException, IOException, DbInitException { if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; @@ -114,7 +113,8 @@ public int getInitialWaitTimeSeconds() { } private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, StorageQueryException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, + StorageQueryException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -151,13 +151,13 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi try { List allProxyStorages = new ArrayList<>(); - TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); for (TenantConfig tenantConfig : tenantConfigs) { allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException + | StorageQueryException e) { throw new StorageTransactionLogicException(e); } } @@ -172,7 +172,7 @@ private void closeAllProxyStorages() throws StorageQueryException { private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportUserUtils bulkImportUserUtils, BulkImportSQLStorage baseTenantStorage) - throws TenantOrAppNotFoundException, StorageQueryException, InvalidConfigException, IOException, + throws TenantOrAppNotFoundException, StorageQueryException, IOException, DbInitException { try { @@ -193,34 +193,36 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage( + firstTenantIdentifier); /* - * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. - * This is necessary because the bulk_import_users entries are always in the public tenant storage, - * but the actual user data could be in a different storage. - * - * If transactions are committed individually, in this order: - * 1. Commit the transaction that imports the user. - * 2. Commit the transaction that deletes the corresponding bulk import entry. - * - * There's a risk where the first commit succeeds, but the second fails. This creates a situation where - * the bulk import entry is re-processed, even though the user has already been imported into the database. - * - * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. - * The processing logic now follows these steps: - * - * 1. Import the user and get the `primaryUserId` (transaction uncommitted). - * 2. Update the `primaryUserId` in the corresponding bulk import entry. - * 3. Commit the import transaction from step 1. - * 4. Delete the bulk import entry. - * - * If step 2 or any earlier step fails, nothing is committed, preventing partial state. - * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the next run. - * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. - * - * The following code implements this logic. - */ + * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. + * This is necessary because the bulk_import_users entries are always in the public tenant storage, + * but the actual user data could be in a different storage. + * + * If transactions are committed individually, in this order: + * 1. Commit the transaction that imports the user. + * 2. Commit the transaction that deletes the corresponding bulk import entry. + * + * There's a risk where the first commit succeeds, but the second fails. This creates a situation where + * the bulk import entry is re-processed, even though the user has already been imported into the database. + * + * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. + * The processing logic now follows these steps: + * + * 1. Import the user and get the `primaryUserId` (transaction uncommitted). + * 2. Update the `primaryUserId` in the corresponding bulk import entry. + * 3. Commit the import transaction from step 1. + * 4. Delete the bulk import entry. + * + * If step 2 or any earlier step fails, nothing is committed, preventing partial state. + * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the + * next run. + * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. + * + * The following code implements this logic. + */ if (user.primaryUserId != null) { AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, user.primaryUserId); @@ -237,12 +239,14 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI BulkImport.processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); } - BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, + primaryLM); Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); BulkImport.createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, + user.loginMethods); BulkImport.createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); BulkImport.createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); BulkImport.createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); @@ -271,7 +275,7 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI closeAllProxyStorages(); } }); - } catch (StorageTransactionLogicException | InvalidBulkImportDataException e) { + } catch (StorageTransactionLogicException | InvalidBulkImportDataException | InvalidConfigException e) { handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); } } @@ -289,6 +293,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport errorMessage[0] = exception.actualException.getMessage(); } else if (e instanceof InvalidBulkImportDataException) { errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); + } else if (e instanceof InvalidConfigException) { + errorMessage[0] = e.getMessage(); } try { diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 23e953ef9..47dfeb682 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; import io.supertokens.webserver.api.bulkimport.BulkImportAPI; +import io.supertokens.webserver.api.bulkimport.CountBulkImportUsersAPI; import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; import io.supertokens.webserver.api.bulkimport.ImportUserAPI; import io.supertokens.webserver.api.core.*; @@ -266,6 +267,7 @@ private void setupRoutes() { addAPI(new BulkImportAPI(main)); addAPI(new DeleteBulkImportUserAPI(main)); addAPI(new ImportUserAPI(main)); + addAPI(new CountBulkImportUsersAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java new file mode 100644 index 000000000..9edb8be57 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/CountBulkImportUsersAPI.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import java.io.IOException; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.multitenancy.exception.BadPermissionException; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +public class CountBulkImportUsersAPI extends WebserverAPI { + public CountBulkImportUsersAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/users/count"; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + String statusString = InputParser.getQueryParamOrThrowError(req, "status", true); + + BULK_IMPORT_USER_STATUS status = null; + if (statusString != null) { + try { + status = BULK_IMPORT_USER_STATUS.valueOf(statusString); + } catch (IllegalArgumentException e) { + throw new ServletException( + new BadRequestException("Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!")); + } + } + + AppIdentifier appIdentifier = null; + Storage storage = null; + + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, status); + + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.addProperty("count", count); + super.sendJsonResponse(200, result, resp); + + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } + } +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index a673b7682..ac298d4ce 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -296,6 +296,72 @@ public void randomPaginationTest() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void testGetBulkImportUsersCount() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Test with status = 'NEW' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.NEW); + assertEquals(10, count); + } + + // Test with status = 'PROCESSING' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to PROCESSING + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.PROCESSING, null); + } + storage.commitTransaction(con); + return null; + }); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.PROCESSING); + assertEquals(10, count); + } + + // Test with status = 'FAILED' + { + List users = generateBulkImportUser(10); + BulkImport.addUsers(appIdentifier, storage, users); + + // Update the users status to FAILED + storage.startTransaction(con -> { + for (BulkImportUser user : users) { + storage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BULK_IMPORT_USER_STATUS.FAILED, null); + } + storage.commitTransaction(con); + return null; + }); + + long count = BulkImport.getBulkImportUsersCount(appIdentifier, storage, BULK_IMPORT_USER_STATUS.FAILED); + assertEquals(10, count); + } + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + @Test public void shouldImportTheUserInTheSameTenant() throws Exception { String[] args = { "../" }; diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java new file mode 100644 index 000000000..6d1e14a61 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/CountBulkImportUsersTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import com.google.gson.JsonObject; + +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; + + +public class CountBulkImportUsersTest { + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void shouldReturn400Error() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + try { + Map params = new HashMap<>(); + params.put("status", "INVALID_STATUS"); + HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); + } catch (io.supertokens.test.httpRequest.HttpResponseException e) { + assertEquals(400, e.statusCode); + assertEquals( + "Http error. Status Code: 400. Message: Invalid value for status. Pass one of NEW, PROCESSING, or FAILED!", + e.getMessage()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldReturn200Response() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + { + Map params = new HashMap<>(); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "NEW"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "PROCESSING"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + { + Map params = new HashMap<>(); + params.put("status", "FAILED"); + JsonObject response = HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + assertEquals("OK", response.get("status").getAsString()); + assertEquals(0, response.get("count").getAsLong()); + } + + process.kill(); + Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + +} From 79a1e67ce9a152462500b08ade467aae709605c4 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Mon, 27 May 2024 17:53:59 +0530 Subject: [PATCH 19/41] fix: PR changes --- .../io/supertokens/bulkimport/BulkImport.java | 8 +- .../bulkimport/ProcessBulkImportUsers.java | 14 +--- .../api/bulkimport/BulkImportAPI.java | 9 ++- .../bulkimport/DeleteBulkImportUserAPI.java | 5 +- .../api/bulkimport/ImportUserAPI.java | 3 +- .../apis/AddBulkImportUsersTest.java | 6 +- .../apis/DeleteBulkImportUsersTest.java | 79 +++++++++++-------- .../test/bulkimport/apis/ImportUserTest.java | 21 +---- 8 files changed, 66 insertions(+), 79 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 8fa6c9e38..7173d93c0 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -237,7 +237,7 @@ private static void processEmailPasswordLoginMethod(TenantIdentifier tenantIdent throw new StorageTransactionLogicException(e); } catch (DuplicateEmailException e) { throw new StorageTransactionLogicException( - new Exception("A user with email " + lm.email + " already exists")); + new Exception("A user with email " + lm.email + " already exists in emailpassword loginMethod.")); } } @@ -253,7 +253,7 @@ private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifi throw new StorageTransactionLogicException(e); } catch (DuplicateThirdPartyUserException e) { throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId - + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists")); + + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists in thirdparty loginMethod.")); } } @@ -266,8 +266,8 @@ private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdenti lm.superTokensUserId = userInfo.getSupertokensUserId(); } catch (RestartFlowException e) { - String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists." - : "A user with phoneNumber " + lm.phoneNumber + " already exists."; + String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists in passwordless loginMethod." + : "A user with phoneNumber " + lm.phoneNumber + " already exists in passwordless loginMethod."; throw new StorageTransactionLogicException(new Exception(errorMessage)); } catch (StorageQueryException | TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(e); diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 1bdf4ebb0..be3dfb744 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -113,8 +113,7 @@ public int getInitialWaitTimeSeconds() { } private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, - StorageQueryException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -133,13 +132,6 @@ private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantId userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); bulkImportProxyStorage.initStorage(false, new ArrayList<>()); - // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. - // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries - // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. - - // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, - // causing other queries to stall indefinitely. - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } } @@ -156,8 +148,7 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException - | StorageQueryException e) { + } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException e) { throw new StorageTransactionLogicException(e); } } @@ -165,7 +156,6 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi private void closeAllProxyStorages() throws StorageQueryException { for (SQLStorage storage : userPoolToStorageMap.values()) { storage.closeConnectionForBulkImportProxyStorage(); - storage.close(); } userPoolToStorageMap.clear(); } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java index c30f63f82..6e8d0157f 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportAPI.java @@ -132,7 +132,14 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S JsonObject input = InputParser.parseJsonObjectOrThrowError(req); JsonArray users = InputParser.parseArrayOrThrowError(input, "users", false); - if (users.size() <= 0 || users.size() > BulkImport.MAX_USERS_TO_ADD) { + if (users.size() == 0) { + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + super.sendJsonResponse(200, result, resp); + return; + } + + if (users.size() > BulkImport.MAX_USERS_TO_ADD) { JsonObject errorResponseJson = new JsonObject(); String errorMsg = users.size() <= 0 ? "You need to add at least one user." : "You can only add " + BulkImport.MAX_USERS_TO_ADD + " users at a time."; diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index 5562fafca..0dd604fcd 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -59,7 +59,10 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S JsonArray arr = InputParser.parseArrayOrThrowError(input, "ids", false); if (arr.size() == 0) { - throw new ServletException(new WebserverAPI.BadRequestException("Field name 'ids' cannot be an empty array")); + JsonObject result = new JsonObject(); + result.add("deletedIds", new JsonArray()); + result.add("invalidIds", new JsonArray()); + super.sendJsonResponse(200, result, resp); } if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java index ae29fdd63..599801680 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -61,8 +61,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); } - JsonObject input = InputParser.parseJsonObjectOrThrowError(req); - JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(input, "user", false); + JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(req); AppIdentifier appIdentifier = null; Storage storage = null; diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index c4ac061d8..641ba3822 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -85,11 +85,7 @@ public void shouldThrow400IfUsersAreMissingInRequestBody() throws Exception { testBadRequest(main, new JsonParser().parse("{\"users\": \"string\"}").getAsJsonObject(), "Field name 'users' is invalid in JSON input"); - // CASE 3: users array is empty - testBadRequest(main, generateUsersJson(0).getAsJsonObject(), - "{\"error\":\"You need to add at least one user.\"}"); - - // CASE 4: users array length is greater than 10000 + // CASE 3: users array length is greater than 10000 testBadRequest(main, generateUsersJson(10001).getAsJsonObject(), "{\"error\":\"You can only add 10000 users at a time.\"}"); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index ebd9b3624..30c07597c 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.List; @@ -80,9 +81,11 @@ public void shouldReturn400Error() throws Exception { HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", e.getMessage()); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' is invalid in JSON input", + e.getMessage()); } } { @@ -91,20 +94,11 @@ public void shouldReturn400Error() throws Exception { HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + fail("The API should have thrown an error"); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", e.getMessage()); - } - } - { - try { - JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/users/remove", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain an empty string", e.getMessage()); + assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", + e.getMessage()); } } { @@ -122,7 +116,9 @@ public void shouldReturn400Error() throws Exception { request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); } catch (io.supertokens.test.httpRequest.HttpResponseException e) { assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", e.getMessage()); + assertEquals( + "Http error. Status Code: 400. Message: Field name 'ids' cannot contain more than 500 elements", + e.getMessage()); } } @@ -142,33 +138,46 @@ public void shouldReturn200Response() throws Exception { return; } - BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); - AppIdentifier appIdentifier = new AppIdentifier(null, null); - - // Insert users - List users = generateBulkImportUser(5); - BulkImport.addUsers(appIdentifier, storage, users); + // Call the API with empty array + { + JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); + JsonObject resonse = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - String invalidId = io.supertokens.utils.Utils.getUUID(); - JsonObject request = new JsonObject(); - JsonArray validIds = new JsonArray(); - for (BulkImportUser user : users) { - validIds.add(new JsonPrimitive(user.id)); + assertEquals(0, resonse.get("deletedIds").getAsJsonArray().size()); + assertEquals(0, resonse.get("invalidIds").getAsJsonArray().size()); } - validIds.add(new JsonPrimitive(invalidId)); - - request.add("ids", validIds); - JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/users/remove", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + { - response.get("deletedIds").getAsJsonArray().forEach(id -> { - assertTrue(validIds.contains(id)); - }); + BulkImportStorage storage = (BulkImportStorage) StorageLayer.getStorage(process.main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); - assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + // Insert users + List users = generateBulkImportUser(5); + BulkImport.addUsers(appIdentifier, storage, users); + + String invalidId = io.supertokens.utils.Utils.getUUID(); + JsonObject request = new JsonObject(); + JsonArray validIds = new JsonArray(); + for (BulkImportUser user : users) { + validIds.add(new JsonPrimitive(user.id)); + } + validIds.add(new JsonPrimitive(invalidId)); + request.add("ids", validIds); + + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users/remove", + request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + + response.get("deletedIds").getAsJsonArray().forEach(id -> { + assertTrue(validIds.contains(id)); + }); + + assertEquals(invalidId, response.get("invalidIds").getAsJsonArray().get(0).getAsString()); + } process.kill(); Assert.assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java index e1d5206a3..5a1b1db85 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/ImportUserTest.java @@ -70,28 +70,13 @@ public void shouldReturn400Error() throws Exception { return; } - { - try { - JsonObject request = new JsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/import", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'user' is invalid in JSON input", - e.getMessage()); - } - } - { FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); try { - JsonObject request = new JsonObject(); List users = BulkImportTestUtils.generateBulkImportUser(1); - request.add("user", users.get(0).toJsonObject()); + JsonObject request = users.get(0).toJsonObject(); HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/import", @@ -132,10 +117,8 @@ public void shouldReturn200Response() throws Exception { UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); } - - JsonObject request = new JsonObject(); List users = BulkImportTestUtils.generateBulkImportUser(1); - request.add("user", users.get(0).toJsonObject()); + JsonObject request = users.get(0).toJsonObject(); JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/import", From 77cb57c08f2016a36a9e24d568e1d0bff44a0cc9 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Wed, 29 May 2024 10:38:44 +0530 Subject: [PATCH 20/41] fix: Add error codes and plainTextPassword import --- .../io/supertokens/bulkimport/BulkImport.java | 216 ++++++++++++------ .../bulkimport/BulkImportUserUtils.java | 16 +- .../bulkimport/ProcessBulkImportUsers.java | 33 ++- .../supertokens/webserver/WebserverAPI.java | 3 +- .../test/bulkimport/BulkImportTest.java | 55 ++++- .../test/bulkimport/BulkImportTestUtils.java | 35 ++- .../ProcessBulkImportUsersCronJobTest.java | 8 +- .../apis/AddBulkImportUsersTest.java | 6 +- 8 files changed, 262 insertions(+), 110 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 7173d93c0..cb8111a78 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -31,6 +31,7 @@ import io.supertokens.config.Config; import io.supertokens.emailpassword.EmailPassword; import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; +import io.supertokens.emailpassword.PasswordHashing; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithEmailAlreadyExistsException; @@ -81,6 +82,9 @@ import com.google.gson.JsonObject; +// Error codes ensure globally unique and identifiable errors in Bulk Import. +// Current range: E001 to E046. + public class BulkImport { // Maximum number of users that can be added in a single /bulk-import/users POST request @@ -170,19 +174,9 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie try { return bulkImportProxyStorage.startTransaction(con -> { try { - for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); - } - - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, primaryLM, + allStoragesForApp); bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); @@ -205,6 +199,21 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie } } + public static void processUserImportSteps(Main main, TransactionConnection con, AppIdentifier appIdentifier, + Storage bulkImportProxyStorage, BulkImportUser user, LoginMethod primaryLM, Storage[] allStoragesForApp) + throws StorageTransactionLogicException { + for (LoginMethod lm : user.loginMethods) { + processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); + } + + createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + } + public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm) throws StorageTransactionLogicException { String firstTenant = lm.tenantIds.get(0); @@ -213,31 +222,41 @@ public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier appIdentifier.getAppId(), firstTenant); if (lm.recipeId.equals("emailpassword")) { - processEmailPasswordLoginMethod(tenantIdentifier, storage, lm); + processEmailPasswordLoginMethod(main, tenantIdentifier, storage, lm); } else if (lm.recipeId.equals("thirdparty")) { processThirdPartyLoginMethod(tenantIdentifier, storage, lm); } else if (lm.recipeId.equals("passwordless")) { processPasswordlessLoginMethod(tenantIdentifier, storage, lm); } else { throw new StorageTransactionLogicException( - new IllegalArgumentException("Unknown recipeId " + lm.recipeId + " for loginMethod ")); + new IllegalArgumentException("E001: Unknown recipeId " + lm.recipeId + " for loginMethod.")); } associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); } - private static void processEmailPasswordLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, + private static void processEmailPasswordLoginMethod(Main main, TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) throws StorageTransactionLogicException { try { + + String passwordHash = lm.passwordHash; + if (passwordHash == null && lm.plainTextPassword != null) { + passwordHash = PasswordHashing.getInstance(main) + .createHashWithSalt(tenantIdentifier.toAppIdentifier(), lm.plainTextPassword); + } + ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, - lm.passwordHash, lm.timeJoinedInMSSinceEpoch); + passwordHash, lm.timeJoinedInMSSinceEpoch); lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E002: " + e.getMessage())); } catch (DuplicateEmailException e) { throw new StorageTransactionLogicException( - new Exception("A user with email " + lm.email + " already exists in emailpassword loginMethod.")); + new Exception( + "E003: A user with email " + lm.email + " already exists in emailpassword loginMethod.")); } } @@ -249,10 +268,12 @@ private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifi lm.timeJoinedInMSSinceEpoch); lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); } catch (DuplicateThirdPartyUserException e) { - throw new StorageTransactionLogicException(new Exception("A user with thirdPartyId " + lm.thirdPartyId + throw new StorageTransactionLogicException(new Exception("E005: A user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists in thirdparty loginMethod.")); } } @@ -266,11 +287,15 @@ private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdenti lm.superTokensUserId = userInfo.getSupertokensUserId(); } catch (RestartFlowException e) { - String errorMessage = lm.email != null ? "A user with email " + lm.email + " already exists in passwordless loginMethod." - : "A user with phoneNumber " + lm.phoneNumber + " already exists in passwordless loginMethod."; + String errorMessage = lm.email != null + ? "E006: A user with email " + lm.email + " already exists in passwordless loginMethod." + : "E007: A user with phoneNumber " + lm.phoneNumber + + " already exists in passwordless loginMethod."; throw new StorageTransactionLogicException(new Exception(errorMessage)); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E008: " + e.getMessage())); } } @@ -285,12 +310,42 @@ private static void associateUserToTenants(Main main, AppIdentifier appIdentifie TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), tenantId); Multitenancy.addUserIdToTenant(main, tenantIdentifier, storage, lm.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | UnknownUserIdException | StorageQueryException - | FeatureNotEnabledException | DuplicateEmailException | DuplicatePhoneNumberException - | DuplicateThirdPartyUserException | AnotherPrimaryUserWithPhoneNumberAlreadyExistsException - | AnotherPrimaryUserWithEmailAlreadyExistsException - | AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E009: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(new Exception("E010: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but it doesn't exist. This should not happen. Please contact support.")); + } catch (AnotherPrimaryUserWithEmailAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E011: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with email " + lm.email + " already exists.")); + } catch (AnotherPrimaryUserWithPhoneNumberAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E012: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with phoneNumber " + lm.phoneNumber + " already exists.")); + } catch (AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException e) { + throw new StorageTransactionLogicException(new Exception("E013: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another primary user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + + lm.thirdPartyUserId + " already exists.")); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException(new Exception("E014: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with email " + lm.email + " already exists.")); + } catch (DuplicatePhoneNumberException e) { + throw new StorageTransactionLogicException(new Exception("E015: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with phoneNumber " + lm.phoneNumber + " already exists.")); + } catch (DuplicateThirdPartyUserException e) { + throw new StorageTransactionLogicException(new Exception("E016: " + "We tried to add the userId " + + lm.getSuperTokenOrExternalUserId() + " to the tenantId " + tenantId + + " but another user with thirdPartyId " + lm.thirdPartyId + " and thirdPartyUserId " + + lm.thirdPartyUserId + " already exists.")); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E017: " + e.getMessage())); } } } @@ -304,16 +359,27 @@ public static void createPrimaryUserAndLinkAccounts(Main main, try { AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E018: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E019: " + e.getMessage())); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException(new Exception( - "We tried to create the primary user for the userId " + primaryLM.getSuperTokenOrExternalUserId() + "E020: We tried to create the primary user for the userId " + + primaryLM.getSuperTokenOrExternalUserId() + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException - | AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); + } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E021: We tried to create the primary user for the userId " + + primaryLM.getSuperTokenOrExternalUserId() + + " but it is already linked with another primary user.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E022: We tried to create the primary user for the userId " + + primaryLM.getSuperTokenOrExternalUserId() + + " but the account info is already associated with another primary user.")); } for (LoginMethod lm : user.loginMethods) { @@ -325,22 +391,32 @@ public static void createPrimaryUserAndLinkAccounts(Main main, AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException | FeatureNotEnabledException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E023: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E024: " + e.getMessage())); } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + new Exception("E025: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); + + " but it doesn't exist.")); } catch (InputUserIdIsNotAPrimaryUserException e) { throw new StorageTransactionLogicException( - new Exception("We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + new Exception("E026: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it is not a primary user. This should not happen. Please contact support.")); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException - | RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException( - new Exception(e.getMessage() + " This should not happen. Please contact support.")); + + " but it is not a primary user.")); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E027: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but the account info is already associated with another primary user.")); + } catch (RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { + throw new StorageTransactionLogicException(new Exception( + "E028: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() + + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() + + " but it is already linked with another primary user.")); } } } @@ -355,14 +431,18 @@ public static void createUserIdMapping(AppIdentifier appIdentifier, null, false, true); primaryLM.externalUserId = user.externalUserId; - } catch (StorageQueryException | ServletException | TenantOrAppNotFoundException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E029: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (ServletException e) { + throw new StorageTransactionLogicException(new Exception("E030: " + e.getMessage())); } catch (UserIdMappingAlreadyExistsException e) { throw new StorageTransactionLogicException( - new Exception("A user with externalId " + user.externalUserId + " already exists")); + new Exception("E031: A user with externalId " + user.externalUserId + " already exists")); } catch (UnknownSuperTokensUserIdException e) { throw new StorageTransactionLogicException( - new Exception("We tried to create the externalUserId mapping for the superTokenUserId " + new Exception("E032: We tried to create the externalUserId mapping for the superTokenUserId " + primaryLM.superTokensUserId + " but it doesn't exist. This should not happen. Please contact support.")); } @@ -375,7 +455,9 @@ public static void createUserMetadata(AppIdentifier appIdentifier, Storage stora try { UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), user.userMetadata); - } catch (StorageQueryException | TenantOrAppNotFoundException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E040: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } } @@ -393,10 +475,12 @@ public static void createUserRoles(Main main, AppIdentifier appIdentifier, Stora UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); } - } catch (TenantOrAppNotFoundException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (UnknownRoleException e) { - throw new StorageTransactionLogicException(new Exception("Role " + userRole.role + throw new StorageTransactionLogicException(new Exception("E034: Role " + userRole.role + " does not exist! You need pre-create the role before assigning it to the user.")); } } @@ -418,7 +502,9 @@ public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Tr emailVerificationSQLStorage .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, lm.getSuperTokenOrExternalUserId(), lm.email, true); - } catch (TenantOrAppNotFoundException | StorageQueryException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E035: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } } @@ -432,11 +518,16 @@ public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Sto Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException | StorageQueryException | FeatureNotEnabledException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E036: " + e.getMessage())); + } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E037: " + e.getMessage())); } catch (DeviceAlreadyExistsException e) { throw new StorageTransactionLogicException( - new Exception("A totp device with name " + totpDevice.deviceName + " already exists")); + new Exception( + "E038: A totp device with name " + totpDevice.deviceName + " already exists")); } } } @@ -459,8 +550,7 @@ public static BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser us } private static synchronized Storage getBulkImportProxyStorage(Main main, TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException, - StorageQueryException { + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); if (userPoolToStorageMap.containsKey(userPoolId)) { return userPoolToStorageMap.get(userPoolId); @@ -479,13 +569,6 @@ private static synchronized Storage getBulkImportProxyStorage(Main main, TenantI userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); bulkImportProxyStorage.initStorage(false, new ArrayList<>()); - // `BulkImportProxyStorage` uses `BulkImportProxyConnection`, which overrides the `.commit()` method on the Connection object. - // The `initStorage()` method runs `select * from table_name limit 1` queries to check if the tables exist but these queries - // don't get committed due to the overridden `.commit()`, so we need to manually commit the transaction to remove any locks on the tables. - - // Without this commit, a call to `select * from bulk_import_users limit 1` in `doesTableExist()` locks the `bulk_import_users` table, - // causing other queries to stall indefinitely. - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); return bulkImportProxyStorage; } } @@ -503,9 +586,14 @@ private static Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appI allProxyStorages.add(getBulkImportProxyStorage(main, tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException - | StorageQueryException e) { - throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E039: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E040: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E041: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E042: " + e.getMessage())); } } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 4c0bc73ba..7c6aac760 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -205,9 +205,15 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); String passwordHash = parseAndValidateFieldType(jsonLoginMethodObj, "passwordHash", ValueType.STRING, - true, String.class, errors, " for an emailpassword recipe."); + false, String.class, errors, " for an emailpassword recipe."); String hashingAlgorithm = parseAndValidateFieldType(jsonLoginMethodObj, "hashingAlgorithm", - ValueType.STRING, true, String.class, errors, " for an emailpassword recipe."); + ValueType.STRING, false, String.class, errors, " for an emailpassword recipe."); + String plainTextPassword = parseAndValidateFieldType(jsonLoginMethodObj, "plainTextPassword", + ValueType.STRING, false, String.class, errors, " for an emailpassword recipe."); + + if ((passwordHash == null || hashingAlgorithm == null) && plainTextPassword == null) { + errors.add("Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe."); + } email = validateAndNormaliseEmail(email, errors); CoreConfig.PASSWORD_HASHING_ALG normalisedHashingAlgorithm = validateAndNormaliseHashingAlgorithm( @@ -218,7 +224,7 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden passwordHash, errors); loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, - timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null)); + timeJoinedInMSSinceEpoch, email, passwordHash, hashingAlgorithm, null, null, null, null)); } else if ("thirdparty".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, true, String.class, errors, " for a thirdparty recipe."); @@ -232,7 +238,7 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden thirdPartyUserId = validateAndNormaliseThirdPartyUserId(thirdPartyUserId, errors); loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, - timeJoinedInMSSinceEpoch, email, null, null, thirdPartyId, thirdPartyUserId, null)); + timeJoinedInMSSinceEpoch, email, null, null, null, thirdPartyId, thirdPartyUserId, null)); } else if ("passwordless".equals(recipeId)) { String email = parseAndValidateFieldType(jsonLoginMethodObj, "email", ValueType.STRING, false, String.class, errors, " for a passwordless recipe."); @@ -247,7 +253,7 @@ private List getParsedLoginMethods(Main main, AppIdentifier appIden } loginMethods.add(new LoginMethod(normalisedTenantIds, recipeId, isVerified, isPrimary, - timeJoinedInMSSinceEpoch, email, null, null, null, null, phoneNumber)); + timeJoinedInMSSinceEpoch, email, null, null, null, null, null, phoneNumber)); } } return loginMethods; diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index be3dfb744..a4b235da0 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -33,6 +33,7 @@ import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.output.Logging; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; @@ -148,8 +149,14 @@ private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifi allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); } return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException | InvalidConfigException | IOException | DbInitException e) { - throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E043: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E044: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E045: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E046: " + e.getMessage())); } } @@ -225,21 +232,8 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI bulkImportProxyStorage.startTransaction(con -> { try { - for (LoginMethod lm : user.loginMethods) { - BulkImport.processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); - } - - BulkImport.createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, - primaryLM); - Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - BulkImport.createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - - BulkImport.verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, - user.loginMethods); - BulkImport.createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - BulkImport.createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - BulkImport.createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, primaryLM, allStoragesForApp); // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this @@ -273,13 +267,18 @@ private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkI private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { - // Java doesn't allow us to reassign local variables inside a lambda expression // so we have to use an array. String[] errorMessage = { e.getMessage() }; if (e instanceof StorageTransactionLogicException) { StorageTransactionLogicException exception = (StorageTransactionLogicException) e; + // If the exception is due to a StorageQueryException, we want to retry the entry after sometime instead + // of marking it as FAILED. We will return early in that case. + if (exception.actualException instanceof StorageQueryException) { + Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); + return; + } errorMessage[0] = exception.actualException.getMessage(); } else if (e instanceof InvalidBulkImportDataException) { errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); diff --git a/src/main/java/io/supertokens/webserver/WebserverAPI.java b/src/main/java/io/supertokens/webserver/WebserverAPI.java index 616413635..4ca52c83c 100644 --- a/src/main/java/io/supertokens/webserver/WebserverAPI.java +++ b/src/main/java/io/supertokens/webserver/WebserverAPI.java @@ -74,10 +74,11 @@ public abstract class WebserverAPI extends HttpServlet { supportedVersions.add(SemVer.v3_0); supportedVersions.add(SemVer.v4_0); supportedVersions.add(SemVer.v5_0); + supportedVersions.add(SemVer.v5_1); } public static SemVer getLatestCDIVersion() { - return SemVer.v5_0; + return SemVer.v5_1; } public SemVer getLatestCDIVersionForRequest(HttpServletRequest req) diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java index ac298d4ce..cd399e61c 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTest.java @@ -47,6 +47,7 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; @@ -391,7 +392,7 @@ public void shouldImportTheUserInTheSameTenant() throws Exception { AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, users.get(0)); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(0), importedUser); process.kill(); @@ -439,10 +440,10 @@ public void shouldImportTheUserInMultipleTenantsWithDifferentStorages() throws E AuthRecipeUserInfo importedUser1 = BulkImport.importUser(main, appIdentifier, bulkImportUserT1); AuthRecipeUserInfo importedUser2 = BulkImport.importUser(main, appIdentifier, bulkImportUserT2); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, bulkImportUserT1, importedUser1); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, bulkImportUserT2, importedUser2); @@ -493,7 +494,7 @@ public void shouldImportUsersConcurrently() throws Exception { for (int i = 0; i < users.size(); i++) { AuthRecipeUserInfo importedUser = futures.get(i).get(); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), users.get(i), importedUser); } @@ -502,4 +503,50 @@ public void shouldImportUsersConcurrently() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldImportWithPlainTextPassword() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, + new EE_FEATURES[] { EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA, EE_FEATURES.ACCOUNT_LINKING }); + + // Create tenants + BulkImportTestUtils.createTenants(main); + + // Create user roles + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + AppIdentifier appIdentifier = new AppIdentifier(null, null); + List users = generateBulkImportUser(1); + BulkImportUser bulkImportUser = users.get(0); + + // Set passwordHash to null and plainTextPassword to a value to ensure we do a plainTextPassword import + for (LoginMethod lm : bulkImportUser.loginMethods) { + if (lm.recipeId == "emailpassword") { + lm.passwordHash = null; + lm.hashingAlgorithm = null; + lm.plainTextPassword = "testPass@123"; + } + } + + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, bulkImportUser); + + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, + appIdentifier.getAsPublicTenantIdentifier(), StorageLayer.getStorage(main), bulkImportUser, importedUser); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 4faea47ab..61740021b 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -28,6 +28,7 @@ import com.google.gson.JsonParser; import io.supertokens.Main; +import io.supertokens.emailpassword.PasswordHashing; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; import io.supertokens.multitenancy.exception.BadPermissionException; @@ -82,11 +83,13 @@ public static List generateBulkImportUser(int numberOfUsers, Lis List loginMethods = new ArrayList<>(); long currentTimeMillis = System.currentTimeMillis(); loginMethods.add(new LoginMethod(tenants, "emailpassword", true, true, currentTimeMillis, email, "$2a", - "BCRYPT", null, null, null)); - loginMethods.add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, - "thirdPartyId" + i, "thirdPartyUserId" + i, null)); - loginMethods.add(new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, - null, null, null)); + "BCRYPT", null, null, null, null)); + loginMethods + .add(new LoginMethod(tenants, "thirdparty", true, false, currentTimeMillis, email, null, null, null, + "thirdPartyId" + i, "thirdPartyUserId" + i, null)); + loginMethods.add( + new LoginMethod(tenants, "passwordless", true, false, currentTimeMillis, email, null, null, null, + null, null, null)); users.add(new BulkImportUser(id, externalId, userMetadata, userRoles, totpDevices, loginMethods)); } return users; @@ -131,15 +134,15 @@ public static void createTenants(Main main) } } - public static void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier appIdentifier, + public static void assertBulkImportUserAndAuthRecipeUserAreEqual(Main main, AppIdentifier appIdentifier, TenantIdentifier tenantIdentifier, Storage storage, BulkImportUser bulkImportUser, AuthRecipeUserInfo authRecipeUser) throws StorageQueryException, TenantOrAppNotFoundException { for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm1 : authRecipeUser.loginMethods) { - bulkImportUser.loginMethods.forEach(lm2 -> { + for (LoginMethod lm2 : bulkImportUser.loginMethods) { if (lm2.recipeId.equals(lm1.recipeId.toString())) { - assertLoginMethodEquals(lm1, lm2); + assertLoginMethodEquals(main, lm1, lm2); } - }); + } } assertEquals(bulkImportUser.externalUserId, authRecipeUser.getSupertokensOrExternalUserId()); assertEquals(bulkImportUser.userMetadata, @@ -155,15 +158,23 @@ public static void assertBulkImportUserAndAuthRecipeUserAreEqual(AppIdentifier a assertTotpDevicesEquals(createdTotpDevices, bulkImportUser.totpDevices.toArray(new TotpDevice[0])); } - private static void assertLoginMethodEquals(io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, - io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) { + private static void assertLoginMethodEquals(Main main, io.supertokens.pluginInterface.authRecipe.LoginMethod lm1, + io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod lm2) + throws TenantOrAppNotFoundException { assertEquals(lm1.email, lm2.email); assertEquals(lm1.verified, lm2.isVerified); assertTrue(lm2.tenantIds.containsAll(lm1.tenantIds) && lm1.tenantIds.containsAll(lm2.tenantIds)); switch (lm2.recipeId) { case "emailpassword": - assertEquals(lm1.passwordHash, lm2.passwordHash); + // If lm2.passwordHash is null then the user was imported using plainTextPassword + // We check if the plainTextPassword matches the stored passwordHash + if (lm2.passwordHash == null) { + assertTrue(PasswordHashing.getInstance(main).verifyPasswordWithHash(lm2.plainTextPassword, + lm1.passwordHash)); + } else { + assertEquals(lm1.passwordHash, lm2.passwordHash); + } break; case "thirdparty": assertEquals(lm1.thirdParty.id, lm2.thirdPartyId); diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index a17f842e7..667287f5a 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -106,7 +106,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { TenantIdentifier publicTenant = new TenantIdentifier(null, null, "public"); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, publicTenant, storage, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, publicTenant, storage, bulkImportUser, container.users[0]); @@ -163,10 +163,10 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t1, storageT1, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t1, storageT1, bulkImportUserT1, containerT1.users[0]); - BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(appIdentifier, t2, storageT2, + BulkImportTestUtils.assertBulkImportUserAndAuthRecipeUserAreEqual(main, appIdentifier, t2, storageT2, bulkImportUserT2, containerT2.users[0]); @@ -205,7 +205,7 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); - assertEquals("Role role1 does not exist! You need pre-create the role before assigning it to the user.", + assertEquals("E034: Role role1 does not exist! You need pre-create the role before assigning it to the user.", usersAfterProcessing.get(0).errorMessage); UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java index 641ba3822..ff7d12f02 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/AddBulkImportUsersTest.java @@ -275,7 +275,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc .getAsJsonObject(); testBadRequest(main, requestBody, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"passwordHash is required for an emailpassword recipe.\",\"hashingAlgorithm is required for an emailpassword recipe.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"email is required for an emailpassword recipe.\",\"Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe.\"]}]}"); // CASE 2: email, passwordHash and hashingAlgorithm field type is incorrect JsonObject requestBody2 = new JsonParser() @@ -284,7 +284,7 @@ public void shouldThrow400IfEmailPasswordRecipeHasInvalidFieldTypes() throws Exc .getAsJsonObject(); testBadRequest(main, requestBody2, "{\"error\":\"" + genericErrMsg - + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\"]}]}"); + + "\",\"users\":[{\"index\":0,\"errors\":[\"email should be of type string for an emailpassword recipe.\",\"passwordHash should be of type string for an emailpassword recipe.\",\"hashingAlgorithm should be of type string for an emailpassword recipe.\",\"Either (passwordHash, hashingAlgorithm) or plainTextPassword is required for an emailpassword recipe.\"]}]}"); // CASE 3: hashingAlgorithm is not one of bcrypt, argon2, firebase_scrypt JsonObject requestBody3 = new JsonParser() @@ -555,7 +555,7 @@ public void shouldFailIfANewFieldWasAddedToBulkImportUser() throws Exception { checkLoginMethodFields(user.loginMethods.get(0), "LoginMethod", Arrays.asList("tenantIds", "isVerified", "isPrimary", "timeJoinedInMSSinceEpoch", - "recipeId", "email", "passwordHash", "hashingAlgorithm", + "recipeId", "email", "passwordHash", "plainTextPassword", "hashingAlgorithm", "phoneNumber", "thirdPartyId", "thirdPartyUserId", "externalUserId", "superTokensUserId")); checkTotpDeviceFields(user.totpDevices.get(0), "TotpDevice", From 78979f4a455883ed5f1af3e0651ad71f31c14d62 Mon Sep 17 00:00:00 2001 From: Ankit Tiwari Date: Tue, 18 Jun 2024 12:05:07 +0530 Subject: [PATCH 21/41] fix: PR changes --- .../bulkimport/DeleteBulkImportUserAPI.java | 1 + .../apis/DeleteBulkImportUsersTest.java | 21 ++++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java index 0dd604fcd..d95e0c268 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/DeleteBulkImportUserAPI.java @@ -63,6 +63,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S result.add("deletedIds", new JsonArray()); result.add("invalidIds", new JsonArray()); super.sendJsonResponse(200, result, resp); + return; } if (arr.size() > BulkImport.DELETE_USERS_MAX_LIMIT) { diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java index 30c07597c..28906ba95 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/DeleteBulkImportUsersTest.java @@ -88,19 +88,6 @@ public void shouldReturn400Error() throws Exception { e.getMessage()); } } - { - try { - JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); - HttpRequestForTesting.sendJsonPOSTRequest(main, "", - "http://localhost:3567/bulk-import/users/remove", - request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - fail("The API should have thrown an error"); - } catch (io.supertokens.test.httpRequest.HttpResponseException e) { - assertEquals(400, e.statusCode); - assertEquals("Http error. Status Code: 400. Message: Field name 'ids' cannot be an empty array", - e.getMessage()); - } - } { try { // Create a string array of 500 uuids @@ -140,13 +127,13 @@ public void shouldReturn200Response() throws Exception { // Call the API with empty array { - JsonObject request = new JsonParser().parse("{\"ids\":[\"\"]}").getAsJsonObject(); - JsonObject resonse = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + JsonObject request = new JsonParser().parse("{\"ids\":[]}").getAsJsonObject(); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/users/remove", request, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); - assertEquals(0, resonse.get("deletedIds").getAsJsonArray().size()); - assertEquals(0, resonse.get("invalidIds").getAsJsonArray().size()); + assertEquals(0, response.get("deletedIds").getAsJsonArray().size()); + assertEquals(0, response.get("invalidIds").getAsJsonArray().size()); } { From 0d707359b97c6004c7572b8a9bbb9c9c81533291 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Fri, 20 Sep 2024 16:28:31 +0200 Subject: [PATCH 22/41] feat: multithreaded bulk import --- config.yaml | 4 + devConfig.yaml | 4 + .../io/supertokens/config/CoreConfig.java | 12 + .../bulkimport/ProcessBulkImportUsers.java | 297 +++------------- .../ProcessBulkUsersImportWorker.java | 321 ++++++++++++++++++ .../multitenancy/Multitenancy.java | 2 +- .../test/bulkimport/BulkImportTestUtils.java | 15 +- .../ProcessBulkImportUsersCronJobTest.java | 185 +++++++++- 8 files changed, 585 insertions(+), 255 deletions(-) create mode 100644 src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java diff --git a/config.yaml b/config.yaml index fdb96d4ba..1207c544e 100644 --- a/config.yaml +++ b/config.yaml @@ -151,3 +151,7 @@ core_config_version: 0 # (OPTIONAL | Default: null) string value. If specified, the supertokens service will only load the specified CUD even # if there are more CUDs in the database and block all other CUDs from being used from this instance. # supertokens_saas_load_only_cud: + +# (DIFFERENT_ACROSS_TENANTS | OPTIONAL | Default: 1) int value. If specified, the supertokens core will use the +# specified number of threads to complete the migration of users. +# bulk_migration_parallelism: diff --git a/devConfig.yaml b/devConfig.yaml index 276b35d42..ee901b068 100644 --- a/devConfig.yaml +++ b/devConfig.yaml @@ -151,3 +151,7 @@ disable_telemetry: true # (OPTIONAL | Default: null) string value. If specified, the supertokens service will only load the specified CUD even # if there are more CUDs in the database and block all other CUDs from being used from this instance. # supertokens_saas_load_only_cud: + +# (DIFFERENT_ACROSS_TENANTS | OPTIONAL | Default: 1) int value. If specified, the supertokens core will use the +# specified number of threads to complete the migration of users. +# bulk_migration_parallelism: diff --git a/src/main/java/io/supertokens/config/CoreConfig.java b/src/main/java/io/supertokens/config/CoreConfig.java index 3de06caa7..ce38880a5 100644 --- a/src/main/java/io/supertokens/config/CoreConfig.java +++ b/src/main/java/io/supertokens/config/CoreConfig.java @@ -209,6 +209,10 @@ public class CoreConfig { @IgnoreForAnnotationCheck private boolean isNormalizedAndValid = false; + @NotConflictingInApp + @JsonProperty + private int bulk_migration_parallelism = 1; + public static Set getValidFields() { CoreConfig coreConfig = new CoreConfig(); JsonObject coreConfigObj = new GsonBuilder().serializeNulls().create().toJsonTree(coreConfig).getAsJsonObject(); @@ -398,6 +402,10 @@ public boolean getHttpsEnabled() { return webserver_https_enabled; } + public int getBulkMigrationParallelism() { + return bulk_migration_parallelism; + } + private String getConfigFileLocation(Main main) { return new File(CLIOptions.get(main).getConfigFilePath() == null ? CLIOptions.get(main).getInstallationPath() + "config.yaml" @@ -590,6 +598,10 @@ void normalizeAndValidate(Main main, boolean includeConfigFilePath) throws Inval } } + if (bulk_migration_parallelism < 1) { + throw new InvalidConfigException("Provided bulk_migration_parallelism must be >= 1"); + } + // Normalize if (ip_allow_regex != null) { ip_allow_regex = ip_allow_regex.trim(); diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index a4b235da0..177ad3da2 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -16,48 +16,37 @@ package io.supertokens.cronjobs.bulkimport; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.google.gson.JsonObject; - import io.supertokens.Main; -import io.supertokens.ResourceDistributor; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserUtils; -import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; import io.supertokens.config.Config; import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; -import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.output.Logging; import io.supertokens.pluginInterface.STORAGE_TYPE; -import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; -import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; -import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.exceptions.DbInitException; -import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; -import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; -import io.supertokens.pluginInterface.sqlStorage.SQLStorage; import io.supertokens.storageLayer.StorageLayer; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + public class ProcessBulkImportUsers extends CronTask { public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; - private Map userPoolToStorageMap = new HashMap<>(); + private ProcessBulkImportUsers(Main main, List> tenantsInfo) { super("ProcessBulkImportUsers", main, tenantsInfo, true); @@ -71,7 +60,7 @@ public static ProcessBulkImportUsers init(Main main, List @Override protected void doTaskPerApp(AppIdentifier app) - throws TenantOrAppNotFoundException, StorageQueryException, IOException, DbInitException { + throws TenantOrAppNotFoundException, StorageQueryException { if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; @@ -86,8 +75,29 @@ protected void doTaskPerApp(AppIdentifier app) String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); - for (BulkImportUser user : users) { - processUser(app, user, bulkImportUserUtils, bulkImportSQLStorage); + //split the loaded users list into smaller chunks + int NUMBER_OF_BATCHES = Config.getConfig(app.getAsPublicTenantIdentifier(), main).getBulkMigrationParallelism(); + List> loadedUsersChunks = makeChunksOf(users, NUMBER_OF_BATCHES); + + //pass the chunks for processing for the workers + ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_BATCHES);; + try { + List> tasks = new ArrayList<>(); + for (List userListChunk : loadedUsersChunks) { + tasks.add(executorService.submit( + new ProcessBulkUsersImportWorker(main, app, userListChunk, bulkImportSQLStorage, + bulkImportUserUtils))); + } + + for (Future task : tasks) { + task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up + } + + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + finally { + executorService.shutdown(); } } @@ -113,230 +123,19 @@ public int getInitialWaitTimeSeconds() { return 0; } - private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) - throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { - String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); - if (userPoolToStorageMap.containsKey(userPoolId)) { - return userPoolToStorageMap.get(userPoolId); - } - - TenantConfig[] allTenants = Multitenancy.getAllTenants(main); - - Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( - allTenants, - Config.getBaseConfigAsJsonObject(main)); - - for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { - if (key.getTenantIdentifier().equals(tenantIdentifier)) { - SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, - normalisedConfigs.get(key), tenantIdentifier, true); - - userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); - bulkImportProxyStorage.initStorage(false, new ArrayList<>()); - return bulkImportProxyStorage; - } - } - throw new TenantOrAppNotFoundException(tenantIdentifier); - } - - private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) - throws StorageTransactionLogicException { - - try { - List allProxyStorages = new ArrayList<>(); - TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); - for (TenantConfig tenantConfig : tenantConfigs) { - allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); - } - return allProxyStorages.toArray(new Storage[0]); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E043: " + e.getMessage())); - } catch (InvalidConfigException e) { - throw new StorageTransactionLogicException(new InvalidConfigException("E044: " + e.getMessage())); - } catch (DbInitException e) { - throw new StorageTransactionLogicException(new DbInitException("E045: " + e.getMessage())); - } catch (IOException e) { - throw new StorageTransactionLogicException(new IOException("E046: " + e.getMessage())); + private List> makeChunksOf(List users, int numberOfChunks) { + List> chunks = new ArrayList<>(); + if (users != null && !users.isEmpty() && numberOfChunks > 0) { + AtomicInteger index = new AtomicInteger(0); + int chunkSize = users.size() / numberOfChunks + 1; + Stream> listStream = users.stream() + .collect(Collectors.groupingBy(x -> index.getAndIncrement() / chunkSize)) + .entrySet().stream() + .sorted(Map.Entry.comparingByKey()).map(Map.Entry::getValue); + + listStream.forEach(chunks::add); } + return chunks; } - private void closeAllProxyStorages() throws StorageQueryException { - for (SQLStorage storage : userPoolToStorageMap.values()) { - storage.closeConnectionForBulkImportProxyStorage(); - } - userPoolToStorageMap.clear(); - } - - private void processUser(AppIdentifier appIdentifier, BulkImportUser user, BulkImportUserUtils bulkImportUserUtils, - BulkImportSQLStorage baseTenantStorage) - throws TenantOrAppNotFoundException, StorageQueryException, IOException, - DbInitException { - - try { - if (Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) { - // Skip validation when the flag is enabled during testing - } else { - // Validate the user - bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id); - } - - // Since all the tenants of a user must share the storage, we will just use the - // storage of the first tenantId of the first loginMethod - - TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); - - SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); - - LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); - - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage( - firstTenantIdentifier); - - /* - * We use two separate storage instances: one for importing the user and another for managing bulk_import_users entries. - * This is necessary because the bulk_import_users entries are always in the public tenant storage, - * but the actual user data could be in a different storage. - * - * If transactions are committed individually, in this order: - * 1. Commit the transaction that imports the user. - * 2. Commit the transaction that deletes the corresponding bulk import entry. - * - * There's a risk where the first commit succeeds, but the second fails. This creates a situation where - * the bulk import entry is re-processed, even though the user has already been imported into the database. - * - * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. - * The processing logic now follows these steps: - * - * 1. Import the user and get the `primaryUserId` (transaction uncommitted). - * 2. Update the `primaryUserId` in the corresponding bulk import entry. - * 3. Commit the import transaction from step 1. - * 4. Delete the bulk import entry. - * - * If step 2 or any earlier step fails, nothing is committed, preventing partial state. - * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't exist in the database—this results in re-processing on the - * next run. - * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by deleting it in the next run. - * - * The following code implements this logic. - */ - if (user.primaryUserId != null) { - AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, - user.primaryUserId); - - if (importedUser != null && isProcessedUserFromSameBulkImportUserEntry(importedUser, user)) { - baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[] { user.id }); - return; - } - } - - bulkImportProxyStorage.startTransaction(con -> { - try { - Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, primaryLM, allStoragesForApp); - - // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle the inconsistent transaction commit. - // If this update statement fails then the outer transaction will fail as well and the user will simpl be processed again. No inconsistency will happen in this - // case. - baseTenantStorage.updateBulkImportUserPrimaryUserId(appIdentifier, user.id, - primaryLM.superTokensUserId); - - // We need to commit the transaction manually because we have overridden that in the proxy storage - // If this fails, the primaryUserId will be updated in the bulkImportUser but it wouldn’t actually exist. - // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user won't exist, and we'll simply re-process it. - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); - - // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a different storage than the baseTenantStorage - // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist in the database. - // When processing the user again, we'll check if primaryUserId exists with the same email. In this case the user will exist, and we'll simply delete the entry. - baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[] { user.id }); - return null; - } catch (StorageTransactionLogicException e) { - // We need to rollback the transaction manually because we have overridden that in the proxy storage - bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); - throw e; - } finally { - closeAllProxyStorages(); - } - }); - } catch (StorageTransactionLogicException | InvalidBulkImportDataException | InvalidConfigException e) { - handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); - } - } - - private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, - BulkImportSQLStorage baseTenantStorage) - throws StorageQueryException { - // Java doesn't allow us to reassign local variables inside a lambda expression - // so we have to use an array. - String[] errorMessage = { e.getMessage() }; - - if (e instanceof StorageTransactionLogicException) { - StorageTransactionLogicException exception = (StorageTransactionLogicException) e; - // If the exception is due to a StorageQueryException, we want to retry the entry after sometime instead - // of marking it as FAILED. We will return early in that case. - if (exception.actualException instanceof StorageQueryException) { - Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); - return; - } - errorMessage[0] = exception.actualException.getMessage(); - } else if (e instanceof InvalidBulkImportDataException) { - errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); - } else if (e instanceof InvalidConfigException) { - errorMessage[0] = e.getMessage(); - } - - try { - baseTenantStorage.startTransaction(con -> { - baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, - BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); - return null; - }); - } catch (StorageTransactionLogicException e1) { - throw new StorageQueryException(e1.actualException); - } - } - - // Checks if the importedUser was processed from the same bulkImportUser entry. - private boolean isProcessedUserFromSameBulkImportUserEntry( - AuthRecipeUserInfo importedUser, BulkImportUser bulkImportEntry) { - if (bulkImportEntry == null || importedUser == null || bulkImportEntry.loginMethods == null || - importedUser.loginMethods == null) { - return false; - } - - for (LoginMethod lm1 : bulkImportEntry.loginMethods) { - for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : importedUser.loginMethods) { - if (lm2.recipeId.toString().equals(lm1.recipeId)) { - if (lm1.email != null && !lm1.email.equals(lm2.email)) { - return false; - } - - switch (lm1.recipeId) { - case "emailpassword": - if (lm1.passwordHash != null && !lm1.passwordHash.equals(lm2.passwordHash)) { - return false; - } - break; - case "thirdparty": - if ((lm1.thirdPartyId != null && !lm1.thirdPartyId.equals(lm2.thirdParty.id)) - || (lm1.thirdPartyUserId != null - && !lm1.thirdPartyUserId.equals(lm2.thirdParty.userId))) { - return false; - } - break; - case "passwordless": - if (lm1.phoneNumber != null && !lm1.phoneNumber.equals(lm2.phoneNumber)) { - return false; - } - break; - default: - return false; - } - } - } - } - - return true; - } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java new file mode 100644 index 000000000..8122553d8 --- /dev/null +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.cronjobs.bulkimport; + +import com.google.gson.JsonObject; +import io.supertokens.Main; +import io.supertokens.ResourceDistributor; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportUserUtils; +import io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException; +import io.supertokens.config.Config; +import io.supertokens.multitenancy.Multitenancy; +import io.supertokens.output.Logging; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.exceptions.DbInitException; +import io.supertokens.pluginInterface.exceptions.InvalidConfigException; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.storageLayer.StorageLayer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ProcessBulkUsersImportWorker implements Runnable { + + private final Map userPoolToStorageMap = new HashMap<>(); + private final Main main; + private final AppIdentifier app; + private final List usersToImport; + private final BulkImportSQLStorage bulkImportSQLStorage; + private final BulkImportUserUtils bulkImportUserUtils; + + ProcessBulkUsersImportWorker(Main main, AppIdentifier app, List userListToImport, BulkImportSQLStorage bulkImportSQLStorage, BulkImportUserUtils bulkImportUserUtils){ + this.main = main; + this.app = app; + this.usersToImport = userListToImport; + this.bulkImportSQLStorage = bulkImportSQLStorage; + this.bulkImportUserUtils = bulkImportUserUtils; + } + + @Override + public void run() { + try { + processMultipleUsers(app, usersToImport, bulkImportUserUtils, bulkImportSQLStorage); + } catch (TenantOrAppNotFoundException | DbInitException | IOException | StorageQueryException e) { + throw new RuntimeException(e); + } + } + + private void processMultipleUsers(AppIdentifier appIdentifier, List users, + BulkImportUserUtils bulkImportUserUtils, + BulkImportSQLStorage baseTenantStorage) + throws TenantOrAppNotFoundException, StorageQueryException, IOException, + DbInitException { + + BulkImportUser lastStartedUser = null; + try { + for (BulkImportUser user : users) { + lastStartedUser = user; + if (Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) { + // Skip validation when the flag is enabled during testing + } else { + // Validate the user + bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id); + } + + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); + + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); + BulkImportUser.LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); + + AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage( + firstTenantIdentifier); + + /* + * We use two separate storage instances: one for importing the user and another for managing + * bulk_import_users entries. + * This is necessary because the bulk_import_users entries are always in the public tenant storage, + * but the actual user data could be in a different storage. + * + * If transactions are committed individually, in this order: + * 1. Commit the transaction that imports the user. + * 2. Commit the transaction that deletes the corresponding bulk import entry. + * + * There's a risk where the first commit succeeds, but the second fails. This creates a situation where + * the bulk import entry is re-processed, even though the user has already been imported into the + * database. + * + * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. + * The processing logic now follows these steps: + * + * 1. Import the user and get the `primaryUserId` (transaction uncommitted). + * 2. Update the `primaryUserId` in the corresponding bulk import entry. + * 3. Commit the import transaction from step 1. + * 4. Delete the bulk import entry. + * + * If step 2 or any earlier step fails, nothing is committed, preventing partial state. + * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't + * exist in the database—this results in re-processing on the + * next run. + * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by + * deleting it in the next run. + * + * The following code implements this logic. + */ + if (user.primaryUserId != null) { + AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, + user.primaryUserId); + + if (importedUser != null && isProcessedUserFromSameBulkImportUserEntry(importedUser, user)) { + baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[]{user.id}); + return; + } + } + + bulkImportProxyStorage.startTransaction(con -> { + try { + Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, + primaryLM, allStoragesForApp); + + // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle + // the inconsistent transaction commit. + // If this update statement fails then the outer transaction will fail as well and the user + // will simpl be processed again. No inconsistency will happen in this + // case. + baseTenantStorage.updateBulkImportUserPrimaryUserId(appIdentifier, user.id, + primaryLM.superTokensUserId); + + // We need to commit the transaction manually because we have overridden that in the proxy + // storage + // If this fails, the primaryUserId will be updated in the bulkImportUser but it wouldn’t + // actually exist. + // When processing the user again, we'll check if primaryUserId exists with the same email. + // In this case the user won't exist, and we'll simply re-process it. + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + + // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a + // different storage than the baseTenantStorage + // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist + // in the database. + // When processing the user again, we'll check if primaryUserId exists with the same email. + // In this case the user will exist, and we'll simply delete the entry. + baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[]{user.id}); + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy + // storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + handleProcessUserExceptions(app, user, e, baseTenantStorage); + } + return null; + }); + + } + } catch (StorageTransactionLogicException | InvalidBulkImportDataException | InvalidConfigException e) { + handleProcessUserExceptions(appIdentifier, lastStartedUser, e, baseTenantStorage); + } finally { + closeAllProxyStorages(); //closing it here to reuse the existing connection with all the users + } + } + + private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImportUser user, Exception e, + BulkImportSQLStorage baseTenantStorage) + throws StorageQueryException { + // Java doesn't allow us to reassign local variables inside a lambda expression + // so we have to use an array. + String[] errorMessage = { e.getMessage() }; + + if (e instanceof StorageTransactionLogicException) { + StorageTransactionLogicException exception = (StorageTransactionLogicException) e; + // If the exception is due to a StorageQueryException, we want to retry the entry after sometime instead + // of marking it as FAILED. We will return early in that case. + if (exception.actualException instanceof StorageQueryException) { + Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); + return; + } + errorMessage[0] = exception.actualException.getMessage(); + } else if (e instanceof InvalidBulkImportDataException) { + errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); + } else if (e instanceof InvalidConfigException) { + errorMessage[0] = e.getMessage(); + } + + try { + baseTenantStorage.startTransaction(con -> { + baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); + return null; + }); + } catch (StorageTransactionLogicException e1) { + throw new StorageQueryException(e1.actualException); + } + } + + private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) + throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { + String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); + if (userPoolToStorageMap.containsKey(userPoolId)) { + return userPoolToStorageMap.get(userPoolId); + } + + TenantConfig[] allTenants = Multitenancy.getAllTenants(main); + + Map normalisedConfigs = Config.getNormalisedConfigsForAllTenants( + allTenants, + Config.getBaseConfigAsJsonObject(main)); + + for (ResourceDistributor.KeyClass key : normalisedConfigs.keySet()) { + if (key.getTenantIdentifier().equals(tenantIdentifier)) { + SQLStorage bulkImportProxyStorage = (SQLStorage) StorageLayer.getNewBulkImportProxyStorageInstance(main, + normalisedConfigs.get(key), tenantIdentifier, true); + + userPoolToStorageMap.put(userPoolId, bulkImportProxyStorage); + bulkImportProxyStorage.initStorage(false, new ArrayList<>()); + return bulkImportProxyStorage; + } + } + throw new TenantOrAppNotFoundException(tenantIdentifier); + } + + private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + + try { + List allProxyStorages = new ArrayList<>(); + TenantConfig[] tenantConfigs = Multitenancy.getAllTenantsForApp(appIdentifier, main); + for (TenantConfig tenantConfig : tenantConfigs) { + allProxyStorages.add(getBulkImportProxyStorage(tenantConfig.tenantIdentifier)); + } + return allProxyStorages.toArray(new Storage[0]); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E043: " + e.getMessage())); + } catch (InvalidConfigException e) { + throw new StorageTransactionLogicException(new InvalidConfigException("E044: " + e.getMessage())); + } catch (DbInitException e) { + throw new StorageTransactionLogicException(new DbInitException("E045: " + e.getMessage())); + } catch (IOException e) { + throw new StorageTransactionLogicException(new IOException("E046: " + e.getMessage())); + } + } + + private void closeAllProxyStorages() throws StorageQueryException { + for (SQLStorage storage : userPoolToStorageMap.values()) { + storage.closeConnectionForBulkImportProxyStorage(); + } + userPoolToStorageMap.clear(); + } + + // Checks if the importedUser was processed from the same bulkImportUser entry. + private boolean isProcessedUserFromSameBulkImportUserEntry( + AuthRecipeUserInfo importedUser, BulkImportUser bulkImportEntry) { + if (bulkImportEntry == null || importedUser == null || bulkImportEntry.loginMethods == null || + importedUser.loginMethods == null) { + return false; + } + + for (BulkImportUser.LoginMethod lm1 : bulkImportEntry.loginMethods) { + for (io.supertokens.pluginInterface.authRecipe.LoginMethod lm2 : importedUser.loginMethods) { + if (lm2.recipeId.toString().equals(lm1.recipeId)) { + if (lm1.email != null && !lm1.email.equals(lm2.email)) { + return false; + } + + switch (lm1.recipeId) { + case "emailpassword": + if (lm1.passwordHash != null && !lm1.passwordHash.equals(lm2.passwordHash)) { + return false; + } + break; + case "thirdparty": + if ((lm1.thirdPartyId != null && !lm1.thirdPartyId.equals(lm2.thirdParty.id)) + || (lm1.thirdPartyUserId != null + && !lm1.thirdPartyUserId.equals(lm2.thirdParty.userId))) { + return false; + } + break; + case "passwordless": + if (lm1.phoneNumber != null && !lm1.phoneNumber.equals(lm2.phoneNumber)) { + return false; + } + break; + default: + return false; + } + } + } + } + + return true; + } +} diff --git a/src/main/java/io/supertokens/multitenancy/Multitenancy.java b/src/main/java/io/supertokens/multitenancy/Multitenancy.java index 2342599dc..88ce910da 100644 --- a/src/main/java/io/supertokens/multitenancy/Multitenancy.java +++ b/src/main/java/io/supertokens/multitenancy/Multitenancy.java @@ -269,7 +269,7 @@ public static boolean addNewOrUpdateAppOrTenant(Main main, TenantConfig newTenan } - public static boolean addNewOrUpdateAppOrTenant(Main main, TenantConfig newTenant, + public synchronized static boolean addNewOrUpdateAppOrTenant(Main main, TenantConfig newTenant, boolean shouldPreventProtectedConfigUpdate, boolean skipThirdPartyConfigValidation, boolean forceReloadResources) diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 61740021b..1788316d0 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -28,6 +28,8 @@ import com.google.gson.JsonParser; import io.supertokens.Main; +import io.supertokens.config.Config; +import io.supertokens.config.CoreConfig; import io.supertokens.emailpassword.PasswordHashing; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; @@ -62,6 +64,10 @@ public static List generateBulkImportUser(int numberOfUsers) { } public static List generateBulkImportUser(int numberOfUsers, List tenants, int startIndex) { + return generateBulkImportUserWithRoles(numberOfUsers, tenants, startIndex, List.of("role1", "role2")); + } + + public static List generateBulkImportUserWithRoles(int numberOfUsers, List tenants, int startIndex, List roles) { List users = new ArrayList<>(); JsonParser parser = new JsonParser(); @@ -74,8 +80,9 @@ public static List generateBulkImportUser(int numberOfUsers, Lis .getAsJsonObject(); List userRoles = new ArrayList<>(); - userRoles.add(new UserRole("role1", tenants)); - userRoles.add(new UserRole("role2", tenants)); + for(String roleName : roles) { + userRoles.add(new UserRole(roleName, tenants)); + } List totpDevices = new ArrayList<>(); totpDevices.add(new TotpDevice("secretKey", 30, 1, "deviceName")); @@ -113,10 +120,10 @@ public static void createTenants(Main main) new EmailPasswordConfig(true), new ThirdPartyConfig(true, null), new PasswordlessConfig(true), - null, null, new JsonObject())); + null, null, Config.getBaseConfigAsJsonObject(main))); } { // tenant 2 - JsonObject config = new JsonObject(); + JsonObject config = Config.getBaseConfigAsJsonObject(main); TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 667287f5a..063362031 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -22,6 +22,7 @@ import io.supertokens.authRecipe.AuthRecipe; import io.supertokens.authRecipe.UserPaginationContainer; import io.supertokens.bulkimport.BulkImport; +import io.supertokens.config.Config; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.featureflag.EE_FEATURES; @@ -47,9 +48,11 @@ import org.junit.rules.TestRule; import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; +import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUserWithRoles; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import java.io.IOException; import java.util.List; public class ProcessBulkImportUsersCronJobTest { @@ -114,6 +117,85 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldProcessBulkImportUsersInNotSoLargeNumbersInTheSameTenant() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 15; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(6000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 1000, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + + @Test + public void shouldProcessBulkImportUsersInLargeNumbersInTheSameTenant() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + int usersCount = 1000; + List users = generateBulkImportUser(usersCount); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(60000); // 1 minute + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + UserPaginationContainer container = AuthRecipe.getUsers(main, 1000, "ASC", null, null, null); + assertEquals(usersCount, container.users.length); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + @Test public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() throws Exception { TestingProcess process = startCronProcess(); @@ -175,7 +257,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() } @Test - public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { + public void shouldDeleteEverythingFromTheDBIfAnythingFails() throws Exception { // Creating a non-existing user role will result in an error. // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. @@ -194,6 +276,8 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); AppIdentifier appIdentifier = new AppIdentifier(null, null); + // note the missing role creation here! + List users = generateBulkImportUser(1); BulkImport.addUsers(appIdentifier, storage, users); @@ -212,6 +296,105 @@ public void shouldDeleteEverythingFromtheDBIfAnythingFails() throws Exception { assertEquals(0, container.users.length); } + + @Test + public void shouldDeleteEverythingFromTheDBIfAnythingFailsOnMultipleThreads() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // note the missing role creation here! + + List users = generateBulkImportUser(100); + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(60000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(100, usersAfterProcessing.size()); + + for(BulkImportUser userAfterProcessing: usersAfterProcessing){ + assertEquals(BULK_IMPORT_USER_STATUS.FAILED, userAfterProcessing.status); // should process every user and every one of them should fail because of the missing role + assertEquals("E034: Role role1 does not exist! You need pre-create the role before assigning it to the user.", + userAfterProcessing.errorMessage); + } + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(0, container.users.length); + } + + @Test + public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + // Creating a non-existing user role will result in an error. + // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. + + // NOTE: We will also need to disable the bulk import user validation in the cron job for this test to work. + Main.isTesting_skipBulkImportUserValidationInCronJob = true; + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + + BulkImportTestUtils.createTenants(main); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + // Create one user role before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + } + + List users = generateBulkImportUserWithRoles(99, List.of("public", "t1"), 0, List.of("role1")); + users.addAll(generateBulkImportUserWithRoles(1, List.of("public", "t1"), 99, List.of("notExistingRole"))); + + BulkImport.addUsers(appIdentifier, storage, users); + + Thread.sleep(60000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, + null, null); + + assertEquals(1, usersAfterProcessing.size()); + + int numberOfFailed = 0; + for(int i = 0; i < usersAfterProcessing.size(); i++){ + if(usersAfterProcessing.get(i).status == BULK_IMPORT_USER_STATUS.FAILED) { + assertEquals( + "E034: Role notExistingRole does not exist! You need pre-create the role before assigning it to the user.", + usersAfterProcessing.get(i).errorMessage); + numberOfFailed++; + } + } + + UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); + assertEquals(99, container.users.length); + assertEquals(1, numberOfFailed); + } + + @Test public void shouldThrowTenantDoesNotExistError() throws Exception { TestingProcess process = startCronProcess(); From b78541eb22d12cd2d1ed14b768b29333834b03bf Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Fri, 20 Sep 2024 16:31:48 +0200 Subject: [PATCH 23/41] fix: changelog update --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f24f9d9d6..693183195 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + - Adds multithreaded worker support for the `ProcessBulkImportUsers` cron job for faster bulk imports + - Add property `bulk_migration_parallelism` for fine-tuning the worker threads number + ## [9.1.0] - 2024-04-25 ### Added From 0c0dea8110a543ae17de614089751c8fbb892c89 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Wed, 25 Sep 2024 11:07:40 +0200 Subject: [PATCH 24/41] fix: add new test --- .../test/bulkimport/BulkImportTestUtils.java | 4 +- .../ProcessBulkImportUsersCronJobTest.java | 63 ++++++++++++++++++- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java index 1788316d0..1f434fa48 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportTestUtils.java @@ -120,10 +120,10 @@ public static void createTenants(Main main) new EmailPasswordConfig(true), new ThirdPartyConfig(true, null), new PasswordlessConfig(true), - null, null, Config.getBaseConfigAsJsonObject(main))); + null, null, new JsonObject())); } { // tenant 2 - JsonObject config = Config.getBaseConfigAsJsonObject(main); + JsonObject config = new JsonObject(); TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t2"); StorageLayer.getStorage(new TenantIdentifier(null, null, null), main) diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 063362031..3e82171a0 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -53,6 +53,7 @@ import static org.junit.Assert.assertNotNull; import java.io.IOException; +import java.util.ArrayList; import java.util.List; public class ProcessBulkImportUsersCronJobTest { @@ -119,7 +120,7 @@ public void shouldProcessBulkImportUsersInTheSameTenant() throws Exception { @Test public void shouldProcessBulkImportUsersInNotSoLargeNumbersInTheSameTenant() throws Exception { - Utils.setValueInConfig("bulk_migration_parallelism", "12"); + Utils.setValueInConfig("bulk_migration_parallelism", "2"); TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -182,7 +183,7 @@ public void shouldProcessBulkImportUsersInLargeNumbersInTheSameTenant() throws E List users = generateBulkImportUser(usersCount); BulkImport.addUsers(appIdentifier, storage, users); - Thread.sleep(60000); // 1 minute + Thread.sleep(2 * 60000); // minute List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, null, null); @@ -256,6 +257,62 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); } + @Test + public void shouldProcessBulkImportUsersInLargeNumberInMultipleTenantsWithDifferentStorages() throws Exception { + Utils.setValueInConfig("bulk_migration_parallelism", "4"); + + TestingProcess process = startCronProcess(); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + BulkImportTestUtils.createTenants(main); + + TenantIdentifier t1 = new TenantIdentifier(null, null, "t1"); + TenantIdentifier t2 = new TenantIdentifier(null, null, "t2"); + + BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); + AppIdentifier appIdentifier = new AppIdentifier(null, null); + + List usersT1 = generateBulkImportUser(500, List.of(t1.getTenantId()), 0); + List usersT2 = generateBulkImportUser(500, List.of(t2.getTenantId()), 500); + + List allUsers = new ArrayList<>(); + allUsers.addAll(usersT1); + allUsers.addAll(usersT2); + + BulkImport.addUsers(appIdentifier, storage, allUsers); + + Thread.sleep(2 * 60000); + + List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, + null, null); + + assertEquals(0, usersAfterProcessing.size()); + + Storage storageT1 = StorageLayer.getStorage(t1, main); + Storage storageT2 = StorageLayer.getStorage(t2, main); + + UserPaginationContainer containerT1 = AuthRecipe.getUsers(t1, storageT1, 500, "ASC", null, null, null); + UserPaginationContainer containerT2 = AuthRecipe.getUsers(t2, storageT2, 500, "ASC", null, null, null); + + assertEquals(usersT1.size() + usersT2.size(), containerT1.users.length + containerT2.users.length); + + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT1, containerT1.users); + UserIdMapping.populateExternalUserIdForUsers(appIdentifier, storageT2, containerT2.users); + + process.kill(); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STOPPED)); + } + @Test public void shouldDeleteEverythingFromTheDBIfAnythingFails() throws Exception { // Creating a non-existing user role will result in an error. @@ -372,7 +429,7 @@ public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() th BulkImport.addUsers(appIdentifier, storage, users); - Thread.sleep(60000); + Thread.sleep(2 * 60000); List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); From 67c42e938ec18aeed101ac354d3c342b0bdabbb3 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Fri, 27 Sep 2024 12:13:38 +0200 Subject: [PATCH 25/41] fix: fixing unreliable mutithreaded bulk import with mysql --- .../bulkimport/BulkImportUserUtils.java | 2 +- .../ProcessBulkUsersImportWorker.java | 44 ++++++++++++++----- .../ProcessBulkImportUsersCronJobTest.java | 12 ++--- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index 7c6aac760..f2df6892e 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -265,7 +265,7 @@ private String validateAndNormaliseExternalUserId(String externalUserId, List 255) { - errors.add("externalUserId " + externalUserId + " is too long. Max length is 128."); + errors.add("externalUserId " + externalUserId + " is too long. Max length is 255."); } if (!allExternalUserIds.add(externalUserId)) { diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index 8122553d8..ef7dc9a6e 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -30,6 +30,7 @@ import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportTransactionRolledBackException; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; @@ -80,12 +81,16 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List { + BulkImportUser finalUser = user; + shouldRetryImmediately = bulkImportProxyStorage.startTransaction(con -> { try { Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, + BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, finalUser, primaryLM, allStoragesForApp); // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle @@ -154,7 +160,7 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List users = generateBulkImportUser(usersCount); BulkImport.addUsers(appIdentifier, storage, users); - Thread.sleep(6000); + Thread.sleep(60000); List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 1000, null, null, null); @@ -159,7 +159,7 @@ public void shouldProcessBulkImportUsersInNotSoLargeNumbersInTheSameTenant() thr @Test public void shouldProcessBulkImportUsersInLargeNumbersInTheSameTenant() throws Exception { - Utils.setValueInConfig("bulk_migration_parallelism", "12"); + Utils.setValueInConfig("bulk_migration_parallelism", "8"); TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -259,7 +259,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() @Test public void shouldProcessBulkImportUsersInLargeNumberInMultipleTenantsWithDifferentStorages() throws Exception { - Utils.setValueInConfig("bulk_migration_parallelism", "4"); + Utils.setValueInConfig("bulk_migration_parallelism", "8"); TestingProcess process = startCronProcess(); Main main = process.getProcess(); @@ -356,7 +356,7 @@ public void shouldDeleteEverythingFromTheDBIfAnythingFails() throws Exception { @Test public void shouldDeleteEverythingFromTheDBIfAnythingFailsOnMultipleThreads() throws Exception { - Utils.setValueInConfig("bulk_migration_parallelism", "12"); + Utils.setValueInConfig("bulk_migration_parallelism", "8"); // Creating a non-existing user role will result in an error. // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. @@ -399,7 +399,7 @@ public void shouldDeleteEverythingFromTheDBIfAnythingFailsOnMultipleThreads() th @Test public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() throws Exception { - Utils.setValueInConfig("bulk_migration_parallelism", "12"); + Utils.setValueInConfig("bulk_migration_parallelism", "8"); // Creating a non-existing user role will result in an error. // Since, user role creation happens at the last step of the bulk import process, everything should be deleted from the DB. From 807f617644c8dfa1d9bbe5b0e387599bb9d3bcdc Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Tue, 1 Oct 2024 16:01:03 +0200 Subject: [PATCH 26/41] fix: review fixes --- config.yaml | 5 +++-- devConfig.yaml | 4 ++-- .../java/io/supertokens/bulkimport/BulkImportUserUtils.java | 4 ++-- src/main/java/io/supertokens/config/CoreConfig.java | 4 +++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config.yaml b/config.yaml index 970c69d5e..11d7d03d9 100644 --- a/config.yaml +++ b/config.yaml @@ -152,6 +152,7 @@ core_config_version: 0 # if there are more CUDs in the database and block all other CUDs from being used from this instance. # supertokens_saas_load_only_cud: -# (DIFFERENT_ACROSS_TENANTS | OPTIONAL | Default: 1) int value. If specified, the supertokens core will use the -# specified number of threads to complete the migration of users. +# (DIFFERENT_ACROSS_TENANTS | OPTIONAL | Default: number of available processor cores) int value. If specified, +# the supertokens core will use the specified number of threads to complete the migration of users. # bulk_migration_parallelism: + diff --git a/devConfig.yaml b/devConfig.yaml index cc47cc807..853cc8b6f 100644 --- a/devConfig.yaml +++ b/devConfig.yaml @@ -152,6 +152,6 @@ disable_telemetry: true # if there are more CUDs in the database and block all other CUDs from being used from this instance. # supertokens_saas_load_only_cud: -# (DIFFERENT_ACROSS_TENANTS | OPTIONAL | Default: 1) int value. If specified, the supertokens core will use the -# specified number of threads to complete the migration of users. +# (DIFFERENT_ACROSS_TENANTS | OPTIONAL | Default: number of available processor cores) int value. If specified, +# the supertokens core will use the specified number of threads to complete the migration of users. # bulk_migration_parallelism: diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java index f2df6892e..7b70d27aa 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImportUserUtils.java @@ -264,8 +264,8 @@ private String validateAndNormaliseExternalUserId(String externalUserId, List 255) { - errors.add("externalUserId " + externalUserId + " is too long. Max length is 255."); + if (externalUserId.length() > 128) { + errors.add("externalUserId " + externalUserId + " is too long. Max length is 128."); } if (!allExternalUserIds.add(externalUserId)) { diff --git a/src/main/java/io/supertokens/config/CoreConfig.java b/src/main/java/io/supertokens/config/CoreConfig.java index d1384ae73..9e6957559 100644 --- a/src/main/java/io/supertokens/config/CoreConfig.java +++ b/src/main/java/io/supertokens/config/CoreConfig.java @@ -307,7 +307,9 @@ public class CoreConfig { @NotConflictingInApp @JsonProperty - private int bulk_migration_parallelism = 1; + @ConfigDescription("If specified, the supertokens core will use the specified number of threads to complete the " + + "migration of users. (Default: number of available processor cores).") + private int bulk_migration_parallelism = Runtime.getRuntime().availableProcessors(); public static Set getValidFields() { CoreConfig coreConfig = new CoreConfig(); From f7cc349f3938edd2c5dfc71ba959b77c26ff0715 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Wed, 2 Oct 2024 10:55:31 +0200 Subject: [PATCH 27/41] fix: fixing failing tests --- src/main/java/io/supertokens/inmemorydb/Start.java | 3 +++ src/test/java/io/supertokens/test/CronjobTest.java | 13 ++++++++----- .../ProcessBulkImportUsersCronJobTest.java | 2 +- .../test/multitenant/AppTenantUserTest.java | 7 +++++-- .../supertokens/test/multitenant/TestAppData.java | 7 +++++++ .../multitenant/api/TestTenantUserAssociation.java | 3 +++ .../test/userIdMapping/UserIdMappingTest.java | 6 ++++-- 7 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 35bf45ce0..2ce5aee4a 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -26,6 +26,7 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.dashboard.DashboardSearchTags; import io.supertokens.pluginInterface.dashboard.DashboardSessionInfo; import io.supertokens.pluginInterface.dashboard.DashboardUser; @@ -733,6 +734,8 @@ public void addInfoToNonAuthRecipesBasedOnUserId(TenantIdentifier tenantIdentifi } } else if (className.equals(JWTRecipeStorage.class.getName())) { /* Since JWT recipe tables do not store userId we do not add any data to them */ + } else if (className.equals(BulkImportStorage.class.getName())){ + //ignore } else if (className.equals(ActiveUsersStorage.class.getName())) { try { ActiveUsersQueries.updateUserLastActive(this, tenantIdentifier.toAppIdentifier(), userId); diff --git a/src/test/java/io/supertokens/test/CronjobTest.java b/src/test/java/io/supertokens/test/CronjobTest.java index 6d72addb8..24e8ce7a5 100644 --- a/src/test/java/io/supertokens/test/CronjobTest.java +++ b/src/test/java/io/supertokens/test/CronjobTest.java @@ -22,6 +22,7 @@ import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.cronjobs.syncCoreConfigWithDb.SyncCoreConfigWithDb; import io.supertokens.exceptions.QuitProgramException; import io.supertokens.featureflag.EE_FEATURES; @@ -965,7 +966,7 @@ public void testThatCronJobsHaveTenantsInfoAfterRestart() throws Exception { { List>> tenantsInfos = Cronjobs.getInstance(process.getProcess()) .getTenantInfos(); - assertEquals(10, tenantsInfos.size()); + assertEquals(11, tenantsInfos.size()); int count = 0; for (List> tenantsInfo : tenantsInfos) { if (tenantsInfo != null) { @@ -975,7 +976,7 @@ public void testThatCronJobsHaveTenantsInfoAfterRestart() throws Exception { count++; } } - assertEquals(9, count); + assertEquals(10, count); } process.kill(false); @@ -992,7 +993,7 @@ public void testThatCronJobsHaveTenantsInfoAfterRestart() throws Exception { { List>> tenantsInfos = Cronjobs.getInstance(process.getProcess()) .getTenantInfos(); - assertEquals(10, tenantsInfos.size()); + assertEquals(11, tenantsInfos.size()); int count = 0; for (List> tenantsInfo : tenantsInfos) { if (tenantsInfo != null) { @@ -1002,7 +1003,7 @@ public void testThatCronJobsHaveTenantsInfoAfterRestart() throws Exception { count++; } } - assertEquals(9, count); + assertEquals(10, count); } process.kill(); @@ -1049,6 +1050,7 @@ public void testThatThereAreTasksOfAllCronTaskClassesAndHaveCorrectIntervals() t intervals.put("io.supertokens.cronjobs.telemetry.Telemetry", 86400); intervals.put("io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys", 86400); + intervals.put("io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers", 60); Map delays = new HashMap<>(); delays.put("io.supertokens.ee.cronjobs.EELicenseCheck", 86400); @@ -1063,9 +1065,10 @@ public void testThatThereAreTasksOfAllCronTaskClassesAndHaveCorrectIntervals() t delays.put("io.supertokens.cronjobs.telemetry.Telemetry", 0); delays.put("io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys", 0); + delays.put("io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers", 0); List allTasks = Cronjobs.getInstance(process.getProcess()).getTasks(); - assertEquals(10, allTasks.size()); + assertEquals(11, allTasks.size()); for (CronTask task : allTasks) { assertEquals(intervals.get(task.getClass().getName()).intValue(), task.getIntervalTimeSeconds()); diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 97f14daf4..c948a9457 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -259,7 +259,7 @@ public void shouldProcessBulkImportUsersInMultipleTenantsWithDifferentStorages() @Test public void shouldProcessBulkImportUsersInLargeNumberInMultipleTenantsWithDifferentStorages() throws Exception { - Utils.setValueInConfig("bulk_migration_parallelism", "8"); + Utils.setValueInConfig("bulk_migration_parallelism", "12"); TestingProcess process = startCronProcess(); Main main = process.getProcess(); diff --git a/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java b/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java index 0dc0f0565..b2913796d 100644 --- a/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java +++ b/src/test/java/io/supertokens/test/multitenant/AppTenantUserTest.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.multitenancy.*; import io.supertokens.pluginInterface.nonAuthRecipe.NonAuthRecipeStorage; import io.supertokens.storageLayer.StorageLayer; @@ -77,7 +78,8 @@ public void testDeletingAppDeleteNonAuthRecipeData() throws Exception { // this list contains the package names for recipes which dont use UserIdMapping ArrayList classesToSkip = new ArrayList<>( - List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName())); + List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName(), + BulkImportStorage.class.getName())); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); Set> classes = reflections.getSubTypesOf(NonAuthRecipeStorage.class); @@ -182,7 +184,8 @@ public void testDisassociationOfUserDeletesNonAuthRecipeData() throws Exception // this list contains the package names for recipes which dont use UserIdMapping ArrayList classesToSkip = new ArrayList<>( - List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName())); + List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName(), + BulkImportStorage.class.getName())); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); Set> classes = reflections.getSubTypesOf(NonAuthRecipeStorage.class); diff --git a/src/test/java/io/supertokens/test/multitenant/TestAppData.java b/src/test/java/io/supertokens/test/multitenant/TestAppData.java index dd2315aed..14047a72e 100644 --- a/src/test/java/io/supertokens/test/multitenant/TestAppData.java +++ b/src/test/java/io/supertokens/test/multitenant/TestAppData.java @@ -21,6 +21,7 @@ import io.supertokens.ActiveUsers; import io.supertokens.Main; import io.supertokens.ProcessState; +import io.supertokens.bulkimport.BulkImport; import io.supertokens.dashboard.Dashboard; import io.supertokens.emailpassword.EmailPassword; import io.supertokens.emailverification.EmailVerification; @@ -31,6 +32,8 @@ import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.multitenancy.*; import io.supertokens.pluginInterface.totp.TOTPDevice; @@ -38,6 +41,7 @@ import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; +import io.supertokens.test.bulkimport.BulkImportTestUtils; import io.supertokens.thirdparty.ThirdParty; import io.supertokens.totp.Totp; import io.supertokens.useridmapping.UserIdMapping; @@ -55,6 +59,7 @@ import java.security.Key; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import static org.junit.Assert.assertEquals; @@ -174,6 +179,8 @@ null, null, new JsonObject() UserIdMapping.createUserIdMapping(process.getProcess(), app.toAppIdentifier(), appStorage, plUser.user.getSupertokensUserId(), "externalid", null, false); + BulkImport.addUsers(app.toAppIdentifier(), appStorage, BulkImportTestUtils.generateBulkImportUser(1)); + String[] tablesThatHaveData = appStorage .getAllTablesInTheDatabaseThatHasDataForAppId(app.getAppId()); tablesThatHaveData = removeStrings(tablesThatHaveData, tablesToIgnore); diff --git a/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java b/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java index 6907017cb..ccc4c6181 100644 --- a/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java +++ b/src/test/java/io/supertokens/test/multitenant/api/TestTenantUserAssociation.java @@ -31,6 +31,7 @@ import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.jwt.JWTRecipeStorage; @@ -199,10 +200,12 @@ public void testUserDisassociationForNotAuthRecipes() throws Exception { if (name.equals(UserMetadataStorage.class.getName()) || name.equals(JWTRecipeStorage.class.getName()) || name.equals(ActiveUsersStorage.class.getName()) + || name.equals(BulkImportStorage.class.getName()) ) { // user metadata is app specific and does not have any tenant specific data // JWT storage does not have any user specific data // Active users storage does not have tenant specific data + // BulkImportStorage continue; } diff --git a/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java b/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java index 014c1c6e9..13868666e 100644 --- a/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java +++ b/src/test/java/io/supertokens/test/userIdMapping/UserIdMappingTest.java @@ -25,6 +25,7 @@ import io.supertokens.pluginInterface.ActiveUsersStorage; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.nonAuthRecipe.NonAuthRecipeStorage; @@ -800,7 +801,8 @@ public void checkThatCreateUserIdMappingHasAllNonAuthRecipeChecks() throws Excep // this list contains the package names for recipes which dont use UserIdMapping ArrayList nonAuthRecipesWhichDontNeedUserIdMapping = new ArrayList<>( - List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName())); + List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName(), + BulkImportStorage.class.getName())); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); Set> classes = reflections.getSubTypesOf(NonAuthRecipeStorage.class); @@ -881,7 +883,7 @@ public void checkThatDeleteUserIdMappingHasAllNonAuthRecipeChecks() throws Excep } ArrayList nonAuthRecipesWhichDontNeedUserIdMapping = new ArrayList<>( - List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName())); + List.of("io.supertokens.pluginInterface.jwt.JWTRecipeStorage", ActiveUsersStorage.class.getName(), BulkImportStorage.class.getName())); Reflections reflections = new Reflections("io.supertokens.pluginInterface"); Set> classes = reflections.getSubTypesOf(NonAuthRecipeStorage.class); List names = classes.stream().map(Class::getCanonicalName).collect(Collectors.toList()); From 1ec5ddea7779f7ef9d0012ca51265caf2de6a469 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Thu, 3 Oct 2024 12:55:32 +0200 Subject: [PATCH 28/41] feat: bulkimport flow tests --- .../test/bulkimport/BulkImportFlowTest.java | 382 ++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java new file mode 100644 index 000000000..5e6298db5 --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -0,0 +1,382 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.config.Config; +import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.featureflag.EE_FEATURES; +import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.test.httpRequest.HttpResponseException; +import io.supertokens.userroles.UserRoles; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class BulkImportFlowTest { + + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void testWithOneMillionUsers() throws Exception { + String[] args = { "../" }; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "14"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + int NUMBER_OF_USERS_TO_UPLOAD = 1000000; //1000000; + int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); + System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); + System.out.println("Worker threads: " + parallelism_set_to); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // upload a bunch of users through the API + { + for (int i = 0; i < (NUMBER_OF_USERS_TO_UPLOAD / 10000); i++) { + JsonObject request = generateUsersJson(10000, i * 10000); // API allows 10k users upload at once + JsonObject response = uploadBulkImportUsersJson(main, request); + assertEquals("OK", response.get("status").getAsString()); + System.out.println(i + " Uploaded 10k users for bulk import"); + } + + } + + + long processingStartedTime = System.currentTimeMillis(); + + + // Starting the processing cronjob here to be able to measure the runtime + startBulkImportCronjob(main); + System.out.println("CronJob started"); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + { + long count = NUMBER_OF_USERS_TO_UPLOAD; + while(count != 0) { + JsonObject response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + count = response.get("count").getAsLong(); + System.out.println("Number of unprocessed users: " + count + "," + response); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int failedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + System.out.println("\t stats: "); + System.out.println("\t\tNEW: \t" + newUsersNumber); + System.out.println("\t\tFAILED: \t" + failedUsersNumber); + System.out.println("\t\tPROCESSING: \t" + processingUsersNumber); + + count = newUsersNumber + processingUsersNumber; + + long elapsedSeconds = (System.currentTimeMillis() - processingStartedTime) / 1000; + System.out.println("Elapsed time: " + elapsedSeconds + " seconds, (" + elapsedSeconds / 3600 + " hours)"); + Thread.sleep(60000); // one minute + } + } + + long processingFinishedTime = System.currentTimeMillis(); + System.out.println("Processing took " + (processingFinishedTime - processingStartedTime) / 1000 + " seconds"); + + //print failed users + { + JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + if(failedUsersLs.has("users") ){ + System.out.println(failedUsersLs.get("users")); + } + } + + // after processing finished, make sure every user got processed correctly + { + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore + failedImportedUsersNumber); + } + + } + + @Test + public void testFirstLazyImportAfterBulkImport() throws Exception { + String[] args = { "../" }; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + int NUMBER_OF_USERS_TO_UPLOAD = 100; + int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); + System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); + System.out.println("Worker threads: " + parallelism_set_to); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + // lazy import most of the users + int successfully_lazy_imported = 0; + for (int i = 0; i < allUsersJson.get("users").getAsJsonArray().size() / 10 * 9; i++) { + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(i).getAsJsonObject(); + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + assertEquals("OK", lazyImportResponse.get("status").getAsString()); + assertNotNull(lazyImportResponse.get("user")); + successfully_lazy_imported++; + System.out.println(i + "th lazy imported"); +// System.out.println("\tOriginal user: " + userToImportLazy); +// System.out.println("\tResponse user: " + lazyImportResponse.get("user")); + } + + // bulk import all of the users + { + JsonObject bulkUploadResponse = uploadBulkImportUsersJson(main, allUsersJson); + assertEquals("OK", bulkUploadResponse.get("status").getAsString()); + System.out.println("Bulk uploaded all of the users"); + } + + long processingStartedTime = System.currentTimeMillis(); + + + // Starting the processing cronjob here to be able to measure the runtime + startBulkImportCronjob(main); + System.out.println("CronJob started"); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + { + long count = NUMBER_OF_USERS_TO_UPLOAD; + while(count != 0) { + JsonObject response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + count = response.get("count").getAsLong(); + System.out.println("Number of unprocessed users: " + count + "," + response); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int failedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + System.out.println("\t stats: "); + System.out.println("\t\tNEW: \t" + newUsersNumber); + System.out.println("\t\tFAILED: \t" + failedUsersNumber); + System.out.println("\t\tPROCESSING: \t" + processingUsersNumber); + + count = newUsersNumber;// + processingUsersNumber; + + Thread.sleep(60000); // one minute + } + } + + long processingFinishedTime = System.currentTimeMillis(); + System.out.println("Processing took " + (processingFinishedTime - processingStartedTime) / 1000 + " seconds"); + + // expect: lazy imported users are already there, duplicate.. errors + // expect: not lazy imported users are imported successfully + { + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD - successfully_lazy_imported, failedImportedUsersNumber); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); // lazy + bulk = all users + } + + + } + + private static JsonObject lazyImportUser(Main main, JsonObject user) + throws HttpResponseException, IOException { + return HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/import", + user, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadBulkImportUsersCountWithStatus(Main main, BulkImportStorage.BULK_IMPORT_USER_STATUS status) + throws HttpResponseException, IOException { + Map params = new HashMap<>(); + if(status!= null) { + params.put("status", status.name()); + } + return HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadBulkImportUsersWithStatus(Main main, BulkImportStorage.BULK_IMPORT_USER_STATUS status) + throws HttpResponseException, IOException { + Map params = new HashMap<>(); + if(status!= null) { + params.put("status", status.name()); + } + return HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/bulk-import/users", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadUsersCount(Main main) throws HttpResponseException, IOException { + Map params = new HashMap<>(); + + return HttpRequestForTesting.sendGETRequest(main, "", + "http://localhost:3567/users/count", + params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject generateUsersJson(int numberOfUsers, int startIndex) { + JsonObject userJsonObject = new JsonObject(); + JsonParser parser = new JsonParser(); + + JsonArray usersArray = new JsonArray(); + for (int i = 0; i < numberOfUsers; i++) { + JsonObject user = new JsonObject(); + + user.addProperty("externalUserId", UUID.randomUUID().toString()); + user.add("userMetadata", parser.parse("{\"key1\":\"value1\",\"key2\":{\"key3\":\"value3\"}}")); + user.add("userRoles", parser.parse( + "[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); + user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); + + JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); + String email = " johndoe+" + (i + startIndex) + "@gmail.com "; + + JsonArray loginMethodsArray = new JsonArray(); + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds, "+910000" + (startIndex + i))); + user.add("loginMethods", loginMethodsArray); + + usersArray.add(user); + } + + userJsonObject.add("users", usersArray); + return userJsonObject; + } + + private static JsonObject createEmailLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "emailpassword"); + loginMethod.addProperty("passwordHash", + "$argon2d$v=19$m=12,t=3,p=1$aGI4enNvMmd0Zm0wMDAwMA$r6p7qbr6HD+8CD7sBi4HVw"); + loginMethod.addProperty("hashingAlgorithm", "argon2"); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", true); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createThirdPartyLoginMethod(String email, JsonArray tenantIds) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("recipeId", "thirdparty"); + loginMethod.addProperty("email", email); + loginMethod.addProperty("thirdPartyId", "google"); + loginMethod.addProperty("thirdPartyUserId", String.valueOf(email.hashCode())); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private static JsonObject createPasswordlessLoginMethod(String email, JsonArray tenantIds, String phoneNumber) { + JsonObject loginMethod = new JsonObject(); + loginMethod.add("tenantIds", tenantIds); + loginMethod.addProperty("email", email); + loginMethod.addProperty("recipeId", "passwordless"); + loginMethod.addProperty("phoneNumber", phoneNumber); + loginMethod.addProperty("isVerified", true); + loginMethod.addProperty("isPrimary", false); + loginMethod.addProperty("timeJoinedInMSSinceEpoch", 0); + return loginMethod; + } + + private void setFeatureFlags(Main main, EE_FEATURES[] features) { + FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); + } + + private static void startBulkImportCronjob(Main main) { + // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer + CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); + CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1); + } + + private static JsonObject uploadBulkImportUsersJson(Main main, JsonObject request) throws IOException, HttpResponseException { + return HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/users", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + +} From 2c148445ddf8f64c12b017a14ae8ac9f473d8c3e Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Sat, 12 Oct 2024 08:58:40 +0200 Subject: [PATCH 29/41] feat: bulk import cron starter api --- src/main/java/io/supertokens/Main.java | 4 +- .../io/supertokens/bulkimport/BulkImport.java | 4 +- .../BulkImportBackgroundJobManager.java | 55 +++++++++ .../io/supertokens/cronjobs/Cronjobs.java | 11 ++ .../bulkimport/ProcessBulkImportUsers.java | 11 +- .../ProcessBulkUsersImportWorker.java | 2 +- .../emailpassword/EmailPassword.java | 4 + .../io/supertokens/webserver/Webserver.java | 6 +- .../BulkImportBackgroundJobManagerAPI.java | 110 +++++++++++++++++ .../test/bulkimport/BulkImportFlowTest.java | 79 +++++++++--- .../ProcessBulkImportUsersCronJobTest.java | 11 +- ...BulkImportBackgroundJobManagerAPITest.java | 114 ++++++++++++++++++ 12 files changed, 383 insertions(+), 28 deletions(-) create mode 100644 src/main/java/io/supertokens/bulkimport/BulkImportBackgroundJobManager.java create mode 100644 src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportBackgroundJobManagerAPI.java create mode 100644 src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java diff --git a/src/main/java/io/supertokens/Main.java b/src/main/java/io/supertokens/Main.java index 5a757face..03120c96c 100644 --- a/src/main/java/io/supertokens/Main.java +++ b/src/main/java/io/supertokens/Main.java @@ -259,8 +259,8 @@ private void init() throws IOException, StorageQueryException { // starts DeleteExpiredAccessTokenSigningKeys cronjob if the access token signing keys can change Cronjobs.addCronjob(this, DeleteExpiredAccessTokenSigningKeys.init(this, uniqueUserPoolIdsTenants)); - // starts ProcessBulkImportUsers cronjob to process bulk import users - Cronjobs.addCronjob(this, ProcessBulkImportUsers.init(this, uniqueUserPoolIdsTenants)); + // initializes ProcessBulkImportUsers cronjob to process bulk import users - start happens via API call @see BulkImportBackgroundJobManager + ProcessBulkImportUsers.init(this, uniqueUserPoolIdsTenants); // this is to ensure tenantInfos are in sync for the new cron job as well MultitenancyHelper.getInstance(this).refreshCronjobs(); diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index cb8111a78..3439750a2 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -96,9 +96,9 @@ public class BulkImport { // Maximum number of users that can be deleted in a single operation public static final int DELETE_USERS_MAX_LIMIT = 500; // Number of users to process in a single batch of ProcessBulkImportUsers Cron Job - public static final int PROCESS_USERS_BATCH_SIZE = 1000; + public static final int PROCESS_USERS_BATCH_SIZE = 10000; // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job - public static final int PROCESS_USERS_INTERVAL_SECONDS = 60; + public static final int PROCESS_USERS_INTERVAL_SECONDS = 1; // This map allows reusing proxy storage for all tenants in the app and closing connections after import. private static Map userPoolToStorageMap = new HashMap<>(); diff --git a/src/main/java/io/supertokens/bulkimport/BulkImportBackgroundJobManager.java b/src/main/java/io/supertokens/bulkimport/BulkImportBackgroundJobManager.java new file mode 100644 index 000000000..e53b9dbe8 --- /dev/null +++ b/src/main/java/io/supertokens/bulkimport/BulkImportBackgroundJobManager.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.bulkimport; + +import io.supertokens.Main; +import io.supertokens.cronjobs.Cronjobs; +import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; + +public class BulkImportBackgroundJobManager { + + public static BULK_IMPORT_BACKGROUND_PROCESS_STATUS startBackgroundJob(Main main, Integer batchSize) throws TenantOrAppNotFoundException { + ProcessBulkImportUsers processBulkImportUsersCron = (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY); + processBulkImportUsersCron.setBatchSize(batchSize); + Cronjobs.addCronjob(main, processBulkImportUsersCron); + return BULK_IMPORT_BACKGROUND_PROCESS_STATUS.ACTIVE; + } + + public static BULK_IMPORT_BACKGROUND_PROCESS_STATUS stopBackgroundJob(Main main) throws TenantOrAppNotFoundException { + ProcessBulkImportUsers processBulkImportUsersCron = (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY); + Cronjobs.removeCronjob(main, processBulkImportUsersCron); + return BULK_IMPORT_BACKGROUND_PROCESS_STATUS.INACTIVE; + } + + public static BULK_IMPORT_BACKGROUND_PROCESS_STATUS checkBackgroundJobStatus(Main main) + throws TenantOrAppNotFoundException { + ProcessBulkImportUsers processBulkImportUsersCron = (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY); + BULK_IMPORT_BACKGROUND_PROCESS_STATUS status; + if(Cronjobs.isCronjobLoaded(main, processBulkImportUsersCron)){ + status = BULK_IMPORT_BACKGROUND_PROCESS_STATUS.ACTIVE; + } else { + status = BULK_IMPORT_BACKGROUND_PROCESS_STATUS.INACTIVE; + } + return status; + } + + public enum BULK_IMPORT_BACKGROUND_PROCESS_STATUS { + ACTIVE, INACTIVE; + } +} diff --git a/src/main/java/io/supertokens/cronjobs/Cronjobs.java b/src/main/java/io/supertokens/cronjobs/Cronjobs.java index 0be582b31..21cd382fe 100644 --- a/src/main/java/io/supertokens/cronjobs/Cronjobs.java +++ b/src/main/java/io/supertokens/cronjobs/Cronjobs.java @@ -100,6 +100,17 @@ public static void addCronjob(Main main, CronTask task) { } } + // TODO test for this + public static boolean isCronjobLoaded(Main main, CronTask task) { + if (getInstance(main) == null) { + init(main); + } + Cronjobs instance = getInstance(main); + synchronized (instance.lock) { + return instance.tasks.contains(task); + } + } + @TestOnly public List getTasks() { return this.tasks; diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 177ad3da2..1a1badb74 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -47,6 +47,7 @@ public class ProcessBulkImportUsers extends CronTask { public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; + private Integer batchSize; private ProcessBulkImportUsers(Main main, List> tenantsInfo) { super("ProcessBulkImportUsers", main, tenantsInfo, true); @@ -70,7 +71,7 @@ protected void doTaskPerApp(AppIdentifier app) .getStorage(app.getAsPublicTenantIdentifier(), main); List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, - BulkImport.PROCESS_USERS_BATCH_SIZE); + this.batchSize); String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); @@ -123,6 +124,14 @@ public int getInitialWaitTimeSeconds() { return 0; } + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + private List> makeChunksOf(List users, int numberOfChunks) { List> chunks = new ArrayList<>(); if (users != null && !users.isEmpty() && numberOfChunks > 0) { diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index ef7dc9a6e..7c2f5f4bb 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -224,7 +224,7 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport // of marking it as FAILED. We will return early in that case. if (exception.actualException instanceof StorageQueryException) { Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); - return; + return; // TODO find out when it's a good idea.. } errorMessage[0] = exception.actualException.getMessage(); } else if (e instanceof InvalidBulkImportDataException) { diff --git a/src/main/java/io/supertokens/emailpassword/EmailPassword.java b/src/main/java/io/supertokens/emailpassword/EmailPassword.java index 83314f0e6..547d34b0c 100644 --- a/src/main/java/io/supertokens/emailpassword/EmailPassword.java +++ b/src/main/java/io/supertokens/emailpassword/EmailPassword.java @@ -32,6 +32,7 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.emailpassword.PasswordResetTokenInfo; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicatePasswordResetTokenException; @@ -224,6 +225,9 @@ public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier ten } catch (DuplicateUserIdException e) { // we retry with a new userId } catch (DuplicateEmailException e) { + if(epStorage instanceof BulkImportStorage){ + throw e; + } AuthRecipeUserInfo[] allUsers = epStorage.listPrimaryUsersByEmail(tenantIdentifier, email); AuthRecipeUserInfo userInfoToBeUpdated = null; LoginMethod loginMethod = null; diff --git a/src/main/java/io/supertokens/webserver/Webserver.java b/src/main/java/io/supertokens/webserver/Webserver.java index 0a2b2a800..ca1ed9de2 100644 --- a/src/main/java/io/supertokens/webserver/Webserver.java +++ b/src/main/java/io/supertokens/webserver/Webserver.java @@ -26,10 +26,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.webserver.api.accountlinking.*; -import io.supertokens.webserver.api.bulkimport.BulkImportAPI; -import io.supertokens.webserver.api.bulkimport.CountBulkImportUsersAPI; -import io.supertokens.webserver.api.bulkimport.DeleteBulkImportUserAPI; -import io.supertokens.webserver.api.bulkimport.ImportUserAPI; +import io.supertokens.webserver.api.bulkimport.*; import io.supertokens.webserver.api.core.*; import io.supertokens.webserver.api.dashboard.*; import io.supertokens.webserver.api.emailpassword.UserAPI; @@ -275,6 +272,7 @@ private void setupRoutes() { addAPI(new DeleteBulkImportUserAPI(main)); addAPI(new ImportUserAPI(main)); addAPI(new CountBulkImportUsersAPI(main)); + addAPI(new BulkImportBackgroundJobManagerAPI(main)); StandardContext context = tomcatReference.getContext(); Tomcat tomcat = tomcatReference.getTomcat(); diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportBackgroundJobManagerAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportBackgroundJobManagerAPI.java new file mode 100644 index 000000000..f94161ed3 --- /dev/null +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/BulkImportBackgroundJobManagerAPI.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.webserver.api.bulkimport; + +import com.google.gson.JsonObject; +import io.supertokens.Main; +import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportBackgroundJobManager; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.webserver.InputParser; +import io.supertokens.webserver.WebserverAPI; +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import java.io.IOException; +import java.io.Serial; + +public class BulkImportBackgroundJobManagerAPI extends WebserverAPI { + + @Serial + private static final long serialVersionUID = 2380841048110043408L; + + public BulkImportBackgroundJobManagerAPI(Main main) { + super(main, ""); + } + + @Override + public String getPath() { + return "/bulk-import/backgroundjob"; + } + + //TODO what's stopping two independent calls interfering with each-other? + // A -> START -> STARTED. + // B -> START -> STARTED + // A -> STOP -> STOPPED + // B -> waits + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException { + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + JsonObject input = InputParser.parseJsonObjectOrThrowError(req); + String commandText = InputParser.parseStringOrThrowError(input, "command", false); + Integer batchSize = InputParser.parseIntOrThrowError(input, "batchSize", true); + + BULK_IMPORT_BACKGROUND_PROCESS_COMMAND command = BULK_IMPORT_BACKGROUND_PROCESS_COMMAND.valueOf(commandText); + BulkImportBackgroundJobManager.BULK_IMPORT_BACKGROUND_PROCESS_STATUS result = null; + try { + if (command.equals(BULK_IMPORT_BACKGROUND_PROCESS_COMMAND.START)){ + if(batchSize == null) { + batchSize = BulkImport.PROCESS_USERS_BATCH_SIZE; + } + result = BulkImportBackgroundJobManager.startBackgroundJob(main, batchSize); + + } else if (command.equals(BULK_IMPORT_BACKGROUND_PROCESS_COMMAND.STOP)){ + result = BulkImportBackgroundJobManager.stopBackgroundJob(main); + } + } catch (TenantOrAppNotFoundException e) { + throw new RuntimeException(e); + } + + JsonObject response = new JsonObject(); + response.addProperty("status", "OK"); + response.addProperty("jobStatus", result.name()); + sendJsonResponse(200, response, resp); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException { + + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } + + BulkImportBackgroundJobManager.BULK_IMPORT_BACKGROUND_PROCESS_STATUS processStatus; + try { + processStatus = BulkImportBackgroundJobManager.checkBackgroundJobStatus(main); + } catch (TenantOrAppNotFoundException e) { + throw new RuntimeException(e); + } + + JsonObject response = new JsonObject(); + response.addProperty("status", "OK"); + response.addProperty("jobStatus", processStatus.name()); + sendJsonResponse(200, response, resp); + } + + public enum BULK_IMPORT_BACKGROUND_PROCESS_COMMAND { + START, STOP; + } + +} diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index 5e6298db5..e2f4e016d 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -17,6 +17,7 @@ package io.supertokens.test.bulkimport; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import io.supertokens.Main; @@ -43,10 +44,10 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Random; import java.util.UUID; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.*; public class BulkImportFlowTest { @@ -77,7 +78,7 @@ public void testWithOneMillionUsers() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - int NUMBER_OF_USERS_TO_UPLOAD = 1000000; //1000000; + int NUMBER_OF_USERS_TO_UPLOAD = 140000; //1000000; int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); System.out.println("Worker threads: " + parallelism_set_to); @@ -103,12 +104,10 @@ public void testWithOneMillionUsers() throws Exception { } - long processingStartedTime = System.currentTimeMillis(); - // Starting the processing cronjob here to be able to measure the runtime - startBulkImportCronjob(main); + startBulkImportCronjob(main, 100000); System.out.println("CronJob started"); // wait for the cron job to process them @@ -163,7 +162,7 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { String[] args = { "../" }; // set processing thread number - Utils.setValueInConfig("bulk_migration_parallelism", "12"); + Utils.setValueInConfig("bulk_migration_parallelism", "14"); TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -214,7 +213,7 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { // Starting the processing cronjob here to be able to measure the runtime - startBulkImportCronjob(main); + startBulkImportCronjob(main, 10000); System.out.println("CronJob started"); // wait for the cron job to process them @@ -236,7 +235,7 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { System.out.println("\t\tFAILED: \t" + failedUsersNumber); System.out.println("\t\tPROCESSING: \t" + processingUsersNumber); - count = newUsersNumber;// + processingUsersNumber; + count = newUsersNumber + processingUsersNumber; // + processingUsersNumber; Thread.sleep(60000); // one minute } @@ -249,12 +248,21 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { // expect: not lazy imported users are imported successfully { int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); - assertEquals(NUMBER_OF_USERS_TO_UPLOAD - successfully_lazy_imported, failedImportedUsersNumber); + assertEquals(successfully_lazy_imported, failedImportedUsersNumber); int usersInCore = loadUsersCount(main).get("count").getAsInt(); assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); // lazy + bulk = all users } + JsonObject failedUsers = loadBulkImportUsersWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + JsonArray faileds = failedUsers.getAsJsonArray("users"); + for (JsonElement failedUser : faileds) { + String errorMessage = failedUser.getAsJsonObject().get("errorMessage").getAsString(); + assertTrue(errorMessage.startsWith("E003:") || errorMessage.startsWith("E005:") + || errorMessage.startsWith("E006:") || errorMessage.startsWith("E007:")); // duplicate email, phone, etc errors + System.out.println(errorMessage); + } + stopBulkImportCronjob(main); } private static JsonObject lazyImportUser(Main main, JsonObject user) @@ -311,10 +319,32 @@ private static JsonObject generateUsersJson(int numberOfUsers, int startIndex) { JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); String email = " johndoe+" + (i + startIndex) + "@gmail.com "; + Random random = new Random(); + JsonArray loginMethodsArray = new JsonArray(); - loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); - loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); - loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds, "+910000" + (startIndex + i))); + if(random.nextInt(2) == 0){ + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + } + if(random.nextInt(2) == 0){ + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + } + if(random.nextInt(2) == 0){ + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds, "+910000" + (startIndex + i))); + } + if(loginMethodsArray.size() == 0) { + int methodNumber = random.nextInt(3); + switch (methodNumber) { + case 0: + loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); + break; + case 1: + loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); + break; + case 2: + loginMethodsArray.add(createPasswordlessLoginMethod(email, tenanatIds, "+911000" + (startIndex + i))); + break; + } + } user.add("loginMethods", loginMethodsArray); usersArray.add(user); @@ -367,10 +397,25 @@ private void setFeatureFlags(Main main, EE_FEATURES[] features) { FeatureFlagTestContent.getInstance(main).setKeyValue(FeatureFlagTestContent.ENABLED_FEATURES, features); } - private static void startBulkImportCronjob(Main main) { - // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer - CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); - CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1); + private static void startBulkImportCronjob(Main main, int batchSize) throws HttpResponseException, IOException { + JsonObject request = new JsonObject(); + request.addProperty("batchSize", batchSize); + request.addProperty("command", "START"); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/backgroundjob", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + System.out.println(response); + assertEquals("ACTIVE", response.get("jobStatus").getAsString()); + } + + private static void stopBulkImportCronjob(Main main) throws HttpResponseException, IOException { + JsonObject request = new JsonObject(); + request.addProperty("command", "STOP"); + JsonObject response = HttpRequestForTesting.sendJsonPOSTRequest(main, "", + "http://localhost:3567/bulk-import/backgroundjob", + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + System.out.println(response); + assertEquals("INACTIVE", response.get("jobStatus").getAsString()); } private static JsonObject uploadBulkImportUsersJson(Main main, JsonObject request) throws IOException, HttpResponseException { diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index c948a9457..3e3446535 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -19,11 +19,15 @@ import io.supertokens.Main; import io.supertokens.ProcessState; +import io.supertokens.ResourceDistributor; import io.supertokens.authRecipe.AuthRecipe; import io.supertokens.authRecipe.UserPaginationContainer; import io.supertokens.bulkimport.BulkImport; +import io.supertokens.bulkimport.BulkImportBackgroundJobManager; import io.supertokens.config.Config; +import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; +import io.supertokens.cronjobs.Cronjobs; import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; @@ -34,7 +38,9 @@ import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; +import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.CronjobTest; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.TestingProcessManager.TestingProcess; import io.supertokens.test.Utils; @@ -519,7 +525,7 @@ public void shouldThrowTenantHaveDifferentStoragesError() throws Exception { usersAfterProcessing.get(0).errorMessage); } - private TestingProcess startCronProcess() throws InterruptedException { + private TestingProcess startCronProcess() throws InterruptedException, TenantOrAppNotFoundException { String[] args = { "../" }; TestingProcessManager.TestingProcess process = TestingProcessManager.start(args, false); @@ -533,9 +539,12 @@ private TestingProcess startCronProcess() throws InterruptedException { // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 100000); + process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Cronjobs.addCronjob(main, (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY)); + BulkImportBackgroundJobManager.startBackgroundJob(main, 1000); if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL) { return null; } diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java b/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java new file mode 100644 index 000000000..606c7451f --- /dev/null +++ b/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens.test.bulkimport.apis; + +import com.google.gson.JsonObject; +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.pluginInterface.STORAGE_TYPE; +import io.supertokens.storageLayer.StorageLayer; +import io.supertokens.test.TestingProcessManager; +import io.supertokens.test.Utils; +import io.supertokens.test.httpRequest.HttpRequestForTesting; +import io.supertokens.test.httpRequest.HttpResponseException; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class BulkImportBackgroundJobManagerAPITest { + + public static final String BACKGROUNDJOB_MANAGER_ENDPOINT = "http://localhost:3567/bulk-import/backgroundjob"; + @Rule + public TestRule watchman = Utils.getOnFailure(); + + @AfterClass + public static void afterTesting() { + Utils.afterTesting(); + } + + @Before + public void beforeEach() { + Utils.reset(); + } + + @Test + public void testByDefaultNotActive() throws Exception { + String[] args = {"../"}; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + //by default inactive + { + JsonObject currentStatus = loadBulkImportCronjobStatus(main); + + assertEquals("OK", currentStatus.get("status").getAsString()); + assertEquals("INACTIVE", currentStatus.get("jobStatus").getAsString()); + } + + //starting it makes it active + { + JsonObject startResponse = startBulkImportCronjob(main, 100); + assertEquals("OK", startResponse.get("status").getAsString()); + assertEquals("ACTIVE", startResponse.get("jobStatus").getAsString()); + + JsonObject currentStatus = loadBulkImportCronjobStatus(main); + assertEquals("OK", currentStatus.get("status").getAsString()); + assertEquals("ACTIVE", currentStatus.get("jobStatus").getAsString()); + } + + } + + private static JsonObject startBulkImportCronjob(Main main, int batchSize) throws HttpResponseException, IOException { + return sendCommandForBulkImportCronjob(main, "START", 100); + } + + private static JsonObject stopBulkImportCronjob(Main main) throws HttpResponseException, IOException { + return sendCommandForBulkImportCronjob(main, "STOP", null); + } + + private static JsonObject sendCommandForBulkImportCronjob(Main main, String command, Integer batchSize) throws HttpResponseException, IOException { + JsonObject request = new JsonObject(); + request.addProperty("command", command); + if (batchSize != null) { + request.addProperty("batchSize", batchSize); + } + return HttpRequestForTesting.sendJsonPOSTRequest(main, "", + BACKGROUNDJOB_MANAGER_ENDPOINT, + request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + private static JsonObject loadBulkImportCronjobStatus(Main main) throws HttpResponseException, IOException { + return HttpRequestForTesting.sendGETRequest(main, "", BACKGROUNDJOB_MANAGER_ENDPOINT, null, + 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + } + + + +} From d0d13f0a98b4b727b95bfe646699baa23f2b634a Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Thu, 17 Oct 2024 10:23:20 +0200 Subject: [PATCH 30/41] fix: tweaking params for faster import --- build.gradle | 1 + .../io/supertokens/bulkimport/BulkImport.java | 2 +- .../bulkimport/ProcessBulkImportUsers.java | 42 +++++++++++-------- .../ProcessBulkUsersImportWorker.java | 12 +++--- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/build.gradle b/build.gradle index 00afc8e7c..0cc0d9899 100644 --- a/build.gradle +++ b/build.gradle @@ -115,6 +115,7 @@ import org.gradle.api.tasks.testing.logging.TestExceptionFormat import org.gradle.api.tasks.testing.logging.TestLogEvent tasks.withType(Test) { + jvmArgs = ['-Xms512m', '-Xmx4g'] testLogging { // set options for log level LIFECYCLE events TestLogEvent.FAILED, diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 3439750a2..8917ce44d 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -98,7 +98,7 @@ public class BulkImport { // Number of users to process in a single batch of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_BATCH_SIZE = 10000; // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job - public static final int PROCESS_USERS_INTERVAL_SECONDS = 1; + public static final int PROCESS_USERS_INTERVAL_SECONDS = 30; // This map allows reusing proxy storage for all tenants in the app and closing connections after import. private static Map userPoolToStorageMap = new HashMap<>(); diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 1a1badb74..13627d79f 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -48,6 +48,7 @@ public class ProcessBulkImportUsers extends CronTask { public static final String RESOURCE_KEY = "io.supertokens.ee.cronjobs.ProcessBulkImportUsers"; private Integer batchSize; + private ExecutorService executorService; private ProcessBulkImportUsers(Main main, List> tenantsInfo) { super("ProcessBulkImportUsers", main, tenantsInfo, true); @@ -70,36 +71,41 @@ protected void doTaskPerApp(AppIdentifier app) BulkImportSQLStorage bulkImportSQLStorage = (BulkImportSQLStorage) StorageLayer .getStorage(app.getAsPublicTenantIdentifier(), main); + String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + + System.out.println(Thread.currentThread().getName() + " ProcessBulkImportUsers: " + " starting to load users " + this.batchSize); List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, this.batchSize); + System.out.println(Thread.currentThread().getName() + " ProcessBulkImportUsers: " + " loaded users"); - String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); - BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + if(users == null || users.isEmpty()) { + return; + } //split the loaded users list into smaller chunks int NUMBER_OF_BATCHES = Config.getConfig(app.getAsPublicTenantIdentifier(), main).getBulkMigrationParallelism(); List> loadedUsersChunks = makeChunksOf(users, NUMBER_OF_BATCHES); //pass the chunks for processing for the workers - ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_OF_BATCHES);; - try { - List> tasks = new ArrayList<>(); - for (List userListChunk : loadedUsersChunks) { - tasks.add(executorService.submit( - new ProcessBulkUsersImportWorker(main, app, userListChunk, bulkImportSQLStorage, - bulkImportUserUtils))); - } + if(executorService == null) { + executorService = Executors.newFixedThreadPool(NUMBER_OF_BATCHES); + } - for (Future task : tasks) { - task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up +// try { + List> tasks = new ArrayList<>(); + for (int i =0; i< NUMBER_OF_BATCHES; i++) { + tasks.add(executorService.submit(new ProcessBulkUsersImportWorker(main, app, loadedUsersChunks.get(i), + bulkImportSQLStorage, bulkImportUserUtils))); } - } catch (ExecutionException | InterruptedException e) { - throw new RuntimeException(e); - } - finally { - executorService.shutdown(); - } +// for (Future task : tasks) { +// task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up +// } +// +// } catch (ExecutionException | InterruptedException e) { +// throw new RuntimeException(e); +// } } @Override diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index 7c2f5f4bb..b789a96b3 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -41,9 +41,11 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.sqlStorage.SQLStorage; +import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; import io.supertokens.storageLayer.StorageLayer; import java.io.IOException; +import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -54,14 +56,14 @@ public class ProcessBulkUsersImportWorker implements Runnable { private final Map userPoolToStorageMap = new HashMap<>(); private final Main main; private final AppIdentifier app; - private final List usersToImport; private final BulkImportSQLStorage bulkImportSQLStorage; private final BulkImportUserUtils bulkImportUserUtils; + private final List usersToProcess; - ProcessBulkUsersImportWorker(Main main, AppIdentifier app, List userListToImport, BulkImportSQLStorage bulkImportSQLStorage, BulkImportUserUtils bulkImportUserUtils){ + ProcessBulkUsersImportWorker(Main main, AppIdentifier app, List usersToProcess, BulkImportSQLStorage bulkImportSQLStorage, BulkImportUserUtils bulkImportUserUtils){ this.main = main; this.app = app; - this.usersToImport = userListToImport; + this.usersToProcess = usersToProcess; this.bulkImportSQLStorage = bulkImportSQLStorage; this.bulkImportUserUtils = bulkImportUserUtils; } @@ -69,7 +71,7 @@ public class ProcessBulkUsersImportWorker implements Runnable { @Override public void run() { try { - processMultipleUsers(app, usersToImport, bulkImportUserUtils, bulkImportSQLStorage); + processMultipleUsers(app, usersToProcess, bulkImportUserUtils, bulkImportSQLStorage); } catch (TenantOrAppNotFoundException | DbInitException | IOException | StorageQueryException e) { throw new RuntimeException(e); } @@ -224,7 +226,7 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport // of marking it as FAILED. We will return early in that case. if (exception.actualException instanceof StorageQueryException) { Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); - return; // TODO find out when it's a good idea.. + return; } errorMessage[0] = exception.actualException.getMessage(); } else if (e instanceof InvalidBulkImportDataException) { From 1d076b57cd23f2b7ee1856025c705c082f4641f3 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Thu, 17 Oct 2024 10:24:48 +0200 Subject: [PATCH 31/41] fix: tests --- .../test/bulkimport/BulkImportFlowTest.java | 10 +++++----- .../apis/BulkImportBackgroundJobManagerAPITest.java | 13 ++++++++++++- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index e2f4e016d..8c8d3cdef 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -78,7 +78,7 @@ public void testWithOneMillionUsers() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - int NUMBER_OF_USERS_TO_UPLOAD = 140000; //1000000; + int NUMBER_OF_USERS_TO_UPLOAD = 1000000; int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); System.out.println("Worker threads: " + parallelism_set_to); @@ -107,7 +107,7 @@ public void testWithOneMillionUsers() throws Exception { long processingStartedTime = System.currentTimeMillis(); // Starting the processing cronjob here to be able to measure the runtime - startBulkImportCronjob(main, 100000); + startBulkImportCronjob(main, 20000); System.out.println("CronJob started"); // wait for the cron job to process them @@ -280,7 +280,7 @@ private static JsonObject loadBulkImportUsersCountWithStatus(Main main, BulkImpo } return HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users/count", - params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + params, 10000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); } private static JsonObject loadBulkImportUsersWithStatus(Main main, BulkImportStorage.BULK_IMPORT_USER_STATUS status) @@ -291,7 +291,7 @@ private static JsonObject loadBulkImportUsersWithStatus(Main main, BulkImportSto } return HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/bulk-import/users", - params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + params, 10000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); } private static JsonObject loadUsersCount(Main main) throws HttpResponseException, IOException { @@ -299,7 +299,7 @@ private static JsonObject loadUsersCount(Main main) throws HttpResponseException return HttpRequestForTesting.sendGETRequest(main, "", "http://localhost:3567/users/count", - params, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + params, 10000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); } private static JsonObject generateUsersJson(int numberOfUsers, int startIndex) { diff --git a/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java b/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java index 606c7451f..f8df51215 100644 --- a/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java +++ b/src/test/java/io/supertokens/test/bulkimport/apis/BulkImportBackgroundJobManagerAPITest.java @@ -83,10 +83,21 @@ public void testByDefaultNotActive() throws Exception { assertEquals("ACTIVE", currentStatus.get("jobStatus").getAsString()); } + + //stopping it makes it inactive + { + JsonObject startResponse = stopBulkImportCronjob(main); + assertEquals("OK", startResponse.get("status").getAsString()); + assertEquals("INACTIVE", startResponse.get("jobStatus").getAsString()); + + JsonObject currentStatus = loadBulkImportCronjobStatus(main); + assertEquals("OK", currentStatus.get("status").getAsString()); + assertEquals("INACTIVE", currentStatus.get("jobStatus").getAsString()); + } } private static JsonObject startBulkImportCronjob(Main main, int batchSize) throws HttpResponseException, IOException { - return sendCommandForBulkImportCronjob(main, "START", 100); + return sendCommandForBulkImportCronjob(main, "START", batchSize); } private static JsonObject stopBulkImportCronjob(Main main) throws HttpResponseException, IOException { From 8070d45373b6b041286a09fb1bfb010f3cc25045 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Tue, 5 Nov 2024 11:07:47 +0100 Subject: [PATCH 32/41] checkpoint --- build.gradle | 1 - .../io/supertokens/authRecipe/AuthRecipe.java | 9 ++-- .../io/supertokens/bulkimport/BulkImport.java | 26 +++++---- .../bulkimport/ProcessBulkImportUsers.java | 24 +++++---- .../test/bulkimport/BulkImportFlowTest.java | 54 +++++++++++++++++-- 5 files changed, 80 insertions(+), 34 deletions(-) diff --git a/build.gradle b/build.gradle index 0cc0d9899..00afc8e7c 100644 --- a/build.gradle +++ b/build.gradle @@ -115,7 +115,6 @@ import org.gradle.api.tasks.testing.logging.TestExceptionFormat import org.gradle.api.tasks.testing.logging.TestLogEvent tasks.withType(Test) { - jvmArgs = ['-Xms512m', '-Xmx4g'] testLogging { // set options for log level LIFECYCLE events TestLogEvent.FAILED, diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index a52a0b6d1..36524d1f6 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -21,11 +21,11 @@ import io.supertokens.authRecipe.exception.InputUserIdIsNotAPrimaryUserException; import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException; import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; -import io.supertokens.featureflag.EE_FEATURES; -import io.supertokens.featureflag.FeatureFlag; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.exception.BadPermissionException; -import io.supertokens.pluginInterface.*; +import io.supertokens.pluginInterface.RECIPE_ID; +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; @@ -43,7 +43,6 @@ import io.supertokens.storageLayer.StorageLayer; import io.supertokens.useridmapping.UserIdType; import io.supertokens.utils.Utils; - import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; @@ -563,7 +562,7 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, return authRecipeStorage.startTransaction(con -> { try { - CreatePrimaryUserResult result = canCreatePrimaryUserHelper(con, appIdentifier, authRecipeStorage, + CreatePrimaryUserResult result = canCreatePrimaryUserHelper(con, appIdentifier, authRecipeStorage, recipeUserId); if (result.wasAlreadyAPrimaryUser) { return result; diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 8917ce44d..cc33abad2 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -16,11 +16,7 @@ package io.supertokens.bulkimport; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; -import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; -import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; -import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; -import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; +import com.google.gson.JsonObject; import io.supertokens.Main; import io.supertokens.ResourceDistributor; import io.supertokens.authRecipe.AuthRecipe; @@ -42,10 +38,15 @@ import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; +import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; +import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -72,16 +73,13 @@ import io.supertokens.utils.Utils; import jakarta.servlet.ServletException; +import javax.annotation.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import javax.annotation.Nullable; - -import com.google.gson.JsonObject; - // Error codes ensure globally unique and identifiable errors in Bulk Import. // Current range: E001 to E046. @@ -207,11 +205,11 @@ public static void processUserImportSteps(Main main, TransactionConnection con, } createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); +// createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); +// verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); +// createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); +// createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); +// createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); } public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 13627d79f..30a42418a 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -88,24 +88,26 @@ protected void doTaskPerApp(AppIdentifier app) List> loadedUsersChunks = makeChunksOf(users, NUMBER_OF_BATCHES); //pass the chunks for processing for the workers - if(executorService == null) { - executorService = Executors.newFixedThreadPool(NUMBER_OF_BATCHES); - } -// try { + executorService = Executors.newFixedThreadPool(NUMBER_OF_BATCHES); + + + try { List> tasks = new ArrayList<>(); for (int i =0; i< NUMBER_OF_BATCHES; i++) { tasks.add(executorService.submit(new ProcessBulkUsersImportWorker(main, app, loadedUsersChunks.get(i), bulkImportSQLStorage, bulkImportUserUtils))); } -// for (Future task : tasks) { -// task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up -// } -// -// } catch (ExecutionException | InterruptedException e) { -// throw new RuntimeException(e); -// } + for (Future task : tasks) { + task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up + } + + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + + bulkImportSQLStorage.doVacuumFull(); } @Override diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index 8c8d3cdef..e2aa496b2 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -23,8 +23,6 @@ import io.supertokens.Main; import io.supertokens.ProcessState; import io.supertokens.config.Config; -import io.supertokens.cronjobs.CronTaskTest; -import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; import io.supertokens.pluginInterface.STORAGE_TYPE; @@ -41,6 +39,8 @@ import org.junit.Test; import org.junit.rules.TestRule; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -107,7 +107,7 @@ public void testWithOneMillionUsers() throws Exception { long processingStartedTime = System.currentTimeMillis(); // Starting the processing cronjob here to be able to measure the runtime - startBulkImportCronjob(main, 20000); + startBulkImportCronjob(main, 5000); System.out.println("CronJob started"); // wait for the cron job to process them @@ -424,4 +424,52 @@ private static JsonObject uploadBulkImportUsersJson(Main main, JsonObject reques request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); } + @Test + public void writeUsersToFile() throws Exception { + String[] args = { "../" }; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "14"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + int NUMBER_OF_USERS_TO_UPLOAD = 1000000; + int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); + System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); + System.out.println("Worker threads: " + parallelism_set_to); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // upload a bunch of users through the API + { + for (int i = 0; i < (NUMBER_OF_USERS_TO_UPLOAD / 10000); i++) { + JsonObject request = generateUsersJson(10000, i * 10000); // API allows 10k users upload at once + FileWriter fileWriter = new FileWriter(new File("/home/prophet/Projects/bulkimport-users-" + i + ".json")); + fileWriter.write(String.valueOf(request)); + fileWriter.flush(); + fileWriter.close(); + } + + } + + System.out.println("setup done, waiting"); + while(true){ + Thread.sleep(10000); + } + } + + } From d2ec5cbec9622e4f08032a0d7d390d2345813f74 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Tue, 5 Nov 2024 15:25:35 +0100 Subject: [PATCH 33/41] fix: remove vacuuming --- .../supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 30a42418a..4f543305b 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -106,8 +106,6 @@ protected void doTaskPerApp(AppIdentifier app) } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } - - bulkImportSQLStorage.doVacuumFull(); } @Override From 16eda706d3653f30f0660cdaa2eafb55caf2e173 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Thu, 7 Nov 2024 16:37:04 +0100 Subject: [PATCH 34/41] fix: minor tweaks --- .../java/io/supertokens/bulkimport/BulkImport.java | 10 +++++----- .../cronjobs/bulkimport/ProcessBulkImportUsers.java | 1 + .../bulkimport/ProcessBulkUsersImportWorker.java | 5 ++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index cc33abad2..ce1d700b7 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -205,11 +205,11 @@ public static void processUserImportSteps(Main main, TransactionConnection con, } createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); -// createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); -// verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); -// createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); -// createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); -// createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); + createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); + verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); + createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); + createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); } public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 4f543305b..6a809636b 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -103,6 +103,7 @@ protected void doTaskPerApp(AppIdentifier app) task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up } + executorService.shutdownNow(); } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index b789a96b3..dc6f71d08 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -41,11 +41,9 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.sqlStorage.SQLStorage; -import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; import io.supertokens.storageLayer.StorageLayer; import java.io.IOException; -import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -85,6 +83,7 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List { try { - Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); + BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, finalUser, primaryLM, allStoragesForApp); From 7aaf42244e3f5abfd537bdf8c17422d26e7635c5 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Fri, 15 Nov 2024 16:56:52 +0100 Subject: [PATCH 35/41] feat: bulk inserting the bulk migration data --- .../io/supertokens/authRecipe/AuthRecipe.java | 339 +++++++++++++++++- .../io/supertokens/bulkimport/BulkImport.java | 304 +++++++++++++++- .../bulkimport/ProcessBulkImportUsers.java | 15 +- .../ProcessBulkUsersImportWorker.java | 138 ++----- .../emailpassword/EmailPassword.java | 11 + .../java/io/supertokens/inmemorydb/Start.java | 132 ++++++- .../queries/EmailVerificationQueries.java | 31 ++ .../inmemorydb/queries/GeneralQueries.java | 114 +++++- .../passwordless/Passwordless.java | 15 +- .../io/supertokens/thirdparty/ThirdParty.java | 20 +- src/main/java/io/supertokens/totp/Totp.java | 26 +- .../usermetadata/UserMetadata.java | 38 ++ .../io/supertokens/userroles/UserRoles.java | 29 +- .../api/bulkimport/ImportUserAPI.java | 5 +- .../test/bulkimport/BulkImportFlowTest.java | 3 +- 15 files changed, 1090 insertions(+), 130 deletions(-) diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 36524d1f6..66234022f 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -47,6 +47,7 @@ import javax.annotation.Nullable; import java.util.*; +import java.util.stream.Collectors; /*This files contains functions that are common for all auth recipes*/ @@ -130,6 +131,18 @@ public static AuthRecipeUserInfo getUserById(AppIdentifier appIdentifier, Storag return StorageUtils.getAuthRecipeStorage(storage).getPrimaryUserById(appIdentifier, userId); } + public static List getUsersById(AppIdentifier appIdentifier, Storage storage, List userIds) + throws StorageQueryException { + AuthRecipeSQLStorage authStorage = StorageUtils.getAuthRecipeStorage(storage); + try { + return authStorage.startTransaction(con -> { + return authStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, userIds); + }); + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e); + } + } + public static class CreatePrimaryUserResult { public AuthRecipeUserInfo user; public boolean wasAlreadyAPrimaryUser; @@ -140,10 +153,21 @@ public CreatePrimaryUserResult(AuthRecipeUserInfo user, boolean wasAlreadyAPrima } } + public static class CreatePrimaryUserBulkResult { + public AuthRecipeUserInfo user; + public boolean wasAlreadyAPrimaryUser; + public Exception error; + + public CreatePrimaryUserBulkResult(AuthRecipeUserInfo user, boolean wasAlreadyAPrimaryUser, Exception error) { + this.user = user; + this.wasAlreadyAPrimaryUser = wasAlreadyAPrimaryUser; + this.error = error; + } + } + public static class CanLinkAccountsResult { public String recipeUserId; public String primaryUserId; - public boolean alreadyLinked; public CanLinkAccountsResult(String recipeUserId, String primaryUserId, boolean alreadyLinked) { @@ -153,6 +177,23 @@ public CanLinkAccountsResult(String recipeUserId, String primaryUserId, boolean } } + public static class CanLinkAccountsBulkResult { + public String recipeUserId; + public String primaryUserId; + public Exception error; + public AuthRecipeUserInfo authRecipeUserInfo; + public boolean alreadyLinked; + + public CanLinkAccountsBulkResult(String recipeUserId, String primaryUserId, boolean alreadyLinked, Exception error, + AuthRecipeUserInfo authRecipeUserInfo) { + this.recipeUserId = recipeUserId; + this.primaryUserId = primaryUserId; + this.alreadyLinked = alreadyLinked; + this.error = error; + this.authRecipeUserInfo = authRecipeUserInfo; + } + } + @TestOnly public static CanLinkAccountsResult canLinkAccounts(Main main, String recipeUserId, String primaryUserId) throws StorageQueryException, UnknownUserIdException, InputUserIdIsNotAPrimaryUserException, @@ -249,6 +290,67 @@ private static CanLinkAccountsResult canLinkAccountsHelper(TransactionConnection return new CanLinkAccountsResult(recipeUser.getSupertokensUserId(), primaryUser.getSupertokensUserId(), false); } + private static List canLinkMultipleAccountsHelper(TransactionConnection con, + AppIdentifier appIdentifier, + Storage storage, + Map recipeUserIdByPrimaryUserId) + throws StorageQueryException { + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + + List results = new ArrayList<>(); + + List primaryUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, + new ArrayList<>(recipeUserIdByPrimaryUserId.values())); + + List recipeUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, + new ArrayList<>(recipeUserIdByPrimaryUserId.keySet())); + + if(recipeUsers != null && primaryUsers != null) { + //collect all the really primary users into a map of userid -> authRecipeUserInfo + Map foundValidPrimaryUsers = primaryUsers.stream().filter(authRecipeUserInfo -> authRecipeUserInfo.isPrimaryUser).collect(Collectors.toMap(AuthRecipeUserInfo::getSupertokensUserId, authRecipeUserInfo -> authRecipeUserInfo)); + Map foundRecipeUsers = recipeUsers.stream().collect(Collectors.toMap(AuthRecipeUserInfo::getSupertokensUserId, authRecipeUserInfo -> authRecipeUserInfo)); + + for(Map.Entry recipeUserByPrimaryUser : recipeUserIdByPrimaryUserId.entrySet()) { + String recipeUserId = recipeUserByPrimaryUser.getKey(); + String primaryUserId = recipeUserByPrimaryUser.getValue(); + AuthRecipeUserInfo primaryUser = foundValidPrimaryUsers.get(primaryUserId); + AuthRecipeUserInfo recipeUser = foundRecipeUsers.get(recipeUserId); + if(primaryUser == null || recipeUser == null) { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, new UnknownUserIdException(), null)); + } else if(recipeUser.isPrimaryUser) { + if (recipeUser.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, true, null, null)); + } else { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, new RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException(recipeUser, "The input recipe user ID is already linked to another user ID"), null)); + } + } else { + if (recipeUser.loginMethods.length == 1) { + Set tenantIds = new HashSet<>(); + tenantIds.addAll(recipeUser.tenantIds); + tenantIds.addAll(primaryUser.tenantIds); + + try { + //TODO (?) this below method still uses multiple DB queries which could be enhanced + checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, + recipeUser.loginMethods[0], primaryUser); + + for (LoginMethod currLoginMethod : primaryUser.loginMethods) { + checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, + currLoginMethod, primaryUser); + } // I don't get why this is needed.. + + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, null, primaryUser)); + + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException exception) { + results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, exception, null)); + } + } + } + } + } + return results; + } + private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection con, AppIdentifier appIdentifier, AuthRecipeSQLStorage authRecipeStorage, Set tenantIds, LoginMethod currLoginMethod, @@ -401,6 +503,66 @@ public static LinkAccountsResult linkAccounts(Main main, AppIdentifier appIdenti } } + public static List linkMultipleAccounts(Main main, AppIdentifier appIdentifier, + Storage storage, Map recipeUserIdToPrimaryUserId) + throws StorageQueryException, TenantOrAppNotFoundException, FeatureNotEnabledException { + + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + throw new FeatureNotEnabledException( + "Account linking feature is not enabled for this app. Please contact support to enable it."); + } + + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + try { + + List linkAccountsResults = authRecipeStorage.startTransaction(con -> { + List canLinkAccounts = canLinkMultipleAccountsHelper(con, appIdentifier, + authRecipeStorage, recipeUserIdToPrimaryUserId); + List results = new ArrayList<>(); + Map recipeUserByPrimaryUserNeedsLinking = new HashMap<>(); + if(!canLinkAccounts.isEmpty()){ + for(CanLinkAccountsBulkResult canLinkAccountsBulkResult : canLinkAccounts) { + if (canLinkAccountsBulkResult.alreadyLinked) { + results.add(new LinkAccountsBulkResult( + canLinkAccountsBulkResult.authRecipeUserInfo, true, null)); + } else if(canLinkAccountsBulkResult.error != null) { + results.add(new LinkAccountsBulkResult( + canLinkAccountsBulkResult.authRecipeUserInfo, false, canLinkAccountsBulkResult.error)); // preparing to return the error + } else { + recipeUserByPrimaryUserNeedsLinking.put(canLinkAccountsBulkResult.recipeUserId, canLinkAccountsBulkResult.primaryUserId); + } + } + // link the remaining + authRecipeStorage.linkMultipleAccounts_Transaction(appIdentifier, con, recipeUserByPrimaryUserNeedsLinking); + List linkedPrimaryUsers = getUsersById(appIdentifier, authRecipeStorage, new ArrayList<>(recipeUserByPrimaryUserNeedsLinking.values())); + + for(AuthRecipeUserInfo linkedUser : linkedPrimaryUsers){ + results.add(new LinkAccountsBulkResult(linkedUser, false, null)); + } + + authRecipeStorage.commitTransaction(con); + } + + return results; + }); + + for(LinkAccountsBulkResult result : linkAccountsResults) { + if (!result.wasAlreadyLinked) { + io.supertokens.pluginInterface.useridmapping.UserIdMapping mappingResult = + io.supertokens.useridmapping.UserIdMapping.getUserIdMapping( + appIdentifier, authRecipeStorage, + result.user.getSupertokensUserId(), UserIdType.SUPERTOKENS); + // finally, we revoke all sessions of the recipeUser Id cause their user ID has changed. + Session.revokeAllSessionsForUser(main, appIdentifier, authRecipeStorage, + mappingResult == null ? result.user.getSupertokensUserId() : mappingResult.externalUserId, false); + } + } + return linkAccountsResults; + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e); + } + } + public static class LinkAccountsResult { public final AuthRecipeUserInfo user; public final boolean wasAlreadyLinked; @@ -411,6 +573,18 @@ public LinkAccountsResult(AuthRecipeUserInfo user, boolean wasAlreadyLinked) { } } + public static class LinkAccountsBulkResult { + public final AuthRecipeUserInfo user; + public final boolean wasAlreadyLinked; + public final Exception error; + + public LinkAccountsBulkResult(AuthRecipeUserInfo user, boolean wasAlreadyLinked, Exception error) { + this.user = user; + this.wasAlreadyLinked = wasAlreadyLinked; + this.error = error; + } + } + @TestOnly public static CreatePrimaryUserResult canCreatePrimaryUser(Main main, String recipeUserId) @@ -528,9 +702,114 @@ private static CreatePrimaryUserResult canCreatePrimaryUserHelper(TransactionCon } } + + return new CreatePrimaryUserResult(targetUser, false); } + private static List canCreatePrimaryUsersHelper(TransactionConnection con, + AppIdentifier appIdentifier, + Storage storage, + List recipeUserIds) + throws StorageQueryException, UnknownUserIdException{ + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + + List targetUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, + recipeUserIds); + if (targetUsers == null || targetUsers.isEmpty()) { + throw new UnknownUserIdException(); + } + List results = new ArrayList<>(); + for(int i = 0; i < targetUsers.size(); i++) { + AuthRecipeUserInfo targetUser = targetUsers.get(i); + if (targetUser.isPrimaryUser) { + if (targetUser.getSupertokensUserId() + .equals(recipeUserIds.get(i))) { // TODO what if there is no i-th recipeUserId? + results.add(new CreatePrimaryUserBulkResult(targetUser, true, null)); + } else { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new RecipeUserIdAlreadyLinkedWithPrimaryUserIdException(targetUser.getSupertokensUserId(), + "This user ID is already linked to another user ID"))); + continue; + } + } + + + // this means that the user has only one login method since it's not a primary user + // nor is it linked to a primary user + assert (targetUser.loginMethods.length == 1); + LoginMethod loginMethod = targetUser.loginMethods[0]; + boolean errorFound = false; + + for (String tenantId : targetUser.tenantIds) { + if (loginMethod.email != null) { + AuthRecipeUserInfo[] usersWithSameEmail = authRecipeStorage + .listPrimaryUsersByEmail_Transaction(appIdentifier, con, + loginMethod.email); + for (AuthRecipeUserInfo user : usersWithSameEmail) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser) { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's email is already associated with another user ID"))); + errorFound = true; + break; + } + } + } + + if (loginMethod.phoneNumber != null) { + AuthRecipeUserInfo[] usersWithSamePhoneNumber = authRecipeStorage + .listPrimaryUsersByPhoneNumber_Transaction(appIdentifier, con, + loginMethod.phoneNumber); + for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser) { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's phone number is already associated with another user" + + " ID"))); + errorFound = true; + break; + } + } + } + + if (loginMethod.thirdParty != null) { + AuthRecipeUserInfo[] usersWithSameThirdParty = authRecipeStorage + .listPrimaryUsersByThirdPartyInfo_Transaction(appIdentifier, con, + loginMethod.thirdParty.id, loginMethod.thirdParty.userId); + for (AuthRecipeUserInfo userWithSameThirdParty : usersWithSameThirdParty) { + if (!userWithSameThirdParty.tenantIds.contains(tenantId)) { + continue; + } + if (userWithSameThirdParty.isPrimaryUser) { + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + userWithSameThirdParty.getSupertokensUserId(), + "This user's third party login is already associated with another" + + " user ID"))); + errorFound = true; + break; + } + } + } + + if(!errorFound){ + results.add(new CreatePrimaryUserBulkResult(targetUser, false, null)); + } + } + } + return results; + } + + @TestOnly public static CreatePrimaryUserResult createPrimaryUser(Main main, String recipeUserId) @@ -592,6 +871,64 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, } } + public static List createPrimaryUsers(Main main, + AppIdentifier appIdentifier, + Storage storage, + List recipeUserIds) + throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, + RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, + FeatureNotEnabledException { + + if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { + throw new FeatureNotEnabledException( + "Account linking feature is not enabled for this app. Please contact support to enable it."); + } + + AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + try { + return authRecipeStorage.startTransaction(con -> { + + try { + List results = canCreatePrimaryUsersHelper(con, appIdentifier, authRecipeStorage, + recipeUserIds); + List canMakePrimaryUsers = new ArrayList<>(); + for(CreatePrimaryUserBulkResult result : results) { + if (result.wasAlreadyAPrimaryUser || result.error != null) { + continue; + } + canMakePrimaryUsers.add(result); + } + authRecipeStorage.makePrimaryUsers_Transaction(appIdentifier, con, + canMakePrimaryUsers.stream().map(canMakePrimaryUser -> canMakePrimaryUser.user.getSupertokensUserId()).collect( + Collectors.toList())); + + authRecipeStorage.commitTransaction(con); + + for(CreatePrimaryUserBulkResult result : results) { + if (result.wasAlreadyAPrimaryUser || result.error != null) { + continue; + } + result.user.isPrimaryUser = true; + } + return results; + + } catch (UnknownUserIdException e) { + throw new StorageTransactionLogicException(e); + } + }); + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof UnknownUserIdException) { + throw (UnknownUserIdException) e.actualException; + } else if (e.actualException instanceof RecipeUserIdAlreadyLinkedWithPrimaryUserIdException) { + throw (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException) e.actualException; + } else if (e.actualException instanceof AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) { + throw (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) e.actualException; + } + throw new StorageQueryException(e); + } + } + + public static AuthRecipeUserInfo[] getUsersByAccountInfo(TenantIdentifier tenantIdentifier, Storage storage, boolean doUnionOfAccountInfo, String email, diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index ce1d700b7..0ab0d89a8 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -43,8 +43,11 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser.LoginMethod; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; +import io.supertokens.pluginInterface.bulkimport.ImportUserBase; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; +import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; +import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateUserIdException; import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; @@ -55,10 +58,13 @@ import io.supertokens.pluginInterface.multitenancy.TenantConfig; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.passwordless.PasswordlessImportUser; import io.supertokens.pluginInterface.passwordless.exception.DuplicatePhoneNumberException; import io.supertokens.pluginInterface.sqlStorage.SQLStorage; import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; +import io.supertokens.pluginInterface.totp.TOTPDevice; import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; @@ -72,6 +78,8 @@ import io.supertokens.userroles.UserRoles; import io.supertokens.utils.Utils; import jakarta.servlet.ServletException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.io.IOException; @@ -79,6 +87,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; // Error codes ensure globally unique and identifiable errors in Bulk Import. // Current range: E001 to E046. @@ -97,6 +106,7 @@ public class BulkImport { public static final int PROCESS_USERS_BATCH_SIZE = 10000; // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_INTERVAL_SECONDS = 30; + private static final Logger log = LoggerFactory.getLogger(BulkImport.class); // This map allows reusing proxy storage for all tenants in the app and closing connections after import. private static Map userPoolToStorageMap = new HashMap<>(); @@ -212,7 +222,147 @@ public static void processUserImportSteps(Main main, TransactionConnection con, createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); } - public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, + public static void processUsersImportSteps(Main main, TransactionConnection connection, AppIdentifier appIdentifier, + Storage bulkImportProxyStorage, List users, Storage[] allStoragesForApp) + throws StorageTransactionLogicException { + processUsersLoginMethods(main, appIdentifier, bulkImportProxyStorage, users); + try { + createPrimaryUsersAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, users); + } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException | + RecipeUserIdAlreadyLinkedWithPrimaryUserIdException | StorageQueryException | FeatureNotEnabledException | + TenantOrAppNotFoundException | UnknownUserIdException e) { + throw new RuntimeException(e); + } + //TODO create user id mapping! + for(BulkImportUser user : users) { + createUserIdMapping(appIdentifier, user, getPrimaryLoginMethod(user), allStoragesForApp); + } + verifyMultipleEmailForAllLoginMethods(appIdentifier, bulkImportProxyStorage, users); + createMultipleTotpDevices(main, appIdentifier, bulkImportProxyStorage, users); + createMultipleUserMetadata(appIdentifier, bulkImportProxyStorage, users); + createMultipleUserRoles(main, appIdentifier, bulkImportProxyStorage, users); + } + + public static void processUsersLoginMethods(Main main, AppIdentifier appIdentifier, Storage storage, + List users) throws StorageTransactionLogicException { + //sort login methods together + System.out.println(Thread.currentThread().getName() + " processUsersLoginMethods"); + Map> sortedLoginMethods = new HashMap<>(); + for (BulkImportUser user: users) { + for(LoginMethod loginMethod : user.loginMethods){ + if(!sortedLoginMethods.containsKey(loginMethod.recipeId)) { + sortedLoginMethods.put(loginMethod.recipeId, new ArrayList<>()); + } + sortedLoginMethods.get(loginMethod.recipeId).add(loginMethod); + } + } + + List importedUsers = new ArrayList<>(); + importedUsers.addAll(processEmailPasswordLoginMethods(main, storage, sortedLoginMethods.get("emailpassword"), appIdentifier)); + importedUsers.addAll(processThirdpartyLoginMethods(main, storage, sortedLoginMethods.get("thirdparty"), appIdentifier)); + importedUsers.addAll(processPasswordlessLoginMethods(appIdentifier, storage, + sortedLoginMethods.get("passwordless"))); + + //TODO + for(Map.Entry> loginMethodEntries : sortedLoginMethods.entrySet()){ + for(LoginMethod loginMethod : loginMethodEntries.getValue()){ + associateUserToTenants(main, appIdentifier, storage, loginMethod, loginMethod.tenantIds.get(0)); + } + } + } + + private static List processPasswordlessLoginMethods(AppIdentifier appIdentifier, Storage storage, + List loginMethods) + throws StorageTransactionLogicException { + try { + List usersToImport = new ArrayList<>(); + for (LoginMethod loginMethod: loginMethods){ + String userId = Utils.getUUID(); + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), loginMethod.tenantIds.get(0)); // the cron runs per app. The app stays the same, the tenant can change + + usersToImport.add(new PasswordlessImportUser(userId, loginMethod.phoneNumber, + loginMethod.email, tenantIdentifierForLoginMethod, loginMethod.timeJoinedInMSSinceEpoch)); + loginMethod.superTokensUserId = userId; + } + + Passwordless.createPasswordlessUsers(storage, usersToImport); + return usersToImport; + } catch (RestartFlowException e) { + String errorMessage =""; // TODO + throw new StorageTransactionLogicException(new Exception(errorMessage)); + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E008: " + e.getMessage())); + } + } + + private static List processThirdpartyLoginMethods(Main main, Storage storage, List loginMethods, + AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + try { + List usersToImport = new ArrayList<>(); + for (LoginMethod loginMethod: loginMethods){ + String userId = Utils.getUUID(); + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), loginMethod.tenantIds.get(0)); // the cron runs per app. The app stays the same, the tenant can change + + usersToImport.add(new ThirdPartyImportUser(loginMethod.email, userId, loginMethod.thirdPartyId, + loginMethod.thirdPartyUserId, tenantIdentifierForLoginMethod, loginMethod.timeJoinedInMSSinceEpoch)); + loginMethod.superTokensUserId = userId; + } + ThirdParty.createThirdPartyUsers(storage, usersToImport); + return usersToImport; + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); +// } catch (TenantOrAppNotFoundException e) { +// throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); +// } catch (DuplicateThirdPartyUserException e) { +// throw new StorageTransactionLogicException(new Exception("E005: A user with thirdPartyId " +// + " and thirdPartyUserId already exists in thirdparty loginMethod.")); // TODO + } + } + + private static List processEmailPasswordLoginMethods(Main main, Storage storage, List loginMethods, + AppIdentifier appIdentifier) + throws StorageTransactionLogicException { + try { + //prepare data for batch import + List usersToImport = new ArrayList<>(); + for(LoginMethod emailPasswordLoginMethod : loginMethods) { + + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), emailPasswordLoginMethod.tenantIds.get(0)); // the cron runs per app. The app stays the same, the tenant can change + + String passwordHash = emailPasswordLoginMethod.passwordHash; + if (passwordHash == null && emailPasswordLoginMethod.plainTextPassword != null) { + passwordHash = PasswordHashing.getInstance(main) + .createHashWithSalt(tenantIdentifierForLoginMethod.toAppIdentifier(), emailPasswordLoginMethod.plainTextPassword); + } + emailPasswordLoginMethod.passwordHash = passwordHash; + String userId = Utils.getUUID(); + usersToImport.add(new EmailPasswordImportUser(userId, emailPasswordLoginMethod.email, + emailPasswordLoginMethod.passwordHash, tenantIdentifierForLoginMethod, emailPasswordLoginMethod.timeJoinedInMSSinceEpoch)); + emailPasswordLoginMethod.superTokensUserId = userId; + } + + EmailPassword.createUsersWithPasswordHash(storage, usersToImport); + return usersToImport; + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E002: " + e.getMessage())); + } catch (DuplicateEmailException e) { + throw new StorageTransactionLogicException( + new Exception( + "E003: A user with email already exists in emailpassword loginMethod.")); + } catch (DuplicateUserIdException e) { + throw new RuntimeException(e); + } + } + + public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, LoginMethod lm) throws StorageTransactionLogicException { String firstTenant = lm.tenantIds.get(0); @@ -230,7 +380,8 @@ public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier new IllegalArgumentException("E001: Unknown recipeId " + lm.recipeId + " for loginMethod.")); } - associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); + associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); //already associated here. Why this? + // found it: the remaining tenants are not associated, only the first one } private static void processEmailPasswordLoginMethod(Main main, TenantIdentifier tenantIdentifier, Storage storage, @@ -348,7 +499,21 @@ private static void associateUserToTenants(Main main, AppIdentifier appIdentifie } } - public static void createPrimaryUserAndLinkAccounts(Main main, + private static void createPrimaryUsersAndLinkAccounts(Main main, + AppIdentifier appIdentifier, Storage storage, List users) + throws StorageTransactionLogicException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, + RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, StorageQueryException, FeatureNotEnabledException, + TenantOrAppNotFoundException, UnknownUserIdException { + System.out.println(Thread.currentThread().getName() + " createPrimaryUsersAndLinkAccounts"); + List userIds = + users.stream().map(bulkImportUser -> getPrimaryLoginMethod(bulkImportUser).getSuperTokenOrExternalUserId()).collect(Collectors.toList()); + + AuthRecipe.createPrimaryUsers(main, appIdentifier, storage, userIds); + linkAccountsForMultipleUser(main, appIdentifier, storage, users); + } + + + public static void createPrimaryUserAndLinkAccounts(Main main, AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { if (user.loginMethods.size() == 1) { @@ -380,6 +545,11 @@ public static void createPrimaryUserAndLinkAccounts(Main main, + " but the account info is already associated with another primary user.")); } + linkAccountsForUser(main, appIdentifier, storage, user, primaryLM); + } + + private static void linkAccountsForUser(Main main, AppIdentifier appIdentifier, Storage storage, BulkImportUser user, + LoginMethod primaryLM) throws StorageTransactionLogicException { for (LoginMethod lm : user.loginMethods) { try { if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { @@ -419,6 +589,28 @@ public static void createPrimaryUserAndLinkAccounts(Main main, } } + private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIdentifier, Storage storage, List users) + throws StorageTransactionLogicException { + Map recipeUserIdByPrimaryUserId = new HashMap<>(); + for(BulkImportUser user: users){ + LoginMethod primaryLM = getPrimaryLoginMethod(user); + for (LoginMethod lm : user.loginMethods) { + if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { + continue; + } + recipeUserIdByPrimaryUserId.put(lm.getSuperTokenOrExternalUserId(), + primaryLM.getSuperTokenOrExternalUserId()); + } + } + + try { + AuthRecipe.linkMultipleAccounts(main, appIdentifier, storage, recipeUserIdByPrimaryUserId); + } catch (StorageQueryException | FeatureNotEnabledException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + //TODO proper error handling + } + public static void createUserIdMapping(AppIdentifier appIdentifier, BulkImportUser user, LoginMethod primaryLM, Storage[] storages) throws StorageTransactionLogicException { if (user.externalUserId != null) { @@ -461,6 +653,26 @@ public static void createUserMetadata(AppIdentifier appIdentifier, Storage stora } } + public static void createMultipleUserMetadata(AppIdentifier appIdentifier, Storage storage, List users) + throws StorageTransactionLogicException { + System.out.println(Thread.currentThread().getName() + " createMultipleUserMetadata"); + + Map usersMetadata = new HashMap<>(); + for(BulkImportUser user: users) { + if (user.userMetadata != null) { + usersMetadata.put(getPrimaryLoginMethod(user).getSuperTokenOrExternalUserId(), user.userMetadata); + } + } + + try { + UserMetadata.updateMultipleUsersMetadata(appIdentifier, storage, usersMetadata); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E040: " + e.getMessage())); + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + public static void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, BulkImportUser user) throws StorageTransactionLogicException { if (user.userRoles != null) { @@ -485,6 +697,39 @@ public static void createUserRoles(Main main, AppIdentifier appIdentifier, Stora } } + public static void createMultipleUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, + List users) throws StorageTransactionLogicException { + Map> rolesToUserByTenant = new HashMap<>(); + System.out.println(Thread.currentThread().getName() + " createMultipleUserRoles"); + for (BulkImportUser user : users) { + + if (user.userRoles != null) { + for (UserRole userRole : user.userRoles) { + for (String tenantId : userRole.tenantIds) { + TenantIdentifier tenantIdentifier = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), + tenantId); + if(!rolesToUserByTenant.containsKey(tenantIdentifier)){ + + rolesToUserByTenant.put(tenantIdentifier, new HashMap<>()); + } + rolesToUserByTenant.get(tenantIdentifier).put(user.externalUserId, userRole.role); + } + } + } + } + try { + + UserRoles.addMultipleRolesToMultipleUsers(main, storage, rolesToUserByTenant); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); + } catch (UnknownRoleException e) { + throw new StorageTransactionLogicException(new Exception("E034: Role " + + " does not exist! You need pre-create the role before assigning it to the user.")); + } + + } + public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, Storage storage, List loginMethods) throws StorageTransactionLogicException { @@ -508,6 +753,35 @@ public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, Tr } } + public static void verifyMultipleEmailForAllLoginMethods(AppIdentifier appIdentifier, Storage storage, + List users) + throws StorageTransactionLogicException { + System.out.println(Thread.currentThread().getName() + " verifyMultipleEmailForAllLoginMethods"); + Map emailToUserId = new HashMap<>(); + for (BulkImportUser user : users) { + for (LoginMethod lm : user.loginMethods) { + emailToUserId.put(lm.email, lm.getSuperTokenOrExternalUserId()); + } + } + + try { + + EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils + .getEmailVerificationStorage(storage); + emailVerificationSQLStorage.startTransaction(con -> { + emailVerificationSQLStorage + .updateMultipleIsEmailVerified_Transaction(appIdentifier, con, + emailToUserId, true); + + emailVerificationSQLStorage.commitTransaction(con); + return null; + }); + + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { if (user.totpDevices != null) { @@ -531,6 +805,30 @@ public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Sto } } + public static void createMultipleTotpDevices(Main main, AppIdentifier appIdentifier, + Storage storage, List users) + throws StorageTransactionLogicException { + System.out.println(Thread.currentThread().getName() + " createMultipleTotpDevices"); + List devices = new ArrayList<>(); + for (BulkImportUser user : users) { + if (user.totpDevices != null) { + for(TotpDevice device : user.totpDevices){ + TOTPDevice totpDevice = new TOTPDevice(getPrimaryLoginMethod(user).getSuperTokenOrExternalUserId(), //TODO getPrimaryLoginMethod call should be done once in the whole process + device.deviceName, device.secretKey, device.period, device.skew, true, + System.currentTimeMillis()); + devices.add(totpDevice); + } + } + } + try { + Totp.createDevices(main, appIdentifier, storage, devices); + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E037: " + e.getMessage())); + } + } + // Returns the primary loginMethod of the user. If no loginMethod is marked as // primary, then the oldest loginMethod is returned. public static BulkImportUser.LoginMethod getPrimaryLoginMethod(BulkImportUser user) { diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 6a809636b..32ce82077 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -35,10 +35,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -100,10 +97,18 @@ protected void doTaskPerApp(AppIdentifier app) } for (Future task : tasks) { - task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up + while(!task.isDone()) { + Thread.sleep(1000); + } + Void result = (Void) task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up + System.out.println("Result: " + result); } + executorService.shutdownNow(); + if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { + System.out.println("Pool did not terminate"); + } } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index dc6f71d08..dd3a9bb09 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -27,7 +27,6 @@ import io.supertokens.output.Logging; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; -import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportTransactionRolledBackException; @@ -86,118 +85,59 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List validUsers = new ArrayList<>(); + while(userIndexPointer < users.size()) { user = users.get(userIndexPointer); - if ((Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) || shouldRetryImmediately) { + if ((Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) || + shouldRetryImmediately) { // Skip validation when the flag is enabled during testing // Skip validation if it's a retry run. This already passed validation. A revalidation triggers // an invalid external user id already exists validation error - which is not true! } else { // Validate the user - bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id); + validUsers.add(bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id)); } + userIndexPointer+=1; + } + // Since all the tenants of a user must share the storage, we will just use the + // storage of the first tenantId of the first loginMethod + TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), validUsers.get(0).loginMethods.get(0).tenantIds.get(0)); - // Since all the tenants of a user must share the storage, we will just use the - // storage of the first tenantId of the first loginMethod - TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), user.loginMethods.get(0).tenantIds.get(0)); - - SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); - BulkImportUser.LoginMethod primaryLM = BulkImport.getPrimaryLoginMethod(user); - - AuthRecipeSQLStorage authRecipeSQLStorage = (AuthRecipeSQLStorage) getBulkImportProxyStorage( - firstTenantIdentifier); - - /* - * We use two separate storage instances: one for importing the user and another for managing - * bulk_import_users entries. - * This is necessary because the bulk_import_users entries are always in the public tenant storage, - * but the actual user data could be in a different storage. - * - * If transactions are committed individually, in this order: - * 1. Commit the transaction that imports the user. - * 2. Commit the transaction that deletes the corresponding bulk import entry. - * - * There's a risk where the first commit succeeds, but the second fails. This creates a situation where - * the bulk import entry is re-processed, even though the user has already been imported into the - * database. - * - * To resolve this, we added a `primaryUserId` field to the `bulk_import_users` table. - * The processing logic now follows these steps: - * - * 1. Import the user and get the `primaryUserId` (transaction uncommitted). - * 2. Update the `primaryUserId` in the corresponding bulk import entry. - * 3. Commit the import transaction from step 1. - * 4. Delete the bulk import entry. - * - * If step 2 or any earlier step fails, nothing is committed, preventing partial state. - * If step 3 fails, the `primaryUserId` in the bulk import entry is updated, but the user doesn't - * exist in the database—this results in re-processing on the - * next run. - * If step 4 fails, the user exists but the bulk import entry remains; this will be handled by - * deleting it in the next run. - * - * The following code implements this logic. - */ - if (user.primaryUserId != null) { - AuthRecipeUserInfo importedUser = authRecipeSQLStorage.getPrimaryUserById(appIdentifier, - user.primaryUserId); - - if (importedUser != null && isProcessedUserFromSameBulkImportUserEntry(importedUser, user)) { - baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[]{user.id}); - return; - } - } + SQLStorage bulkImportProxyStorage = (SQLStorage) getBulkImportProxyStorage(firstTenantIdentifier); + + BulkImportUser finalUser = user; + shouldRetryImmediately = bulkImportProxyStorage.startTransaction(con -> { + try { + + BulkImport.processUsersImportSteps(main, con, appIdentifier, bulkImportProxyStorage, validUsers, allStoragesForApp); - BulkImportUser finalUser = user; - shouldRetryImmediately = bulkImportProxyStorage.startTransaction(con -> { - try { - - BulkImport.processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, finalUser, - primaryLM, allStoragesForApp); - - // We are updating the primaryUserId in the bulkImportUser entry. This will help us handle - // the inconsistent transaction commit. - // If this update statement fails then the outer transaction will fail as well and the user - // will simpl be processed again. No inconsistency will happen in this - // case. - baseTenantStorage.updateBulkImportUserPrimaryUserId(appIdentifier, finalUser.id, - primaryLM.superTokensUserId); - - // We need to commit the transaction manually because we have overridden that in the proxy - // storage - // If this fails, the primaryUserId will be updated in the bulkImportUser but it wouldn’t - // actually exist. - // When processing the user again, we'll check if primaryUserId exists with the same email. - // In this case the user won't exist, and we'll simply re-process it. - bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); - - // NOTE: We need to use the baseTenantStorage as bulkImportProxyStorage could have a - // different storage than the baseTenantStorage - // If this fails, the primaryUserId will be updated in the bulkImportUser and it would exist - // in the database. - // When processing the user again, we'll check if primaryUserId exists with the same email. - // In this case the user will exist, and we'll simply delete the entry. - baseTenantStorage.deleteBulkImportUsers(appIdentifier, new String[]{finalUser.id}); - } catch (StorageTransactionLogicException e) { - // We need to rollback the transaction manually because we have overridden that in the proxy - // storage - bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); - if(isBulkImportTransactionRolledBackIsTheRealCause(e)){ - return true; - //@see BulkImportTransactionRolledBackException for explanation - } - handleProcessUserExceptions(app, finalUser, e, baseTenantStorage); + bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); + + String[] toDelete = new String[validUsers.size()]; + for(int i = 0; i < validUsers.size(); i++) { + toDelete[i] = validUsers.get(i).id; } - return false; - }); - if(!shouldRetryImmediately){ - userIndexPointer++; + baseTenantStorage.deleteBulkImportUsers(appIdentifier, toDelete); + } catch (StorageTransactionLogicException e) { + // We need to rollback the transaction manually because we have overridden that in the proxy + // storage + bulkImportProxyStorage.rollbackTransactionForBulkImportProxyStorage(); + if(isBulkImportTransactionRolledBackIsTheRealCause(e)){ + return true; + //@see BulkImportTransactionRolledBackException for explanation + } + handleProcessUserExceptions(app, finalUser, e, baseTenantStorage); } + return false; + }); + + if(!shouldRetryImmediately){ + userIndexPointer++; } } catch (StorageTransactionLogicException | InvalidBulkImportDataException | InvalidConfigException e) { - handleProcessUserExceptions(appIdentifier, user, e, baseTenantStorage); + throw new RuntimeException(e); } finally { closeAllProxyStorages(); //closing it here to reuse the existing connection with all the users } diff --git a/src/main/java/io/supertokens/emailpassword/EmailPassword.java b/src/main/java/io/supertokens/emailpassword/EmailPassword.java index 547d34b0c..7026d3d27 100644 --- a/src/main/java/io/supertokens/emailpassword/EmailPassword.java +++ b/src/main/java/io/supertokens/emailpassword/EmailPassword.java @@ -33,6 +33,7 @@ import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; import io.supertokens.pluginInterface.emailpassword.PasswordResetTokenInfo; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicatePasswordResetTokenException; @@ -56,6 +57,7 @@ import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.security.spec.InvalidKeySpecException; +import java.util.List; public class EmailPassword { @@ -210,6 +212,7 @@ public static ImportUserResponse importUserWithPasswordHash(TenantIdentifier ten } return response; } + public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier tenantIdentifier, Storage storage, @Nonnull String email, @Nonnull String passwordHash, @Nullable long timeJoined) @@ -255,6 +258,14 @@ public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier ten } } + public static void createUsersWithPasswordHash(Storage storage, + List usersToImport) + throws StorageQueryException, DuplicateEmailException, TenantOrAppNotFoundException, + DuplicateUserIdException, StorageTransactionLogicException { + EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); + epStorage.signUpMultiple(usersToImport); + } + @TestOnly public static ImportUserResponse importUserWithPasswordHash(Main main, @Nonnull String email, @Nonnull String passwordHash) diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 534daf779..5a27c7d27 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -32,6 +32,7 @@ import io.supertokens.pluginInterface.dashboard.DashboardUser; import io.supertokens.pluginInterface.dashboard.exceptions.UserIdNotFoundException; import io.supertokens.pluginInterface.dashboard.sqlStorage.DashboardSQLStorage; +import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; import io.supertokens.pluginInterface.emailpassword.PasswordResetTokenInfo; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicatePasswordResetTokenException; @@ -66,12 +67,14 @@ import io.supertokens.pluginInterface.oauth.exception.OAuthClientNotFoundException; import io.supertokens.pluginInterface.passwordless.PasswordlessCode; import io.supertokens.pluginInterface.passwordless.PasswordlessDevice; +import io.supertokens.pluginInterface.passwordless.PasswordlessImportUser; import io.supertokens.pluginInterface.passwordless.exception.*; import io.supertokens.pluginInterface.passwordless.sqlStorage.PasswordlessSQLStorage; import io.supertokens.pluginInterface.session.SessionInfo; import io.supertokens.pluginInterface.session.SessionStorage; import io.supertokens.pluginInterface.session.sqlStorage.SessionSQLStorage; import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; +import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; import io.supertokens.pluginInterface.thirdparty.sqlStorage.ThirdPartySQLStorage; import io.supertokens.pluginInterface.totp.TOTPDevice; @@ -102,10 +105,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.SQLTransactionRollbackException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Set; +import java.util.*; public class Start implements SessionSQLStorage, EmailPasswordSQLStorage, EmailVerificationSQLStorage, ThirdPartySQLStorage, @@ -233,7 +233,7 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev tries++; try { return startTransactionHelper(logic); - } catch (SQLException | StorageQueryException | StorageTransactionLogicException e) { + } catch (SQLException | StorageQueryException | StorageTransactionLogicException | TenantOrAppNotFoundException e) { if ((e instanceof SQLTransactionRollbackException || (e.getMessage() != null && e.getMessage().toLowerCase().contains("deadlock"))) && tries < 3) { @@ -245,6 +245,8 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev throw (StorageQueryException) e; } else if (e instanceof StorageTransactionLogicException) { throw (StorageTransactionLogicException) e; + } else if (e instanceof TenantOrAppNotFoundException) { // TODO this should not be here. + throw new StorageTransactionLogicException(e); } throw new StorageQueryException(e); } @@ -252,7 +254,7 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev } private T startTransactionHelper(TransactionLogic logic) - throws StorageQueryException, StorageTransactionLogicException, SQLException { + throws StorageQueryException, StorageTransactionLogicException, SQLException, TenantOrAppNotFoundException { Connection con = null; try { con = ConnectionPool.getConnection(this); @@ -809,6 +811,13 @@ public AuthRecipeUserInfo signUp(TenantIdentifier tenantIdentifier, String id, S } } + @Override + public void signUpMultiple(List users) + throws StorageQueryException, DuplicateUserIdException, DuplicateEmailException, + TenantOrAppNotFoundException, StorageTransactionLogicException { + // TODO + } + @Override public void addPasswordResetToken(AppIdentifier appIdentifier, PasswordResetTokenInfo passwordResetTokenInfo) throws StorageQueryException, UnknownUserIdException, DuplicatePasswordResetTokenException { @@ -993,6 +1002,40 @@ public void updateIsEmailVerified_Transaction(AppIdentifier appIdentifier, Trans } } + @Override + public void updateMultipleIsEmailVerified_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + Map emailToUserId, boolean isEmailVerified) + throws StorageQueryException, TenantOrAppNotFoundException { + Connection sqlCon = (Connection) con.getConnection(); + try { + EmailVerificationQueries.updateMultipleUsersIsEmailVerified_Transaction(this, sqlCon, appIdentifier, + emailToUserId, isEmailVerified); + } catch (SQLException e) { + if (e instanceof SQLiteException) { + SQLiteConfig config = Config.getConfig(this); + String serverMessage = e.getMessage(); + + if (isForeignKeyConstraintError( + serverMessage, + config.getTenantsTable(), + new String[]{"app_id"}, + new Object[]{appIdentifier.getAppId()})) { + throw new TenantOrAppNotFoundException(appIdentifier); + } + } + + boolean isPSQLPrimKeyError = e instanceof SQLiteException && isPrimaryKeyError( + e.getMessage(), + Config.getConfig(this).getEmailVerificationTable(), + new String[]{"app_id", "user_id", "email"}); + + if (!isEmailVerified || !isPSQLPrimKeyError) { + throw new StorageQueryException(e); + } + // we do not throw an error since the email is already verified + } + } + @Override public void deleteEmailVerificationUserInfo_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -1137,6 +1180,20 @@ public void deleteThirdPartyUser_Transaction(TransactionConnection con, AppIdent } } + @Override + public void importThirdPartyUsers_Transaction(TransactionConnection con, + Collection usersToImport) + throws StorageQueryException { + // TODO + } + + @Override + public void importPasswordlessUsers_Transaction(TransactionConnection con, + Collection users) + throws StorageQueryException { + //todo + } + @Override public AuthRecipeUserInfo signUp( TenantIdentifier tenantIdentifier, String id, String email, @@ -1836,6 +1893,16 @@ public JsonObject getUserMetadata_Transaction(AppIdentifier appIdentifier, Trans } } + @Override + public Map getMultipleUsersMetadatas_Transaction(AppIdentifier appIdentifier, + TransactionConnection con, + List userIds) + throws StorageQueryException { + return Map.of(); // TODO + } + + + @Override public int setUserMetadata_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String userId, JsonObject metadata) @@ -1861,6 +1928,13 @@ public int setUserMetadata_Transaction(AppIdentifier appIdentifier, TransactionC } } + @Override + public void setMultipleUsersMetadatas_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + Map metadataByUserId) + throws StorageQueryException, TenantOrAppNotFoundException { + //TODO + } + @Override public int deleteUserMetadata_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -2116,6 +2190,13 @@ public void deleteAllRolesForUser_Transaction(TransactionConnection con, AppIden } } + @Override + public void addRolesToUsers_Transaction(TransactionConnection connection, + Map> rolesToUserByTenants) + throws StorageQueryException { + // TODO + } + @Override public void createUserIdMapping(AppIdentifier appIdentifier, String superTokensUserId, String externalUserId, @org.jetbrains.annotations.Nullable String externalUserIdInfo) @@ -2657,6 +2738,13 @@ public TOTPDevice createDevice_Transaction(TransactionConnection con, AppIdentif } } + @Override + public void createDevices_Transaction(TransactionConnection con, AppIdentifier appIdentifier, + List devices) + throws StorageQueryException, TenantOrAppNotFoundException { + // TODO + } + @Override public TOTPDevice getDeviceByName_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId, String deviceName) throws StorageQueryException { @@ -2865,6 +2953,18 @@ public AuthRecipeUserInfo getPrimaryUserById_Transaction(AppIdentifier appIdenti } } + @Override + public List getPrimaryUsersByIds_Transaction(AppIdentifier appIdentifier, + TransactionConnection con, List userIds) + throws StorageQueryException { + try { + Connection sqlCon = (Connection) con.getConnection(); + return GeneralQueries.getPrimaryUsersInfoForUserIds_Transaction(this, sqlCon, appIdentifier, userIds); + } catch (SQLException e) { + throw new StorageQueryException(e); + } + } + @Override public AuthRecipeUserInfo[] listPrimaryUsersByEmail_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String email) @@ -2932,6 +3032,19 @@ public void makePrimaryUser_Transaction(AppIdentifier appIdentifier, Transaction } } + @Override + public void makePrimaryUsers_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + List userIds) throws StorageQueryException { + try { + Connection sqlCon = (Connection) con.getConnection(); + // we do not bother returning if a row was updated here or not, cause it's happening + // in a transaction anyway. + GeneralQueries.makePrimaryUsers_Transaction(this, sqlCon, appIdentifier, userIds); + } catch (SQLException e) { + throw new StorageQueryException(e); + } + } + @Override public void linkAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String recipeUserId, String primaryUserId) throws StorageQueryException { @@ -2945,6 +3058,13 @@ public void linkAccounts_Transaction(AppIdentifier appIdentifier, TransactionCon } } + @Override + public void linkMultipleAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + Map recipeUserIdByPrimaryUserId) + throws StorageQueryException { + // TODO + } + @Override public void unlinkAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, String primaryUserId, String recipeUserId) diff --git a/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java index 8d5ccc7d8..5df8a518d 100644 --- a/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java +++ b/src/main/java/io/supertokens/inmemorydb/queries/EmailVerificationQueries.java @@ -29,6 +29,7 @@ import io.supertokens.pluginInterface.sqlStorage.TransactionConnection; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; @@ -103,6 +104,36 @@ public static void updateUsersIsEmailVerified_Transaction(Start start, Connectio } } + public static void updateMultipleUsersIsEmailVerified_Transaction(Start start, Connection con, AppIdentifier appIdentifier, + Map emailToUserIds, + boolean isEmailVerified) + throws SQLException, StorageQueryException { + + if (isEmailVerified) { + String QUERY = "INSERT INTO " + getConfig(start).getEmailVerificationTable() + + "(app_id, user_id, email) VALUES(?, ?, ?)"; + PreparedStatement insertQuery = con.prepareStatement(QUERY); + for(Map.Entry emailToUser : emailToUserIds.entrySet()){ + insertQuery.setString(1, appIdentifier.getAppId()); + insertQuery.setString(2, emailToUser.getValue()); + insertQuery.setString(3, emailToUser.getKey()); + insertQuery.addBatch(); + } + insertQuery.executeBatch(); + } else { + String QUERY = "DELETE FROM " + getConfig(start).getEmailVerificationTable() + + " WHERE app_id = ? AND user_id = ? AND email = ?"; + PreparedStatement deleteQuery = con.prepareStatement(QUERY); + for (Map.Entry emailToUser : emailToUserIds.entrySet()) { + deleteQuery.setString(1, appIdentifier.getAppId()); + deleteQuery.setString(2, emailToUser.getValue()); + deleteQuery.setString(3, emailToUser.getKey()); + deleteQuery.addBatch(); + } + deleteQuery.executeBatch(); + } + } + public static void deleteAllEmailVerificationTokensForUser_Transaction(Start start, Connection con, TenantIdentifier tenantIdentifier, String userId, diff --git a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java index eb2fe4809..5e8df292f 100644 --- a/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java +++ b/src/main/java/io/supertokens/inmemorydb/queries/GeneralQueries.java @@ -35,10 +35,7 @@ import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.TestOnly; -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.ResultSet; -import java.sql.SQLException; +import java.sql.*; import java.util.*; import java.util.stream.Collectors; @@ -960,6 +957,37 @@ public static void makePrimaryUser_Transaction(Start start, Connection sqlCon, A } } + public static void makePrimaryUsers_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, + List userIds) + throws SQLException, StorageQueryException { + + String users_update_QUERY = "UPDATE " + getConfig(start).getUsersTable() + + " SET is_linked_or_is_a_primary_user = true WHERE app_id = ? AND user_id = ?"; + String appid_to_userid_update_QUERY = "UPDATE " + getConfig(start).getAppIdToUserIdTable() + + " SET is_linked_or_is_a_primary_user = true WHERE app_id = ? AND user_id = ?"; + + PreparedStatement usersUpdateStatement = sqlCon.prepareStatement(users_update_QUERY); + PreparedStatement appIdToUserIdUpdateStatement = sqlCon.prepareStatement(appid_to_userid_update_QUERY); + int counter = 0; + for(String userId: userIds){ + usersUpdateStatement.setString(1, appIdentifier.getAppId()); + usersUpdateStatement.setString(2, userId); + usersUpdateStatement.addBatch(); + + appIdToUserIdUpdateStatement.setString(1, appIdentifier.getAppId()); + appIdToUserIdUpdateStatement.setString(2, userId); + appIdToUserIdUpdateStatement.addBatch(); + + counter++; + if(counter % 100 == 0) { + usersUpdateStatement.executeBatch(); + appIdToUserIdUpdateStatement.executeBatch(); + } + } + usersUpdateStatement.executeBatch(); + appIdToUserIdUpdateStatement.executeBatch(); + } + public static void linkAccounts_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, String recipeUserId, String primaryUserId) throws SQLException, StorageQueryException { @@ -990,6 +1018,54 @@ public static void linkAccounts_Transaction(Start start, Connection sqlCon, AppI } } + public static void linkMultipleAccounts_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, + Map recipeUserIdToPrimaryUserId) + throws SQLException, StorageQueryException { + + if(recipeUserIdToPrimaryUserId == null || recipeUserIdToPrimaryUserId.isEmpty()){ + return; + } + + String update_users_QUERY = "UPDATE " + getConfig(start).getUsersTable() + + " SET is_linked_or_is_a_primary_user = true, primary_or_recipe_user_id = ? WHERE app_id = ? AND " + + "user_id = ?"; + + String update_appid_to_userid_QUERY = "UPDATE " + getConfig(start).getAppIdToUserIdTable() + + " SET is_linked_or_is_a_primary_user = true, primary_or_recipe_user_id = ? WHERE app_id = ? AND " + + "user_id = ?"; + + PreparedStatement updateUsers = sqlCon.prepareStatement(update_users_QUERY); + PreparedStatement updateAppIdToUserId = sqlCon.prepareStatement(update_appid_to_userid_QUERY); + + int counter = 0; + for(Map.Entry linkEntry : recipeUserIdToPrimaryUserId.entrySet()) { + String primaryUserId = linkEntry.getValue(); + String recipeUserId = linkEntry.getKey(); + + updateUsers.setString(1, primaryUserId); + updateUsers.setString(2, appIdentifier.getAppId()); + updateUsers.setString(3, recipeUserId); + updateUsers.addBatch(); + + updateAppIdToUserId.setString(1, primaryUserId); + updateAppIdToUserId.setString(2, appIdentifier.getAppId()); + updateAppIdToUserId.setString(3, recipeUserId); + updateAppIdToUserId.addBatch(); + + counter++; + if (counter % 100 == 0) { + updateUsers.executeBatch(); + updateAppIdToUserId.executeBatch(); + } + } + + updateUsers.executeBatch(); + updateAppIdToUserId.executeBatch(); + + updateTimeJoinedForPrimaryUsers_Transaction(start, sqlCon, appIdentifier, + new ArrayList<>(recipeUserIdToPrimaryUserId.values())); + } + public static void unlinkAccounts_Transaction(Start start, Connection sqlCon, AppIdentifier appIdentifier, String primaryUserId, String recipeUserId) throws SQLException, StorageQueryException { @@ -1221,6 +1297,17 @@ public static AuthRecipeUserInfo getPrimaryUserInfoForUserId_Transaction(Start s return result.get(0); } + public static List getPrimaryUsersInfoForUserIds_Transaction(Start start, Connection con, + AppIdentifier appIdentifier, List ids) + throws SQLException, StorageQueryException { + + List result = getPrimaryUserInfoForUserIds_Transaction(start, con, appIdentifier, ids); + if (result.isEmpty()) { + return null; + } + return result; + } + private static List getPrimaryUserInfoForUserIds(Start start, AppIdentifier appIdentifier, List userIds) @@ -1670,6 +1757,25 @@ public static void updateTimeJoinedForPrimaryUser_Transaction(Start start, Conne }); } + public static void updateTimeJoinedForPrimaryUsers_Transaction(Start start, Connection sqlCon, + AppIdentifier appIdentifier, List primaryUserIds) + throws SQLException, StorageQueryException { + String QUERY = "UPDATE " + getConfig(start).getUsersTable() + + " SET primary_or_recipe_user_time_joined = (SELECT MIN(time_joined) FROM " + + getConfig(start).getUsersTable() + " WHERE app_id = ? AND primary_or_recipe_user_id = ?) WHERE " + + " app_id = ? AND primary_or_recipe_user_id = ?"; + PreparedStatement updateStatement = sqlCon.prepareStatement(QUERY); + for(String primaryUserId : primaryUserIds) { + updateStatement.setString(1, appIdentifier.getAppId()); + updateStatement.setString(2, primaryUserId); + updateStatement.setString(3, appIdentifier.getAppId()); + updateStatement.setString(4, primaryUserId); + updateStatement.addBatch(); + } + + updateStatement.executeBatch(); + } + private static class AllAuthRecipeUsersResultHolder { String userId; String tenantId; diff --git a/src/main/java/io/supertokens/passwordless/Passwordless.java b/src/main/java/io/supertokens/passwordless/Passwordless.java index 23afba33a..d2e6c1ef0 100644 --- a/src/main/java/io/supertokens/passwordless/Passwordless.java +++ b/src/main/java/io/supertokens/passwordless/Passwordless.java @@ -21,7 +21,6 @@ import io.supertokens.config.Config; import io.supertokens.emailpassword.exceptions.EmailChangeNotAllowedException; import io.supertokens.multitenancy.Multitenancy; -import io.supertokens.multitenancy.MultitenancyHelper; import io.supertokens.multitenancy.exception.BadPermissionException; import io.supertokens.passwordless.exceptions.*; import io.supertokens.pluginInterface.RECIPE_ID; @@ -42,6 +41,7 @@ import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.passwordless.PasswordlessCode; import io.supertokens.pluginInterface.passwordless.PasswordlessDevice; +import io.supertokens.pluginInterface.passwordless.PasswordlessImportUser; import io.supertokens.pluginInterface.passwordless.exception.*; import io.supertokens.pluginInterface.passwordless.sqlStorage.PasswordlessSQLStorage; import io.supertokens.storageLayer.StorageLayer; @@ -550,6 +550,19 @@ public static AuthRecipeUserInfo createPasswordlessUser(TenantIdentifier tenantI } } + public static void createPasswordlessUsers(Storage storage, + List importUsers) + throws TenantOrAppNotFoundException, StorageQueryException, RestartFlowException, + StorageTransactionLogicException { + PasswordlessSQLStorage passwordlessStorage = StorageUtils.getPasswordlessStorage(storage); + + passwordlessStorage.startTransaction(con -> { + passwordlessStorage.importPasswordlessUsers_Transaction(con, importUsers); + passwordlessStorage.commitTransaction(con); + return null; + }); + } + @TestOnly public static void removeCode(Main main, String codeId) throws StorageQueryException, StorageTransactionLogicException { diff --git a/src/main/java/io/supertokens/thirdparty/ThirdParty.java b/src/main/java/io/supertokens/thirdparty/ThirdParty.java index 44247337d..bde6ff129 100644 --- a/src/main/java/io/supertokens/thirdparty/ThirdParty.java +++ b/src/main/java/io/supertokens/thirdparty/ThirdParty.java @@ -34,6 +34,7 @@ import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.ThirdPartyConfig; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; +import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateUserIdException; import io.supertokens.pluginInterface.thirdparty.sqlStorage.ThirdPartySQLStorage; @@ -42,10 +43,7 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nonnull; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import java.util.*; public class ThirdParty { @@ -355,6 +353,20 @@ public static SignInUpResponse createThirdPartyUser(TenantIdentifier tenantIdent } } + public static void createThirdPartyUsers(Storage storage, + Collection usersToImport) + throws StorageQueryException, StorageTransactionLogicException { + ThirdPartySQLStorage tpStorage = StorageUtils.getThirdPartyStorage(storage); + + tpStorage.startTransaction(con -> { + tpStorage.importThirdPartyUsers_Transaction(con, usersToImport); + tpStorage.commitTransaction(con); + return null; + }); + + // TODO error handling + } + @Deprecated public static AuthRecipeUserInfo getUser(AppIdentifier appIdentifier, Storage storage, String userId) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/totp/Totp.java b/src/main/java/io/supertokens/totp/Totp.java index d2afce084..61158bce2 100644 --- a/src/main/java/io/supertokens/totp/Totp.java +++ b/src/main/java/io/supertokens/totp/Totp.java @@ -5,10 +5,10 @@ import io.supertokens.config.Config; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.mfa.Mfa; -import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.exceptions.StorageQueryException; +import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; @@ -33,6 +33,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; +import java.util.List; public class Totp { private static String generateSecret() throws NoSuchAlgorithmException { @@ -143,6 +144,27 @@ public static TOTPDevice createDevice(Main main, AppIdentifier appIdentifier, St } } + public static void createDevices(Main main, AppIdentifier appIdentifier, Storage storage, List devices) + throws StorageQueryException, FeatureNotEnabledException, + StorageTransactionLogicException { + + try { + Mfa.checkForMFAFeature(appIdentifier, main); + + TOTPSQLStorage totpStorage = StorageUtils.getTOTPStorage(storage); + + totpStorage.startTransaction(con -> { + totpStorage.createDevices_Transaction(con, appIdentifier, devices); + totpStorage.commitTransaction(con); + System.out.println("Created TOTP devices"); + return null; + }); + + } catch (TenantOrAppNotFoundException e ) { + throw new StorageTransactionLogicException(e); + } + } + public static TOTPDevice registerDevice(AppIdentifier appIdentifier, Storage storage, Main main, String userId, String deviceName, int skew, int period) throws StorageQueryException, DeviceAlreadyExistsException, NoSuchAlgorithmException, diff --git a/src/main/java/io/supertokens/usermetadata/UserMetadata.java b/src/main/java/io/supertokens/usermetadata/UserMetadata.java index 938f6f749..545e5eaa1 100644 --- a/src/main/java/io/supertokens/usermetadata/UserMetadata.java +++ b/src/main/java/io/supertokens/usermetadata/UserMetadata.java @@ -30,6 +30,8 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Map; public class UserMetadata { @@ -76,6 +78,42 @@ public static JsonObject updateUserMetadata(AppIdentifier appIdentifier, Storage } } + public static void updateMultipleUsersMetadata(AppIdentifier appIdentifier, Storage storage, + @Nonnull Map metadataToUpdateByUserId) + throws StorageQueryException, StorageTransactionLogicException, TenantOrAppNotFoundException { + UserMetadataSQLStorage umdStorage = StorageUtils.getUserMetadataStorage(storage); + + try { + umdStorage.startTransaction(con -> { + Map originalMetadatas = umdStorage.getMultipleUsersMetadatas_Transaction(appIdentifier, con, + new ArrayList<>(metadataToUpdateByUserId.keySet())); + + // updating only the already existing ones. The others don't need update + for(Map.Entry metadataByUserId : originalMetadatas.entrySet()){ + JsonObject originalMetadata = metadataByUserId.getValue(); + String userId = metadataByUserId.getKey(); + JsonObject updatedMetadata = originalMetadata == null ? new JsonObject() : originalMetadata; + MetadataUtils.shallowMergeMetadataUpdate(updatedMetadata, metadataToUpdateByUserId.get(userId)); + metadataToUpdateByUserId.put(userId, updatedMetadata); + } + + try { + umdStorage.setMultipleUsersMetadatas_Transaction(appIdentifier, con, metadataToUpdateByUserId); + umdStorage.commitTransaction(con); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + + return null; + }); + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof TenantOrAppNotFoundException) { + throw (TenantOrAppNotFoundException) e.actualException; + } + throw e; + } + } + @TestOnly public static JsonObject getUserMetadata(Main main, @Nonnull String userId) throws StorageQueryException { Storage storage = StorageLayer.getStorage(main); diff --git a/src/main/java/io/supertokens/userroles/UserRoles.java b/src/main/java/io/supertokens/userroles/UserRoles.java index 21b132aa1..41811f1e5 100644 --- a/src/main/java/io/supertokens/userroles/UserRoles.java +++ b/src/main/java/io/supertokens/userroles/UserRoles.java @@ -31,7 +31,7 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; -import java.util.Arrays; +import java.util.Map; public class UserRoles { // add a role to a user and return true, if the role is already mapped to the user return false, but if @@ -57,6 +57,33 @@ public static boolean addRoleToUser(Main main, TenantIdentifier tenantIdentifier } } + public static void addMultipleRolesToMultipleUsers(Main main, Storage storage, Map> rolesToUserByTenant) + throws StorageTransactionLogicException, UnknownRoleException, TenantOrAppNotFoundException { + + // Roles are stored in public tenant storage and role to user mapping is stored in the tenant's storage + // We do this because it's not straight forward to replicate roles to all storages of an app + for(TenantIdentifier tenantIdentifier : rolesToUserByTenant.keySet()){ + Storage appStorage = StorageLayer.getStorage( + tenantIdentifier.toAppIdentifier().getAsPublicTenantIdentifier(), main); + // TODO!! +// if (!doesRoleExist(tenantIdentifier.toAppIdentifier(), appStorage, role)) { +// throw new UnknownRoleException(); +// } + + try { + UserRolesSQLStorage userRolesStorage = StorageUtils.getUserRolesStorage(storage); + userRolesStorage.startTransaction(con -> { + userRolesStorage.addRolesToUsers_Transaction(con, rolesToUserByTenant); + userRolesStorage.commitTransaction(con); + return null; + }); + + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); + } + } + } + @TestOnly public static boolean addRoleToUser(Main main, String userId, String role) throws StorageQueryException, UnknownRoleException { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java index 599801680..a6f9d3c8f 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -16,12 +16,9 @@ package io.supertokens.webserver.api.bulkimport; -import java.io.IOException; - import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; - import io.supertokens.Main; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportUserUtils; @@ -43,6 +40,8 @@ import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; + public class ImportUserAPI extends WebserverAPI { public ImportUserAPI(Main main) { super(main, ""); diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index e2aa496b2..a135582d6 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -78,7 +78,8 @@ public void testWithOneMillionUsers() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - int NUMBER_OF_USERS_TO_UPLOAD = 1000000; + //int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million + int NUMBER_OF_USERS_TO_UPLOAD = 10000; int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); System.out.println("Worker threads: " + parallelism_set_to); From d33eb5e66bd6b1bb6e9c0c34b335870e4f764520 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Tue, 19 Nov 2024 15:10:43 +0100 Subject: [PATCH 36/41] fix: fast as a lightning --- .../io/supertokens/authRecipe/AuthRecipe.java | 144 +++++++----- .../io/supertokens/bulkimport/BulkImport.java | 87 ++++++-- .../java/io/supertokens/inmemorydb/Start.java | 42 ++++ .../storageLayer/StorageLayer.java | 46 +++- .../useridmapping/UserIdMapping.java | 211 ++++++++++++++++++ src/main/java/io/supertokens/utils/Utils.java | 1 + .../test/bulkimport/BulkImportFlowTest.java | 8 +- 7 files changed, 457 insertions(+), 82 deletions(-) diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 66234022f..5fa1f4616 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -281,19 +281,22 @@ private static CanLinkAccountsResult canLinkAccountsHelper(TransactionConnection tenantIds.addAll(recipeUser.tenantIds); tenantIds.addAll(primaryUser.tenantIds); - checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, recipeUser.loginMethods[0], primaryUser); - - for (LoginMethod currLoginMethod : primaryUser.loginMethods) { - checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, currLoginMethod, primaryUser); - } - +// checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, recipeUser.loginMethods[0], primaryUser); +// +// for (LoginMethod currLoginMethod : primaryUser.loginMethods) { +// checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, currLoginMethod, primaryUser); +// } +// TODO !!! return new CanLinkAccountsResult(recipeUser.getSupertokensUserId(), primaryUser.getSupertokensUserId(), false); } private static List canLinkMultipleAccountsHelper(TransactionConnection con, AppIdentifier appIdentifier, Storage storage, - Map recipeUserIdByPrimaryUserId) + Map recipeUserIdByPrimaryUserId, + List allDistinctEmailAddresses, + List phones, + Map thirdpartyUserIdToId) throws StorageQueryException { AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); @@ -305,6 +308,10 @@ private static List canLinkMultipleAccountsHelper(Tra List recipeUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, new ArrayList<>(recipeUserIdByPrimaryUserId.keySet())); + List allUsersWithExtraData = + List.of(authRecipeStorage.listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction + (appIdentifier, con, allDistinctEmailAddresses, phones, thirdpartyUserIdToId)); + if(recipeUsers != null && primaryUsers != null) { //collect all the really primary users into a map of userid -> authRecipeUserInfo Map foundValidPrimaryUsers = primaryUsers.stream().filter(authRecipeUserInfo -> authRecipeUserInfo.isPrimaryUser).collect(Collectors.toMap(AuthRecipeUserInfo::getSupertokensUserId, authRecipeUserInfo -> authRecipeUserInfo)); @@ -331,12 +338,13 @@ private static List canLinkMultipleAccountsHelper(Tra try { //TODO (?) this below method still uses multiple DB queries which could be enhanced + //TODO update: this below method takes a significant time. enhance it! 20 mins vs 8 hours checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, - recipeUser.loginMethods[0], primaryUser); + recipeUser.loginMethods[0], primaryUser, allUsersWithExtraData); for (LoginMethod currLoginMethod : primaryUser.loginMethods) { checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, - currLoginMethod, primaryUser); + currLoginMethod, primaryUser, allUsersWithExtraData); } // I don't get why this is needed.. results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, null, primaryUser)); @@ -354,7 +362,8 @@ private static List canLinkMultipleAccountsHelper(Tra private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection con, AppIdentifier appIdentifier, AuthRecipeSQLStorage authRecipeStorage, Set tenantIds, LoginMethod currLoginMethod, - AuthRecipeUserInfo primaryUser) + AuthRecipeUserInfo primaryUser, + List allUsersWithExtraData) throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException { // we loop through the union of both the user's tenantIds and check that the criteria for // linking accounts is not violated in any of them. We do a union and not an intersection @@ -372,9 +381,10 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection // tenants of the same storage - therefore, the storage will be the same. if (currLoginMethod.email != null) { - AuthRecipeUserInfo[] usersWithSameEmail = authRecipeStorage - .listPrimaryUsersByEmail_Transaction(appIdentifier, con, - currLoginMethod.email); + List usersWithSameEmail = + allUsersWithExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod -> loginMethod.email).collect( + Collectors.toList()).contains(currLoginMethod.email)).collect(Collectors.toList()); for (AuthRecipeUserInfo user : usersWithSameEmail) { if (!user.tenantIds.contains(tenantId)) { continue; @@ -388,9 +398,10 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection } if (currLoginMethod.phoneNumber != null) { - AuthRecipeUserInfo[] usersWithSamePhoneNumber = authRecipeStorage - .listPrimaryUsersByPhoneNumber_Transaction(appIdentifier, con, - currLoginMethod.phoneNumber); + List usersWithSamePhoneNumber = + allUsersWithExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod -> loginMethod.phoneNumber).collect( + Collectors.toList()).contains(currLoginMethod.phoneNumber)).collect(Collectors.toList()); for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { if (!user.tenantIds.contains(tenantId)) { continue; @@ -405,22 +416,24 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection } if (currLoginMethod.thirdParty != null) { - AuthRecipeUserInfo[] usersWithSameThirdParty = authRecipeStorage - .listPrimaryUsersByThirdPartyInfo_Transaction(appIdentifier, con, - currLoginMethod.thirdParty.id, currLoginMethod.thirdParty.userId); - for (AuthRecipeUserInfo userWithSameThirdParty : usersWithSameThirdParty) { - if (!userWithSameThirdParty.tenantIds.contains(tenantId)) { - continue; - } - if (userWithSameThirdParty.isPrimaryUser && - !userWithSameThirdParty.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { - throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( - userWithSameThirdParty.getSupertokensUserId(), - "This user's third party login is already associated with another" + - " user ID"); + List extraUsersWithThirdParty = allUsersWithExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).anyMatch(loginMethod1 -> loginMethod1.thirdParty != null)).collect(Collectors.toList()); + for(AuthRecipeUserInfo extraUser : extraUsersWithThirdParty) { + if(extraUser.isPrimaryUser && extraUser.tenantIds.contains(tenantId) + && !extraUser.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + for (LoginMethod loginMethodExtra : extraUser.loginMethods) { + if (loginMethodExtra.thirdParty != null && + loginMethodExtra.thirdParty.userId.equals(currLoginMethod.thirdParty.userId) + && loginMethodExtra.thirdParty.id.equals(currLoginMethod.thirdParty.id)) { + + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + extraUser.getSupertokensUserId(), + "This user's third party login is already associated with another" + + " user ID"); + } + } } } - } } } @@ -504,7 +517,9 @@ public static LinkAccountsResult linkAccounts(Main main, AppIdentifier appIdenti } public static List linkMultipleAccounts(Main main, AppIdentifier appIdentifier, - Storage storage, Map recipeUserIdToPrimaryUserId) + Storage storage, Map recipeUserIdToPrimaryUserId, + List allDistinctEmailAddresses, List allDistinctPhones, + Map allThirdpartyUserIdsToThirdpartyIds) throws StorageQueryException, TenantOrAppNotFoundException, FeatureNotEnabledException { if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { @@ -517,7 +532,8 @@ public static List linkMultipleAccounts(Main main, AppId List linkAccountsResults = authRecipeStorage.startTransaction(con -> { List canLinkAccounts = canLinkMultipleAccountsHelper(con, appIdentifier, - authRecipeStorage, recipeUserIdToPrimaryUserId); + authRecipeStorage, recipeUserIdToPrimaryUserId, allDistinctEmailAddresses, allDistinctPhones, + allThirdpartyUserIdsToThirdpartyIds); List results = new ArrayList<>(); Map recipeUserByPrimaryUserNeedsLinking = new HashMap<>(); if(!canLinkAccounts.isEmpty()){ @@ -710,7 +726,10 @@ private static CreatePrimaryUserResult canCreatePrimaryUserHelper(TransactionCon private static List canCreatePrimaryUsersHelper(TransactionConnection con, AppIdentifier appIdentifier, Storage storage, - List recipeUserIds) + List recipeUserIds, + List allDistinctEmails, + List allPhones, + Map thirdpartyUserIdToThirdpartyId) throws StorageQueryException, UnknownUserIdException{ AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); @@ -720,11 +739,16 @@ private static List canCreatePrimaryUsersHelper(Tra throw new UnknownUserIdException(); } List results = new ArrayList<>(); + List allUsersWithProvidedExtraData = + List.of(authRecipeStorage. + listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction(appIdentifier, con, + allDistinctEmails, allPhones, thirdpartyUserIdToThirdpartyId)); + for(int i = 0; i < targetUsers.size(); i++) { AuthRecipeUserInfo targetUser = targetUsers.get(i); if (targetUser.isPrimaryUser) { if (targetUser.getSupertokensUserId() - .equals(recipeUserIds.get(i))) { // TODO what if there is no i-th recipeUserId? + .equals(recipeUserIds.get(i))) { results.add(new CreatePrimaryUserBulkResult(targetUser, true, null)); } else { results.add(new CreatePrimaryUserBulkResult(targetUser, false, @@ -743,9 +767,9 @@ private static List canCreatePrimaryUsersHelper(Tra for (String tenantId : targetUser.tenantIds) { if (loginMethod.email != null) { - AuthRecipeUserInfo[] usersWithSameEmail = authRecipeStorage - .listPrimaryUsersByEmail_Transaction(appIdentifier, con, - loginMethod.email); + List usersWithSameEmail = allUsersWithProvidedExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod1 -> loginMethod1.email).collect(Collectors.toList()).contains(loginMethod.email)).collect( + Collectors.toList()); for (AuthRecipeUserInfo user : usersWithSameEmail) { if (!user.tenantIds.contains(tenantId)) { continue; @@ -762,9 +786,9 @@ private static List canCreatePrimaryUsersHelper(Tra } if (loginMethod.phoneNumber != null) { - AuthRecipeUserInfo[] usersWithSamePhoneNumber = authRecipeStorage - .listPrimaryUsersByPhoneNumber_Transaction(appIdentifier, con, - loginMethod.phoneNumber); + List usersWithSamePhoneNumber = allUsersWithProvidedExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).map(loginMethod1 -> loginMethod1.phoneNumber).collect(Collectors.toList()).contains(loginMethod.phoneNumber)).collect( + Collectors.toList()); for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { if (!user.tenantIds.contains(tenantId)) { continue; @@ -782,21 +806,24 @@ private static List canCreatePrimaryUsersHelper(Tra } if (loginMethod.thirdParty != null) { - AuthRecipeUserInfo[] usersWithSameThirdParty = authRecipeStorage - .listPrimaryUsersByThirdPartyInfo_Transaction(appIdentifier, con, - loginMethod.thirdParty.id, loginMethod.thirdParty.userId); - for (AuthRecipeUserInfo userWithSameThirdParty : usersWithSameThirdParty) { - if (!userWithSameThirdParty.tenantIds.contains(tenantId)) { - continue; - } - if (userWithSameThirdParty.isPrimaryUser) { - results.add(new CreatePrimaryUserBulkResult(targetUser, false, - new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( - userWithSameThirdParty.getSupertokensUserId(), - "This user's third party login is already associated with another" + - " user ID"))); - errorFound = true; - break; + List extraUsersWithThirdParty = allUsersWithProvidedExtraData.stream().filter(authRecipeUserInfo -> Arrays.stream( + authRecipeUserInfo.loginMethods).anyMatch(loginMethod1 -> loginMethod1.thirdParty != null)).collect(Collectors.toList()); + for(AuthRecipeUserInfo extraUser : extraUsersWithThirdParty) { + if(extraUser.isPrimaryUser && extraUser.tenantIds.contains(tenantId)) { + for (LoginMethod loginMethodExtra : extraUser.loginMethods) { + if (loginMethodExtra.thirdParty != null && + loginMethodExtra.thirdParty.userId.equals(loginMethod.thirdParty.userId) + && loginMethodExtra.thirdParty.id.equals(loginMethod.thirdParty.id)) { + + results.add(new CreatePrimaryUserBulkResult(targetUser, false, + new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + extraUser.getSupertokensUserId(), + "This user's third party login is already associated with another" + + " user ID"))); + errorFound = true; + break; + } + } } } } @@ -874,7 +901,10 @@ public static CreatePrimaryUserResult createPrimaryUser(Main main, public static List createPrimaryUsers(Main main, AppIdentifier appIdentifier, Storage storage, - List recipeUserIds) + List recipeUserIds, + List allDistinctEmails, + List allDistinctPhones, + Map thirdpartyUserIdsToThirdpartyIds) throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { @@ -890,7 +920,7 @@ public static List createPrimaryUsers(Main main, try { List results = canCreatePrimaryUsersHelper(con, appIdentifier, authRecipeStorage, - recipeUserIds); + recipeUserIds, allDistinctEmails, allDistinctPhones, thirdpartyUserIdsToThirdpartyIds); List canMakePrimaryUsers = new ArrayList<>(); for(CreatePrimaryUserBulkResult result : results) { if (result.wasAlreadyAPrimaryUser || result.error != null) { diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 0ab0d89a8..37b5ea23f 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -78,15 +78,13 @@ import io.supertokens.userroles.UserRoles; import io.supertokens.utils.Utils; import jakarta.servlet.ServletException; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; // Error codes ensure globally unique and identifiable errors in Bulk Import. @@ -105,7 +103,7 @@ public class BulkImport { // Number of users to process in a single batch of ProcessBulkImportUsers Cron Job public static final int PROCESS_USERS_BATCH_SIZE = 10000; // Time interval in seconds between two consecutive runs of ProcessBulkImportUsers Cron Job - public static final int PROCESS_USERS_INTERVAL_SECONDS = 30; + public static final int PROCESS_USERS_INTERVAL_SECONDS = 1; private static final Logger log = LoggerFactory.getLogger(BulkImport.class); // This map allows reusing proxy storage for all tenants in the app and closing connections after import. @@ -233,10 +231,8 @@ public static void processUsersImportSteps(Main main, TransactionConnection conn TenantOrAppNotFoundException | UnknownUserIdException e) { throw new RuntimeException(e); } - //TODO create user id mapping! - for(BulkImportUser user : users) { - createUserIdMapping(appIdentifier, user, getPrimaryLoginMethod(user), allStoragesForApp); - } + + createMultipleUserIdMapping(appIdentifier, users, allStoragesForApp); verifyMultipleEmailForAllLoginMethods(appIdentifier, bulkImportProxyStorage, users); createMultipleTotpDevices(main, appIdentifier, bulkImportProxyStorage, users); createMultipleUserMetadata(appIdentifier, bulkImportProxyStorage, users); @@ -507,9 +503,26 @@ private static void createPrimaryUsersAndLinkAccounts(Main main, System.out.println(Thread.currentThread().getName() + " createPrimaryUsersAndLinkAccounts"); List userIds = users.stream().map(bulkImportUser -> getPrimaryLoginMethod(bulkImportUser).getSuperTokenOrExternalUserId()).collect(Collectors.toList()); + Set allEmails = new HashSet<>(); + Set allPhoneNumber = new HashSet<>(); + Map allThirdParty = new HashMap<>(); + for(BulkImportUser user: users){ + for(LoginMethod loginMethod : user.loginMethods) { + if(loginMethod.email != null) { + allEmails.add(loginMethod.email); + } + if(loginMethod.phoneNumber != null){ + allPhoneNumber.add(loginMethod.phoneNumber); + } + if(loginMethod.thirdPartyId != null && loginMethod.thirdPartyUserId != null){ + allThirdParty.put(loginMethod.thirdPartyUserId, loginMethod.thirdPartyId); + } - AuthRecipe.createPrimaryUsers(main, appIdentifier, storage, userIds); - linkAccountsForMultipleUser(main, appIdentifier, storage, users); + } + } + + AuthRecipe.createPrimaryUsers(main, appIdentifier, storage, userIds, new ArrayList<>(allEmails), new ArrayList<>(allPhoneNumber), allThirdParty); + linkAccountsForMultipleUser(main, appIdentifier, storage, users, new ArrayList<>(allEmails), new ArrayList<>(allPhoneNumber), allThirdParty); } @@ -589,8 +602,24 @@ private static void linkAccountsForUser(Main main, AppIdentifier appIdentifier, } } - private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIdentifier, Storage storage, List users) + private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIdentifier, Storage storage, + List users, + List allDistinctEmails, + List allDistinctPhones, + Map thirdpartyUserIdsToThirdpartyIds) throws StorageTransactionLogicException { + Map recipeUserIdByPrimaryUserId = collectRecipeIdsToPrimaryIds(users); + try { + AuthRecipe.linkMultipleAccounts(main, appIdentifier, storage, recipeUserIdByPrimaryUserId, + allDistinctEmails, allDistinctPhones, thirdpartyUserIdsToThirdpartyIds); + } catch (StorageQueryException | FeatureNotEnabledException | TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); + } + //TODO proper error handling + } + + @NotNull + private static Map collectRecipeIdsToPrimaryIds(List users) { Map recipeUserIdByPrimaryUserId = new HashMap<>(); for(BulkImportUser user: users){ LoginMethod primaryLM = getPrimaryLoginMethod(user); @@ -602,13 +631,7 @@ private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIden primaryLM.getSuperTokenOrExternalUserId()); } } - - try { - AuthRecipe.linkMultipleAccounts(main, appIdentifier, storage, recipeUserIdByPrimaryUserId); - } catch (StorageQueryException | FeatureNotEnabledException | TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(e); - } - //TODO proper error handling + return recipeUserIdByPrimaryUserId; } public static void createUserIdMapping(AppIdentifier appIdentifier, @@ -639,6 +662,32 @@ public static void createUserIdMapping(AppIdentifier appIdentifier, } } + public static void createMultipleUserIdMapping(AppIdentifier appIdentifier, + List users, Storage[] storages) throws StorageTransactionLogicException { + System.out.println(Thread.currentThread().getName() + " createMultipleUserIdMapping"); + Map superTokensUserIdToExternalUserId = new HashMap<>(); + for(BulkImportUser user: users) { + if(user.externalUserId != null) { + LoginMethod primaryLoginMethod = getPrimaryLoginMethod(user); + superTokensUserIdToExternalUserId.put(primaryLoginMethod.superTokensUserId, user.externalUserId); + } + } + try { + List mappingResults = UserIdMapping.createMultipleUserIdMappings( + appIdentifier, storages, + superTokensUserIdToExternalUserId, + false, true); + +// for(UserIdMapping.UserIdBulkMappingResult mappingResult : mappingResults){ +// if(mappingResult.error == null) { // no error means successful mapping +// // TODO +// } +// } + } catch (Exception e) { + //TODO proper error handling + } + } + public static void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { if (user.userMetadata != null) { diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 5a27c7d27..4572049ad 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -655,6 +655,13 @@ public boolean isUserIdBeingUsedInNonAuthRecipe(AppIdentifier appIdentifier, Str } } + @Override + public Map> findNonAuthRecipesWhereForUserIdsUsed(AppIdentifier appIdentifier, + List userIds) + throws StorageQueryException { + return Map.of(); + } + @TestOnly @Override public void addInfoToNonAuthRecipesBasedOnUserId(TenantIdentifier tenantIdentifier, String className, String userId) @@ -1146,6 +1153,13 @@ public void updateIsEmailVerifiedToExternalUserId(AppIdentifier appIdentifier, S externalUserId); } + @Override + public void updateMultipleIsEmailVerifiedToExternalUserIds(AppIdentifier appIdentifier, + Map supertokensUserIdToExternalUserId) + throws StorageQueryException { + + } + @Override public void deleteExpiredPasswordResetTokens() throws StorageQueryException { try { @@ -1334,6 +1348,12 @@ public boolean doesUserIdExist(TenantIdentifier tenantIdentifier, String userId) } } + @Override + public List findExistingUserIds(AppIdentifier appIdentifier, List userIds) + throws StorageQueryException { + return List.of(); // TODO + } + @Override public AuthRecipeUserInfo getPrimaryUserById(AppIdentifier appIdentifier, String userId) throws StorageQueryException { @@ -2237,6 +2257,13 @@ public void createUserIdMapping(AppIdentifier appIdentifier, String superTokensU } } + @Override + public void createBulkUserIdMapping(AppIdentifier appIdentifier, + Map superTokensUserIdToExternalUserId) + throws StorageQueryException { + + } + @Override public boolean deleteUserIdMapping(AppIdentifier appIdentifier, String userId, boolean isSuperTokensUserId) throws StorageQueryException { @@ -2977,6 +3004,13 @@ public AuthRecipeUserInfo[] listPrimaryUsersByEmail_Transaction(AppIdentifier ap } } + @Override + public AuthRecipeUserInfo[] listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction( + AppIdentifier appIdentifier, TransactionConnection con, List emails, List phones, + Map thirdpartyIdToThirdpartyUserId) throws StorageQueryException { + return new AuthRecipeUserInfo[0]; // TODO + } + @Override public AuthRecipeUserInfo[] listPrimaryUsersByPhoneNumber_Transaction(AppIdentifier appIdentifier, TransactionConnection con, @@ -3142,6 +3176,14 @@ public UserIdMapping[] getUserIdMapping_Transaction(TransactionConnection con, A } } + @Override + public List getMultipleUserIdMapping_Transaction(TransactionConnection connection, + AppIdentifier appIdentifier, List userIds, + boolean isSupertokensIds) + throws StorageQueryException { + return List.of(); // TODO + } + @Override public int getUsersCountWithMoreThanOneLoginMethodOrTOTPEnabled(AppIdentifier appIdentifier) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index e4c725bad..e1ef206f9 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -17,7 +17,10 @@ package io.supertokens.storageLayer; import com.google.gson.JsonObject; -import io.supertokens.*; +import io.supertokens.Main; +import io.supertokens.ProcessState; +import io.supertokens.ResourceDistributor; +import io.supertokens.StorageAndUserIdMapping; import io.supertokens.cliOptions.CLIOptions; import io.supertokens.config.Config; import io.supertokens.exceptions.QuitProgramException; @@ -31,7 +34,10 @@ import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; -import io.supertokens.pluginInterface.multitenancy.*; +import io.supertokens.pluginInterface.multitenancy.AppIdentifier; +import io.supertokens.pluginInterface.multitenancy.MultitenancyStorage; +import io.supertokens.pluginInterface.multitenancy.TenantConfig; +import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.pluginInterface.useridmapping.UserIdMapping; import io.supertokens.useridmapping.UserIdType; @@ -572,4 +578,40 @@ public static StorageAndUserIdMapping findStorageAndUserIdMappingForUser( throw new IllegalStateException("should never come here"); } } + + public static List findStorageAndUserIdMappingForBulkUserImport( + AppIdentifier appIdentifier, Storage[] storages, List userIds, + UserIdType userIdType) throws StorageQueryException { + + if (storages.length == 0) { + throw new IllegalStateException("No storages were provided!"); + } + + if (storages[0].getType() != STORAGE_TYPE.SQL) { + // for non sql plugin, there will be only one storage as multitenancy is not supported + assert storages.length == 1; + return Collections.singletonList(new StorageAndUserIdMapping(storages[0], null)); + } + List allMappingsFromAllStorages = new ArrayList<>(); + if (userIdType != UserIdType.ANY) { + for (Storage storage : storages) { + List existingIdsInStorage = ((AuthRecipeStorage)storage).findExistingUserIds(appIdentifier, userIds); + List mappingsFromThisStorage = io.supertokens.useridmapping.UserIdMapping.getMultipleUserIdMapping( + appIdentifier, storage, + userIds, userIdType); + + for(String existingId : existingIdsInStorage) { + UserIdMapping mappingForId = mappingsFromThisStorage.stream() + .filter(userIdMapping -> (userIdType == UserIdType.SUPERTOKENS && userIdMapping.superTokensUserId.equals(existingId)) + || (userIdType == UserIdType.EXTERNAL && userIdMapping.externalUserId.equals(existingId)) ) + .findFirst().orElse(null); + allMappingsFromAllStorages.add(new StorageAndUserIdMapping(storage, mappingForId)); + } + } + } else { + throw new IllegalStateException("UserIdType.ANY is not supported for this method"); + } + return allMappingsFromAllStorages; + } + } diff --git a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java index 5f81c81ee..d2c35ddf4 100644 --- a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java +++ b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java @@ -45,9 +45,22 @@ import javax.annotation.Nullable; import java.util.*; +import java.util.stream.Collectors; public class UserIdMapping { + public static class UserIdBulkMappingResult { + public String supertokensUserId; + public String externalUserId; + public Exception error; + + public UserIdBulkMappingResult(String supertokensUserId, String externalUserId, Exception error) { + this.supertokensUserId = supertokensUserId; + this.error = error; + this.externalUserId = externalUserId; + } + } + @TestOnly public static void createUserIdMapping(AppIdentifier appIdentifier, Storage[] storages, String superTokensUserId, String externalUserId, @@ -158,11 +171,189 @@ public static void createUserIdMapping(AppIdentifier appIdentifier, Storage[] st } } + StorageUtils.getUserIdMappingStorage(userStorage) .createUserIdMapping(appIdentifier, superTokensUserId, externalUserId, externalUserIdInfo); } + //support method for the primary intention of bulk importing users. + public static List createMultipleUserIdMappings(AppIdentifier appIdentifier, Storage[] storages, + Map superTokensUserIdToExternalUserId, boolean force, + boolean makeExceptionForEmailVerification) + throws UnknownSuperTokensUserIdException, + UserIdMappingAlreadyExistsException, StorageQueryException, ServletException, + TenantOrAppNotFoundException { + + // We first need to check if the external user id exists across all app storages because we do not want + // 2 users from different user pool but same app to point to same external user id. + // We may still end up having that situation due to race conditions, as we are not taking any app level lock, + // but we are okay with it as of now, by returning prioritized mapping based on which the tenant the request + // came from. + // This issue - https://github.com/supertokens/supertokens-core/issues/610 - must be resolved when the + // race condition is fixed. + + List mappingResults = new ArrayList<>(); + + // with external id + List mappingAndStorageWithExternal = + StorageLayer.findStorageAndUserIdMappingForBulkUserImport( + appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.values()), UserIdType.EXTERNAL); + + // with supertokens id + List mappingAndStorageWithSupertokens = + StorageLayer.findStorageAndUserIdMappingForBulkUserImport( + appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.keySet()), UserIdType.SUPERTOKENS); + + //with external id treated as supertokens id - should not happen + List mappingAndStoragesAsInvalid = StorageLayer.findStorageAndUserIdMappingForBulkUserImport( + appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.values()), UserIdType.SUPERTOKENS); + + //TODO does it matter which storage? + Map> userIdsUsedInNonAuthRecipes = + storages[0].findNonAuthRecipesWhereForUserIdsUsed(appIdentifier, new ArrayList<>(superTokensUserIdToExternalUserId.keySet())); + + //for collecting which users needs to be updated + Map supertokensToExternalUserIdsToUpdateEmailVerified = new HashMap<>(); + List noErrorFound = new ArrayList<>(); + + for(Map.Entry supertokensIdToExternalId : superTokensUserIdToExternalUserId.entrySet()) { + String supertokensId = supertokensIdToExternalId.getKey(); + String externalId = supertokensIdToExternalId.getValue(); + StorageAndUserIdMapping mappingByExternal = findStorageAndUserIdMappingForUser(externalId, mappingAndStorageWithExternal, false); + if (mappingByExternal != null && mappingByExternal.userIdMapping != null ){ + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, + new UserIdMappingAlreadyExistsException(supertokensId.equals(mappingByExternal.userIdMapping.superTokensUserId), + externalId.equals(mappingByExternal.userIdMapping.externalUserId)))); + continue; + } + StorageAndUserIdMapping mappingBySupertokens = findStorageAndUserIdMappingForUser(supertokensId, mappingAndStorageWithSupertokens, true); + if(mappingBySupertokens == null) { + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, new UnknownSuperTokensUserIdException())); + continue; + } + Storage userStorage = mappingBySupertokens.storage; + + // if a userIdMapping is created with force, then we skip the following checks + if (!force) { + // We do not allow for a UserIdMapping to be created when the externalUserId is a SuperTokens userId. + // There could be a case where User_1 has a userId mapping and a new SuperTokens User, User_2 is created + // whose userId is equal to the User_1's externalUserId. + // Theoretically this could happen but the likelihood of generating a non-unique UUID is low enough that we + // ignore it. + + { + if (findStorageAndUserIdMappingForUser(externalId, mappingAndStoragesAsInvalid, true) != null) { + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, new ServletException(new WebserverAPI.BadRequestException( + "Cannot create a userId mapping where the externalId is also a SuperTokens userID")))); + continue; + } + } + + List storageClasses; + if(userIdsUsedInNonAuthRecipes.containsKey(supertokensId)){ + storageClasses = userIdsUsedInNonAuthRecipes.get(supertokensId); + } else { + storageClasses = new ArrayList<>(); + } + + if (makeExceptionForEmailVerification) { + // check that none of the non-auth recipes are using the superTokensUserId + + if (storageClasses.size() == 1 && + storageClasses.get(0).equals(EmailVerificationStorage.class.getName())) { + // if the userId is used in email verification, then we do an exception and update the + // isEmailVerified + // to the externalUserId. We do this because we automatically set the isEmailVerified to true for + // passwordless + // and third party sign in up when the user info from provider says the email is verified and If + // we don't make + // an exception, then the creation of userIdMapping for the user will be blocked. And, to + // overcome that the + // email will have to be unverified first, then the userIdMapping should be created and then the + // email must be + // verified again on the externalUserId, which is not a good user experience. + supertokensToExternalUserIdsToUpdateEmailVerified.put(supertokensId, externalId); + + } else if (!storageClasses.isEmpty()) { + createBulkIdMappingErrorForNonAuthRecipeUsage(storageClasses, mappingResults, supertokensId, + externalId); + continue; + } + } else { + //if we are not making any exceptions, then having the id used is an error! + if(!storageClasses.isEmpty()) { + createBulkIdMappingErrorForNonAuthRecipeUsage(storageClasses, mappingResults, supertokensId, externalId); + continue; + } + } + + noErrorFound.add(mappingBySupertokens); + } + + //userstorage - group users by storage + Map> partitionedMappings = partitionUsersByStorage(noErrorFound); + for(Storage storage : partitionedMappings.keySet()){ + List mappingsForCurrentStorage = partitionedMappings.get(storage); + + Map supertokensIdToExternalIdInCurrentStorage = new HashMap<>(); + for(StorageAndUserIdMapping storageAndUserIdMapping: mappingsForCurrentStorage) { + supertokensIdToExternalIdInCurrentStorage.put(storageAndUserIdMapping.userIdMapping.superTokensUserId, + superTokensUserIdToExternalUserId.get(storageAndUserIdMapping.userIdMapping.superTokensUserId)); + } + + EmailVerificationStorage emailVerificationStorage = StorageUtils.getEmailVerificationStorage(storage); + emailVerificationStorage.updateMultipleIsEmailVerifiedToExternalUserIds(appIdentifier, supertokensIdToExternalIdInCurrentStorage); + + StorageUtils.getUserIdMappingStorage(storage).createBulkUserIdMapping(appIdentifier, supertokensIdToExternalIdInCurrentStorage); + for(String supertokensIdForResult : supertokensIdToExternalIdInCurrentStorage.keySet()) { + mappingResults.add(new UserIdBulkMappingResult(supertokensIdForResult, supertokensIdToExternalIdInCurrentStorage.get(supertokensIdForResult), null)); + } + } + } + return mappingResults; + } + + private static void createBulkIdMappingErrorForNonAuthRecipeUsage(List storageClasses, + List mappingResults, + String supertokensId, String externalId) { + String recipeName = storageClasses.get(0); + String[] parts = recipeName.split("[.]"); + recipeName = parts[parts.length - 1]; + recipeName = recipeName.replace("Storage", ""); + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, new ServletException(new WebserverAPI.BadRequestException( + "UserId is already in use in " + recipeName + " recipe")))); + } + + private static Map> partitionUsersByStorage(List storageAndMappings){ + Map> results = new HashMap<>(); + for(StorageAndUserIdMapping storageAndUserIdMapping : storageAndMappings) { + if(!results.containsKey(storageAndUserIdMapping.storage)){ + results.put(storageAndUserIdMapping.storage, new ArrayList<>()); + } + results.get(storageAndUserIdMapping.storage).add(storageAndUserIdMapping); + } + return results; + } + + private static StorageAndUserIdMapping findStorageAndUserIdMappingForUser(String userId, List findIn, boolean supertokensId) { + List mappings = findIn.stream().filter(storageAndUserIdMapping -> { + if(storageAndUserIdMapping.userIdMapping != null) { + if(supertokensId) { + return storageAndUserIdMapping.userIdMapping.superTokensUserId.equals(userId); + } else { + return storageAndUserIdMapping.userIdMapping.externalUserId.equals(userId); + } + } + return false; + }).collect(Collectors.toList()); // theoretically it shouldn't happen that there are more than one element in the list + if(mappings.size() > 1) { + throw new IllegalStateException("more than one mapping exists for Id."); + } + return mappings.isEmpty() ? null : mappings.get(0); + } + + @TestOnly public static void createUserIdMapping(Main main, String superTokensUserId, String externalUserId, @@ -208,6 +399,26 @@ public static io.supertokens.pluginInterface.useridmapping.UserIdMapping getUser } } + public static List getMultipleUserIdMapping( + AppIdentifier appIdentifier, Storage storage, List userIds, + UserIdType userIdType) + throws StorageQueryException { + UserIdMappingSQLStorage uidMappingStorage = + (UserIdMappingSQLStorage) storage; + + try { + return uidMappingStorage.startTransaction(con -> { + return uidMappingStorage.getMultipleUserIdMapping_Transaction(con, appIdentifier, userIds, userIdType == UserIdType.SUPERTOKENS); + }); + } catch (StorageTransactionLogicException e) { + if (e.actualException instanceof StorageQueryException) { + throw (StorageQueryException) e.actualException; + } else { + throw new IllegalStateException(e.actualException); + } + } + } + public static io.supertokens.pluginInterface.useridmapping.UserIdMapping getUserIdMapping( TransactionConnection con, AppIdentifier appIdentifier, Storage storage, String userId, diff --git a/src/main/java/io/supertokens/utils/Utils.java b/src/main/java/io/supertokens/utils/Utils.java index b64b80a0a..d02ebce5b 100644 --- a/src/main/java/io/supertokens/utils/Utils.java +++ b/src/main/java/io/supertokens/utils/Utils.java @@ -475,4 +475,5 @@ public static String snakeCaseToCamelCase(String toCamelCase) { } return toCamelCase; } + } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index a135582d6..233c7e1f8 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -69,7 +69,7 @@ public void testWithOneMillionUsers() throws Exception { String[] args = { "../" }; // set processing thread number - Utils.setValueInConfig("bulk_migration_parallelism", "14"); + Utils.setValueInConfig("bulk_migration_parallelism", "12"); TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); @@ -78,8 +78,8 @@ public void testWithOneMillionUsers() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - //int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million - int NUMBER_OF_USERS_TO_UPLOAD = 10000; + int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million + //int NUMBER_OF_USERS_TO_UPLOAD = 10000; int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); System.out.println("Worker threads: " + parallelism_set_to); @@ -108,7 +108,7 @@ public void testWithOneMillionUsers() throws Exception { long processingStartedTime = System.currentTimeMillis(); // Starting the processing cronjob here to be able to measure the runtime - startBulkImportCronjob(main, 5000); + startBulkImportCronjob(main, 6000); System.out.println("CronJob started"); // wait for the cron job to process them From a41f9514c7a8930d92e14fada583bdc2702974da Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Tue, 19 Nov 2024 17:06:34 +0100 Subject: [PATCH 37/41] fix: restoring lost method --- .../io/supertokens/authRecipe/AuthRecipe.java | 100 +++++++++++++++--- 1 file changed, 85 insertions(+), 15 deletions(-) diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 5fa1f4616..a0a406d73 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -281,12 +281,12 @@ private static CanLinkAccountsResult canLinkAccountsHelper(TransactionConnection tenantIds.addAll(recipeUser.tenantIds); tenantIds.addAll(primaryUser.tenantIds); -// checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, recipeUser.loginMethods[0], primaryUser); -// -// for (LoginMethod currLoginMethod : primaryUser.loginMethods) { -// checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, currLoginMethod, primaryUser); -// } -// TODO !!! + checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, recipeUser.loginMethods[0], primaryUser); + + for (LoginMethod currLoginMethod : primaryUser.loginMethods) { + checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, currLoginMethod, primaryUser); + } + return new CanLinkAccountsResult(recipeUser.getSupertokensUserId(), primaryUser.getSupertokensUserId(), false); } @@ -337,15 +337,13 @@ private static List canLinkMultipleAccountsHelper(Tra tenantIds.addAll(primaryUser.tenantIds); try { - //TODO (?) this below method still uses multiple DB queries which could be enhanced - //TODO update: this below method takes a significant time. enhance it! 20 mins vs 8 hours - checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, + bulkCheckIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, recipeUser.loginMethods[0], primaryUser, allUsersWithExtraData); for (LoginMethod currLoginMethod : primaryUser.loginMethods) { - checkIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, + bulkCheckIfLoginMethodCanBeLinkedOnTenant(con, appIdentifier, authRecipeStorage, tenantIds, currLoginMethod, primaryUser, allUsersWithExtraData); - } // I don't get why this is needed.. + } results.add(new CanLinkAccountsBulkResult(recipeUserId, primaryUserId, false, null, primaryUser)); @@ -360,10 +358,82 @@ private static List canLinkMultipleAccountsHelper(Tra } private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection con, AppIdentifier appIdentifier, - AuthRecipeSQLStorage authRecipeStorage, - Set tenantIds, LoginMethod currLoginMethod, - AuthRecipeUserInfo primaryUser, - List allUsersWithExtraData) + AuthRecipeSQLStorage authRecipeStorage, + Set tenantIds, LoginMethod currLoginMethod, + AuthRecipeUserInfo primaryUser) + throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException { + // we loop through the union of both the user's tenantIds and check that the criteria for + // linking accounts is not violated in any of them. We do a union and not an intersection + // cause if we did an intersection, and that yields that account linking is allowed, it could + // result in one tenant having two primary users with the same email. For example: + // - tenant1 has u1 with email e, and u2 with email e, primary user (one is ep, one is tp) + // - tenant2 has u3 with email e, primary user (passwordless) + // now if we want to link u3 with u1, we have to deny it cause if we don't, it will result in + // u1 and u2 to be primary users with the same email in the same tenant. If we do an + // intersection, we will get an empty set, but if we do a union, we will get both the tenants and + // do the checks in both. + for (String tenantId : tenantIds) { + // we do not bother with getting the storage for each tenant here because + // we get the tenants from the user itself, and the user can only be shared across + // tenants of the same storage - therefore, the storage will be the same. + + if (currLoginMethod.email != null) { + AuthRecipeUserInfo[] usersWithSameEmail = + authRecipeStorage.listPrimaryUsersByEmail_Transaction(appIdentifier, con, currLoginMethod.email); + for (AuthRecipeUserInfo user : usersWithSameEmail) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser && !user.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's email is already associated with another user ID"); + } + } + } + + if (currLoginMethod.phoneNumber != null) { + AuthRecipeUserInfo[] usersWithSamePhoneNumber = + authRecipeStorage.listPrimaryUsersByPhoneNumber_Transaction(appIdentifier, con, + currLoginMethod.phoneNumber); + for (AuthRecipeUserInfo user : usersWithSamePhoneNumber) { + if (!user.tenantIds.contains(tenantId)) { + continue; + } + if (user.isPrimaryUser && !user.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + user.getSupertokensUserId(), + "This user's phone number is already associated with another user" + + " ID"); + } + } + } + + if (currLoginMethod.thirdParty != null) { + AuthRecipeUserInfo[] usersWithSameThirdParty = authRecipeStorage + .listPrimaryUsersByThirdPartyInfo_Transaction(appIdentifier, con, + currLoginMethod.thirdParty.id, currLoginMethod.thirdParty.userId); + for (AuthRecipeUserInfo userWithSameThirdParty : usersWithSameThirdParty) { + if (!userWithSameThirdParty.tenantIds.contains(tenantId)) { + continue; + } + if (userWithSameThirdParty.isPrimaryUser) { + throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( + userWithSameThirdParty.getSupertokensUserId(), + "This user's third party login is already associated with another" + + " user ID"); + + } + } + } + } + } + + private static void bulkCheckIfLoginMethodCanBeLinkedOnTenant(TransactionConnection con, AppIdentifier appIdentifier, + AuthRecipeSQLStorage authRecipeStorage, + Set tenantIds, LoginMethod currLoginMethod, + AuthRecipeUserInfo primaryUser, + List allUsersWithExtraData) throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException { // we loop through the union of both the user's tenantIds and check that the criteria for // linking accounts is not violated in any of them. We do a union and not an intersection From 96f6e0a374169c57b2bcba2fec090d5ebc019d7d Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Fri, 22 Nov 2024 17:08:26 +0100 Subject: [PATCH 38/41] fix: reworked error handling to comform previous approach with messages --- .../io/supertokens/authRecipe/AuthRecipe.java | 39 +- .../io/supertokens/bulkimport/BulkImport.java | 557 +++++++----------- .../ProcessBulkUsersImportWorker.java | 32 +- .../emailpassword/EmailPassword.java | 12 +- .../java/io/supertokens/inmemorydb/Start.java | 34 +- .../passwordless/Passwordless.java | 2 +- .../storageLayer/StorageLayer.java | 9 +- .../io/supertokens/thirdparty/ThirdParty.java | 13 +- src/main/java/io/supertokens/totp/Totp.java | 1 - .../useridmapping/UserIdMapping.java | 15 +- .../io/supertokens/userroles/UserRoles.java | 22 +- .../api/bulkimport/ImportUserAPI.java | 86 +-- .../test/bulkimport/BulkImportFlowTest.java | 225 ++++++- 13 files changed, 583 insertions(+), 464 deletions(-) diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index a0a406d73..1820a32c9 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -29,6 +29,7 @@ import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; import io.supertokens.pluginInterface.authRecipe.sqlStorage.AuthRecipeSQLStorage; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.dashboard.DashboardSearchTags; import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -598,6 +599,7 @@ public static List linkMultipleAccounts(Main main, AppId } AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + Map errorByUserId = new HashMap<>(); try { List linkAccountsResults = authRecipeStorage.startTransaction(con -> { @@ -614,6 +616,7 @@ public static List linkMultipleAccounts(Main main, AppId } else if(canLinkAccountsBulkResult.error != null) { results.add(new LinkAccountsBulkResult( canLinkAccountsBulkResult.authRecipeUserInfo, false, canLinkAccountsBulkResult.error)); // preparing to return the error + errorByUserId.put(canLinkAccountsBulkResult.recipeUserId, canLinkAccountsBulkResult.error); } else { recipeUserByPrimaryUserNeedsLinking.put(canLinkAccountsBulkResult.recipeUserId, canLinkAccountsBulkResult.primaryUserId); } @@ -628,7 +631,9 @@ public static List linkMultipleAccounts(Main main, AppId authRecipeStorage.commitTransaction(con); } - + if(!errorByUserId.isEmpty()) { + throw new StorageQueryException(new BulkImportBatchInsertException("link accounts errors", errorByUserId)); + } return results; }); @@ -802,7 +807,6 @@ private static List canCreatePrimaryUsersHelper(Tra Map thirdpartyUserIdToThirdpartyId) throws StorageQueryException, UnknownUserIdException{ AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); - List targetUsers = authRecipeStorage.getPrimaryUsersByIds_Transaction(appIdentifier, con, recipeUserIds); if (targetUsers == null || targetUsers.isEmpty()) { @@ -978,13 +982,13 @@ public static List createPrimaryUsers(Main main, throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, FeatureNotEnabledException { - if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( "Account linking feature is not enabled for this app. Please contact support to enable it."); } AuthRecipeSQLStorage authRecipeStorage = StorageUtils.getAuthRecipeStorage(storage); + Map errorsByUserId = new HashMap<>(); try { return authRecipeStorage.startTransaction(con -> { @@ -993,7 +997,11 @@ public static List createPrimaryUsers(Main main, recipeUserIds, allDistinctEmails, allDistinctPhones, thirdpartyUserIdsToThirdpartyIds); List canMakePrimaryUsers = new ArrayList<>(); for(CreatePrimaryUserBulkResult result : results) { - if (result.wasAlreadyAPrimaryUser || result.error != null) { + if (result.wasAlreadyAPrimaryUser) { + continue; + } + if(result.error != null) { + errorsByUserId.put(result.user.getSupertokensUserId(), result.error); continue; } canMakePrimaryUsers.add(result); @@ -1005,26 +1013,27 @@ public static List createPrimaryUsers(Main main, authRecipeStorage.commitTransaction(con); for(CreatePrimaryUserBulkResult result : results) { - if (result.wasAlreadyAPrimaryUser || result.error != null) { + if (result.wasAlreadyAPrimaryUser) { + continue; + } + if(result.error != null) { + errorsByUserId.put(result.user.getSupertokensUserId(), result.error); continue; } result.user.isPrimaryUser = true; } - return results; + if(!errorsByUserId.isEmpty()) { + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("create primary users errors", errorsByUserId)); + } + + return results; } catch (UnknownUserIdException e) { throw new StorageTransactionLogicException(e); } }); - } catch (StorageTransactionLogicException e) { - if (e.actualException instanceof UnknownUserIdException) { - throw (UnknownUserIdException) e.actualException; - } else if (e.actualException instanceof RecipeUserIdAlreadyLinkedWithPrimaryUserIdException) { - throw (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException) e.actualException; - } else if (e.actualException instanceof AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) { - throw (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) e.actualException; - } - throw new StorageQueryException(e); + } catch (StorageTransactionLogicException e) { + throw new StorageQueryException(e.actualException); } } diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index 37b5ea23f..d148f340b 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -26,7 +26,6 @@ import io.supertokens.authRecipe.exception.RecipeUserIdAlreadyLinkedWithPrimaryUserIdException; import io.supertokens.config.Config; import io.supertokens.emailpassword.EmailPassword; -import io.supertokens.emailpassword.EmailPassword.ImportUserResponse; import io.supertokens.emailpassword.PasswordHashing; import io.supertokens.featureflag.exceptions.FeatureNotEnabledException; import io.supertokens.multitenancy.Multitenancy; @@ -34,7 +33,6 @@ import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithPhoneNumberAlreadyExistsException; import io.supertokens.multitenancy.exception.AnotherPrimaryUserWithThirdPartyInfoAlreadyExistsException; import io.supertokens.passwordless.Passwordless; -import io.supertokens.passwordless.exceptions.RestartFlowException; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; @@ -44,10 +42,10 @@ import io.supertokens.pluginInterface.bulkimport.BulkImportUser.TotpDevice; import io.supertokens.pluginInterface.bulkimport.BulkImportUser.UserRole; import io.supertokens.pluginInterface.bulkimport.ImportUserBase; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.emailpassword.EmailPasswordImportUser; import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateEmailException; -import io.supertokens.pluginInterface.emailpassword.exceptions.DuplicateUserIdException; import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; import io.supertokens.pluginInterface.emailverification.sqlStorage.EmailVerificationSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; @@ -65,20 +63,17 @@ import io.supertokens.pluginInterface.thirdparty.ThirdPartyImportUser; import io.supertokens.pluginInterface.thirdparty.exception.DuplicateThirdPartyUserException; import io.supertokens.pluginInterface.totp.TOTPDevice; -import io.supertokens.pluginInterface.totp.exception.DeviceAlreadyExistsException; import io.supertokens.pluginInterface.useridmapping.exception.UnknownSuperTokensUserIdException; import io.supertokens.pluginInterface.useridmapping.exception.UserIdMappingAlreadyExistsException; import io.supertokens.pluginInterface.userroles.exception.UnknownRoleException; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.thirdparty.ThirdParty; -import io.supertokens.thirdparty.ThirdParty.SignInUpResponse; import io.supertokens.totp.Totp; import io.supertokens.useridmapping.UserIdMapping; import io.supertokens.usermetadata.UserMetadata; import io.supertokens.userroles.UserRoles; import io.supertokens.utils.Utils; import jakarta.servlet.ServletException; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -181,8 +176,8 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie return bulkImportProxyStorage.startTransaction(con -> { try { Storage[] allStoragesForApp = getAllProxyStoragesForApp(main, appIdentifier); - processUserImportSteps(main, con, appIdentifier, bulkImportProxyStorage, user, primaryLM, - allStoragesForApp); + + processUsersImportSteps(main, con, appIdentifier, bulkImportProxyStorage, List.of(user), allStoragesForApp); bulkImportProxyStorage.commitTransactionForBulkImportProxyStorage(); @@ -205,21 +200,6 @@ public static synchronized AuthRecipeUserInfo importUser(Main main, AppIdentifie } } - public static void processUserImportSteps(Main main, TransactionConnection con, AppIdentifier appIdentifier, - Storage bulkImportProxyStorage, BulkImportUser user, LoginMethod primaryLM, Storage[] allStoragesForApp) - throws StorageTransactionLogicException { - for (LoginMethod lm : user.loginMethods) { - processUserLoginMethod(main, appIdentifier, bulkImportProxyStorage, lm); - } - - createPrimaryUserAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserIdMapping(appIdentifier, user, primaryLM, allStoragesForApp); - verifyEmailForAllLoginMethods(appIdentifier, con, bulkImportProxyStorage, user.loginMethods); - createTotpDevices(main, appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserMetadata(appIdentifier, bulkImportProxyStorage, user, primaryLM); - createUserRoles(main, appIdentifier, bulkImportProxyStorage, user); - } - public static void processUsersImportSteps(Main main, TransactionConnection connection, AppIdentifier appIdentifier, Storage bulkImportProxyStorage, List users, Storage[] allStoragesForApp) throws StorageTransactionLogicException { @@ -242,7 +222,6 @@ public static void processUsersImportSteps(Main main, TransactionConnection conn public static void processUsersLoginMethods(Main main, AppIdentifier appIdentifier, Storage storage, List users) throws StorageTransactionLogicException { //sort login methods together - System.out.println(Thread.currentThread().getName() + " processUsersLoginMethods"); Map> sortedLoginMethods = new HashMap<>(); for (BulkImportUser user: users) { for(LoginMethod loginMethod : user.loginMethods){ @@ -253,18 +232,42 @@ public static void processUsersLoginMethods(Main main, AppIdentifier appIdentifi } } - List importedUsers = new ArrayList<>(); - importedUsers.addAll(processEmailPasswordLoginMethods(main, storage, sortedLoginMethods.get("emailpassword"), appIdentifier)); - importedUsers.addAll(processThirdpartyLoginMethods(main, storage, sortedLoginMethods.get("thirdparty"), appIdentifier)); - importedUsers.addAll(processPasswordlessLoginMethods(appIdentifier, storage, - sortedLoginMethods.get("passwordless"))); + List importedUsers = new ArrayList<>(); + if (sortedLoginMethods.containsKey("emailpassword")) { + importedUsers.addAll( + processEmailPasswordLoginMethods(main, storage, sortedLoginMethods.get("emailpassword"), + appIdentifier)); + } + if (sortedLoginMethods.containsKey("thirdparty")) { + importedUsers.addAll( + processThirdpartyLoginMethods(main, storage, sortedLoginMethods.get("thirdparty"), + appIdentifier)); + } + if (sortedLoginMethods.containsKey("passwordless")) { + importedUsers.addAll(processPasswordlessLoginMethods(appIdentifier, storage, + sortedLoginMethods.get("passwordless"))); + } + Set actualKeys = new HashSet<>(sortedLoginMethods.keySet()); + List.of("emailpassword", "thirdparty", "passwordless").forEach(actualKeys::remove); + if(!actualKeys.isEmpty()){ + throw new StorageTransactionLogicException( + new IllegalArgumentException("E001: Unknown recipeId(s) [" + + actualKeys.stream().map(s -> s+" ") + "] for loginMethod.")); + } - //TODO - for(Map.Entry> loginMethodEntries : sortedLoginMethods.entrySet()){ - for(LoginMethod loginMethod : loginMethodEntries.getValue()){ - associateUserToTenants(main, appIdentifier, storage, loginMethod, loginMethod.tenantIds.get(0)); + Map errorsById = new HashMap<>(); + for (Map.Entry> loginMethodEntries : sortedLoginMethods.entrySet()) { + for (LoginMethod loginMethod : loginMethodEntries.getValue()) { + try { + associateUserToTenants(main, appIdentifier, storage, loginMethod, loginMethod.tenantIds.get(0)); + } catch (StorageTransactionLogicException e){ + errorsById.put(loginMethod.superTokensUserId, e.actualException); + } + } + } + if(!errorsById.isEmpty()){ + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("tenant association errors", errorsById)); } - } } private static List processPasswordlessLoginMethods(AppIdentifier appIdentifier, Storage storage, @@ -272,10 +275,12 @@ private static List processPasswordlessLoginMethods(Ap throws StorageTransactionLogicException { try { List usersToImport = new ArrayList<>(); - for (LoginMethod loginMethod: loginMethods){ + for (LoginMethod loginMethod : loginMethods) { String userId = Utils.getUUID(); - TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), loginMethod.tenantIds.get(0)); // the cron runs per app. The app stays the same, the tenant can change + TenantIdentifier tenantIdentifierForLoginMethod = new TenantIdentifier( + appIdentifier.getConnectionUriDomain(), + appIdentifier.getAppId(), loginMethod.tenantIds.get( + 0)); // the cron runs per app. The app stays the same, the tenant can change usersToImport.add(new PasswordlessImportUser(userId, loginMethod.phoneNumber, loginMethod.email, tenantIdentifierForLoginMethod, loginMethod.timeJoinedInMSSinceEpoch)); @@ -283,11 +288,29 @@ private static List processPasswordlessLoginMethods(Ap } Passwordless.createPasswordlessUsers(storage, usersToImport); - return usersToImport; - } catch (RestartFlowException e) { - String errorMessage =""; // TODO - throw new StorageTransactionLogicException(new Exception(errorMessage)); - } catch (StorageQueryException e) { + return usersToImport; + } catch (StorageQueryException | StorageTransactionLogicException e) { + if (e.getCause() instanceof BulkImportBatchInsertException) { + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof DuplicateEmailException) { + String message = "E006: A user with email " + + loginMethods.stream() + .filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get().email + " already exists in passwordless loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof DuplicatePhoneNumberException) { + String message = "E007: A user with phoneNumber " + + loginMethods.stream() + .filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get().phoneNumber + " already exists in passwordless loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } throw new StorageTransactionLogicException(e); } catch (TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(new Exception("E008: " + e.getMessage())); @@ -308,15 +331,29 @@ private static List processThirdpartyLoginMethods(Main loginMethod.thirdPartyUserId, tenantIdentifierForLoginMethod, loginMethod.timeJoinedInMSSinceEpoch)); loginMethod.superTokensUserId = userId; } - ThirdParty.createThirdPartyUsers(storage, usersToImport); + ThirdParty.createMultipleThirdPartyUsers(storage, usersToImport); return usersToImport; - } catch (StorageQueryException e) { + } catch (StorageQueryException | StorageTransactionLogicException e) { + if (e.getCause() instanceof BulkImportBatchInsertException) { + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof DuplicateThirdPartyUserException) { + LoginMethod loginMethodForError = loginMethods.stream() + .filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get(); + String message = "E005: A user with thirdPartyId " + loginMethodForError.thirdPartyId + + " and thirdPartyUserId " + loginMethodForError.thirdPartyUserId + + " already exists in thirdparty loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } throw new StorageTransactionLogicException(e); -// } catch (TenantOrAppNotFoundException e) { -// throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); -// } catch (DuplicateThirdPartyUserException e) { -// throw new StorageTransactionLogicException(new Exception("E005: A user with thirdPartyId " -// + " and thirdPartyUserId already exists in thirdparty loginMethod.")); // TODO + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); } } @@ -343,104 +380,25 @@ private static List processEmailPasswordLoginMethods( emailPasswordLoginMethod.superTokensUserId = userId; } - EmailPassword.createUsersWithPasswordHash(storage, usersToImport); + EmailPassword.createMultipleUsersWithPasswordHash(storage, usersToImport); return usersToImport; - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E002: " + e.getMessage())); - } catch (DuplicateEmailException e) { - throw new StorageTransactionLogicException( - new Exception( - "E003: A user with email already exists in emailpassword loginMethod.")); - } catch (DuplicateUserIdException e) { - throw new RuntimeException(e); - } - } - - public static void processUserLoginMethod(Main main, AppIdentifier appIdentifier, Storage storage, - LoginMethod lm) throws StorageTransactionLogicException { - String firstTenant = lm.tenantIds.get(0); - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), firstTenant); - - if (lm.recipeId.equals("emailpassword")) { - processEmailPasswordLoginMethod(main, tenantIdentifier, storage, lm); - } else if (lm.recipeId.equals("thirdparty")) { - processThirdPartyLoginMethod(tenantIdentifier, storage, lm); - } else if (lm.recipeId.equals("passwordless")) { - processPasswordlessLoginMethod(tenantIdentifier, storage, lm); - } else { - throw new StorageTransactionLogicException( - new IllegalArgumentException("E001: Unknown recipeId " + lm.recipeId + " for loginMethod.")); - } - - associateUserToTenants(main, appIdentifier, storage, lm, firstTenant); //already associated here. Why this? - // found it: the remaining tenants are not associated, only the first one - } - - private static void processEmailPasswordLoginMethod(Main main, TenantIdentifier tenantIdentifier, Storage storage, - LoginMethod lm) throws StorageTransactionLogicException { - try { - - String passwordHash = lm.passwordHash; - if (passwordHash == null && lm.plainTextPassword != null) { - passwordHash = PasswordHashing.getInstance(main) - .createHashWithSalt(tenantIdentifier.toAppIdentifier(), lm.plainTextPassword); + } catch (StorageQueryException | StorageTransactionLogicException e) { + if(e.getCause() instanceof BulkImportBatchInsertException){ + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for(String userid : errorsByPosition.keySet()){ + Exception exception = errorsByPosition.get(userid); + if(exception instanceof DuplicateEmailException){ + String message = "E003: A user with email " + + loginMethods.stream().filter(loginMethod -> loginMethod.superTokensUserId.equals(userid)) + .findFirst().get().email + " already exists in emailpassword loginMethod."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("translated", errorsByPosition)); } - - ImportUserResponse userInfo = EmailPassword.createUserWithPasswordHash(tenantIdentifier, storage, lm.email, - passwordHash, lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException e) { throw new StorageTransactionLogicException(e); } catch (TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(new Exception("E002: " + e.getMessage())); - } catch (DuplicateEmailException e) { - throw new StorageTransactionLogicException( - new Exception( - "E003: A user with email " + lm.email + " already exists in emailpassword loginMethod.")); - } - } - - private static void processThirdPartyLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, LoginMethod lm) - throws StorageTransactionLogicException { - try { - SignInUpResponse userInfo = ThirdParty.createThirdPartyUser( - tenantIdentifier, storage, lm.thirdPartyId, lm.thirdPartyUserId, lm.email, - lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.user.getSupertokensUserId(); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E004: " + e.getMessage())); - } catch (DuplicateThirdPartyUserException e) { - throw new StorageTransactionLogicException(new Exception("E005: A user with thirdPartyId " + lm.thirdPartyId - + " and thirdPartyUserId " + lm.thirdPartyUserId + " already exists in thirdparty loginMethod.")); - } - } - - private static void processPasswordlessLoginMethod(TenantIdentifier tenantIdentifier, Storage storage, - LoginMethod lm) - throws StorageTransactionLogicException { - try { - AuthRecipeUserInfo userInfo = Passwordless.createPasswordlessUser(tenantIdentifier, storage, lm.email, - lm.phoneNumber, lm.timeJoinedInMSSinceEpoch); - - lm.superTokensUserId = userInfo.getSupertokensUserId(); - } catch (RestartFlowException e) { - String errorMessage = lm.email != null - ? "E006: A user with email " + lm.email + " already exists in passwordless loginMethod." - : "E007: A user with phoneNumber " + lm.phoneNumber - + " already exists in passwordless loginMethod."; - throw new StorageTransactionLogicException(new Exception(errorMessage)); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E008: " + e.getMessage())); } } @@ -496,110 +454,70 @@ private static void associateUserToTenants(Main main, AppIdentifier appIdentifie } private static void createPrimaryUsersAndLinkAccounts(Main main, - AppIdentifier appIdentifier, Storage storage, List users) + AppIdentifier appIdentifier, Storage storage, + List users) throws StorageTransactionLogicException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, StorageQueryException, FeatureNotEnabledException, TenantOrAppNotFoundException, UnknownUserIdException { - System.out.println(Thread.currentThread().getName() + " createPrimaryUsersAndLinkAccounts"); List userIds = - users.stream().map(bulkImportUser -> getPrimaryLoginMethod(bulkImportUser).getSuperTokenOrExternalUserId()).collect(Collectors.toList()); + users.stream() + .map(bulkImportUser -> getPrimaryLoginMethod(bulkImportUser).getSuperTokenOrExternalUserId()) + .collect(Collectors.toList()); Set allEmails = new HashSet<>(); Set allPhoneNumber = new HashSet<>(); Map allThirdParty = new HashMap<>(); - for(BulkImportUser user: users){ - for(LoginMethod loginMethod : user.loginMethods) { - if(loginMethod.email != null) { + for (BulkImportUser user : users) { + for (LoginMethod loginMethod : user.loginMethods) { + if (loginMethod.email != null) { allEmails.add(loginMethod.email); } - if(loginMethod.phoneNumber != null){ + if (loginMethod.phoneNumber != null) { allPhoneNumber.add(loginMethod.phoneNumber); } - if(loginMethod.thirdPartyId != null && loginMethod.thirdPartyUserId != null){ + if (loginMethod.thirdPartyId != null && loginMethod.thirdPartyUserId != null) { allThirdParty.put(loginMethod.thirdPartyUserId, loginMethod.thirdPartyId); } } } - AuthRecipe.createPrimaryUsers(main, appIdentifier, storage, userIds, new ArrayList<>(allEmails), new ArrayList<>(allPhoneNumber), allThirdParty); - linkAccountsForMultipleUser(main, appIdentifier, storage, users, new ArrayList<>(allEmails), new ArrayList<>(allPhoneNumber), allThirdParty); - } - - - public static void createPrimaryUserAndLinkAccounts(Main main, - AppIdentifier appIdentifier, Storage storage, BulkImportUser user, LoginMethod primaryLM) - throws StorageTransactionLogicException { - if (user.loginMethods.size() == 1) { - return; - } - try { - AuthRecipe.createPrimaryUser(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId()); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E018: " + e.getMessage())); + AuthRecipe.createPrimaryUsers(main, appIdentifier, storage, userIds, new ArrayList<>(allEmails), + new ArrayList<>(allPhoneNumber), allThirdParty); } catch (StorageQueryException e) { + if(e.getCause() instanceof BulkImportBatchInsertException){ + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof UnknownUserIdException) { + String message = "E020: We tried to create the primary user for the userId " + + userid + + " but it doesn't exist. This should not happen. Please contact support."; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof RecipeUserIdAlreadyLinkedWithPrimaryUserIdException) { + String message = "E021: We tried to create the primary user for the userId " + + userid + + " but it is already linked with another primary user."; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) { + String message = "E022: We tried to create the primary user for the userId " + + userid + + " but the account info is already associated with another primary user."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); + } throw new StorageTransactionLogicException(e); + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E018: " + e.getMessage())); } catch (FeatureNotEnabledException e) { throw new StorageTransactionLogicException(new Exception("E019: " + e.getMessage())); - } catch (UnknownUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "E020: We tried to create the primary user for the userId " - + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist. This should not happen. Please contact support.")); - } catch (RecipeUserIdAlreadyLinkedWithPrimaryUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "E021: We tried to create the primary user for the userId " - + primaryLM.getSuperTokenOrExternalUserId() - + " but it is already linked with another primary user.")); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "E022: We tried to create the primary user for the userId " - + primaryLM.getSuperTokenOrExternalUserId() - + " but the account info is already associated with another primary user.")); } - linkAccountsForUser(main, appIdentifier, storage, user, primaryLM); - } - - private static void linkAccountsForUser(Main main, AppIdentifier appIdentifier, Storage storage, BulkImportUser user, - LoginMethod primaryLM) throws StorageTransactionLogicException { - for (LoginMethod lm : user.loginMethods) { - try { - if (lm.getSuperTokenOrExternalUserId().equals(primaryLM.getSuperTokenOrExternalUserId())) { - continue; - } - - AuthRecipe.linkAccounts(main, appIdentifier, storage, lm.getSuperTokenOrExternalUserId(), - primaryLM.getSuperTokenOrExternalUserId()); - - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E023: " + e.getMessage())); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (FeatureNotEnabledException e) { - throw new StorageTransactionLogicException(new Exception("E024: " + e.getMessage())); - } catch (UnknownUserIdException e) { - throw new StorageTransactionLogicException( - new Exception("E025: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it doesn't exist.")); - } catch (InputUserIdIsNotAPrimaryUserException e) { - throw new StorageTransactionLogicException( - new Exception("E026: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it is not a primary user.")); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "E027: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but the account info is already associated with another primary user.")); - } catch (RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException e) { - throw new StorageTransactionLogicException(new Exception( - "E028: We tried to link the userId " + lm.getSuperTokenOrExternalUserId() - + " to the primary userId " + primaryLM.getSuperTokenOrExternalUserId() - + " but it is already linked with another primary user.")); - } - } + linkAccountsForMultipleUser(main, appIdentifier, storage, users, new ArrayList<>(allEmails), + new ArrayList<>(allPhoneNumber), allThirdParty); } private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIdentifier, Storage storage, @@ -612,13 +530,45 @@ private static void linkAccountsForMultipleUser(Main main, AppIdentifier appIden try { AuthRecipe.linkMultipleAccounts(main, appIdentifier, storage, recipeUserIdByPrimaryUserId, allDistinctEmails, allDistinctPhones, thirdpartyUserIdsToThirdpartyIds); - } catch (StorageQueryException | FeatureNotEnabledException | TenantOrAppNotFoundException e) { + } catch (TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(new Exception("E023: " + e.getMessage())); + } catch (FeatureNotEnabledException e) { + throw new StorageTransactionLogicException(new Exception("E024: " + e.getMessage())); + } catch (StorageQueryException e) { + if (e.getCause() instanceof BulkImportBatchInsertException) { + Map errorByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userId : errorByPosition.keySet()) { + Exception currentException = errorByPosition.get(userId); + String recipeUID = recipeUserIdByPrimaryUserId.get(userId); + if (currentException instanceof UnknownUserIdException) { + String message = "E025: We tried to link the userId " + recipeUID + + " to the primary userId " + userId + + " but it doesn't exist."; + errorByPosition.put(userId, new Exception(message)); + } else if (currentException instanceof InputUserIdIsNotAPrimaryUserException) { + String message = "E026: We tried to link the userId " + recipeUID + + " to the primary userId " + userId + + " but it is not a primary user."; + errorByPosition.put(userId, new Exception(message)); + } else if (currentException instanceof AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException) { + String message = "E027: We tried to link the userId " + userId + + " to the primary userId " + recipeUID + + " but the account info is already associated with another primary user."; + errorByPosition.put(userId, new Exception(message)); + } else if (currentException instanceof RecipeUserIdAlreadyLinkedWithAnotherPrimaryUserIdException) { + String message = "E028: We tried to link the userId " + recipeUID + + " to the primary userId " + userId + + " but it is already linked with another primary user."; + errorByPosition.put(userId, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("link accounts translated", errorByPosition)); + } throw new StorageTransactionLogicException(e); } - //TODO proper error handling } - @NotNull private static Map collectRecipeIdsToPrimaryIds(List users) { Map recipeUserIdByPrimaryUserId = new HashMap<>(); for(BulkImportUser user: users){ @@ -634,37 +584,8 @@ private static Map collectRecipeIdsToPrimaryIds(List users, Storage[] storages) throws StorageTransactionLogicException { - System.out.println(Thread.currentThread().getName() + " createMultipleUserIdMapping"); Map superTokensUserIdToExternalUserId = new HashMap<>(); for(BulkImportUser user: users) { if(user.externalUserId != null) { @@ -678,33 +599,33 @@ public static void createMultipleUserIdMapping(AppIdentifier appIdentifier, superTokensUserIdToExternalUserId, false, true); -// for(UserIdMapping.UserIdBulkMappingResult mappingResult : mappingResults){ -// if(mappingResult.error == null) { // no error means successful mapping -// // TODO -// } -// } - } catch (Exception e) { - //TODO proper error handling - } - } - - public static void createUserMetadata(AppIdentifier appIdentifier, Storage storage, BulkImportUser user, - LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.userMetadata != null) { - try { - UserMetadata.updateUserMetadata(appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - user.userMetadata); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E040: " + e.getMessage())); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); + } catch (StorageQueryException e) { + if(e.getCause() instanceof BulkImportBatchInsertException) { + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof ServletException) { + String message = "E030: " + e.getMessage(); + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof UserIdMappingAlreadyExistsException) { + String message = "E031: A user with externalId " + superTokensUserIdToExternalUserId.get(userid) + " already exists"; + errorsByPosition.put(userid, new Exception(message)); + } else if (exception instanceof UnknownSuperTokensUserIdException) { + String message = "E032: We tried to create the externalUserId mapping for the superTokenUserId " + + userid + + " but it doesn't exist. This should not happen. Please contact support."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("translated", errorsByPosition)); } + throw new StorageTransactionLogicException(e); } } public static void createMultipleUserMetadata(AppIdentifier appIdentifier, Storage storage, List users) throws StorageTransactionLogicException { - System.out.println(Thread.currentThread().getName() + " createMultipleUserMetadata"); Map usersMetadata = new HashMap<>(); for(BulkImportUser user: users) { @@ -722,34 +643,9 @@ public static void createMultipleUserMetadata(AppIdentifier appIdentifier, Stora } } - public static void createUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, - BulkImportUser user) throws StorageTransactionLogicException { - if (user.userRoles != null) { - for (UserRole userRole : user.userRoles) { - try { - for (String tenantId : userRole.tenantIds) { - TenantIdentifier tenantIdentifier = new TenantIdentifier( - appIdentifier.getConnectionUriDomain(), appIdentifier.getAppId(), - tenantId); - - UserRoles.addRoleToUser(main, tenantIdentifier, storage, user.externalUserId, userRole.role); - } - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (UnknownRoleException e) { - throw new StorageTransactionLogicException(new Exception("E034: Role " + userRole.role - + " does not exist! You need pre-create the role before assigning it to the user.")); - } - } - } - } - public static void createMultipleUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, List users) throws StorageTransactionLogicException { Map> rolesToUserByTenant = new HashMap<>(); - System.out.println(Thread.currentThread().getName() + " createMultipleUserRoles"); for (BulkImportUser user : users) { if (user.userRoles != null) { @@ -772,40 +668,21 @@ public static void createMultipleUserRoles(Main main, AppIdentifier appIdentifie UserRoles.addMultipleRolesToMultipleUsers(main, storage, rolesToUserByTenant); } catch (TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); - } catch (UnknownRoleException e) { - throw new StorageTransactionLogicException(new Exception("E034: Role " - + " does not exist! You need pre-create the role before assigning it to the user.")); - } - - } - - public static void verifyEmailForAllLoginMethods(AppIdentifier appIdentifier, TransactionConnection con, - Storage storage, - List loginMethods) throws StorageTransactionLogicException { - - for (LoginMethod lm : loginMethods) { - try { - - TenantIdentifier tenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), - appIdentifier.getAppId(), lm.tenantIds.get(0)); - - EmailVerificationSQLStorage emailVerificationSQLStorage = StorageUtils - .getEmailVerificationStorage(storage); - emailVerificationSQLStorage - .updateIsEmailVerified_Transaction(tenantIdentifier.toAppIdentifier(), con, - lm.getSuperTokenOrExternalUserId(), lm.email, true); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E035: " + e.getMessage())); - } catch (StorageQueryException e) { + } catch (StorageTransactionLogicException e) { + if(e.actualException instanceof UnknownRoleException){ + throw new StorageTransactionLogicException(new Exception("E034: Role " + + " does not exist! You need pre-create the role before assigning it to the user.")); + } else { throw new StorageTransactionLogicException(e); } + } + } public static void verifyMultipleEmailForAllLoginMethods(AppIdentifier appIdentifier, Storage storage, List users) throws StorageTransactionLogicException { - System.out.println(Thread.currentThread().getName() + " verifyMultipleEmailForAllLoginMethods"); Map emailToUserId = new HashMap<>(); for (BulkImportUser user : users) { for (LoginMethod lm : user.loginMethods) { @@ -831,38 +708,14 @@ public static void verifyMultipleEmailForAllLoginMethods(AppIdentifier appIdenti } } - public static void createTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, - BulkImportUser user, LoginMethod primaryLM) throws StorageTransactionLogicException { - if (user.totpDevices != null) { - for (TotpDevice totpDevice : user.totpDevices) { - try { - Totp.createDevice(main, appIdentifier, storage, primaryLM.getSuperTokenOrExternalUserId(), - totpDevice.deviceName, totpDevice.skew, totpDevice.period, totpDevice.secretKey, - true, System.currentTimeMillis()); - } catch (TenantOrAppNotFoundException e) { - throw new StorageTransactionLogicException(new Exception("E036: " + e.getMessage())); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } catch (FeatureNotEnabledException e) { - throw new StorageTransactionLogicException(new Exception("E037: " + e.getMessage())); - } catch (DeviceAlreadyExistsException e) { - throw new StorageTransactionLogicException( - new Exception( - "E038: A totp device with name " + totpDevice.deviceName + " already exists")); - } - } - } - } - public static void createMultipleTotpDevices(Main main, AppIdentifier appIdentifier, Storage storage, List users) throws StorageTransactionLogicException { - System.out.println(Thread.currentThread().getName() + " createMultipleTotpDevices"); List devices = new ArrayList<>(); for (BulkImportUser user : users) { if (user.totpDevices != null) { for(TotpDevice device : user.totpDevices){ - TOTPDevice totpDevice = new TOTPDevice(getPrimaryLoginMethod(user).getSuperTokenOrExternalUserId(), //TODO getPrimaryLoginMethod call should be done once in the whole process + TOTPDevice totpDevice = new TOTPDevice(getPrimaryLoginMethod(user).getSuperTokenOrExternalUserId(), device.deviceName, device.secretKey, device.period, device.skew, true, System.currentTimeMillis()); devices.add(totpDevice); @@ -872,7 +725,7 @@ public static void createMultipleTotpDevices(Main main, AppIdentifier appIdentif try { Totp.createDevices(main, appIdentifier, storage, devices); } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); + throw new StorageTransactionLogicException(new Exception("E036: " + e.getMessage())); } catch (FeatureNotEnabledException e) { throw new StorageTransactionLogicException(new Exception("E037: " + e.getMessage())); } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index dd3a9bb09..5b8da78e9 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -27,8 +27,8 @@ import io.supertokens.output.Logging; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; -import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportTransactionRolledBackException; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.exceptions.DbInitException; @@ -93,6 +93,7 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List { try { @@ -128,7 +128,7 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List usersBatch, Exception e, BulkImportSQLStorage baseTenantStorage) throws StorageQueryException { // Java doesn't allow us to reassign local variables inside a lambda expression // so we have to use an array. String[] errorMessage = { e.getMessage() }; + Map bulkImportUserIdToErrorMessage = new HashMap<>(); if (e instanceof StorageTransactionLogicException) { StorageTransactionLogicException exception = (StorageTransactionLogicException) e; @@ -167,7 +168,24 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport Logging.error(main, null, "We got an StorageQueryException while processing a bulk import user entry. It will be retried again. Error Message: " + e.getMessage(), true); return; } - errorMessage[0] = exception.actualException.getMessage(); + if(exception.actualException instanceof BulkImportBatchInsertException){ + Map userIndexToError = ((BulkImportBatchInsertException) exception.actualException).exceptionByUserId; + for(String userid : userIndexToError.keySet()){ + String id = usersBatch.stream() + .filter(bulkImportUser -> + bulkImportUser.loginMethods.stream() + .map(loginMethod -> loginMethod.superTokensUserId) + .anyMatch(s -> s.equals(userid))).findFirst().get().id; + bulkImportUserIdToErrorMessage.put(id, userIndexToError.get(userid).getMessage()); + } + } else { + //fail the whole batch + errorMessage[0] = exception.actualException.getMessage(); + for(BulkImportUser user : usersBatch){ + bulkImportUserIdToErrorMessage.put(user.id, errorMessage[0]); + } + } + } else if (e instanceof InvalidBulkImportDataException) { errorMessage[0] = ((InvalidBulkImportDataException) e).errors.toString(); } else if (e instanceof InvalidConfigException) { @@ -176,8 +194,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, BulkImport try { baseTenantStorage.startTransaction(con -> { - baseTenantStorage.updateBulkImportUserStatus_Transaction(appIdentifier, con, user.id, - BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED, errorMessage[0]); + baseTenantStorage.updateMultipleBulkImportUsersStatusToError_Transaction(appIdentifier, con, + bulkImportUserIdToErrorMessage); return null; }); } catch (StorageTransactionLogicException e1) { diff --git a/src/main/java/io/supertokens/emailpassword/EmailPassword.java b/src/main/java/io/supertokens/emailpassword/EmailPassword.java index 7026d3d27..11b3ec431 100644 --- a/src/main/java/io/supertokens/emailpassword/EmailPassword.java +++ b/src/main/java/io/supertokens/emailpassword/EmailPassword.java @@ -258,12 +258,14 @@ public static ImportUserResponse createUserWithPasswordHash(TenantIdentifier ten } } - public static void createUsersWithPasswordHash(Storage storage, - List usersToImport) - throws StorageQueryException, DuplicateEmailException, TenantOrAppNotFoundException, - DuplicateUserIdException, StorageTransactionLogicException { + public static void createMultipleUsersWithPasswordHash(Storage storage, + List usersToImport) + throws StorageQueryException, TenantOrAppNotFoundException, StorageTransactionLogicException { EmailPasswordSQLStorage epStorage = StorageUtils.getEmailPasswordStorage(storage); - epStorage.signUpMultiple(usersToImport); + epStorage.startTransaction(con -> { + epStorage.signUpMultipleViaBulkImport_Transaction(con, usersToImport); + return null; + }); } @TestOnly diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 4572049ad..5ca0ecbb1 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -233,7 +233,8 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev tries++; try { return startTransactionHelper(logic); - } catch (SQLException | StorageQueryException | StorageTransactionLogicException | TenantOrAppNotFoundException e) { + } catch (SQLException | StorageQueryException | StorageTransactionLogicException | + TenantOrAppNotFoundException e) { if ((e instanceof SQLTransactionRollbackException || (e.getMessage() != null && e.getMessage().toLowerCase().contains("deadlock"))) && tries < 3) { @@ -245,8 +246,6 @@ public T startTransaction(TransactionLogic logic, TransactionIsolationLev throw (StorageQueryException) e; } else if (e instanceof StorageTransactionLogicException) { throw (StorageTransactionLogicException) e; - } else if (e instanceof TenantOrAppNotFoundException) { // TODO this should not be here. - throw new StorageTransactionLogicException(e); } throw new StorageQueryException(e); } @@ -818,13 +817,6 @@ public AuthRecipeUserInfo signUp(TenantIdentifier tenantIdentifier, String id, S } } - @Override - public void signUpMultiple(List users) - throws StorageQueryException, DuplicateUserIdException, DuplicateEmailException, - TenantOrAppNotFoundException, StorageTransactionLogicException { - // TODO - } - @Override public void addPasswordResetToken(AppIdentifier appIdentifier, PasswordResetTokenInfo passwordResetTokenInfo) throws StorageQueryException, UnknownUserIdException, DuplicatePasswordResetTokenException { @@ -937,6 +929,13 @@ public void deleteEmailPasswordUser_Transaction(TransactionConnection con, AppId } } + @Override + public void signUpMultipleViaBulkImport_Transaction(TransactionConnection connection, + List users) + throws StorageQueryException, StorageTransactionLogicException { + //TODO + } + @Override public void deleteExpiredEmailVerificationTokens() throws StorageQueryException { try { @@ -1196,16 +1195,16 @@ public void deleteThirdPartyUser_Transaction(TransactionConnection con, AppIdent @Override public void importThirdPartyUsers_Transaction(TransactionConnection con, - Collection usersToImport) - throws StorageQueryException { + List usersToImport) + throws StorageQueryException, StorageTransactionLogicException { // TODO } @Override public void importPasswordlessUsers_Transaction(TransactionConnection con, - Collection users) + List users) throws StorageQueryException { - //todo + // TODO } @Override @@ -2199,6 +2198,13 @@ public boolean doesRoleExist_Transaction(AppIdentifier appIdentifier, Transactio } } + @Override + public List doesMultipleRoleExist_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + List roles) throws StorageQueryException { + // TODO + return List.of(); + } + @Override public void deleteAllRolesForUser_Transaction(TransactionConnection con, AppIdentifier appIdentifier, String userId) throws StorageQueryException { diff --git a/src/main/java/io/supertokens/passwordless/Passwordless.java b/src/main/java/io/supertokens/passwordless/Passwordless.java index d2e6c1ef0..a317c6716 100644 --- a/src/main/java/io/supertokens/passwordless/Passwordless.java +++ b/src/main/java/io/supertokens/passwordless/Passwordless.java @@ -552,7 +552,7 @@ public static AuthRecipeUserInfo createPasswordlessUser(TenantIdentifier tenantI public static void createPasswordlessUsers(Storage storage, List importUsers) - throws TenantOrAppNotFoundException, StorageQueryException, RestartFlowException, + throws TenantOrAppNotFoundException, StorageQueryException, StorageTransactionLogicException { PasswordlessSQLStorage passwordlessStorage = StorageUtils.getPasswordlessStorage(storage); diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index e1ef206f9..630110a03 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -590,7 +590,11 @@ public static List findStorageAndUserIdMappingForBulkUs if (storages[0].getType() != STORAGE_TYPE.SQL) { // for non sql plugin, there will be only one storage as multitenancy is not supported assert storages.length == 1; - return Collections.singletonList(new StorageAndUserIdMapping(storages[0], null)); + List results = new ArrayList<>(); + for(String userId : userIds) { + results.add(new StorageAndUserIdMapping(storages[0], new UserIdMapping(userId, null, null))); + } + return results; } List allMappingsFromAllStorages = new ArrayList<>(); if (userIdType != UserIdType.ANY) { @@ -605,6 +609,9 @@ public static List findStorageAndUserIdMappingForBulkUs .filter(userIdMapping -> (userIdType == UserIdType.SUPERTOKENS && userIdMapping.superTokensUserId.equals(existingId)) || (userIdType == UserIdType.EXTERNAL && userIdMapping.externalUserId.equals(existingId)) ) .findFirst().orElse(null); + if(mappingForId == null && userIdType == UserIdType.SUPERTOKENS) { + mappingForId = new UserIdMapping(existingId, null, null); + } allMappingsFromAllStorages.add(new StorageAndUserIdMapping(storage, mappingForId)); } } diff --git a/src/main/java/io/supertokens/thirdparty/ThirdParty.java b/src/main/java/io/supertokens/thirdparty/ThirdParty.java index bde6ff129..57aa43819 100644 --- a/src/main/java/io/supertokens/thirdparty/ThirdParty.java +++ b/src/main/java/io/supertokens/thirdparty/ThirdParty.java @@ -43,7 +43,10 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nonnull; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; public class ThirdParty { @@ -353,9 +356,9 @@ public static SignInUpResponse createThirdPartyUser(TenantIdentifier tenantIdent } } - public static void createThirdPartyUsers(Storage storage, - Collection usersToImport) - throws StorageQueryException, StorageTransactionLogicException { + public static void createMultipleThirdPartyUsers(Storage storage, + List usersToImport) + throws StorageQueryException, StorageTransactionLogicException, TenantOrAppNotFoundException { ThirdPartySQLStorage tpStorage = StorageUtils.getThirdPartyStorage(storage); tpStorage.startTransaction(con -> { @@ -363,8 +366,6 @@ public static void createThirdPartyUsers(Storage storage, tpStorage.commitTransaction(con); return null; }); - - // TODO error handling } @Deprecated diff --git a/src/main/java/io/supertokens/totp/Totp.java b/src/main/java/io/supertokens/totp/Totp.java index 61158bce2..c8b705668 100644 --- a/src/main/java/io/supertokens/totp/Totp.java +++ b/src/main/java/io/supertokens/totp/Totp.java @@ -156,7 +156,6 @@ public static void createDevices(Main main, AppIdentifier appIdentifier, Storage totpStorage.startTransaction(con -> { totpStorage.createDevices_Transaction(con, appIdentifier, devices); totpStorage.commitTransaction(con); - System.out.println("Created TOTP devices"); return null; }); diff --git a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java index d2c35ddf4..1e9f2c117 100644 --- a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java +++ b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java @@ -22,6 +22,7 @@ import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.authRecipe.LoginMethod; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.emailpassword.exceptions.UnknownUserIdException; import io.supertokens.pluginInterface.emailverification.EmailVerificationStorage; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -181,9 +182,7 @@ public static void createUserIdMapping(AppIdentifier appIdentifier, Storage[] st public static List createMultipleUserIdMappings(AppIdentifier appIdentifier, Storage[] storages, Map superTokensUserIdToExternalUserId, boolean force, boolean makeExceptionForEmailVerification) - throws UnknownSuperTokensUserIdException, - UserIdMappingAlreadyExistsException, StorageQueryException, ServletException, - TenantOrAppNotFoundException { + throws StorageQueryException { // We first need to check if the external user id exists across all app storages because we do not want // 2 users from different user pool but same app to point to same external user id. @@ -209,7 +208,6 @@ public static List createMultipleUserIdMappings(AppIden List mappingAndStoragesAsInvalid = StorageLayer.findStorageAndUserIdMappingForBulkUserImport( appIdentifier, storages, new ArrayList<>(superTokensUserIdToExternalUserId.values()), UserIdType.SUPERTOKENS); - //TODO does it matter which storage? Map> userIdsUsedInNonAuthRecipes = storages[0].findNonAuthRecipesWhereForUserIdsUsed(appIdentifier, new ArrayList<>(superTokensUserIdToExternalUserId.keySet())); @@ -311,6 +309,15 @@ public static List createMultipleUserIdMappings(AppIden } } } + Map errors = new HashMap<>(); + for(UserIdBulkMappingResult result : mappingResults){ + if(result.error != null) { + errors.put(result.supertokensUserId, result.error); + } + } + if(!errors.isEmpty()) { + throw new StorageQueryException(new BulkImportBatchInsertException("useridmapping errors", errors)); + } return mappingResults; } diff --git a/src/main/java/io/supertokens/userroles/UserRoles.java b/src/main/java/io/supertokens/userroles/UserRoles.java index 41811f1e5..478ca23c8 100644 --- a/src/main/java/io/supertokens/userroles/UserRoles.java +++ b/src/main/java/io/supertokens/userroles/UserRoles.java @@ -31,6 +31,8 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; import java.util.Map; public class UserRoles { @@ -58,24 +60,28 @@ public static boolean addRoleToUser(Main main, TenantIdentifier tenantIdentifier } public static void addMultipleRolesToMultipleUsers(Main main, Storage storage, Map> rolesToUserByTenant) - throws StorageTransactionLogicException, UnknownRoleException, TenantOrAppNotFoundException { + throws StorageTransactionLogicException, TenantOrAppNotFoundException { // Roles are stored in public tenant storage and role to user mapping is stored in the tenant's storage // We do this because it's not straight forward to replicate roles to all storages of an app for(TenantIdentifier tenantIdentifier : rolesToUserByTenant.keySet()){ Storage appStorage = StorageLayer.getStorage( tenantIdentifier.toAppIdentifier().getAsPublicTenantIdentifier(), main); - // TODO!! -// if (!doesRoleExist(tenantIdentifier.toAppIdentifier(), appStorage, role)) { -// throw new UnknownRoleException(); -// } try { UserRolesSQLStorage userRolesStorage = StorageUtils.getUserRolesStorage(storage); userRolesStorage.startTransaction(con -> { - userRolesStorage.addRolesToUsers_Transaction(con, rolesToUserByTenant); - userRolesStorage.commitTransaction(con); - return null; + + List rolesFound = ((UserRolesSQLStorage) appStorage).doesMultipleRoleExist_Transaction( + tenantIdentifier.toAppIdentifier().getAsPublicTenantIdentifier().toAppIdentifier(), + con, new ArrayList<>(rolesToUserByTenant.get(tenantIdentifier).values())); + + if(rolesFound.contains(Boolean.FALSE)){ + throw new StorageTransactionLogicException(new UnknownRoleException()); + } + userRolesStorage.addRolesToUsers_Transaction(con, rolesToUserByTenant); + userRolesStorage.commitTransaction(con); + return null; }); } catch (StorageQueryException e) { diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java index a6f9d3c8f..f21dd809c 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -43,59 +43,59 @@ import java.io.IOException; public class ImportUserAPI extends WebserverAPI { - public ImportUserAPI(Main main) { - super(main, ""); - } + public ImportUserAPI(Main main) { + super(main, ""); + } - @Override - public String getPath() { - return "/bulk-import/import"; - } + @Override + public String getPath() { + return "/bulk-import/import"; + } - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - // API is app specific + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + // API is app specific - if (StorageLayer.isInMemDb(main)) { - throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); - } + if (StorageLayer.isInMemDb(main)) { + throw new ServletException(new BadRequestException("This API is not supported in the in-memory database.")); + } - JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(req); + JsonObject jsonUser = InputParser.parseJsonObjectOrThrowError(req); - AppIdentifier appIdentifier = null; - Storage storage = null; - String[] allUserRoles = null; + AppIdentifier appIdentifier = null; + Storage storage = null; + String[] allUserRoles = null; - try { - appIdentifier = getAppIdentifier(req); - storage = enforcePublicTenantAndGetPublicTenantStorage(req); - allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); - } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { - throw new ServletException(e); - } + try { + appIdentifier = getAppIdentifier(req); + storage = enforcePublicTenantAndGetPublicTenantStorage(req); + allUserRoles = StorageUtils.getUserRolesStorage(storage).getRoles(appIdentifier); + } catch (TenantOrAppNotFoundException | BadPermissionException | StorageQueryException e) { + throw new ServletException(e); + } - BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); + BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); - try { - BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, jsonUser, - Utils.getUUID()); + try { + BulkImportUser user = bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, jsonUser, + Utils.getUUID()); - AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, user); + AuthRecipeUserInfo importedUser = BulkImport.importUser(main, appIdentifier, user); - JsonObject result = new JsonObject(); - result.addProperty("status", "OK"); - result.add("user", importedUser.toJson()); - super.sendJsonResponse(200, result, resp); - } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { - JsonArray errors = e.errors.stream() - .map(JsonPrimitive::new) - .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + JsonObject result = new JsonObject(); + result.addProperty("status", "OK"); + result.add("user", importedUser.toJson()); + super.sendJsonResponse(200, result, resp); + } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { + JsonArray errors = e.errors.stream() + .map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); - JsonObject errorResponseJson = new JsonObject(); - errorResponseJson.add("errors", errors); - throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); - } catch (StorageQueryException | TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { - throw new ServletException(e); + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (StorageQueryException | TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { + throw new ServletException(e); + } } - } } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index 233c7e1f8..509fa5e23 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -78,8 +78,8 @@ public void testWithOneMillionUsers() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million - //int NUMBER_OF_USERS_TO_UPLOAD = 10000; + //int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million + int NUMBER_OF_USERS_TO_UPLOAD = 10000; int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); System.out.println("Worker threads: " + parallelism_set_to); @@ -108,7 +108,7 @@ public void testWithOneMillionUsers() throws Exception { long processingStartedTime = System.currentTimeMillis(); // Starting the processing cronjob here to be able to measure the runtime - startBulkImportCronjob(main, 6000); + startBulkImportCronjob(main, 8000); System.out.println("CronJob started"); // wait for the cron job to process them @@ -117,7 +117,7 @@ public void testWithOneMillionUsers() throws Exception { // Note2: the successfully processed users get deleted from the bulk_import_users table { long count = NUMBER_OF_USERS_TO_UPLOAD; - while(count != 0) { + while(true) { JsonObject response = loadBulkImportUsersCountWithStatus(main, null); assertEquals("OK", response.get("status").getAsString()); count = response.get("count").getAsLong(); @@ -134,6 +134,9 @@ public void testWithOneMillionUsers() throws Exception { long elapsedSeconds = (System.currentTimeMillis() - processingStartedTime) / 1000; System.out.println("Elapsed time: " + elapsedSeconds + " seconds, (" + elapsedSeconds / 3600 + " hours)"); + if(count == 0 ){ + break; + } Thread.sleep(60000); // one minute } } @@ -154,10 +157,91 @@ public void testWithOneMillionUsers() throws Exception { int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); int usersInCore = loadUsersCount(main).get("count").getAsInt(); assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore + failedImportedUsersNumber); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); } } + @Test + public void testBatchWithDuplicates() throws Exception { + String[] args = {"../"}; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA}); + + int NUMBER_OF_USERS_TO_UPLOAD = 2; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + usersJson.get("users").getAsJsonArray().add(generateUsersJson(1, 0).get("users").getAsJsonArray().get(0).getAsJsonObject()); + usersJson.get("users").getAsJsonArray().add(generateUsersJson(1, 1).get("users").getAsJsonArray().get(0).getAsJsonObject()); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // Starting the processing cronjob here to be able to measure the runtime + startBulkImportCronjob(main, 8000); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + int failedUsersNumber = 0; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + count = response.get("count").getAsLong(); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + failedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); // 5 seconds + } + + //print failed users + JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + if (failedUsersLs.has("users")) { + System.out.println(failedUsersLs.get("users")); + } + System.out.println("Failed Users: " + failedUsersLs); + System.out.println("Failed Users Number: " + failedUsersNumber); + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + System.out.println("Users in core: " + usersInCore); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD + 2, usersInCore + failedImportedUsersNumber); + assertEquals(2, failedImportedUsersNumber); + + } + @Test public void testFirstLazyImportAfterBulkImport() throws Exception { String[] args = { "../" }; @@ -172,7 +256,7 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - int NUMBER_OF_USERS_TO_UPLOAD = 100; + int NUMBER_OF_USERS_TO_UPLOAD = 1000; int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); System.out.println("Worker threads: " + parallelism_set_to); @@ -266,6 +350,133 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { stopBulkImportCronjob(main); } + @Test + public void testLazyImport() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + int NUMBER_OF_USERS_TO_UPLOAD = 100; + int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); + System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); + System.out.println("Worker threads: " + parallelism_set_to); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + // lazy import most of the users + int successfully_lazy_imported = 0; + for (int i = 0; i < allUsersJson.get("users").getAsJsonArray().size(); i++) { + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(i).getAsJsonObject(); + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + assertEquals("OK", lazyImportResponse.get("status").getAsString()); + assertNotNull(lazyImportResponse.get("user")); + successfully_lazy_imported++; + System.out.println(i + "th lazy imported"); + } + + // expect: lazy imported users are already there, duplicate.. errors + // expect: not lazy imported users are imported successfully + { + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, successfully_lazy_imported ); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); + } + } + + @Test + public void testLazyImportUnknownRecipeLoginMethod() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(1, 0); + allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject().get("loginMethods") + .getAsJsonArray().get(0).getAsJsonObject().addProperty("recipeId", "not-existing-recipe"); + + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); + try { + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + } catch (HttpResponseException expected) { + assertEquals(400, expected.statusCode); + assertNotNull(expected.getMessage()); + assertEquals("Http error. Status Code: 400. Message: {\"errors\":[\"Invalid recipeId for loginMethod. Pass one of emailpassword, thirdparty or, passwordless!\"]}", + expected.getMessage()); + } + } + + @Test + public void testLazyImportDuplicatesFail() throws Exception { + String[] args = { "../" }; + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[] { + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + { + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + } + + // create users + JsonObject allUsersJson = generateUsersJson(1, 0); + + JsonObject userToImportLazy = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); + JsonObject lazyImportResponse = lazyImportUser(main, userToImportLazy); + assertEquals("OK", lazyImportResponse.get("status").getAsString()); + assertNotNull(lazyImportResponse.get("user")); + + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(1, usersInCore); + + JsonObject userToImportLazyAgain = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); + try { + JsonObject lazyImportResponseTwo = lazyImportUser(main, userToImportLazy); + } catch (HttpResponseException expected) { + assertEquals(400, expected.statusCode); + System.out.println(expected.getMessage()); + } + } + private static JsonObject lazyImportUser(Main main, JsonObject user) throws HttpResponseException, IOException { return HttpRequestForTesting.sendJsonPOSTRequest(main, "", @@ -323,9 +534,9 @@ private static JsonObject generateUsersJson(int numberOfUsers, int startIndex) { Random random = new Random(); JsonArray loginMethodsArray = new JsonArray(); - if(random.nextInt(2) == 0){ + //if(random.nextInt(2) == 0){ loginMethodsArray.add(createEmailLoginMethod(email, tenanatIds)); - } + //} if(random.nextInt(2) == 0){ loginMethodsArray.add(createThirdPartyLoginMethod(email, tenanatIds)); } From 21e91fca989c98f2f1ec891a1485a7c4f6282ebc Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Tue, 26 Nov 2024 21:08:37 +0100 Subject: [PATCH 39/41] fix: fixing tests --- .../StorageAndUserIdMappingForBulkImport.java | 31 ++ .../io/supertokens/authRecipe/AuthRecipe.java | 3 +- .../io/supertokens/bulkimport/BulkImport.java | 50 ++- .../bulkimport/ProcessBulkImportUsers.java | 16 +- .../ProcessBulkUsersImportWorker.java | 73 ++-- .../java/io/supertokens/inmemorydb/Start.java | 37 +- .../storageLayer/StorageLayer.java | 10 +- .../useridmapping/UserIdMapping.java | 84 ++-- .../io/supertokens/userroles/UserRoles.java | 65 +-- .../api/bulkimport/ImportUserAPI.java | 21 +- .../test/bulkimport/BulkImportFlowTest.java | 406 +++++++++++++----- .../ProcessBulkImportUsersCronJobTest.java | 27 +- 12 files changed, 551 insertions(+), 272 deletions(-) create mode 100644 src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java diff --git a/src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java b/src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java new file mode 100644 index 000000000..0daeedf96 --- /dev/null +++ b/src/main/java/io/supertokens/StorageAndUserIdMappingForBulkImport.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2024, VRAI Labs and/or its affiliates. All rights reserved. + * + * This software is licensed under the Apache License, Version 2.0 (the + * "License") as published by the Apache Software Foundation. + * + * You may not use this file except in compliance with the License. You may + * obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package io.supertokens; + +import io.supertokens.pluginInterface.Storage; +import io.supertokens.pluginInterface.useridmapping.UserIdMapping; + +public class StorageAndUserIdMappingForBulkImport extends StorageAndUserIdMapping { + + public String userIdInQuestion; + + public StorageAndUserIdMappingForBulkImport(Storage storage, + UserIdMapping userIdMapping, String userIdInQuestion) { + super(storage, userIdMapping); + this.userIdInQuestion = userIdInQuestion; + } +} diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index 1820a32c9..d8eb407ed 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -979,8 +979,7 @@ public static List createPrimaryUsers(Main main, List allDistinctEmails, List allDistinctPhones, Map thirdpartyUserIdsToThirdpartyIds) - throws StorageQueryException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, - RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, UnknownUserIdException, TenantOrAppNotFoundException, + throws StorageQueryException, TenantOrAppNotFoundException, FeatureNotEnabledException { if (!Utils.isAccountLinkingEnabled(main, appIdentifier)) { throw new FeatureNotEnabledException( diff --git a/src/main/java/io/supertokens/bulkimport/BulkImport.java b/src/main/java/io/supertokens/bulkimport/BulkImport.java index d148f340b..abea35442 100644 --- a/src/main/java/io/supertokens/bulkimport/BulkImport.java +++ b/src/main/java/io/supertokens/bulkimport/BulkImport.java @@ -206,17 +206,15 @@ public static void processUsersImportSteps(Main main, TransactionConnection conn processUsersLoginMethods(main, appIdentifier, bulkImportProxyStorage, users); try { createPrimaryUsersAndLinkAccounts(main, appIdentifier, bulkImportProxyStorage, users); - } catch (AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException | - RecipeUserIdAlreadyLinkedWithPrimaryUserIdException | StorageQueryException | FeatureNotEnabledException | - TenantOrAppNotFoundException | UnknownUserIdException e) { - throw new RuntimeException(e); + createMultipleUserIdMapping(appIdentifier, users, allStoragesForApp); + verifyMultipleEmailForAllLoginMethods(appIdentifier, bulkImportProxyStorage, users); + createMultipleTotpDevices(main, appIdentifier, bulkImportProxyStorage, users); + createMultipleUserMetadata(appIdentifier, bulkImportProxyStorage, users); + createMultipleUserRoles(main, appIdentifier, bulkImportProxyStorage, users); + } catch ( StorageQueryException | FeatureNotEnabledException | + TenantOrAppNotFoundException e) { + throw new StorageTransactionLogicException(e); } - - createMultipleUserIdMapping(appIdentifier, users, allStoragesForApp); - verifyMultipleEmailForAllLoginMethods(appIdentifier, bulkImportProxyStorage, users); - createMultipleTotpDevices(main, appIdentifier, bulkImportProxyStorage, users); - createMultipleUserMetadata(appIdentifier, bulkImportProxyStorage, users); - createMultipleUserRoles(main, appIdentifier, bulkImportProxyStorage, users); } public static void processUsersLoginMethods(Main main, AppIdentifier appIdentifier, Storage storage, @@ -456,9 +454,8 @@ private static void associateUserToTenants(Main main, AppIdentifier appIdentifie private static void createPrimaryUsersAndLinkAccounts(Main main, AppIdentifier appIdentifier, Storage storage, List users) - throws StorageTransactionLogicException, AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException, - RecipeUserIdAlreadyLinkedWithPrimaryUserIdException, StorageQueryException, FeatureNotEnabledException, - TenantOrAppNotFoundException, UnknownUserIdException { + throws StorageTransactionLogicException, StorageQueryException, FeatureNotEnabledException, + TenantOrAppNotFoundException { List userIds = users.stream() .map(bulkImportUser -> getPrimaryLoginMethod(bulkImportUser).getSuperTokenOrExternalUserId()) @@ -591,6 +588,7 @@ public static void createMultipleUserIdMapping(AppIdentifier appIdentifier, if(user.externalUserId != null) { LoginMethod primaryLoginMethod = getPrimaryLoginMethod(user); superTokensUserIdToExternalUserId.put(primaryLoginMethod.superTokensUserId, user.externalUserId); + primaryLoginMethod.externalUserId = user.externalUserId; } } try { @@ -645,7 +643,7 @@ public static void createMultipleUserMetadata(AppIdentifier appIdentifier, Stora public static void createMultipleUserRoles(Main main, AppIdentifier appIdentifier, Storage storage, List users) throws StorageTransactionLogicException { - Map> rolesToUserByTenant = new HashMap<>(); + Map>> rolesToUserByTenant = new HashMap<>(); for (BulkImportUser user : users) { if (user.userRoles != null) { @@ -658,24 +656,34 @@ public static void createMultipleUserRoles(Main main, AppIdentifier appIdentifie rolesToUserByTenant.put(tenantIdentifier, new HashMap<>()); } - rolesToUserByTenant.get(tenantIdentifier).put(user.externalUserId, userRole.role); + if(!rolesToUserByTenant.get(tenantIdentifier).containsKey(user.externalUserId)){ + rolesToUserByTenant.get(tenantIdentifier).put(user.externalUserId, new ArrayList<>()); + } + rolesToUserByTenant.get(tenantIdentifier).get(user.externalUserId).add(userRole.role); } } } } try { - UserRoles.addMultipleRolesToMultipleUsers(main, storage, rolesToUserByTenant); + UserRoles.addMultipleRolesToMultipleUsers(main, appIdentifier, storage, rolesToUserByTenant); } catch (TenantOrAppNotFoundException e) { throw new StorageTransactionLogicException(new Exception("E033: " + e.getMessage())); } catch (StorageTransactionLogicException e) { - if(e.actualException instanceof UnknownRoleException){ - throw new StorageTransactionLogicException(new Exception("E034: Role " - + " does not exist! You need pre-create the role before assigning it to the user.")); + if(e.actualException instanceof BulkImportBatchInsertException){ + Map errorsByPosition = ((BulkImportBatchInsertException) e.getCause()).exceptionByUserId; + for (String userid : errorsByPosition.keySet()) { + Exception exception = errorsByPosition.get(userid); + if (exception instanceof UnknownRoleException) { + String message = "E034: Role does not exist! You need to pre-create the role before " + + "assigning it to the user."; + errorsByPosition.put(userid, new Exception(message)); + } + } + throw new StorageTransactionLogicException(new BulkImportBatchInsertException("roles errors translated", errorsByPosition)); } else { throw new StorageTransactionLogicException(e); } - } } @@ -686,7 +694,7 @@ public static void verifyMultipleEmailForAllLoginMethods(AppIdentifier appIdenti Map emailToUserId = new HashMap<>(); for (BulkImportUser user : users) { for (LoginMethod lm : user.loginMethods) { - emailToUserId.put(lm.email, lm.getSuperTokenOrExternalUserId()); + emailToUserId.put(lm.getSuperTokenOrExternalUserId(), lm.email); } } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java index 32ce82077..74e73a2ae 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkImportUsers.java @@ -35,7 +35,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -71,10 +74,8 @@ protected void doTaskPerApp(AppIdentifier app) String[] allUserRoles = StorageUtils.getUserRolesStorage(bulkImportSQLStorage).getRoles(app); BulkImportUserUtils bulkImportUserUtils = new BulkImportUserUtils(allUserRoles); - System.out.println(Thread.currentThread().getName() + " ProcessBulkImportUsers: " + " starting to load users " + this.batchSize); List users = bulkImportSQLStorage.getBulkImportUsersAndChangeStatusToProcessing(app, this.batchSize); - System.out.println(Thread.currentThread().getName() + " ProcessBulkImportUsers: " + " loaded users"); if(users == null || users.isEmpty()) { return; @@ -91,7 +92,7 @@ protected void doTaskPerApp(AppIdentifier app) try { List> tasks = new ArrayList<>(); - for (int i =0; i< NUMBER_OF_BATCHES; i++) { + for (int i =0; i< NUMBER_OF_BATCHES && i < loadedUsersChunks.size(); i++) { tasks.add(executorService.submit(new ProcessBulkUsersImportWorker(main, app, loadedUsersChunks.get(i), bulkImportSQLStorage, bulkImportUserUtils))); } @@ -101,14 +102,9 @@ protected void doTaskPerApp(AppIdentifier app) Thread.sleep(1000); } Void result = (Void) task.get(); //to know if there were any errors while executing and for waiting in this thread for all the other threads to finish up - System.out.println("Result: " + result); } - - executorService.shutdownNow(); - if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) { - System.out.println("Pool did not terminate"); - } + } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } diff --git a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java index 5b8da78e9..855f616a8 100644 --- a/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java +++ b/src/main/java/io/supertokens/cronjobs/bulkimport/ProcessBulkUsersImportWorker.java @@ -43,10 +43,7 @@ import io.supertokens.storageLayer.StorageLayer; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class ProcessBulkUsersImportWorker implements Runnable { @@ -83,23 +80,32 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List validUsers = new ArrayList<>(); + Map validationErrorsBeforeActualProcessing = new HashMap<>(); while(userIndexPointer < users.size()) { user = users.get(userIndexPointer); - if ((Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) || - shouldRetryImmediately) { + if (Main.isTesting && Main.isTesting_skipBulkImportUserValidationInCronJob) { // Skip validation when the flag is enabled during testing // Skip validation if it's a retry run. This already passed validation. A revalidation triggers // an invalid external user id already exists validation error - which is not true! - //TODO set invalid users status to failed + validUsers.add(user); } else { // Validate the user - validUsers.add(bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, user.toJsonObject(), user.id)); + try { + validUsers.add(bulkImportUserUtils.createBulkImportUserFromJSON(main, appIdentifier, + user.toJsonObject(), user.id)); + } catch (InvalidBulkImportDataException exception) { + validationErrorsBeforeActualProcessing.put(user.id, new Exception( + String.valueOf(exception.errors))); + } } userIndexPointer+=1; } + + if(!validationErrorsBeforeActualProcessing.isEmpty()) { + throw new BulkImportBatchInsertException("Invalid input data", validationErrorsBeforeActualProcessing); + } // Since all the tenants of a user must share the storage, we will just use the // storage of the first tenantId of the first loginMethod TenantIdentifier firstTenantIdentifier = new TenantIdentifier(appIdentifier.getConnectionUriDomain(), @@ -107,7 +113,7 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List { + bulkImportProxyStorage.startTransaction(con -> { try { BulkImport.processUsersImportSteps(main, con, appIdentifier, bulkImportProxyStorage, validUsers, allStoragesForApp); @@ -132,12 +138,10 @@ private void processMultipleUsers(AppIdentifier appIdentifier, List userIndexToError = ((BulkImportBatchInsertException) exception.actualException).exceptionByUserId; - for(String userid : userIndexToError.keySet()){ - String id = usersBatch.stream() - .filter(bulkImportUser -> - bulkImportUser.loginMethods.stream() - .map(loginMethod -> loginMethod.superTokensUserId) - .anyMatch(s -> s.equals(userid))).findFirst().get().id; - bulkImportUserIdToErrorMessage.put(id, userIndexToError.get(userid).getMessage()); - } + handleBulkImportException(usersBatch, (BulkImportBatchInsertException) exception.actualException, bulkImportUserIdToErrorMessage); } else { //fail the whole batch errorMessage[0] = exception.actualException.getMessage(); @@ -190,6 +186,8 @@ private void handleProcessUserExceptions(AppIdentifier appIdentifier, List usersBatch, BulkImportBatchInsertException exception, + Map bulkImportUserIdToErrorMessage) { + Map userIndexToError = exception.exceptionByUserId; + for(String userid : userIndexToError.keySet()){ + Optional userWithId = usersBatch.stream() + .filter(bulkImportUser -> bulkImportUser.id.equals(userid) || bulkImportUser.externalUserId.equals(userid)).findFirst(); + String id = null; + if(userWithId.isPresent()){ + id = userWithId.get().id; + } + + if(id == null) { + userWithId = usersBatch.stream() + .filter(bulkImportUser -> + bulkImportUser.loginMethods.stream() + .map(loginMethod -> loginMethod.superTokensUserId) + .anyMatch(s -> s.equals(userid))).findFirst(); + if(userWithId.isPresent()){ + id = userWithId.get().id; + } + } + bulkImportUserIdToErrorMessage.put(id, userIndexToError.get(userid).getMessage()); + } + } + private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantIdentifier) throws InvalidConfigException, IOException, TenantOrAppNotFoundException, DbInitException { String userPoolId = StorageLayer.getStorage(tenantIdentifier, main).getUserPoolId(); @@ -229,7 +252,7 @@ private synchronized Storage getBulkImportProxyStorage(TenantIdentifier tenantId throw new TenantOrAppNotFoundException(tenantIdentifier); } - private Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) + private synchronized Storage[] getAllProxyStoragesForApp(Main main, AppIdentifier appIdentifier) throws StorageTransactionLogicException { try { diff --git a/src/main/java/io/supertokens/inmemorydb/Start.java b/src/main/java/io/supertokens/inmemorydb/Start.java index 5ca0ecbb1..18f290816 100644 --- a/src/main/java/io/supertokens/inmemorydb/Start.java +++ b/src/main/java/io/supertokens/inmemorydb/Start.java @@ -658,7 +658,8 @@ public boolean isUserIdBeingUsedInNonAuthRecipe(AppIdentifier appIdentifier, Str public Map> findNonAuthRecipesWhereForUserIdsUsed(AppIdentifier appIdentifier, List userIds) throws StorageQueryException { - return Map.of(); + throw new UnsupportedOperationException("'findNonAuthRecipesWhereForUserIdsUsed' is not supported for in-memory db"); + } @TestOnly @@ -933,7 +934,7 @@ public void deleteEmailPasswordUser_Transaction(TransactionConnection con, AppId public void signUpMultipleViaBulkImport_Transaction(TransactionConnection connection, List users) throws StorageQueryException, StorageTransactionLogicException { - //TODO + throw new UnsupportedOperationException("'signUpMultipleViaBulkImport_Transaction' is not supported for in-memory db"); } @Override @@ -1156,7 +1157,7 @@ public void updateIsEmailVerifiedToExternalUserId(AppIdentifier appIdentifier, S public void updateMultipleIsEmailVerifiedToExternalUserIds(AppIdentifier appIdentifier, Map supertokensUserIdToExternalUserId) throws StorageQueryException { - + throw new UnsupportedOperationException("'updateMultipleIsEmailVerifiedToExternalUserIds' is not supported for in-memory db"); } @Override @@ -1197,14 +1198,14 @@ public void deleteThirdPartyUser_Transaction(TransactionConnection con, AppIdent public void importThirdPartyUsers_Transaction(TransactionConnection con, List usersToImport) throws StorageQueryException, StorageTransactionLogicException { - // TODO + throw new UnsupportedOperationException("'importThirdPartyUsers_Transaction' is not supported for in-memory db"); } @Override public void importPasswordlessUsers_Transaction(TransactionConnection con, List users) throws StorageQueryException { - // TODO + throw new UnsupportedOperationException("'importPasswordlessUsers_Transaction' is not supported for in-memory db"); } @Override @@ -1350,7 +1351,7 @@ public boolean doesUserIdExist(TenantIdentifier tenantIdentifier, String userId) @Override public List findExistingUserIds(AppIdentifier appIdentifier, List userIds) throws StorageQueryException { - return List.of(); // TODO + throw new UnsupportedOperationException("'findExistingUserIds' is not supported for in-memory db"); } @Override @@ -1917,7 +1918,7 @@ public Map getMultipleUsersMetadatas_Transaction(AppIdentifi TransactionConnection con, List userIds) throws StorageQueryException { - return Map.of(); // TODO + throw new UnsupportedOperationException("'getMultipleUsersMetadatas_Transaction' is not supported for in-memory db"); } @@ -1951,7 +1952,7 @@ public int setUserMetadata_Transaction(AppIdentifier appIdentifier, TransactionC public void setMultipleUsersMetadatas_Transaction(AppIdentifier appIdentifier, TransactionConnection con, Map metadataByUserId) throws StorageQueryException, TenantOrAppNotFoundException { - //TODO + throw new UnsupportedOperationException("'setMultipleUsersMetadatas_Transaction' is not supported for in-memory db"); } @Override @@ -2199,10 +2200,10 @@ public boolean doesRoleExist_Transaction(AppIdentifier appIdentifier, Transactio } @Override - public List doesMultipleRoleExist_Transaction(AppIdentifier appIdentifier, TransactionConnection con, + public List doesMultipleRoleExist_Transaction(AppIdentifier appIdentifier, TransactionConnection con, List roles) throws StorageQueryException { - // TODO - return List.of(); + throw new UnsupportedOperationException("'doesMultipleRoleExist_Transaction' is not supported for in-memory db"); + } @Override @@ -2218,9 +2219,9 @@ public void deleteAllRolesForUser_Transaction(TransactionConnection con, AppIden @Override public void addRolesToUsers_Transaction(TransactionConnection connection, - Map> rolesToUserByTenants) + Map>> rolesToUserByTenants) throws StorageQueryException { - // TODO + throw new UnsupportedOperationException("'addRolesToUsers_Transaction' is not supported for in-memory db"); } @Override @@ -2267,7 +2268,7 @@ public void createUserIdMapping(AppIdentifier appIdentifier, String superTokensU public void createBulkUserIdMapping(AppIdentifier appIdentifier, Map superTokensUserIdToExternalUserId) throws StorageQueryException { - + throw new UnsupportedOperationException("'createBulkUserIdMapping' is not supported for in-memory db"); } @Override @@ -2775,7 +2776,7 @@ public TOTPDevice createDevice_Transaction(TransactionConnection con, AppIdentif public void createDevices_Transaction(TransactionConnection con, AppIdentifier appIdentifier, List devices) throws StorageQueryException, TenantOrAppNotFoundException { - // TODO + throw new UnsupportedOperationException("'createDevices_Transaction' is not supported for in-memory db"); } @Override @@ -3014,7 +3015,7 @@ public AuthRecipeUserInfo[] listPrimaryUsersByEmail_Transaction(AppIdentifier ap public AuthRecipeUserInfo[] listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction( AppIdentifier appIdentifier, TransactionConnection con, List emails, List phones, Map thirdpartyIdToThirdpartyUserId) throws StorageQueryException { - return new AuthRecipeUserInfo[0]; // TODO + throw new UnsupportedOperationException("'listPrimaryUsersByMultipleEmailsOrPhoneNumbersOrThirdparty_Transaction' is not supported for in-memory db"); } @Override @@ -3102,7 +3103,7 @@ public void linkAccounts_Transaction(AppIdentifier appIdentifier, TransactionCon public void linkMultipleAccounts_Transaction(AppIdentifier appIdentifier, TransactionConnection con, Map recipeUserIdByPrimaryUserId) throws StorageQueryException { - // TODO + throw new UnsupportedOperationException("'linkMultipleAccounts_Transaction' is not supported for in-memory db"); } @Override @@ -3187,7 +3188,7 @@ public List getMultipleUserIdMapping_Transaction(TransactionConne AppIdentifier appIdentifier, List userIds, boolean isSupertokensIds) throws StorageQueryException { - return List.of(); // TODO + throw new UnsupportedOperationException("'getMultipleUserIdMapping_Transaction' is not supported for in-memory db"); } @Override diff --git a/src/main/java/io/supertokens/storageLayer/StorageLayer.java b/src/main/java/io/supertokens/storageLayer/StorageLayer.java index 630110a03..c054b7feb 100644 --- a/src/main/java/io/supertokens/storageLayer/StorageLayer.java +++ b/src/main/java/io/supertokens/storageLayer/StorageLayer.java @@ -17,10 +17,7 @@ package io.supertokens.storageLayer; import com.google.gson.JsonObject; -import io.supertokens.Main; -import io.supertokens.ProcessState; -import io.supertokens.ResourceDistributor; -import io.supertokens.StorageAndUserIdMapping; +import io.supertokens.*; import io.supertokens.cliOptions.CLIOptions; import io.supertokens.config.Config; import io.supertokens.exceptions.QuitProgramException; @@ -609,10 +606,7 @@ public static List findStorageAndUserIdMappingForBulkUs .filter(userIdMapping -> (userIdType == UserIdType.SUPERTOKENS && userIdMapping.superTokensUserId.equals(existingId)) || (userIdType == UserIdType.EXTERNAL && userIdMapping.externalUserId.equals(existingId)) ) .findFirst().orElse(null); - if(mappingForId == null && userIdType == UserIdType.SUPERTOKENS) { - mappingForId = new UserIdMapping(existingId, null, null); - } - allMappingsFromAllStorages.add(new StorageAndUserIdMapping(storage, mappingForId)); + allMappingsFromAllStorages.add(new StorageAndUserIdMappingForBulkImport(storage, mappingForId, existingId)); } } } else { diff --git a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java index 1e9f2c117..453ec6e54 100644 --- a/src/main/java/io/supertokens/useridmapping/UserIdMapping.java +++ b/src/main/java/io/supertokens/useridmapping/UserIdMapping.java @@ -18,6 +18,7 @@ import io.supertokens.Main; import io.supertokens.StorageAndUserIdMapping; +import io.supertokens.StorageAndUserIdMappingForBulkImport; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; @@ -60,6 +61,15 @@ public UserIdBulkMappingResult(String supertokensUserId, String externalUserId, this.error = error; this.externalUserId = externalUserId; } + + @Override + public String toString() { + return "UserIdBulkMappingResult{" + + "supertokensUserId='" + supertokensUserId + '\'' + + ", externalUserId='" + externalUserId + '\'' + + ", error=" + error + + '}'; + } } @TestOnly @@ -218,16 +228,20 @@ public static List createMultipleUserIdMappings(AppIden for(Map.Entry supertokensIdToExternalId : superTokensUserIdToExternalUserId.entrySet()) { String supertokensId = supertokensIdToExternalId.getKey(); String externalId = supertokensIdToExternalId.getValue(); - StorageAndUserIdMapping mappingByExternal = findStorageAndUserIdMappingForUser(externalId, mappingAndStorageWithExternal, false); - if (mappingByExternal != null && mappingByExternal.userIdMapping != null ){ + StorageAndUserIdMapping mappingByExternal = findStorageAndUserIdMappingForUser(externalId, + mappingAndStorageWithExternal, false); + if (mappingByExternal != null && mappingByExternal.userIdMapping != null) { mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, - new UserIdMappingAlreadyExistsException(supertokensId.equals(mappingByExternal.userIdMapping.superTokensUserId), + new UserIdMappingAlreadyExistsException( + supertokensId.equals(mappingByExternal.userIdMapping.superTokensUserId), externalId.equals(mappingByExternal.userIdMapping.externalUserId)))); continue; } - StorageAndUserIdMapping mappingBySupertokens = findStorageAndUserIdMappingForUser(supertokensId, mappingAndStorageWithSupertokens, true); - if(mappingBySupertokens == null) { - mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, new UnknownSuperTokensUserIdException())); + StorageAndUserIdMapping mappingBySupertokens = findStorageAndUserIdMappingForUser(supertokensId, + mappingAndStorageWithSupertokens, true); + if (mappingBySupertokens == null) { + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, + new UnknownSuperTokensUserIdException())); continue; } Storage userStorage = mappingBySupertokens.storage; @@ -242,14 +256,15 @@ public static List createMultipleUserIdMappings(AppIden { if (findStorageAndUserIdMappingForUser(externalId, mappingAndStoragesAsInvalid, true) != null) { - mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, new ServletException(new WebserverAPI.BadRequestException( - "Cannot create a userId mapping where the externalId is also a SuperTokens userID")))); + mappingResults.add(new UserIdBulkMappingResult(supertokensId, externalId, + new ServletException(new WebserverAPI.BadRequestException( + "Cannot create a userId mapping where the externalId is also a SuperTokens userID")))); continue; } } List storageClasses; - if(userIdsUsedInNonAuthRecipes.containsKey(supertokensId)){ + if (userIdsUsedInNonAuthRecipes.containsKey(supertokensId)) { storageClasses = userIdsUsedInNonAuthRecipes.get(supertokensId); } else { storageClasses = new ArrayList<>(); @@ -280,35 +295,44 @@ public static List createMultipleUserIdMappings(AppIden } } else { //if we are not making any exceptions, then having the id used is an error! - if(!storageClasses.isEmpty()) { - createBulkIdMappingErrorForNonAuthRecipeUsage(storageClasses, mappingResults, supertokensId, externalId); + if (!storageClasses.isEmpty()) { + createBulkIdMappingErrorForNonAuthRecipeUsage(storageClasses, mappingResults, supertokensId, + externalId); continue; } } noErrorFound.add(mappingBySupertokens); } + } + //userstorage - group users by storage + Map> partitionedMappings = partitionUsersByStorage(noErrorFound); + for(Storage storage : partitionedMappings.keySet()){ - //userstorage - group users by storage - Map> partitionedMappings = partitionUsersByStorage(noErrorFound); - for(Storage storage : partitionedMappings.keySet()){ - List mappingsForCurrentStorage = partitionedMappings.get(storage); + List mappingsForCurrentStorage = partitionedMappings.get(storage); + Map mappingInCurrentStorageThatNeedsToBeDone = new HashMap<>(); + Map supertokensIdToExternalIdInCurrentStorageForEmailUpdate = new HashMap<>(); - Map supertokensIdToExternalIdInCurrentStorage = new HashMap<>(); - for(StorageAndUserIdMapping storageAndUserIdMapping: mappingsForCurrentStorage) { - supertokensIdToExternalIdInCurrentStorage.put(storageAndUserIdMapping.userIdMapping.superTokensUserId, - superTokensUserIdToExternalUserId.get(storageAndUserIdMapping.userIdMapping.superTokensUserId)); + for(StorageAndUserIdMapping storageAndUserIdMapping: mappingsForCurrentStorage) { + String userIdInQuestion = ((StorageAndUserIdMappingForBulkImport)storageAndUserIdMapping).userIdInQuestion; + + if(supertokensToExternalUserIdsToUpdateEmailVerified.keySet().contains(userIdInQuestion)){ + supertokensIdToExternalIdInCurrentStorageForEmailUpdate.put(userIdInQuestion, + superTokensUserIdToExternalUserId.get(userIdInQuestion)); } + mappingInCurrentStorageThatNeedsToBeDone.put(userIdInQuestion, superTokensUserIdToExternalUserId.get(userIdInQuestion)); + } - EmailVerificationStorage emailVerificationStorage = StorageUtils.getEmailVerificationStorage(storage); - emailVerificationStorage.updateMultipleIsEmailVerifiedToExternalUserIds(appIdentifier, supertokensIdToExternalIdInCurrentStorage); + StorageUtils.getUserIdMappingStorage(storage).createBulkUserIdMapping(appIdentifier, mappingInCurrentStorageThatNeedsToBeDone); - StorageUtils.getUserIdMappingStorage(storage).createBulkUserIdMapping(appIdentifier, supertokensIdToExternalIdInCurrentStorage); - for(String supertokensIdForResult : supertokensIdToExternalIdInCurrentStorage.keySet()) { - mappingResults.add(new UserIdBulkMappingResult(supertokensIdForResult, supertokensIdToExternalIdInCurrentStorage.get(supertokensIdForResult), null)); - } + EmailVerificationStorage emailVerificationStorage = StorageUtils.getEmailVerificationStorage(storage); + emailVerificationStorage.updateMultipleIsEmailVerifiedToExternalUserIds(appIdentifier, supertokensIdToExternalIdInCurrentStorageForEmailUpdate); + + for(String supertokensIdForResult : mappingInCurrentStorageThatNeedsToBeDone.keySet()) { + mappingResults.add(new UserIdBulkMappingResult(supertokensIdForResult, mappingInCurrentStorageThatNeedsToBeDone.get(supertokensIdForResult), null)); } } + Map errors = new HashMap<>(); for(UserIdBulkMappingResult result : mappingResults){ if(result.error != null) { @@ -345,16 +369,18 @@ private static Map> partitionUsersByStora private static StorageAndUserIdMapping findStorageAndUserIdMappingForUser(String userId, List findIn, boolean supertokensId) { List mappings = findIn.stream().filter(storageAndUserIdMapping -> { - if(storageAndUserIdMapping.userIdMapping != null) { + if(storageAndUserIdMapping instanceof StorageAndUserIdMappingForBulkImport && ((StorageAndUserIdMappingForBulkImport) storageAndUserIdMapping).userIdInQuestion != null) { + return ((StorageAndUserIdMappingForBulkImport)storageAndUserIdMapping).userIdInQuestion.equals(userId); + } else if(storageAndUserIdMapping.userIdMapping != null) { if(supertokensId) { - return storageAndUserIdMapping.userIdMapping.superTokensUserId.equals(userId); + return userId.equals(storageAndUserIdMapping.userIdMapping.superTokensUserId); } else { - return storageAndUserIdMapping.userIdMapping.externalUserId.equals(userId); + return userId.equals(storageAndUserIdMapping.userIdMapping.externalUserId); } } return false; }).collect(Collectors.toList()); // theoretically it shouldn't happen that there are more than one element in the list - if(mappings.size() > 1) { + if(mappings.size() > 1 && !(mappings.get(0) instanceof StorageAndUserIdMappingForBulkImport)) { throw new IllegalStateException("more than one mapping exists for Id."); } return mappings.isEmpty() ? null : mappings.get(0); diff --git a/src/main/java/io/supertokens/userroles/UserRoles.java b/src/main/java/io/supertokens/userroles/UserRoles.java index 478ca23c8..8d88b637a 100644 --- a/src/main/java/io/supertokens/userroles/UserRoles.java +++ b/src/main/java/io/supertokens/userroles/UserRoles.java @@ -19,6 +19,7 @@ import io.supertokens.Main; import io.supertokens.pluginInterface.Storage; import io.supertokens.pluginInterface.StorageUtils; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; import io.supertokens.pluginInterface.exceptions.StorageTransactionLogicException; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; @@ -31,9 +32,7 @@ import org.jetbrains.annotations.TestOnly; import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; public class UserRoles { // add a role to a user and return true, if the role is already mapped to the user return false, but if @@ -59,34 +58,52 @@ public static boolean addRoleToUser(Main main, TenantIdentifier tenantIdentifier } } - public static void addMultipleRolesToMultipleUsers(Main main, Storage storage, Map> rolesToUserByTenant) + public static void addMultipleRolesToMultipleUsers(Main main, AppIdentifier appIdentifier, Storage storage, + Map>> rolesToUserByTenant) throws StorageTransactionLogicException, TenantOrAppNotFoundException { // Roles are stored in public tenant storage and role to user mapping is stored in the tenant's storage // We do this because it's not straight forward to replicate roles to all storages of an app - for(TenantIdentifier tenantIdentifier : rolesToUserByTenant.keySet()){ - Storage appStorage = StorageLayer.getStorage( - tenantIdentifier.toAppIdentifier().getAsPublicTenantIdentifier(), main); - - try { - UserRolesSQLStorage userRolesStorage = StorageUtils.getUserRolesStorage(storage); - userRolesStorage.startTransaction(con -> { - - List rolesFound = ((UserRolesSQLStorage) appStorage).doesMultipleRoleExist_Transaction( - tenantIdentifier.toAppIdentifier().getAsPublicTenantIdentifier().toAppIdentifier(), - con, new ArrayList<>(rolesToUserByTenant.get(tenantIdentifier).values())); + Storage appStorage = StorageLayer.getStorage( + appIdentifier.getAsPublicTenantIdentifier(), main); - if(rolesFound.contains(Boolean.FALSE)){ - throw new StorageTransactionLogicException(new UnknownRoleException()); + try { + UserRolesSQLStorage userRolesStorage = StorageUtils.getUserRolesStorage(storage); + UserRolesSQLStorage publicRoleStorage = StorageUtils.getUserRolesStorage(appStorage); + Map errorsByUser = new HashMap<>(); + publicRoleStorage.startTransaction(con -> { + Set rolesToSearchFor = new HashSet<>(); + for (TenantIdentifier tenantIdentifier : rolesToUserByTenant.keySet()) { + for(String userId : rolesToUserByTenant.get(tenantIdentifier).keySet()){ + rolesToSearchFor.addAll(rolesToUserByTenant.get(tenantIdentifier).get(userId)); } - userRolesStorage.addRolesToUsers_Transaction(con, rolesToUserByTenant); - userRolesStorage.commitTransaction(con); - return null; - }); + } + List rolesFound = ((UserRolesSQLStorage) appStorage).doesMultipleRoleExist_Transaction( + appIdentifier, con, + new ArrayList<>(rolesToSearchFor)); + + for (Map> rolesToUsers : rolesToUserByTenant.values()) { + for (String userId : rolesToUsers.keySet()) { + List rolesOfUser = rolesToUsers.get(userId); + if (!new HashSet<>(rolesFound).containsAll(rolesOfUser)) { //wrapping in hashset for performance reasons + errorsByUser.put(userId, new UnknownRoleException()); + } + } + } + if (!errorsByUser.isEmpty()) { + throw new StorageTransactionLogicException( + new BulkImportBatchInsertException("Roles errors", errorsByUser)); + } + return null; + }); + userRolesStorage.startTransaction(con -> { + userRolesStorage.addRolesToUsers_Transaction(con, rolesToUserByTenant); + userRolesStorage.commitTransaction(con); + return null; + }); - } catch (StorageQueryException e) { - throw new StorageTransactionLogicException(e); - } + } catch (StorageQueryException e) { + throw new StorageTransactionLogicException(e); } } diff --git a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java index f21dd809c..cc8bcd6f8 100644 --- a/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java +++ b/src/main/java/io/supertokens/webserver/api/bulkimport/ImportUserAPI.java @@ -27,6 +27,7 @@ import io.supertokens.pluginInterface.StorageUtils; import io.supertokens.pluginInterface.authRecipe.AuthRecipeUserInfo; import io.supertokens.pluginInterface.bulkimport.BulkImportUser; +import io.supertokens.pluginInterface.bulkimport.exceptions.BulkImportBatchInsertException; import io.supertokens.pluginInterface.exceptions.DbInitException; import io.supertokens.pluginInterface.exceptions.InvalidConfigException; import io.supertokens.pluginInterface.exceptions.StorageQueryException; @@ -86,16 +87,30 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws S result.addProperty("status", "OK"); result.add("user", importedUser.toJson()); super.sendJsonResponse(200, result, resp); + } catch (StorageQueryException e) { + JsonArray errors = new JsonArray(); + if(e.getCause() instanceof BulkImportBatchInsertException){ + BulkImportBatchInsertException insertException = (BulkImportBatchInsertException) e.getCause(); + errors.addAll( + insertException.exceptionByUserId.values().stream().map(exc -> exc.getMessage()).map(JsonPrimitive::new) + .collect(JsonArray::new, JsonArray::add, JsonArray::addAll) + ); + } else { + errors.add(new JsonPrimitive(e.getMessage())); + } + + JsonObject errorResponseJson = new JsonObject(); + errorResponseJson.add("errors", errors); + throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); + } catch (TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { + throw new ServletException(e); } catch (io.supertokens.bulkimport.exceptions.InvalidBulkImportDataException e) { JsonArray errors = e.errors.stream() .map(JsonPrimitive::new) .collect(JsonArray::new, JsonArray::add, JsonArray::addAll); - JsonObject errorResponseJson = new JsonObject(); errorResponseJson.add("errors", errors); throw new ServletException(new WebserverAPI.BadRequestException(errorResponseJson.toString())); - } catch (StorageQueryException | TenantOrAppNotFoundException | InvalidConfigException | DbInitException e) { - throw new ServletException(e); } } } diff --git a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java index 509fa5e23..bbf16e3b9 100644 --- a/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/BulkImportFlowTest.java @@ -22,11 +22,12 @@ import com.google.gson.JsonParser; import io.supertokens.Main; import io.supertokens.ProcessState; -import io.supertokens.config.Config; import io.supertokens.featureflag.EE_FEATURES; import io.supertokens.featureflag.FeatureFlagTestContent; +import io.supertokens.multitenancy.Multitenancy; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage; +import io.supertokens.pluginInterface.multitenancy.*; import io.supertokens.storageLayer.StorageLayer; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.Utils; @@ -39,8 +40,6 @@ import org.junit.Test; import org.junit.rules.TestRule; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -78,11 +77,7 @@ public void testWithOneMillionUsers() throws Exception { setFeatureFlags(main, new EE_FEATURES[] { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - //int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million - int NUMBER_OF_USERS_TO_UPLOAD = 10000; - int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); - System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); - System.out.println("Worker threads: " + parallelism_set_to); + int NUMBER_OF_USERS_TO_UPLOAD = 1000000; // million if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; @@ -100,16 +95,13 @@ public void testWithOneMillionUsers() throws Exception { JsonObject request = generateUsersJson(10000, i * 10000); // API allows 10k users upload at once JsonObject response = uploadBulkImportUsersJson(main, request); assertEquals("OK", response.get("status").getAsString()); - System.out.println(i + " Uploaded 10k users for bulk import"); } } - long processingStartedTime = System.currentTimeMillis(); - + long processingStarted = System.currentTimeMillis(); // Starting the processing cronjob here to be able to measure the runtime startBulkImportCronjob(main, 8000); - System.out.println("CronJob started"); // wait for the cron job to process them // periodically check the remaining unprocessed users @@ -121,36 +113,21 @@ public void testWithOneMillionUsers() throws Exception { JsonObject response = loadBulkImportUsersCountWithStatus(main, null); assertEquals("OK", response.get("status").getAsString()); count = response.get("count").getAsLong(); - System.out.println("Number of unprocessed users: " + count + "," + response); int newUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); - int failedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); - System.out.println("\t stats: "); - System.out.println("\t\tNEW: \t" + newUsersNumber); - System.out.println("\t\tFAILED: \t" + failedUsersNumber); - System.out.println("\t\tPROCESSING: \t" + processingUsersNumber); count = newUsersNumber + processingUsersNumber; - long elapsedSeconds = (System.currentTimeMillis() - processingStartedTime) / 1000; - System.out.println("Elapsed time: " + elapsedSeconds + " seconds, (" + elapsedSeconds / 3600 + " hours)"); if(count == 0 ){ break; } - Thread.sleep(60000); // one minute + Thread.sleep(5000); // one minute } } - long processingFinishedTime = System.currentTimeMillis(); - System.out.println("Processing took " + (processingFinishedTime - processingStartedTime) / 1000 + " seconds"); - - //print failed users - { - JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); - if(failedUsersLs.has("users") ){ - System.out.println(failedUsersLs.get("users")); - } - } + long processingFinished = System.currentTimeMillis(); + System.out.println("Processed " + NUMBER_OF_USERS_TO_UPLOAD + " users in " + (processingFinished - processingStarted) / 1000 + + " seconds ( or " + (processingFinished - processingStarted) / 60000 + " minutes)"); // after processing finished, make sure every user got processed correctly { @@ -163,7 +140,7 @@ public void testWithOneMillionUsers() throws Exception { } @Test - public void testBatchWithDuplicates() throws Exception { + public void testBatchWithDuplicate() throws Exception { String[] args = {"../"}; // set processing thread number @@ -226,20 +203,291 @@ public void testBatchWithDuplicates() throws Exception { //print failed users JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); - if (failedUsersLs.has("users")) { - System.out.println(failedUsersLs.get("users")); - } - System.out.println("Failed Users: " + failedUsersLs); - System.out.println("Failed Users Number: " + failedUsersNumber); // after processing finished, make sure every user got processed correctly int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); int usersInCore = loadUsersCount(main).get("count").getAsInt(); - System.out.println("Users in core: " + usersInCore); assertEquals(NUMBER_OF_USERS_TO_UPLOAD + 2, usersInCore + failedImportedUsersNumber); assertEquals(2, failedImportedUsersNumber); + + for(JsonElement userJson : failedUsersLs.get("users").getAsJsonArray()) { + String errorMessage = userJson.getAsJsonObject().get("errorMessage").getAsString(); + assertTrue(errorMessage.startsWith("E003:")); + } + + } + + @Test + public void testBatchWithDuplicateUserIdMappingWithInputValidation() throws Exception { + String[] args = {"../"}; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA}); + + int NUMBER_OF_USERS_TO_UPLOAD = 20; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + //set the first and last users' externalId to the same value + usersJson.get("users").getAsJsonArray().get(0).getAsJsonObject().addProperty("externalUserId", + "some-text-external-id"); + usersJson.get("users").getAsJsonArray().get(19).getAsJsonObject().addProperty("externalUserId", + "some-text-external-id"); + + try { + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + } catch (HttpResponseException expected) { + assertEquals(400, expected.statusCode); + assertEquals("Http error. Status Code: 400. Message: {\"error\":\"Data has missing or invalid fields. Please check the users field for more details.\",\"users\":[{\"index\":19,\"errors\":[\"externalUserId some-text-external-id is not unique. It is already used by another user.\"]}]}", + expected.getMessage()); + } + } + + @Test + public void testBatchWithInvalidInput() throws Exception { + String[] args = {"../"}; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA}); + + int NUMBER_OF_USERS_TO_UPLOAD = 2; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + usersJson.get("users").getAsJsonArray().get(0).getAsJsonObject().addProperty("externalUserId", + Boolean.FALSE); // invalid, should be string + + try { + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + } catch (HttpResponseException exception) { + assertEquals(400, exception.statusCode); + assertEquals("Http error. Status Code: 400. Message: {\"error\":\"Data has missing or invalid " + + "fields. Please check the users field for more details.\",\"users\":[{\"index\":0,\"errors\":" + + "[\"externalUserId should be of type string.\"]}]}", exception.getMessage()); + } + } + + @Test + public void testBatchWithMissingRole() throws Exception { + String[] args = {"../"}; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "12"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA}); + + int NUMBER_OF_USERS_TO_UPLOAD = 2; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Creating only one user role before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + try { + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + } catch (HttpResponseException exception) { + assertEquals(400, exception.statusCode); + assertEquals(400, exception.statusCode); + assertEquals("Http error. Status Code: 400. Message: {\"error\":\"Data has missing or " + + "invalid fields. Please check the users field for more details.\",\"users\":[{\"index\":0,\"errors\"" + + ":[\"Role role2 does not exist.\"]},{\"index\":1,\"errors\":[\"Role role2 does not exist.\"]}]}", + exception.getMessage()); + } + } + + @Test + public void testBatchWithOnlyOneWithDuplicate() throws Exception { + String[] args = {"../"}; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "2"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA}); + + int NUMBER_OF_USERS_TO_UPLOAD = 9; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + //create tenant t1 + TenantIdentifier tenantIdentifier = new TenantIdentifier(null, null, "t1"); + + Multitenancy.addNewOrUpdateAppOrTenant( + main, + new TenantIdentifier(null, null, null), + new TenantConfig( + tenantIdentifier, + new EmailPasswordConfig(true), + new ThirdPartyConfig(true, null), + new PasswordlessConfig(true), + null, null, new JsonObject())); + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + usersJson.get("users").getAsJsonArray().add(generateUsersJson(1, 0).get("users").getAsJsonArray().get(0).getAsJsonObject()); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // Starting the processing cronjob here to be able to measure the runtime + startBulkImportCronjob(main, 10); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + int failedUsersNumber = 0; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); + } + + //print failed users + JsonObject failedUsersLs = loadBulkImportUsersWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED); + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD + 1, usersInCore + failedImportedUsersNumber); + assertEquals(1, failedImportedUsersNumber); + + + for(JsonElement userJson : failedUsersLs.get("users").getAsJsonArray()) { + String errorMessage = userJson.getAsJsonObject().get("errorMessage").getAsString(); + assertTrue(errorMessage.startsWith("E003:")); + } + + } + + @Test + public void testBatchWithOneThreadWorks() throws Exception { + String[] args = {"../"}; + + // set processing thread number + Utils.setValueInConfig("bulk_migration_parallelism", "1"); + + TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); + assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); + Main main = process.getProcess(); + + setFeatureFlags(main, new EE_FEATURES[]{ + EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA}); + + int NUMBER_OF_USERS_TO_UPLOAD = 5; + + if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { + return; + } + + // Create user roles before inserting bulk users + UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); + UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); + + // upload a bunch of users through the API + JsonObject usersJson = generateUsersJson(NUMBER_OF_USERS_TO_UPLOAD, 0); + + JsonObject response = uploadBulkImportUsersJson(main, usersJson); + assertEquals("OK", response.get("status").getAsString()); + + // Starting the processing cronjob here to be able to measure the runtime + startBulkImportCronjob(main, 8000); + + // wait for the cron job to process them + // periodically check the remaining unprocessed users + // Note1: the cronjob starts the processing automatically + // Note2: the successfully processed users get deleted from the bulk_import_users table + + long count = NUMBER_OF_USERS_TO_UPLOAD; + while (true) { + response = loadBulkImportUsersCountWithStatus(main, null); + assertEquals("OK", response.get("status").getAsString()); + int newUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); + int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); + + count = newUsersNumber + processingUsersNumber; + if(count == 0) { + break; + } + Thread.sleep(5000); // 5 seconds + } + + // after processing finished, make sure every user got processed correctly + int failedImportedUsersNumber = loadBulkImportUsersCountWithStatus(main, + BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); + int usersInCore = loadUsersCount(main).get("count").getAsInt(); + assertEquals(NUMBER_OF_USERS_TO_UPLOAD, usersInCore); + assertEquals(0, failedImportedUsersNumber); } @Test @@ -257,9 +505,6 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); int NUMBER_OF_USERS_TO_UPLOAD = 1000; - int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); - System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); - System.out.println("Worker threads: " + parallelism_set_to); if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; @@ -282,24 +527,17 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { assertEquals("OK", lazyImportResponse.get("status").getAsString()); assertNotNull(lazyImportResponse.get("user")); successfully_lazy_imported++; - System.out.println(i + "th lazy imported"); -// System.out.println("\tOriginal user: " + userToImportLazy); -// System.out.println("\tResponse user: " + lazyImportResponse.get("user")); } // bulk import all of the users { JsonObject bulkUploadResponse = uploadBulkImportUsersJson(main, allUsersJson); assertEquals("OK", bulkUploadResponse.get("status").getAsString()); - System.out.println("Bulk uploaded all of the users"); } - long processingStartedTime = System.currentTimeMillis(); - - // Starting the processing cronjob here to be able to measure the runtime startBulkImportCronjob(main, 10000); - System.out.println("CronJob started"); + // wait for the cron job to process them // periodically check the remaining unprocessed users @@ -310,15 +548,8 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { while(count != 0) { JsonObject response = loadBulkImportUsersCountWithStatus(main, null); assertEquals("OK", response.get("status").getAsString()); - count = response.get("count").getAsLong(); - System.out.println("Number of unprocessed users: " + count + "," + response); int newUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.NEW).get("count").getAsInt(); - int failedUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.FAILED).get("count").getAsInt(); int processingUsersNumber = loadBulkImportUsersCountWithStatus(main, BulkImportStorage.BULK_IMPORT_USER_STATUS.PROCESSING).get("count").getAsInt(); - System.out.println("\t stats: "); - System.out.println("\t\tNEW: \t" + newUsersNumber); - System.out.println("\t\tFAILED: \t" + failedUsersNumber); - System.out.println("\t\tPROCESSING: \t" + processingUsersNumber); count = newUsersNumber + processingUsersNumber; // + processingUsersNumber; @@ -326,8 +557,6 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { } } - long processingFinishedTime = System.currentTimeMillis(); - System.out.println("Processing took " + (processingFinishedTime - processingStartedTime) / 1000 + " seconds"); // expect: lazy imported users are already there, duplicate.. errors // expect: not lazy imported users are imported successfully @@ -344,7 +573,6 @@ public void testFirstLazyImportAfterBulkImport() throws Exception { String errorMessage = failedUser.getAsJsonObject().get("errorMessage").getAsString(); assertTrue(errorMessage.startsWith("E003:") || errorMessage.startsWith("E005:") || errorMessage.startsWith("E006:") || errorMessage.startsWith("E007:")); // duplicate email, phone, etc errors - System.out.println(errorMessage); } stopBulkImportCronjob(main); @@ -362,9 +590,6 @@ public void testLazyImport() throws Exception { EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); int NUMBER_OF_USERS_TO_UPLOAD = 100; - int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); - System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); - System.out.println("Worker threads: " + parallelism_set_to); if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { return; @@ -387,7 +612,6 @@ public void testLazyImport() throws Exception { assertEquals("OK", lazyImportResponse.get("status").getAsString()); assertNotNull(lazyImportResponse.get("user")); successfully_lazy_imported++; - System.out.println(i + "th lazy imported"); } // expect: lazy imported users are already there, duplicate.. errors @@ -470,10 +694,10 @@ public void testLazyImportDuplicatesFail() throws Exception { JsonObject userToImportLazyAgain = allUsersJson.get("users").getAsJsonArray().get(0).getAsJsonObject(); try { - JsonObject lazyImportResponseTwo = lazyImportUser(main, userToImportLazy); + JsonObject lazyImportResponseTwo = lazyImportUser(main, userToImportLazyAgain); } catch (HttpResponseException expected) { - assertEquals(400, expected.statusCode); System.out.println(expected.getMessage()); + assertEquals(400, expected.statusCode); } } @@ -481,7 +705,7 @@ private static JsonObject lazyImportUser(Main main, JsonObject user) throws HttpResponseException, IOException { return HttpRequestForTesting.sendJsonPOSTRequest(main, "", "http://localhost:3567/bulk-import/import", - user, 1000, 1000, null, Utils.getCdiVersionStringLatestForTests(), null); + user, 100000, 100000, null, Utils.getCdiVersionStringLatestForTests(), null); } private static JsonObject loadBulkImportUsersCountWithStatus(Main main, BulkImportStorage.BULK_IMPORT_USER_STATUS status) @@ -528,6 +752,7 @@ private static JsonObject generateUsersJson(int numberOfUsers, int startIndex) { "[{\"role\":\"role1\", \"tenantIds\": [\"public\"]},{\"role\":\"role2\", \"tenantIds\": [\"public\"]}]")); user.add("totpDevices", parser.parse("[{\"secretKey\":\"secretKey\",\"deviceName\":\"deviceName\"}]")); + //JsonArray tenanatIds = parser.parse("[\"public\", \"t1\"]").getAsJsonArray(); JsonArray tenanatIds = parser.parse("[\"public\"]").getAsJsonArray(); String email = " johndoe+" + (i + startIndex) + "@gmail.com "; @@ -635,53 +860,4 @@ private static JsonObject uploadBulkImportUsersJson(Main main, JsonObject reques "http://localhost:3567/bulk-import/users", request, 1000, 10000, null, Utils.getCdiVersionStringLatestForTests(), null); } - - @Test - public void writeUsersToFile() throws Exception { - String[] args = { "../" }; - - // set processing thread number - Utils.setValueInConfig("bulk_migration_parallelism", "14"); - - TestingProcessManager.TestingProcess process = TestingProcessManager.start(args); - assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); - Main main = process.getProcess(); - - setFeatureFlags(main, new EE_FEATURES[] { - EE_FEATURES.ACCOUNT_LINKING, EE_FEATURES.MULTI_TENANCY, EE_FEATURES.MFA }); - - int NUMBER_OF_USERS_TO_UPLOAD = 1000000; - int parallelism_set_to = Config.getConfig(main).getBulkMigrationParallelism(); - System.out.println("Number of users to be imported with bulk import: " + NUMBER_OF_USERS_TO_UPLOAD); - System.out.println("Worker threads: " + parallelism_set_to); - - if (StorageLayer.getBaseStorage(main).getType() != STORAGE_TYPE.SQL || StorageLayer.isInMemDb(main)) { - return; - } - - // Create user roles before inserting bulk users - { - UserRoles.createNewRoleOrModifyItsPermissions(main, "role1", null); - UserRoles.createNewRoleOrModifyItsPermissions(main, "role2", null); - } - - // upload a bunch of users through the API - { - for (int i = 0; i < (NUMBER_OF_USERS_TO_UPLOAD / 10000); i++) { - JsonObject request = generateUsersJson(10000, i * 10000); // API allows 10k users upload at once - FileWriter fileWriter = new FileWriter(new File("/home/prophet/Projects/bulkimport-users-" + i + ".json")); - fileWriter.write(String.valueOf(request)); - fileWriter.flush(); - fileWriter.close(); - } - - } - - System.out.println("setup done, waiting"); - while(true){ - Thread.sleep(10000); - } - } - - } diff --git a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java index 3e3446535..0bf2647b2 100644 --- a/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java +++ b/src/test/java/io/supertokens/test/bulkimport/ProcessBulkImportUsersCronJobTest.java @@ -19,13 +19,10 @@ import io.supertokens.Main; import io.supertokens.ProcessState; -import io.supertokens.ResourceDistributor; import io.supertokens.authRecipe.AuthRecipe; import io.supertokens.authRecipe.UserPaginationContainer; import io.supertokens.bulkimport.BulkImport; import io.supertokens.bulkimport.BulkImportBackgroundJobManager; -import io.supertokens.config.Config; -import io.supertokens.cronjobs.CronTask; import io.supertokens.cronjobs.CronTaskTest; import io.supertokens.cronjobs.Cronjobs; import io.supertokens.cronjobs.bulkimport.ProcessBulkImportUsers; @@ -33,35 +30,32 @@ import io.supertokens.featureflag.FeatureFlagTestContent; import io.supertokens.pluginInterface.STORAGE_TYPE; import io.supertokens.pluginInterface.Storage; -import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.BulkImportStorage.BULK_IMPORT_USER_STATUS; +import io.supertokens.pluginInterface.bulkimport.BulkImportUser; import io.supertokens.pluginInterface.bulkimport.sqlStorage.BulkImportSQLStorage; import io.supertokens.pluginInterface.multitenancy.AppIdentifier; import io.supertokens.pluginInterface.multitenancy.TenantIdentifier; import io.supertokens.pluginInterface.multitenancy.exceptions.TenantOrAppNotFoundException; import io.supertokens.storageLayer.StorageLayer; -import io.supertokens.test.CronjobTest; import io.supertokens.test.TestingProcessManager; import io.supertokens.test.TestingProcessManager.TestingProcess; import io.supertokens.test.Utils; import io.supertokens.useridmapping.UserIdMapping; import io.supertokens.userroles.UserRoles; - import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; +import java.util.ArrayList; +import java.util.List; + import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUser; import static io.supertokens.test.bulkimport.BulkImportTestUtils.generateBulkImportUserWithRoles; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - public class ProcessBulkImportUsersCronJobTest { @Rule public TestRule watchman = Utils.getOnFailure(); @@ -352,7 +346,7 @@ public void shouldDeleteEverythingFromTheDBIfAnythingFails() throws Exception { assertEquals(1, usersAfterProcessing.size()); assertEquals(BULK_IMPORT_USER_STATUS.FAILED, usersAfterProcessing.get(0).status); - assertEquals("E034: Role role1 does not exist! You need pre-create the role before assigning it to the user.", + assertEquals("E034: Role does not exist! You need to pre-create the role before assigning it to the user.", usersAfterProcessing.get(0).errorMessage); UserPaginationContainer container = AuthRecipe.getUsers(main, 100, "ASC", null, null, null); @@ -395,7 +389,7 @@ public void shouldDeleteEverythingFromTheDBIfAnythingFailsOnMultipleThreads() th for(BulkImportUser userAfterProcessing: usersAfterProcessing){ assertEquals(BULK_IMPORT_USER_STATUS.FAILED, userAfterProcessing.status); // should process every user and every one of them should fail because of the missing role - assertEquals("E034: Role role1 does not exist! You need pre-create the role before assigning it to the user.", + assertEquals("E034: Role does not exist! You need to pre-create the role before assigning it to the user.", userAfterProcessing.errorMessage); } @@ -419,7 +413,6 @@ public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() th return; } - BulkImportTestUtils.createTenants(main); BulkImportSQLStorage storage = (BulkImportSQLStorage) StorageLayer.getStorage(main); @@ -435,7 +428,7 @@ public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() th BulkImport.addUsers(appIdentifier, storage, users); - Thread.sleep(2 * 60000); + Thread.sleep(60000); // one minute List usersAfterProcessing = storage.getBulkImportUsers(appIdentifier, 100, null, null, null); @@ -446,7 +439,7 @@ public void shouldDeleteOnlyFailedFromTheDBIfAnythingFailsOnMultipleThreads() th for(int i = 0; i < usersAfterProcessing.size(); i++){ if(usersAfterProcessing.get(i).status == BULK_IMPORT_USER_STATUS.FAILED) { assertEquals( - "E034: Role notExistingRole does not exist! You need pre-create the role before assigning it to the user.", + "E034: Role does not exist! You need to pre-create the role before assigning it to the user.", usersAfterProcessing.get(i).errorMessage); numberOfFailed++; } @@ -538,13 +531,13 @@ private TestingProcess startCronProcess() throws InterruptedException, TenantOrA // We are setting a non-zero initial wait for tests to avoid race condition with the beforeTest process that deletes data in the storage layer CronTaskTest.getInstance(main).setInitialWaitTimeInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 5); - CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 100000); + CronTaskTest.getInstance(main).setIntervalInSeconds(ProcessBulkImportUsers.RESOURCE_KEY, 1); process.startProcess(); assertNotNull(process.checkOrWaitForEvent(ProcessState.PROCESS_STATE.STARTED)); Cronjobs.addCronjob(main, (ProcessBulkImportUsers) main.getResourceDistributor().getResource(new TenantIdentifier(null, null, null), ProcessBulkImportUsers.RESOURCE_KEY)); - BulkImportBackgroundJobManager.startBackgroundJob(main, 1000); + BulkImportBackgroundJobManager.startBackgroundJob(main, 8000); if (StorageLayer.getStorage(main).getType() != STORAGE_TYPE.SQL) { return null; } From 4fe694ac7d963e510798d4e27eb2ce591824d493 Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Wed, 27 Nov 2024 10:00:54 +0100 Subject: [PATCH 40/41] fix: fixing failing tests, changing version --- build.gradle | 2 +- src/main/java/io/supertokens/authRecipe/AuthRecipe.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index c4779d56c..17e593dbc 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ compileTestJava { options.encoding = "UTF-8" } // } //} -version = "9.3.0" +version = "9.4.0" repositories { mavenCentral() diff --git a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java index d8eb407ed..1b0da61a8 100644 --- a/src/main/java/io/supertokens/authRecipe/AuthRecipe.java +++ b/src/main/java/io/supertokens/authRecipe/AuthRecipe.java @@ -418,7 +418,8 @@ private static void checkIfLoginMethodCanBeLinkedOnTenant(TransactionConnection if (!userWithSameThirdParty.tenantIds.contains(tenantId)) { continue; } - if (userWithSameThirdParty.isPrimaryUser) { + if (userWithSameThirdParty.isPrimaryUser && + !userWithSameThirdParty.getSupertokensUserId().equals(primaryUser.getSupertokensUserId())) { throw new AccountInfoAlreadyAssociatedWithAnotherPrimaryUserIdException( userWithSameThirdParty.getSupertokensUserId(), "This user's third party login is already associated with another" + From 535d00b3c6fd540d39b68abea47fb014336352be Mon Sep 17 00:00:00 2001 From: tamassoltesz Date: Wed, 27 Nov 2024 11:56:43 +0100 Subject: [PATCH 41/41] chore: update changelog --- CHANGELOG.md | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e05b6d60..88a37174a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,46 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [9.4.0] + +### Added +- Adds property `bulk_migration_parallelism` for fine-tuning the worker threads number +- Adds APIs to bulk import users + - GET `/bulk-import/users` + - POST `/bulk-import/users` + - GET `/bulk-import/users/count` + - POST `/bulk-import/users/remove` + - POST `/bulk-import/users/import` + - POST `/bulk-import/backgroundjob` + - GET `/bulk-import/backgroundjob` +- Adds `ProcessBulkImportUsers` cron job to process bulk import users +- Adds multithreaded worker support for the `ProcessBulkImportUsers` cron job for faster bulk imports +- Adds support for lazy importing users + +### Migrations + +```sql +"CREATE TABLE IF NOT EXISTS bulk_import_users ( + id CHAR(36), + app_id VARCHAR(64) NOT NULL DEFAULT 'public', + primary_user_id VARCHAR(36), + raw_data TEXT NOT NULL, + status VARCHAR(128) DEFAULT 'NEW', + error_msg TEXT, + created_at BIGINT NOT NULL, + updated_at BIGINT NOT NULL, + CONSTRAINT bulk_import_users_pkey PRIMARY KEY(app_id, id), + CONSTRAINT bulk_import_users__app_id_fkey FOREIGN KEY(app_id) REFERENCES apps(app_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS bulk_import_users_status_updated_at_index ON bulk_import_users (app_id, status, updated_at); + +CREATE INDEX IF NOT EXISTS bulk_import_users_pagination_index1 ON bulk_import_users (app_id, status, created_at DESC, + id DESC); + +CREATE INDEX IF NOT EXISTS bulk_import_users_pagination_index2 ON bulk_import_users (app_id, created_at DESC, id DESC); +``` + ## [9.3.0] ### Changes @@ -156,13 +196,6 @@ CREATE INDEX oauth_logout_challenges_time_created_index ON oauth_logout_challeng - Adds validation to firstFactors and requiredSecondaryFactors names while creating tenants/apps/etc. to not allow special chars. -### Added - -- Adds multithreaded worker support for the `ProcessBulkImportUsers` cron job for faster bulk imports -- Adds property `bulk_migration_parallelism` for fine-tuning the worker threads number -- Adds APIs to bulk import users -- Adds `ProcessBulkImportUsers` cron job to process bulk import users - ## [9.2.2] - 2024-09-04 - Adds index on `last_active_time` for `user_last_active` table to improve the performance of MAU computation. @@ -260,7 +293,6 @@ If using MySQL ALTER TABLE tenant_configs ADD COLUMN is_first_factors_null BOOLEAN DEFAULT TRUE; ALTER TABLE tenant_configs ALTER COLUMN is_first_factors_null DROP DEFAULT; ``` ->>>>>>> master ## [9.0.2] - 2024-04-17