chore: reactive migration

app/server/appsmith-server/src/main/java/com/appsmith/server/applications/git/ApplicationGitFileUtilsCEImpl.java

@@ -362,7 +362,8 @@ public Mono<ArtifactExchangeJson> reconstructArtifactExchangeJsonFromFilesInRepo
                     getApplicationResource(applicationReference.getMetadata(), ApplicationJson.class);
             ApplicationJson applicationJson = getApplicationJsonFromGitReference(applicationReference);
             copyNestedNonNullProperties(metadata, applicationJson);
-            return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(applicationJson);
+            return jsonSchemaMigration.migrateApplicationJsonToLatestSchema(
+                    applicationJson, baseArtifactId, branchName);
         });
     }
 

app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaMigration.java

@@ -6,6 +6,7 @@
 import com.appsmith.server.exceptions.AppsmithError;
 import com.appsmith.server.exceptions.AppsmithException;
 import com.appsmith.server.helpers.CollectionUtils;
+import com.appsmith.server.migrations.utils.JsonSchemaMigrationHelper;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
@@ -17,24 +18,13 @@
 public class JsonSchemaMigration {
 
     private final JsonSchemaVersions jsonSchemaVersions;
+    private final JsonSchemaMigrationHelper jsonSchemaMigrationHelper;
 
     private boolean isCompatible(ApplicationJson applicationJson) {
         return (applicationJson.getClientSchemaVersion() <= jsonSchemaVersions.getClientVersion())
                 && (applicationJson.getServerSchemaVersion() <= jsonSchemaVersions.getServerVersion());
     }
 
-    /**
-     * This is a temporary check which is being placed for the compatibility of server versions in scenarios
-     * where user is moving a json from an instance which has
-     * release_autocommit_feature_enabled true to an instance which has the flag as false. In that case the server
-     * version number of json would be 8 and in new instance it would be not compatible.
-     * @param applicationJson
-     * @return
-     */
-    private boolean isAutocommitVersionBump(ApplicationJson applicationJson) {
-        return jsonSchemaVersions.getServerVersion() == 7 && applicationJson.getServerSchemaVersion() == 8;
-    }
-
     private void setSchemaVersions(ApplicationJson applicationJson) {
         applicationJson.setServerSchemaVersion(getCorrectSchemaVersion(applicationJson.getServerSchemaVersion()));
         applicationJson.setClientSchemaVersion(getCorrectSchemaVersion(applicationJson.getClientSchemaVersion()));
@@ -53,67 +43,47 @@ public Mono<? extends ArtifactExchangeJson> migrateArtifactExchangeJsonToLatestS
             ArtifactExchangeJson artifactExchangeJson) {
 
         if (ArtifactType.APPLICATION.equals(artifactExchangeJson.getArtifactJsonType())) {
-            return migrateApplicationJsonToLatestSchema((ApplicationJson) artifactExchangeJson);
+            return migrateApplicationJsonToLatestSchema((ApplicationJson) artifactExchangeJson, null, null);
         }
 
         return Mono.fromCallable(() -> artifactExchangeJson);
     }
 
-    public Mono<ApplicationJson> migrateApplicationJsonToLatestSchema(ApplicationJson applicationJson) {
+    public Mono<ApplicationJson> migrateApplicationJsonToLatestSchema(
+            ApplicationJson applicationJson, String baseApplicationId, String branchName) {
         return Mono.fromCallable(() -> {
                     setSchemaVersions(applicationJson);
-                    if (isCompatible(applicationJson)) {
-                        return migrateServerSchema(applicationJson);
-                    }
-
-                    if (isAutocommitVersionBump(applicationJson)) {
-                        return migrateServerSchema(applicationJson);
+                    return applicationJson;
+                })
+                .flatMap(appJson -> {
+                    if (!isCompatible(appJson)) {
+                        return Mono.empty();
                     }
 
-                    return null;
+                    return migrateServerSchema(applicationJson, baseApplicationId, branchName);
                 })
                 .switchIfEmpty(Mono.error(new AppsmithException(AppsmithError.INCOMPATIBLE_IMPORTED_JSON)));
     }
 
     /**
-     * migrate artifacts to latest schema by adding the right DTOs, or any migration.
-     * This method would be deprecated soon enough
-     * @param artifactExchangeJson : the json to be imported
-     * @return transformed artifact exchange json
-     */
-    @Deprecated
-    public ArtifactExchangeJson migrateArtifactToLatestSchema(ArtifactExchangeJson artifactExchangeJson) {
-
-        if (!ArtifactType.APPLICATION.equals(artifactExchangeJson.getArtifactJsonType())) {
-            return artifactExchangeJson;
-        }
-
-        ApplicationJson applicationJson = (ApplicationJson) artifactExchangeJson;
-        setSchemaVersions(applicationJson);
-        if (!isCompatible(applicationJson)) {
-            if (!isAutocommitVersionBump(applicationJson)) {
-                throw new AppsmithException(AppsmithError.INCOMPATIBLE_IMPORTED_JSON);
-            }
-        }
-        return migrateServerSchema(applicationJson);
-    }
-
-    /**
-     * This method may be moved to the publisher chain itself
-     * @param applicationJson : applicationJson which needs to be transformed
-     * @return : transformed applicationJson
+     * Migrates ApplicationJson to latest schema.
+     * @param applicationJson : Application json for which migration is to be performed
+     * @param baseApplicationId : base applicationId
+     * @param branchName : branch name in case it is a git connected application.
+     * @return : A publisher of migrated json
      */
-    private ApplicationJson migrateServerSchema(ApplicationJson applicationJson) {
+    private Mono<ApplicationJson> migrateServerSchema(ApplicationJson applicationJson, String baseApplicationId, String branchName) {
+        Mono<ApplicationJson> migrateApplicationJsonMono = Mono.just(applicationJson);
+        // No need to run server side migration
         if (jsonSchemaVersions.getServerVersion().equals(applicationJson.getServerSchemaVersion())) {
-            // No need to run server side migration
-            return applicationJson;
+            return migrateApplicationJsonMono;
         }
+
         // Run migration linearly
         // Updating the schema version after each migration is not required as we are not exiting by breaking the switch
         // cases, but this keeps the version number and the migration in sync
         switch (applicationJson.getServerSchemaVersion()) {
             case 0:
-
             case 1:
                 // Migration for deprecating archivedAt field in ActionDTO
                 if (!CollectionUtils.isNullOrEmpty(applicationJson.getActionList())) {
@@ -130,7 +100,7 @@ private ApplicationJson migrateServerSchema(ApplicationJson applicationJson) {
             case 4:
                 // Remove unwanted fields from DTO and allow serialization for JsonIgnore fields
                 if (!CollectionUtils.isNullOrEmpty(applicationJson.getPageList())
-                        && applicationJson.getExportedApplication() != null) {
+                    && applicationJson.getExportedApplication() != null) {
                     MigrationHelperMethods.arrangeApplicationPagesAsPerImportedPageOrder(applicationJson);
                     MigrationHelperMethods.updateMongoEscapedWidget(applicationJson);
                 }
@@ -145,18 +115,23 @@ private ApplicationJson migrateServerSchema(ApplicationJson applicationJson) {
                 MigrationHelperMethods.ensureXmlParserPresenceInCustomJsLibList(applicationJson);
                 applicationJson.setServerSchemaVersion(7);
             case 7:
+                applicationJson.setServerSchemaVersion(8);
             case 8:
                 MigrationHelperMethods.migrateThemeSettingsForAnvil(applicationJson);
                 applicationJson.setServerSchemaVersion(9);
+            case 9:
+                if (Boolean.TRUE.equals(MigrationHelperMethods.doesRestApiRequireMigration(applicationJson))) {
+                    migrateApplicationJsonMono = migrateApplicationJsonMono
+                        .flatMap(migratedJson -> jsonSchemaMigrationHelper
+                            .addDatasourceConfigurationToDefaultRestApiActions(baseApplicationId, branchName, migratedJson));
+                }
+                applicationJson.setServerSchemaVersion(10);
+                // Moving forward all case implementation should be wrapped in the Mono even if they are non reactive
+                // code in order to perform linear migrations.
             default:
-                // Unable to detect the serverSchema
-        }
-
-        if (applicationJson.getServerSchemaVersion().equals(jsonSchemaVersions.getServerVersion())) {
-            return applicationJson;
         }
 
         applicationJson.setServerSchemaVersion(jsonSchemaVersions.getServerVersion());
-        return applicationJson;
+        return migrateApplicationJsonMono;
     }
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/JsonSchemaVersionsFallback.java

@@ -4,7 +4,7 @@
 
 @Component
 public class JsonSchemaVersionsFallback {
-    private static final Integer serverVersion = 9;
+    private static final Integer serverVersion = 10;
     public static final Integer clientVersion = 1;
 
     public Integer getServerVersion() {

app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/MigrationHelperMethods.java

@@ -1,8 +1,11 @@
 package com.appsmith.server.migrations;
 
+import com.appsmith.external.constants.PluginConstants;
 import com.appsmith.external.helpers.MustacheHelper;
 import com.appsmith.external.models.ActionDTO;
 import com.appsmith.external.models.BaseDomain;
+import com.appsmith.external.models.Datasource;
+import com.appsmith.external.models.DatasourceConfiguration;
 import com.appsmith.external.models.InvisibleActionFields;
 import com.appsmith.external.models.Property;
 import com.appsmith.server.constants.ApplicationConstants;
@@ -1231,4 +1234,106 @@ public static void setThemeSettings(Application.ThemeSetting themeSetting) {
             themeSetting.setSizing(1);
         }
     }
+
+    private static boolean conditionForDefaultRestDatasourceMigration(NewAction action) {
+        Datasource actionDatasource = action.getUnpublishedAction().getDatasource();
+
+        // condition to check if the action is default rest datasource.
+        // it has no datasource id and name is equal to DEFAULT_REST_DATASOURCE
+        boolean isActionDefaultRestDatasource = !org.springframework.util.StringUtils.hasText(actionDatasource.getId())
+                && PluginConstants.DEFAULT_REST_DATASOURCE.equals(actionDatasource.getName());
+
+        // condition to check if the action has missing url or has no config at all
+        boolean isDatasourceConfigurationOrUrlMissing = actionDatasource.getDatasourceConfiguration() == null
+                || !org.springframework.util.StringUtils.hasText(
+                        actionDatasource.getDatasourceConfiguration().getUrl());
+
+        return isActionDefaultRestDatasource && isDatasourceConfigurationOrUrlMissing;
+    }
+
+    /**
+     * Adds datasource configuration and relevant url to the embedded datasource actions.
+     * @param applicationJson: ApplicationJson for which the migration has to be performed
+     * @param defaultDatasourceActionMap: gitSyncId to actions with default rest datasource map
+     */
+    public static void migrateApplicationJsonToVersionTen(
+            ApplicationJson applicationJson, Map<String, NewAction> defaultDatasourceActionMap) {
+        List<NewAction> actionList = applicationJson.getActionList();
+        if (CollectionUtils.isNullOrEmpty(actionList)) {
+            return;
+        }
+
+        for (NewAction action : actionList) {
+            if (action.getUnpublishedAction() == null
+                    || action.getUnpublishedAction().getDatasource() == null) {
+                continue;
+            }
+
+            Datasource actionDatasource = action.getUnpublishedAction().getDatasource();
+            if (conditionForDefaultRestDatasourceMigration(action)) {
+                // Idea is to add datasourceConfiguration to existing DEFAULT_REST_DATASOURCE apis,
+                // for which the datasource configuration is missing
+                // the url would be set to empty string as right url is not present over here.
+                setDatasourceConfigDetailsInDefaultRestDatasourceForActions(action, defaultDatasourceActionMap);
+            }
+        }
+    }
+
+    /**
+     * Finds if the applicationJson has any default rest datasource which has a null datasource configuration
+     * or an unset url.
+     * @param applicationJson : Application Json for which requirement is to be checked.
+     * @return true if the application has a rest api which doesn't have a valid datasource configuration.
+     */
+    public static Boolean doesRestApiRequireMigration(ApplicationJson applicationJson) {
+        List<NewAction> actionList = applicationJson.getActionList();
+        if (CollectionUtils.isNullOrEmpty(actionList)) {
+            return Boolean.FALSE;
+        }
+
+        for (NewAction action : actionList) {
+            if (action.getUnpublishedAction() == null
+                    || action.getUnpublishedAction().getDatasource() == null) {
+                continue;
+            }
+
+            Datasource actionDatasource = action.getUnpublishedAction().getDatasource();
+            if (conditionForDefaultRestDatasourceMigration(action)) {
+                return Boolean.TRUE;
+            }
+        }
+
+        return Boolean.FALSE;
+    }
+
+    /**
+     * Adds the relevant url in the default rest datasource for the given action from an action in the db
+     * otherwise sets the url to empty
+     * it's established that action doesn't have the datasource.
+     * @param action : default rest datasource actions which doesn't have valid datasource configuration.
+     * @param defaultDatasourceActionMap : gitSyncId to actions with default rest datasource map
+     */
+    public static void setDatasourceConfigDetailsInDefaultRestDatasourceForActions(
+            NewAction action, Map<String, NewAction> defaultDatasourceActionMap) {
+
+        ActionDTO actionDTO = action.getUnpublishedAction();
+        Datasource actionDatasource = actionDTO.getDatasource();
+        DatasourceConfiguration datasourceConfiguration = new DatasourceConfiguration();
+
+        if (defaultDatasourceActionMap.containsKey(action.getGitSyncId())) {
+            NewAction actionFromMap = defaultDatasourceActionMap.get(action.getGitSyncId());
+            DatasourceConfiguration datasourceConfigurationFromDBAction =
+                    actionFromMap.getUnpublishedAction().getDatasource().getDatasourceConfiguration();
+
+            if (datasourceConfigurationFromDBAction != null) {
+                datasourceConfiguration.setUrl(datasourceConfigurationFromDBAction.getUrl());
+            }
+        }
+
+        if (!org.springframework.util.StringUtils.hasText(datasourceConfiguration.getUrl())) {
+            datasourceConfiguration.setUrl("");
+        }
+
+        actionDatasource.setDatasourceConfiguration(datasourceConfiguration);
+    }
 }

app/server/appsmith-server/src/main/java/com/appsmith/server/migrations/utils/JsonSchemaMigrationHelper.java

@@ -0,0 +1,75 @@
+package com.appsmith.server.migrations.utils;
+
+import com.appsmith.external.constants.PluginConstants;
+import com.appsmith.server.applications.base.ApplicationService;
+import com.appsmith.server.domains.Application;
+import com.appsmith.server.domains.NewAction;
+import com.appsmith.server.dtos.ApplicationJson;
+import com.appsmith.server.migrations.MigrationHelperMethods;
+import com.appsmith.server.newactions.base.NewActionService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+import reactor.core.publisher.Mono;
+
+import java.util.Map;
+import java.util.Optional;
+
+@Component
+@Slf4j
+@RequiredArgsConstructor
+public class JsonSchemaMigrationHelper {
+
+    private final ApplicationService applicationService;
+    private final NewActionService newActionService;
+
+    public Mono<ApplicationJson> addDatasourceConfigurationToDefaultRestApiActions(
+            String baseApplicationId, String branchName, ApplicationJson applicationJson) {
+
+        Mono<ApplicationJson> contingencyMigrationJson = Mono.defer(() -> Mono.fromCallable(() -> {
+            MigrationHelperMethods.migrateApplicationJsonToVersionTen(applicationJson, Map.of());
+            return applicationJson;
+        }));
+
+        if (!StringUtils.hasText(baseApplicationId) || !StringUtils.hasText(branchName)) {
+            return contingencyMigrationJson;
+        }
+
+        Mono<Application> applicationMono = applicationService
+                .findByBranchNameAndBaseApplicationId(branchName, baseApplicationId, null)
+                .cache();
+
+        return applicationMono
+                .flatMap(branchedApplication -> {
+                    return newActionService
+                            .findAllByApplicationIdAndViewMode(
+                                    branchedApplication.getId(), Boolean.FALSE, Optional.empty(), Optional.empty())
+                            .filter(action -> {
+                                if (action.getUnpublishedAction() == null
+                                        || action.getUnpublishedAction().getDatasource() == null) {
+                                    return false;
+                                }
+
+                                boolean reverseFlag = StringUtils.hasText(action.getUnpublishedAction()
+                                                .getDatasource()
+                                                .getId())
+                                        || !PluginConstants.DEFAULT_REST_DATASOURCE.equals(action.getUnpublishedAction()
+                                                .getDatasource()
+                                                .getName());
+
+                                return !reverseFlag;
+                            })
+                            .collectMap(NewAction::getGitSyncId);
+                })
+                .map(newActionMap -> {
+                    MigrationHelperMethods.migrateApplicationJsonToVersionTen(applicationJson, newActionMap);
+                    return applicationJson;
+                })
+                .switchIfEmpty(contingencyMigrationJson)
+                .onErrorResume(error -> {
+                    log.error("Error occurred while migrating actions of application json. {}", error.getMessage());
+                    return contingencyMigrationJson;
+                });
+    }
+}

app/server/appsmith-server/src/test/java/com/appsmith/server/git/autocommit/AutoCommitEventHandlerImplTest.java

@@ -442,7 +442,8 @@ public void autoCommitServerMigration_WhenServerHasNoChanges_NoCommitMade() thro
 
         doReturn(Mono.just(applicationJson1))
                 .when(jsonSchemaMigration)
-                .migrateApplicationJsonToLatestSchema(applicationJson);
+                .migrateApplicationJsonToLatestSchema(
+                        Mockito.eq(applicationJson), Mockito.anyString(), Mockito.anyString());
 
         doReturn(Mono.just("success"))
                 .when(gitExecutor)
@@ -574,7 +575,9 @@ public void autocommitServerMigration_WhenJsonSchemaMigrationPresent_CommitSucce
         AppsmithBeanUtils.copyNewFieldValuesIntoOldObject(applicationJson, applicationJson1);
         applicationJson1.setServerSchemaVersion(jsonSchemaVersions.getServerVersion() + 1);
 
-        doReturn(Mono.just(applicationJson1)).when(jsonSchemaMigration).migrateApplicationJsonToLatestSchema(any());
+        doReturn(Mono.just(applicationJson1))
+                .when(jsonSchemaMigration)
+                .migrateApplicationJsonToLatestSchema(any(), Mockito.anyString(), Mockito.anyString());
 
         gitFileSystemTestHelper.setupGitRepository(autoCommitEvent, applicationJson);