[#12048] Optimse account request - createdAt patch script (#13107)

* Optimise account request patch script

* Fix incorrect console prefix

* fix buffer not flushing

src/client/java/teammates/client/scripts/sql/DataMigrationEntitiesBaseScriptSql.java

@@ -231,6 +231,8 @@ protected void doOperation() {
                 log(String.format("Number Of Entity Key Scanned: %d", numberOfScannedKey.get()));
                 log(String.format("Number Of Entity affected: %d", numberOfAffectedEntities.get()));
                 log(String.format("Number Of Entity updated: %d", numberOfUpdatedEntities.get()));
+            } else {
+                flushEntitiesSavingBuffer();
             }
         }
 

src/client/java/teammates/client/scripts/sql/DataMigrationForAccountAndReadNotificationSql.java

@@ -223,6 +223,8 @@ protected void doOperation() {
                 log(String.format("Number Of Entity Key Scanned: %d", numberOfScannedKey.get()));
                 log(String.format("Number Of Entity affected: %d", numberOfAffectedEntities.get()));
                 log(String.format("Number Of Entity updated: %d", numberOfUpdatedEntities.get()));
+            } else {
+                flushEntitiesSavingBuffer();
             }
         }
 

src/client/java/teammates/client/scripts/sql/PatchCreatedAtAccountRequest.java

@@ -8,6 +8,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.atomic.AtomicLong;
@@ -24,6 +25,7 @@
 import teammates.client.util.ClientProperties;
 import teammates.common.util.Const;
 import teammates.common.util.HibernateUtil;
+import teammates.storage.sqlentity.AccountRequest;
 import teammates.test.FileHelper;
 
 // CHECKSTYLE.ON:ImportOrder
@@ -41,7 +43,7 @@ public class PatchCreatedAtAccountRequest extends DatastoreClient {
     // cannot set number greater than 300
     // see
     // https://stackoverflow.com/questions/41499505/objectify-queries-setting-limit-above-300-does-not-work
-    private static final int BATCH_SIZE = 100;
+    private static final int BATCH_SIZE = 500;
 
     // Creates the folder that will contain the stored log.
     static {
@@ -52,10 +54,13 @@ public class PatchCreatedAtAccountRequest extends DatastoreClient {
     AtomicLong numberOfAffectedEntities;
     AtomicLong numberOfUpdatedEntities;
 
+    private List<AccountRequest> entitiesSavingBuffer;
+
     private PatchCreatedAtAccountRequest() {
         numberOfScannedKey = new AtomicLong();
         numberOfAffectedEntities = new AtomicLong();
         numberOfUpdatedEntities = new AtomicLong();
+        entitiesSavingBuffer = new LinkedList<AccountRequest>();
 
         String connectionUrl = ClientProperties.SCRIPT_API_URL;
         String username = ClientProperties.SCRIPT_API_NAME;
@@ -94,38 +99,31 @@ protected Query<teammates.storage.entity.AccountRequest> getFilterQuery() {
     }
 
     private void doMigration(teammates.storage.entity.AccountRequest entity) {
-        try {
-            if (!isMigrationNeeded(entity)) {
-                return;
-            }
-            if (!isPreview()) {
-                migrateEntity(entity);
-            }
-        } catch (Exception e) {
-            logError("Problem patching account request " + entity);
-            logError(e.getMessage());
+        if (!isMigrationNeeded(entity)) {
+            return;
+        }
+        if (!isPreview()) {
+            migrateEntity(entity);
         }
     }
 
     /**
      * Migrates the entity. In this case, add entity to buffer.
      */
     protected void migrateEntity(teammates.storage.entity.AccountRequest oldEntity) {
-        HibernateUtil.beginTransaction();
-
+
         CriteriaBuilder cb = HibernateUtil.getCriteriaBuilder();
         CriteriaQuery<teammates.storage.sqlentity.AccountRequest> cr = cb.createQuery(teammates.storage.sqlentity.AccountRequest.class);
         Root<teammates.storage.sqlentity.AccountRequest> root = cr.from(teammates.storage.sqlentity.AccountRequest.class);
-
         cr.select(root).where(cb.equal(root.get("institute"), oldEntity.getInstitute()))
             .where(cb.equal(root.get("email"), oldEntity.getEmail()))
             .where(cb.equal(root.get("name"), oldEntity.getName()))
             .where(cb.equal(root.get("registrationKey"), oldEntity.getRegistrationKey()));
 
         List<teammates.storage.sqlentity.AccountRequest> matchingAccounts = HibernateUtil.createQuery(cr).getResultList();
-        
+
         if (matchingAccounts.size() == 0){
-            throw new Error("No matching account found");
+            throw new Error("No matching account request found");
         }
 
         if (matchingAccounts.size() > 1) {
@@ -135,7 +133,9 @@ protected void migrateEntity(teammates.storage.entity.AccountRequest oldEntity)
         // Get first items since there is guaranteed to be one
         teammates.storage.sqlentity.AccountRequest newAccountReq = matchingAccounts.get(0);
         newAccountReq.setCreatedAt(oldEntity.getCreatedAt());
-        HibernateUtil.commitTransaction();
+
+        saveEntityDeferred(newAccountReq);
+
         numberOfAffectedEntities.incrementAndGet();
         numberOfUpdatedEntities.incrementAndGet();
     }
@@ -154,6 +154,7 @@ protected void doOperation() {
 
         boolean shouldContinue = true;
         while (shouldContinue) {
+            HibernateUtil.beginTransaction();
             shouldContinue = false;
             Query<teammates.storage.entity.AccountRequest> filterQueryKeys = getFilterQuery().limit(BATCH_SIZE);
             if (cursor != null) {
@@ -165,14 +166,22 @@ protected void doOperation() {
 
             while (iterator.hasNext()) {
                 shouldContinue = true;
-
-                doMigration(iterator.next());
+                teammates.storage.entity.AccountRequest accReq = iterator.next();
+                try {
+                    doMigration(accReq);
+                } catch (Exception e) {
+                    logError("Problem patching account request " + accReq.getId());
+                    logError(e.getMessage());
+                    e.getStackTrace();
+                    return;
+                }
 
                 numberOfScannedKey.incrementAndGet();
             }
 
             if (shouldContinue) {
                 cursor = iterator.getCursorAfter();
+                flushEntitiesSavingBuffer();
                 savePositionOfCursorToFile(cursor);
                 log(String.format("Cursor Position: %s", cursor.toUrlSafe()));
                 log(String.format("Number Of Entity Key Scanned: %d", numberOfScannedKey.get()));
@@ -246,4 +255,32 @@ protected void logError(String logLine) {
         log("[ERROR]" + logLine);
     }
 
+    /**
+     * Stores the entity to save in a buffer and saves it later.
+     */
+    protected void saveEntityDeferred(AccountRequest entity) {
+        entitiesSavingBuffer.add(entity);
+    }
+
+    /**
+     * Flushes the saving buffer by issuing Cloud SQL save request.
+     */
+    private void flushEntitiesSavingBuffer() {
+        if (!entitiesSavingBuffer.isEmpty() && !isPreview()) {
+            log("Saving entities in batch..." + entitiesSavingBuffer.size());
+
+            long startTime = System.currentTimeMillis();
+            for (AccountRequest entity : entitiesSavingBuffer) {
+                HibernateUtil.persist(entity);
+            }
+
+            HibernateUtil.flushSession();
+            HibernateUtil.clearSession();
+            HibernateUtil.commitTransaction();
+            long endTime = System.currentTimeMillis();
+            log("Flushing " + entitiesSavingBuffer.size() + " took " + (endTime - startTime) + " milliseconds");
+        }
+        entitiesSavingBuffer.clear();
+    }
+
 }

src/client/java/teammates/client/scripts/sql/VerifyNonCourseEntityAttributesBaseScript.java

@@ -212,7 +212,7 @@ protected void runCheckAllEntities(Class<T> sqlEntityClass,
      * @param logLine the line to log
      */
     protected void log(String logLine) {
-        System.out.println(String.format("[ERROR IN VALIDATION] %s %s", getLogPrefix(), logLine));
+        System.out.println(String.format("%s %s", getLogPrefix(), logLine));
     }
 
     /**