feat: Introduce backup and restore for postgres

deploy/docker/fs/opt/appsmith/utils/bin/backup.js

@@ -125,7 +125,15 @@ function getEncryptionPasswordFromUser(){
 
 async function exportDatabase(destFolder) {
   console.log('Exporting database');
-  await executeMongoDumpCMD(destFolder, utils.getDburl())
+  const dbUrl = utils.getDburl();
+  // Check the DB url
+  if (dbUrl.startsWith('mongodb')) {
+    await executeMongoDumpCMD(destFolder, dbUrl);
+  } else if (dbUrl.startsWith('postgresql')) {
+    await executePostgresDumpCMD(destFolder, dbUrl);
+  } else {
+    throw new Error(`Unsupported database type in URL: ${dbUrl}`);
+  }
   console.log('Exporting database done.');
 }
 
@@ -141,7 +149,7 @@ async function createGitStorageArchive(destFolder) {
 
 async function createManifestFile(path) {
   const version = await utils.getCurrentAppsmithVersion()
-  const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(utils.getDburl()) }
+  const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromDBURI(utils.getDburl()) }
   await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
 }
 
@@ -157,8 +165,12 @@ async function exportDockerEnvFile(destFolder, encryptArchive) {
   console.log('Exporting docker environment file done.');
 }
 
-async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
-  return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
+async function executeMongoDumpCMD(destFolder, dbUrl) {
+  return await utils.execCommand(['mongodump', `--uri=${dbUrl}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
+}
+
+async function executePostgresDumpCMD(destFolder, dbUrl) {
+  return await utils.execCommand(['pg_dump', dbUrl, '-n', 'appsmith','-Fc', '-f', destFolder + '/pg-data.archive']);
 }
 
 async function createFinalArchive(destFolder, timestamp) {
@@ -253,6 +265,7 @@ module.exports = {
   generateBackupRootPath,
   getBackupContentsPath,
   executeMongoDumpCMD,
+  executePostgresDumpCMD,
   getGitRoot,
   executeCopyCMD,
   removeSensitiveEnvData,

deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js

@@ -55,6 +55,16 @@ test('Test mongodump CMD generaton', async () => {
   console.log(res)
 })
 
+test('Test postgresdump CMD generaton', async () => {
+  var dest = '/dest'
+  var appsmithMongoURI = 'postgresql://username:password@host/appsmith'
+  var cmd = 'pg_dump postgresql://username:password@host/appsmith -Fc -f /dest/pg-data.archive'
+  utils.execCommand =  jest.fn().mockImplementation(async (a) => a.join(' '));
+  const res = await backup.executePostgresDumpCMD(dest, appsmithMongoURI)
+  expect(res).toBe(cmd)
+  console.log(res)
+})
+
 test('Test get gitRoot path when APPSMITH_GIT_ROOT is \'\' ', () => {
   expect(backup.getGitRoot('')).toBe('/appsmith-stacks/git-storage')
 });
@@ -228,28 +238,42 @@ test('Test backup encryption function', async () => {
 test('Get DB name from Mongo URI 1', async () => {
   var mongodb_uri = "mongodb+srv://admin:[email protected]/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"
   var expectedDBName = 'my_db_name'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 
 test('Get DB name from Mongo URI 2', async () => {
   var mongodb_uri = "mongodb+srv://admin:[email protected]/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"
   var expectedDBName = 'test123'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 
 test('Get DB name from Mongo URI 3', async () => {
   var mongodb_uri = "mongodb+srv://admin:[email protected]/test123"
   var expectedDBName = 'test123'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 
 test('Get DB name from Mongo URI 4', async () => {
   var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"
   var expectedDBName = 'appsmith'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
+  expect(dbName).toEqual(expectedDBName)
+})
+
+test('Get DB name from PostgreSQL URI', async () => {
+  var pg_uri = "postgresql://user:password@host:5432/postgres_db"
+  var expectedDBName = 'postgres_db'
+  const dbName = utils.getDatabaseNameFromDBURI(pg_uri)
+  expect(dbName).toEqual(expectedDBName)
+})
+
+test('Get DB name from PostgreSQL URI with query params', async () => {
+  var pg_uri = "postgresql://user:password@host:5432/postgres_db?sslmode=disable"
+  var expectedDBName = 'postgres_db'
+  const dbName = utils.getDatabaseNameFromDBURI(pg_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 

deploy/docker/fs/opt/appsmith/utils/bin/export_db.js

@@ -1,4 +1,4 @@
-// Init function export mongodb
+// Init function export db
 const shell = require('shelljs');
 const Constants = require('./constants');
 const utils = require('./utils');
@@ -7,7 +7,15 @@ function export_database() {
   console.log('export_database  ....');
   dbUrl = utils.getDburl();
   shell.mkdir('-p', [Constants.BACKUP_PATH]);
-  const cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`;
+  let cmd;
+  if (dbUrl.startsWith('mongodb')) {
+    cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`;
+  } else if (dbUrl.startsWith('postgresql')) {
+    // Dump only the appsmith schema with custom format
+    cmd = `pg_dump ${dbUrl} -n appsmith -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`;
+  } else {
+    throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported');
+  }
   shell.exec(cmd);
   console.log('export_database done');
 }

deploy/docker/fs/opt/appsmith/utils/bin/import_db.js

@@ -9,11 +9,32 @@ const utils = require('./utils');
 function import_database() {
   console.log('import_database  ....')
   dbUrl = utils.getDburl();
-  const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`
-  shell.exec(cmd)
+  if (dbUrl.startsWith('mongodb')) {
+    restore_mongo_db(dbUrl);
+  } else if (dbUrl.startsWith('postgresql')) {
+    restore_postgres_db(dbUrl);
+  } else {
+    throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported');
+  }
   console.log('import_database done')
 }
 
+restore_mongo_db = (dbUrl) => {
+  const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`;
+  shell.exec(cmd);
+}
+
+restore_postgres_db = (dbUrl) => {
+  let cmd;
+  if (dbUrl.includes('localhost') || dbUrl.includes('127.0.0.1')) {
+    const toDbName = utils.getDatabaseNameFromDBURI(dbUrl);
+    cmd = `pg_restore -U postgres -d 'postgresql://localhost:5432/${toDbName}' --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`;
+  } else {
+    cmd = `pg_restore -d ${dbUrl} --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`;
+  }
+  shell.exec(cmd);
+}
+
 function stop_application() {
   shell.exec('/usr/bin/supervisorctl stop backend rts')
 }
@@ -22,6 +43,19 @@ function start_application() {
   shell.exec('/usr/bin/supervisorctl start backend rts')
 }
 
+function get_table_or_collection_len() {
+  let count;
+  const dbUrl = utils.getDburl();
+  if (dbUrl.startsWith('mongodb')) {
+    count = shell.exec(`mongo ${dbUrl} --quiet --eval "db.getCollectionNames().length"`)
+  } else if (dbUrl.startsWith('postgresql')) {
+    count = shell.exec(`psql -d ${dbUrl} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`)
+  } else {
+    throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported');
+  }
+  return parseInt(count.stdout.toString().trimEnd());
+}
+
 // Main application workflow
 const main = (forceOption) => {
   let errorCode = 0
@@ -37,8 +71,7 @@ const main = (forceOption) => {
 
     shell.echo('stop backend & rts application before import database')
     stop_application()
-    const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_DB_URL} --quiet --eval "db.getCollectionNames().length"`)
-    const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd())
+    const collectionsLen = get_table_or_collection_len();
     if (collectionsLen > 0) {
       if (forceOption) {
         import_database()

deploy/docker/fs/opt/appsmith/utils/bin/restore.js

@@ -59,17 +59,49 @@ async function extractArchive(backupFilePath, restoreRootPath) {
 
 async function restoreDatabase(restoreContentsPath, dbUrl) {
   console.log('Restoring database...');
+  if (dbUrl.startsWith('mongodb')) {
+    await restore_mongo_db(restoreContentsPath, dbUrl);
+  } else if (dbUrl.includes('postgresql')) {
+    await restore_postgres_db(restoreContentsPath, dbUrl);
+  } else {
+    throw new Error('Unsupported database type, only MongoDB and PostgreSQL are supported');
+  }
+  console.log('Restoring database completed');
+}
+
+async function restore_mongo_db(restoreContentsPath, dbUrl) {
   const cmd = ['mongorestore', `--uri=${dbUrl}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip']
   try {
     const fromDbName = await getBackupDatabaseName(restoreContentsPath);
-    const toDbName = utils.getDatabaseNameFromMongoURI(dbUrl);
+    const toDbName = utils.getDatabaseNameFromDBURI(dbUrl);
     console.log("Restoring database from " + fromDbName + " to " + toDbName)
     cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`)
   } catch (error) {
     console.warn('Error reading manifest file. Assuming same database name.', error);
   }
   await utils.execCommand(cmd);
-  console.log('Restoring database completed');
+}
+
+async function restore_postgres_db(restoreContentsPath, dbUrl) {
+  const cmd = ['pg_restore', '--verbose', '--clean', `${restoreContentsPath}/pg-data.archive`];
+  const url = new URL(dbUrl);
+  const isLocalhost = ['localhost', '127.0.0.1'].includes(url.hostname);
+  if (isLocalhost) {
+    let dbName;
+    try {
+      dbName = utils.getDatabaseNameFromDBURI(dbUrl);
+      console.log("Restoring database to " + dbName);
+    } catch (error) {
+      console.warn('Error reading manifest file. Assuming same database name as appsmith.', error);
+      dbName = 'appsmith';
+    }
+    cmd.push('-d' , 'postgresql://localhost:5432/' + dbName);
+    // Use default user for local postgres
+    cmd.push('-U', 'postgres');
+  } else {
+    cmd.push('-d', dbUrl);
+  }
+  await utils.execCommand(cmd);
 }
 
 async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys) {

deploy/docker/fs/opt/appsmith/utils/bin/utils.js

@@ -179,7 +179,13 @@ function execCommandSilent(cmd, options) {
   });
 }
 
-function getDatabaseNameFromMongoURI(uri) {
+/**
+ * Extracts database name from MongoDB or PostgreSQL connection URI
+ * @param {string} uri - Database connection URI
+ * @returns {string} Database name 
+ * @returns 
+ */
+function getDatabaseNameFromDBURI(uri) {
   const uriParts = uri.split("/");
   return uriParts[uriParts.length - 1].split("?")[0];
 }
@@ -195,6 +201,6 @@ module.exports = {
   getCurrentAppsmithVersion,
   preprocessMongoDBURI,
   execCommandSilent,
-  getDatabaseNameFromMongoURI,
+  getDatabaseNameFromDBURI,
   getDburl
 };