Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: Format appsmithctl code #37532

Merged
merged 2 commits into from
Nov 19, 2024
Merged

chore: Format appsmithctl code #37532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f743a48
chore: Format appsmithctl code
sharat87 Nov 18, 2024
5c83801
Remove duplicate test case
sharat87 Nov 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
262 changes: 171 additions & 91 deletions deploy/docker/fs/opt/appsmith/utils/bin/backup.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
const fsPromises = require('fs/promises');
const path = require('path');
const os = require('os');
const utils = require('./utils');
const Constants = require('./constants');
const logger = require('./logger');
const mailer = require('./mailer');
const tty = require('tty');
const readlineSync = require('readline-sync');
const fsPromises = require("fs/promises");
const path = require("path");
const os = require("os");
const utils = require("./utils");
const Constants = require("./constants");
const logger = require("./logger");
const mailer = require("./mailer");
const tty = require("tty");
const readlineSync = require("readline-sync");

const command_args = process.argv.slice(3);

Expand All @@ -19,9 +19,10 @@ async function run() {
await utils.ensureSupervisorIsRunning();

try {
console.log('Available free space at /appsmith-stacks');
const availSpaceInBytes = getAvailableBackupSpaceInBytes("/appsmith-stacks");
console.log('\n');
console.log("Available free space at /appsmith-stacks");
const availSpaceInBytes =
getAvailableBackupSpaceInBytes("/appsmith-stacks");
console.log("\n");

checkAvailableBackupSpace(availSpaceInBytes);

Expand All @@ -36,44 +37,67 @@ async function run() {

await createManifestFile(backupContentsPath);

if (!command_args.includes('--non-interactive') && (tty.isatty(process.stdout.fd))){
if (
!command_args.includes("--non-interactive") &&
tty.isatty(process.stdout.fd)
) {
encryptionPassword = getEncryptionPasswordFromUser();
if (encryptionPassword == -1){
throw new Error('Backup process aborted because a valid enctyption password could not be obtained from the user');
if (encryptionPassword == -1) {
throw new Error(
"Backup process aborted because a valid enctyption password could not be obtained from the user",
);
}
encryptArchive = true;
}
await exportDockerEnvFile(backupContentsPath, encryptArchive);

archivePath = await createFinalArchive(backupRootPath, timestamp);
// shell.exec("openssl enc -aes-256-cbc -pbkdf2 -iter 100000 -in " + archivePath + " -out " + archivePath + ".enc");
if (encryptArchive){
const encryptedArchivePath = await encryptBackupArchive(archivePath,encryptionPassword);
await logger.backup_info('Finished creating an encrypted a backup archive at ' + encryptedArchivePath);
if (archivePath != null) {
await fsPromises.rm(archivePath, { recursive: true, force: true });
}
}
else {
await logger.backup_info('Finished creating a backup archive at ' + archivePath);
console.log('********************************************************* IMPORTANT!!! *************************************************************');
console.log('*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **')
console.log('*** These values are not included in the backup export. **');
console.log('************************************************************************************************************************************');
if (encryptArchive) {
const encryptedArchivePath = await encryptBackupArchive(
archivePath,
encryptionPassword,
);
await logger.backup_info(
"Finished creating an encrypted a backup archive at " +
encryptedArchivePath,
);
if (archivePath != null) {
await fsPromises.rm(archivePath, { recursive: true, force: true });
}
} else {
await logger.backup_info(
"Finished creating a backup archive at " + archivePath,
);
console.log(
"********************************************************* IMPORTANT!!! *************************************************************",
);
console.log(
"*** Please ensure you have saved the APPSMITH_ENCRYPTION_SALT and APPSMITH_ENCRYPTION_PASSWORD variables from the docker.env file **",
);
console.log(
"*** These values are not included in the backup export. **",
);
console.log(
"************************************************************************************************************************************",
);
}

await fsPromises.rm(backupRootPath, { recursive: true, force: true });

await logger.backup_info('Finished taking a backup at ' + archivePath);

await logger.backup_info("Finished taking a backup at " + archivePath);
} catch (err) {
errorCode = 1;
await logger.backup_error(err.stack);

if (command_args.includes('--error-mail')) {
if (command_args.includes("--error-mail")) {
const currentTS = new Date().getTime();
const lastMailTS = await utils.getLastBackupErrorMailSentInMilliSec();
if ((lastMailTS + Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC) < currentTS) {
if (
lastMailTS +
Constants.DURATION_BETWEEN_BACKUP_ERROR_MAILS_IN_MILLI_SEC <
currentTS
) {
await mailer.sendBackupErrorToAdmins(err, timestamp);
await utils.updateLastBackupErrorMailSentInMilliSec(currentTS);
}
Expand All @@ -92,137 +116,193 @@ async function run() {
}
}

async function encryptBackupArchive(archivePath, encryptionPassword){
const encryptedArchivePath = archivePath + '.enc';
await utils.execCommand(['openssl', 'enc', '-aes-256-cbc', '-pbkdf2', '-iter', 100000, '-in', archivePath, '-out', encryptedArchivePath, '-k', encryptionPassword ])
async function encryptBackupArchive(archivePath, encryptionPassword) {
const encryptedArchivePath = archivePath + ".enc";
await utils.execCommand([
"openssl",
"enc",
"-aes-256-cbc",
"-pbkdf2",
"-iter",
100000,
"-in",
archivePath,
"-out",
encryptedArchivePath,
"-k",
encryptionPassword,
]);
return encryptedArchivePath;
}

function getEncryptionPasswordFromUser(){
for (const _ of [1, 2, 3])
{
const encryptionPwd1 = readlineSync.question('Enter a password to encrypt the backup archive: ', { hideEchoBack: true });
const encryptionPwd2 = readlineSync.question('Enter the above password again: ', { hideEchoBack: true });
if (encryptionPwd1 === encryptionPwd2){
if (encryptionPwd1){
function getEncryptionPasswordFromUser() {
for (const _ of [1, 2, 3]) {
const encryptionPwd1 = readlineSync.question(
"Enter a password to encrypt the backup archive: ",
{ hideEchoBack: true },
);
const encryptionPwd2 = readlineSync.question(
"Enter the above password again: ",
{ hideEchoBack: true },
);
if (encryptionPwd1 === encryptionPwd2) {
if (encryptionPwd1) {
return encryptionPwd1;
}
console.error("Invalid input. Empty password is not allowed, please try again.")
}
else {
}
console.error(
"Invalid input. Empty password is not allowed, please try again.",
);
} else {
console.error("The passwords do not match, please try again.");
}
}
console.error("Aborting backup process, failed to obtain valid encryption password.");
return -1
console.error(
"Aborting backup process, failed to obtain valid encryption password.",
);
return -1;
}

async function exportDatabase(destFolder) {
console.log('Exporting database');
await executeMongoDumpCMD(destFolder, utils.getDburl())
console.log('Exporting database done.');
console.log("Exporting database");
await executeMongoDumpCMD(destFolder, utils.getDburl());
console.log("Exporting database done.");
}

async function createGitStorageArchive(destFolder) {
console.log('Creating git-storage archive');
console.log("Creating git-storage archive");

const gitRoot = getGitRoot(process.env.APPSMITH_GIT_ROOT);

await executeCopyCMD(gitRoot, destFolder)
await executeCopyCMD(gitRoot, destFolder);

console.log('Created git-storage archive');
console.log("Created git-storage archive");
}

async function createManifestFile(path) {
const version = await utils.getCurrentAppsmithVersion()
const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(utils.getDburl()) }
await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
const version = await utils.getCurrentAppsmithVersion();
const manifest_data = {
appsmithVersion: version,
dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()),
};
await fsPromises.writeFile(
path + "/manifest.json",
JSON.stringify(manifest_data),
);
}

async function exportDockerEnvFile(destFolder, encryptArchive) {
console.log('Exporting docker environment file');
const content = await fsPromises.readFile('/appsmith-stacks/configuration/docker.env', { encoding: 'utf8' });
console.log("Exporting docker environment file");
const content = await fsPromises.readFile(
"/appsmith-stacks/configuration/docker.env",
{ encoding: "utf8" },
);
let cleaned_content = removeSensitiveEnvData(content);
if (encryptArchive){
cleaned_content += '\nAPPSMITH_ENCRYPTION_SALT=' + process.env.APPSMITH_ENCRYPTION_SALT +
'\nAPPSMITH_ENCRYPTION_PASSWORD=' + process.env.APPSMITH_ENCRYPTION_PASSWORD
if (encryptArchive) {
cleaned_content +=
"\nAPPSMITH_ENCRYPTION_SALT=" +
process.env.APPSMITH_ENCRYPTION_SALT +
"\nAPPSMITH_ENCRYPTION_PASSWORD=" +
process.env.APPSMITH_ENCRYPTION_PASSWORD;
}
await fsPromises.writeFile(destFolder + '/docker.env', cleaned_content);
console.log('Exporting docker environment file done.');
await fsPromises.writeFile(destFolder + "/docker.env", cleaned_content);
console.log("Exporting docker environment file done.");
}

async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
return await utils.execCommand([
"mongodump",
`--uri=${appsmithMongoURI}`,
`--archive=${destFolder}/mongodb-data.gz`,
"--gzip",
]); // generate cmd
}

async function createFinalArchive(destFolder, timestamp) {
console.log('Creating final archive');
console.log("Creating final archive");

const archive = `${Constants.BACKUP_PATH}/appsmith-backup-${timestamp}.tar.gz`;
await utils.execCommand(['tar', '-cah', '-C', destFolder, '-f', archive, '.']);

console.log('Created final archive');
await utils.execCommand([
"tar",
"-cah",
"-C",
destFolder,
"-f",
archive,
".",
]);

console.log("Created final archive");

return archive;
}

async function postBackupCleanup() {
console.log('Starting the cleanup task after taking a backup.');
let backupArchivesLimit = getBackupArchiveLimit(process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT);
console.log("Starting the cleanup task after taking a backup.");
let backupArchivesLimit = getBackupArchiveLimit(
process.env.APPSMITH_BACKUP_ARCHIVE_LIMIT,
);
const backupFiles = await utils.listLocalBackupFiles();
while (backupFiles.length > backupArchivesLimit) {
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName);
}
console.log('Cleanup task completed.');

console.log("Cleanup task completed.");
}
async function executeCopyCMD(srcFolder, destFolder) {
return await utils.execCommand(['ln', '-s', srcFolder, destFolder + '/git-storage'])
return await utils.execCommand([
"ln",
"-s",
srcFolder,
destFolder + "/git-storage",
]);
}

function getGitRoot(gitRoot) {
if (gitRoot == null || gitRoot === '') {
gitRoot = '/appsmith-stacks/git-storage';
if (gitRoot == null || gitRoot === "") {
gitRoot = "/appsmith-stacks/git-storage";
}
return gitRoot
return gitRoot;
}

function generateBackupRootPath() {
return fsPromises.mkdtemp(path.join(os.tmpdir(), 'appsmithctl-backup-'));
return fsPromises.mkdtemp(path.join(os.tmpdir(), "appsmithctl-backup-"));
}

function getBackupContentsPath(backupRootPath, timestamp) {
return backupRootPath + '/appsmith-backup-' + timestamp;
return backupRootPath + "/appsmith-backup-" + timestamp;
}

function removeSensitiveEnvData(content) {
// Remove encryption and Mongodb data from docker.env
const output_lines = []
content.split(/\r?\n/).forEach(line => {
if (!line.startsWith("APPSMITH_ENCRYPTION") && !line.startsWith("APPSMITH_MONGODB") && !line.startsWith("APPSMITH_DB_URL=")) {
const output_lines = [];
content.split(/\r?\n/).forEach((line) => {
if (
!line.startsWith("APPSMITH_ENCRYPTION") &&
!line.startsWith("APPSMITH_MONGODB") &&
!line.startsWith("APPSMITH_DB_URL=")
) {
output_lines.push(line);
}
});
return output_lines.join('\n')
return output_lines.join("\n");
}

function getBackupArchiveLimit(backupArchivesLimit) {
if (!backupArchivesLimit)
backupArchivesLimit = Constants.APPSMITH_DEFAULT_BACKUP_ARCHIVE_LIMIT;
return backupArchivesLimit
return backupArchivesLimit;
}

async function removeOldBackups(backupFiles, backupArchivesLimit) {
while (backupFiles.length > backupArchivesLimit) {
const fileName = backupFiles.shift();
await fsPromises.rm(Constants.BACKUP_PATH + '/' + fileName);
await fsPromises.rm(Constants.BACKUP_PATH + "/" + fileName);
}
return backupFiles
return backupFiles;
}

function getTimeStampInISO() {
return new Date().toISOString().replace(/:/g, '-')
return new Date().toISOString().replace(/:/g, "-");
}

async function getAvailableBackupSpaceInBytes(path) {
Expand All @@ -232,12 +312,12 @@ async function getAvailableBackupSpaceInBytes(path) {

function checkAvailableBackupSpace(availSpaceInBytes) {
if (availSpaceInBytes < Constants.MIN_REQUIRED_DISK_SPACE_IN_BYTES) {
throw new Error("Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.");
throw new Error(
"Not enough space available at /appsmith-stacks. Please ensure availability of at least 2GB to backup successfully.",
);
}
}



module.exports = {
run,
getTimeStampInISO,
Expand Down
Loading
Loading