feat: Introduce backup and restore for postgres

app/server/appsmith-server/src/main/java/com/appsmith/server/configurations/CommonDBConfig.java

@@ -33,7 +33,7 @@ public MongoProperties configureMongoDB() {
         if (!appsmithDbUrl.startsWith("mongodb")) {
             return null;
         }
-        log.info("Found MongoDB uri configuring now");
+        log.info("Found MongoDB uri configuring now.");
         MongoProperties mongoProperties = new MongoProperties();
         mongoProperties.setUri(appsmithDbUrl);
         return mongoProperties;

deploy/docker/fs/opt/appsmith/utils/bin/backup.js

@@ -125,7 +125,12 @@ function getEncryptionPasswordFromUser(){
 
 async function exportDatabase(destFolder) {
   console.log('Exporting database');
-  await executeMongoDumpCMD(destFolder, utils.getDburl())
+  // Check the DB url
+  if (utils.getDburl().startsWith('mongodb')) {
+    await executeMongoDumpCMD(destFolder, utils.getDburl())
+  } else if (utils.getDburl().startsWith('postgresql')) {
+    await executePostgresDumpCMD(destFolder, utils.getDburl());
+  }
   console.log('Exporting database done.');
 }
 
@@ -141,7 +146,7 @@ async function createGitStorageArchive(destFolder) {
 
 async function createManifestFile(path) {
   const version = await utils.getCurrentAppsmithVersion()
-  const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromMongoURI(utils.getDburl()) }
+  const manifest_data = { "appsmithVersion": version, "dbName": utils.getDatabaseNameFromDBURI(utils.getDburl()) }
   await fsPromises.writeFile(path + '/manifest.json', JSON.stringify(manifest_data));
 }
 
@@ -161,6 +166,10 @@ async function executeMongoDumpCMD(destFolder, appsmithMongoURI) {
   return await utils.execCommand(['mongodump', `--uri=${appsmithMongoURI}`, `--archive=${destFolder}/mongodb-data.gz`, '--gzip']);// generate cmd
 }
 
+async function executePostgresDumpCMD(destFolder, appsmithDBURI) {
+  return await utils.execCommand(['pg_dump', appsmithDBURI, '-Fc', '-f', destFolder + '/pg-data.archive']);
+}
+
 async function createFinalArchive(destFolder, timestamp) {
   console.log('Creating final archive');
 
@@ -252,6 +261,7 @@ module.exports = {
   generateBackupRootPath,
   getBackupContentsPath,
   executeMongoDumpCMD,
+  executePostgresDumpCMD,
   getGitRoot,
   executeCopyCMD,
   removeSensitiveEnvData,

deploy/docker/fs/opt/appsmith/utils/bin/backup.test.js

@@ -56,6 +56,16 @@ test('Test mongodump CMD generaton', async () => {
   console.log(res)
 })
 
+test('Test postgresdump CMD generaton', async () => {
+  var dest = '/dest'
+  var appsmithMongoURI = 'postgresql://username:password@host/appsmith'
+  var cmd = 'pg_dump postgresql://username:password@host/appsmith -Fc -f /dest/pg-data.archive'
+  utils.execCommand =  jest.fn().mockImplementation(async (a) => a.join(' '));
+  const res = await backup.executePostgresDumpCMD(dest, appsmithMongoURI)
+  expect(res).toBe(cmd)
+  console.log(res)
+})
+
 test('Test get gitRoot path when APPSMITH_GIT_ROOT is \'\' ', () => {
   expect(backup.getGitRoot('')).toBe('/appsmith-stacks/git-storage')
 });
@@ -229,28 +239,28 @@ test('Test backup encryption function', async () => {
 test('Get DB name from Mongo URI 1', async () => {
   var mongodb_uri = "mongodb+srv://admin:[email protected]/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"
   var expectedDBName = 'my_db_name'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 
 test('Get DB name from Mongo URI 2', async () => {
   var mongodb_uri = "mongodb+srv://admin:[email protected]/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin"
   var expectedDBName = 'test123'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 
 test('Get DB name from Mongo URI 3', async () => {
   var mongodb_uri = "mongodb+srv://admin:[email protected]/test123"
   var expectedDBName = 'test123'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 
 test('Get DB name from Mongo URI 4', async () => {
   var mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith"
   var expectedDBName = 'appsmith'
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri)
+  const dbName = utils.getDatabaseNameFromDBURI(mongodb_uri)
   expect(dbName).toEqual(expectedDBName)
 })
 

deploy/docker/fs/opt/appsmith/utils/bin/export_db.js

@@ -7,7 +7,14 @@ function export_database() {
   console.log('export_database  ....');
   dbUrl = utils.getDburl();
   shell.mkdir('-p', [Constants.BACKUP_PATH]);
-  const cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`;
+  let cmd;
+  if (dbUrl.startsWith('mongodb')) {
+    cmd = `mongodump --uri='${dbUrl}' --archive='${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`;
+  } else if (dbUrl.startsWith('postgresql')) {
+    cmd = `pg_dump ${dbUrl} -Fc -f '${Constants.BACKUP_PATH}/${Constants.DUMP_FILE_NAME}'`;
+  } else {
+    throw new Error('Unsupported database URL');
+  }
   shell.exec(cmd);
   console.log('export_database done');
 }

deploy/docker/fs/opt/appsmith/utils/bin/import_db.js

@@ -9,11 +9,24 @@ const utils = require('./utils');
 function import_database() {
   console.log('import_database  ....')
   dbUrl = utils.getDburl();
-  const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`
-  shell.exec(cmd)
+  if (utils.getDburl().startsWith('mongodb')) {
+    restore_mongo_db();
+  } else if (utils.getDburl().startsWith('postgresql')) {
+    restore_postgres_db();
+  }
   console.log('import_database done')
 }
 
+restore_mongo_db = () => {
+  const cmd = `mongorestore --uri='${dbUrl}' --drop --archive='${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}' --gzip`;
+  shell.exec(cmd);
+}
+
+restore_postgres_db = () => {
+  const cmd = `pg_restore -U postgres -d appsmith --verbose --clean ${Constants.RESTORE_PATH}/${Constants.DUMP_FILE_NAME}`;
+  shell.exec(cmd);
+}
+
 function stop_application() {
   shell.exec('/usr/bin/supervisorctl stop backend rts')
 }
@@ -22,6 +35,16 @@ function start_application() {
   shell.exec('/usr/bin/supervisorctl start backend rts')
 }
 
+function get_table_or_collection_len() {
+  let count;
+  if (utils.getDburl().startsWith('mongodb')) {
+    count = shell.exec(`mongo ${utils.getDburl()} --quiet --eval "db.getCollectionNames().length"`)
+  } else if (utils.getDburl().startsWith('postgresql')) {
+    count = shell.exec(`psql -U postgres -d ${utils.getDburl()} -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'appsmith';"`)
+  }
+  return parseInt(count.stdout.toString().trimEnd());
+}
+
 // Main application workflow
 const main = (forceOption) => {
   let errorCode = 0
@@ -37,8 +60,7 @@ const main = (forceOption) => {
 
     shell.echo('stop backend & rts application before import database')
     stop_application()
-    const shellCmdResult = shell.exec(`mongo ${process.env.APPSMITH_DB_URL} --quiet --eval "db.getCollectionNames().length"`)
-    const collectionsLen = parseInt(shellCmdResult.stdout.toString().trimEnd())
+    const collectionsLen = get_table_or_collection_len();
     if (collectionsLen > 0) {
       if (forceOption) {
         import_database()

deploy/docker/fs/opt/appsmith/utils/bin/restore.js

@@ -59,17 +59,37 @@ async function extractArchive(backupFilePath, restoreRootPath) {
 
 async function restoreDatabase(restoreContentsPath, dbUrl) {
   console.log('Restoring database...');
+  if (dbUrl.startsWith('mongodb')) {
+    await restore_mongo_db(restoreContentsPath, dbUrl);
+  } else if (dbUrl.includes('postgresql')) {
+    await restore_postgres_db(restoreContentsPath, dbUrl);
+  }
+  console.log('Restoring database completed');
+}
+
+async function restore_mongo_db(restoreContentsPath, dbUrl) {
   const cmd = ['mongorestore', `--uri=${dbUrl}`, '--drop', `--archive=${restoreContentsPath}/mongodb-data.gz`, '--gzip']
   try {
     const fromDbName = await getBackupDatabaseName(restoreContentsPath);
-    const toDbName = utils.getDatabaseNameFromMongoURI(dbUrl);
+    const toDbName = utils.getDatabaseNameFromDBURI(dbUrl);
     console.log("Restoring database from " + fromDbName + " to " + toDbName)
     cmd.push('--nsInclude=*', `--nsFrom=${fromDbName}.*`, `--nsTo=${toDbName}.*`)
   } catch (error) {
     console.warn('Error reading manifest file. Assuming same database name.', error);
   }
   await utils.execCommand(cmd);
-  console.log('Restoring database completed');
+}
+
+async function restore_postgres_db(restoreContentsPath, dbUrl) {
+  const cmd = ['pg_restore', '-U', 'postgres', '-c', `${restoreContentsPath}/pg-data.archive`];
+  try {
+    const toDbName = utils.getDatabaseNameFromDBURI(dbUrl);
+    console.log("Restoring database to " + toDbName);
+    cmd.push('-d' , toDbName);
+  } catch (error) {
+    console.warn('Error reading manifest file. Assuming same database name.', error);
+  }
+  await utils.execCommand(cmd);
 }
 
 async function restoreDockerEnvFile(restoreContentsPath, backupName, overwriteEncryptionKeys) {

deploy/docker/fs/opt/appsmith/utils/bin/utils.js

@@ -179,7 +179,7 @@ function execCommandSilent(cmd, options) {
   });
 }
 
-function getDatabaseNameFromMongoURI(uri) {
+function getDatabaseNameFromDBURI(uri) {
   const uriParts = uri.split("/");
   return uriParts[uriParts.length - 1].split("?")[0];
 }
@@ -195,6 +195,6 @@ module.exports = {
   getCurrentAppsmithVersion,
   preprocessMongoDBURI,
   execCommandSilent,
-  getDatabaseNameFromMongoURI,
+  getDatabaseNameFromDBURI,
   getDburl
 };