feat: Introduce backup and restore for postgres

app/client/packages/rts/src/ctl/backup/BackupState.ts

@@ -1,5 +1,7 @@
 export class BackupState {
   readonly args: readonly string[];
+  readonly dbUrl: string;
+
   readonly initAt: string = new Date().toISOString().replace(/:/g, "-");
   readonly errors: string[] = [];
 
@@ -8,8 +10,9 @@ export class BackupState {
 
   isEncryptionEnabled: boolean = false;
 
-  constructor(args: string[]) {
+  constructor(args: string[], url: string) {
     this.args = Object.freeze([...args]);
+    this.dbUrl = url;
 
     // We seal `this` so that no link in the chain can "add" new properties to the state. This is intentional. If any
     // link wants to save data in the `BackupState`, which shouldn't even be needed in most cases, it should do so by

app/client/packages/rts/src/ctl/backup/backup.test.ts

@@ -8,6 +8,7 @@ import {
   encryptBackupArchive,
   executeCopyCMD,
   executeMongoDumpCMD,
+  executePostgresDumpCMD,
   getAvailableBackupSpaceInBytes,
   getEncryptionPasswordFromUser,
   getGitRoot,
@@ -66,6 +67,34 @@ describe("Backup Tests", () => {
     console.log(res);
   });
 
+  test("Test postgres dump CMD generation", async () => {
+    const dest = "/dest";
+    const url = "postgresql://username:password@host/appsmith";
+    const cmd = [
+      "pg_dump --host=host",
+      "--port=5432",
+      "--username=username",
+      "--dbname=appsmith",
+      "--schema=appsmith",
+      "--schema=public",
+      "--schema=temporal",
+      "--file=/dest/pg-data.sql",
+      "--verbose",
+      "--serializable-deferrable",
+    ].join(" ");
+
+    const res = await executePostgresDumpCMD(dest, {
+      host: "host",
+      port: 5432,
+      username: "username",
+      password: "password",
+      database: "appsmith",
+    });
+
+    expect(res).toBe(cmd);
+    console.log(res);
+  });
+
   test("Test get gitRoot path when APPSMITH_GIT_ROOT is '' ", () => {
     expect(getGitRoot("")).toBe("/appsmith-stacks/git-storage");
   });
@@ -246,7 +275,7 @@ test("Get DB name from Mongo URI 1", async () => {
   const mongodb_uri =
     "mongodb+srv://admin:[email protected]/my_db_name?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin";
   const expectedDBName = "my_db_name";
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri);
+  const dbName = utils.getDatabaseNameFromUrl(mongodb_uri);
 
   expect(dbName).toEqual(expectedDBName);
 });
@@ -255,7 +284,7 @@ test("Get DB name from Mongo URI 2", async () => {
   const mongodb_uri =
     "mongodb+srv://admin:[email protected]/test123?retryWrites=true&minPoolSize=1&maxPoolSize=10&maxIdleTimeMS=900000&authSource=admin";
   const expectedDBName = "test123";
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri);
+  const dbName = utils.getDatabaseNameFromUrl(mongodb_uri);
 
   expect(dbName).toEqual(expectedDBName);
 });
@@ -264,15 +293,31 @@ test("Get DB name from Mongo URI 3", async () => {
   const mongodb_uri =
     "mongodb+srv://admin:[email protected]/test123";
   const expectedDBName = "test123";
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri);
+  const dbName = utils.getDatabaseNameFromUrl(mongodb_uri);
 
   expect(dbName).toEqual(expectedDBName);
 });
 
 test("Get DB name from Mongo URI 4", async () => {
   const mongodb_uri = "mongodb://appsmith:pAssW0rd!@localhost:27017/appsmith";
   const expectedDBName = "appsmith";
-  const dbName = utils.getDatabaseNameFromMongoURI(mongodb_uri);
+  const dbName = utils.getDatabaseNameFromUrl(mongodb_uri);
 
   expect(dbName).toEqual(expectedDBName);
 });
+
+test("Get DB name from Postgres URL", async () => {
+  const dbName = utils.getDatabaseNameFromUrl(
+    "postgresql://user:password@host:5432/postgres_db",
+  );
+
+  expect(dbName).toEqual("postgres_db");
+});
+
+test("Get DB name from Postgres URL with query params", async () => {
+  const dbName = utils.getDatabaseNameFromUrl(
+    "postgresql://user:password@host:5432/postgres_db?sslmode=disable",
+  );
+
+  expect(dbName).toEqual("postgres_db");
+});

app/client/packages/rts/src/ctl/backup/index.ts

@@ -8,15 +8,25 @@ import * as linkClasses from "./links";
 import { BackupState } from "./BackupState";
 
 export async function run(args: string[]) {
+  const url = utils.getDburl();
+
+  if (!url.startsWith("mongodb") && !url.startsWith("postgresql")) {
+    console.error("Only MongoDB and Postgres databases are supported.");
+    process.exitCode = 1;
+
+    return;
+  }
+
   await utils.ensureSupervisorIsRunning();
 
-  const state: BackupState = new BackupState(args);
+  const state: BackupState = new BackupState(args, url);
 
   const chain: Link[] = [
     new linkClasses.BackupFolderLink(state),
     new linkClasses.DiskSpaceLink(),
     new linkClasses.ManifestLink(state),
     new linkClasses.MongoDumpLink(state),
+    new linkClasses.PostgresDumpLink(state),
     new linkClasses.GitStorageLink(state),
     new linkClasses.EnvFileLink(state),
 

app/client/packages/rts/src/ctl/backup/links/ManifestLink.ts

@@ -14,7 +14,7 @@ export class ManifestLink implements Link {
     const version = await utils.getCurrentAppsmithVersion();
     const manifestData = {
       appsmithVersion: version,
-      dbName: utils.getDatabaseNameFromMongoURI(utils.getDburl()),
+      dbName: utils.getDatabaseNameFromUrl(utils.getDburl()),
     };
 
     await fsPromises.writeFile(

app/client/packages/rts/src/ctl/backup/links/MongoDumpLink.ts

@@ -9,9 +9,13 @@ export class MongoDumpLink implements Link {
   constructor(private readonly state: BackupState) {}
 
   async doBackup() {
-    console.log("Exporting database");
-    await executeMongoDumpCMD(this.state.backupRootPath, utils.getDburl());
-    console.log("Exporting database done.");
+    const url = this.state.dbUrl;
+
+    if (url.startsWith("mongodb")) {
+      console.log("Exporting database");
+      await executeMongoDumpCMD(this.state.backupRootPath, url);
+      console.log("Exporting database done.");
+    }
   }
 }
 

app/client/packages/rts/src/ctl/backup/links/PostgresDumpLink.ts

@@ -0,0 +1,150 @@
+import type { Link } from ".";
+import type { BackupState } from "../BackupState";
+import * as utils from "../../utils";
+
+interface ConnectionDetails {
+  host: string;
+  port: number;
+  username: string;
+  password: string;
+  database: string;
+}
+
+/**
+ * Backup & restore for Postgres database data using `pg_dump` and `psql`.
+ */
+export class PostgresDumpLink implements Link {
+  private postgresUrl: null | ConnectionDetails = null;
+
+  constructor(private readonly state: BackupState) {}
+
+  async preBackup() {
+    const url = this.state.dbUrl;
+
+    if (url.startsWith("postgresql")) {
+      this.postgresUrl = parsePostgresUrl(url);
+
+      return;
+    }
+
+    if (process.env.APPSMITH_KEYCLOAK_DB_URL) {
+      const dbUrlFromEnv = process.env.APPSMITH_KEYCLOAK_DB_URL;
+
+      if (dbUrlFromEnv.startsWith("postgresql://")) {
+        this.postgresUrl = parsePostgresUrl(dbUrlFromEnv);
+      } else if (dbUrlFromEnv.includes("/")) {
+        // then it's just the hostname and database in there
+        const [host, database] = dbUrlFromEnv.split("/");
+        this.postgresUrl = {
+          host,
+          port: 5432,
+          username: process.env.APPSMITH_KEYCLOAK_DB_USERNAME,
+          password: process.env.APPSMITH_KEYCLOAK_DB_PASSWORD,
+          database,
+        };
+      } else {
+        // Identify this as an invalid value for this env variable.
+        // But we ignore this fact for now, since Postgres itself is not a critical component yet.
+        console.warn(
+          "APPSMITH_KEYCLOAK_DB_URL is set, but it doesn't start with postgresql://. This is not a valid value for this env variable. But we ignore this fact for now, since Postgres itself is not a critical component yet.",
+        );
+      }
+    } else if (process.env.APPSMITH_ENABLE_EMBEDDED_DB !== "0") {
+      this.postgresUrl = {
+        // Get unix_socket_directories from postgresql.conf, like in pg-utils.sh/get_unix_socket_directory.
+        // Unix socket directory
+        host: "/var/run/postgresql",
+        port: 5432,
+        username: "postgres",
+        password: process.env.APPSMITH_TEMPORAL_PASSWORD,
+        database: "appsmith",
+      };
+    } else {
+      throw new Error("No Postgres DB URL found");
+    }
+  }
+
+  async doBackup() {
+    if (this.postgresUrl) {
+      await executePostgresDumpCMD(this.state.backupRootPath, this.postgresUrl);
+    }
+  }
+
+  async doRestore(restoreContentsPath: string) {
+    const env = {
+      ...process.env,
+    };
+
+    const cmd = ["psql", "-v", "ON_ERROR_STOP=1"];
+
+    const isLocalhost = ["localhost", "127.0.0.1"].includes(
+      this.postgresUrl.host,
+    );
+
+    if (isLocalhost) {
+      env.PGHOST = "/var/run/postgresql";
+      env.PGPORT = "5432";
+      env.PGUSER = "postgres";
+      env.PGPASSWORD = process.env.APPSMITH_TEMPORAL_PASSWORD;
+      env.PGDATABASE = this.postgresUrl.database;
+    } else {
+      env.PGHOST = this.postgresUrl.host;
+      env.PGPORT = this.postgresUrl.port.toString();
+      env.PGUSER = this.postgresUrl.username;
+      env.PGPASSWORD = this.postgresUrl.password;
+      env.PGDATABASE = this.postgresUrl.database;
+    }
+
+    await utils.execCommand(
+      [
+        ...cmd,
+        "--command=DROP SCHEMA IF EXISTS public CASCADE; DROP SCHEMA IF EXISTS appsmith CASCADE; DROP SCHEMA IF EXISTS temporal CASCADE;",
+      ],
+      { env },
+    );
+
+    await utils.execCommand(
+      [...cmd, `--file=${restoreContentsPath}/pg-data.sql`],
+      { env },
+    );
+    console.log("Restoring Postgres database completed");
+  }
+}
+
+function parsePostgresUrl(url: string): ConnectionDetails {
+  const parsed = new URL(url);
+  return {
+    host: parsed.hostname,
+    port: parseInt(parsed.port || "5432"),
+    username: parsed.username,
+    password: decodeURIComponent(parsed.password),
+    database: parsed.pathname.substring(1),
+  };
+}
+
+export async function executePostgresDumpCMD(
+  destFolder: string,
+  details: ConnectionDetails,
+) {
+  const args = [
+    "pg_dump",
+    `--host=${details.host}`,
+    `--port=${details.port || "5432"}`,
+    `--username=${details.username}`,
+    `--dbname=${details.database}`,
+    "--schema=appsmith",
+    "--schema=public", // Keycloak
+    "--schema=temporal",
+    `--file=${destFolder}/pg-data.sql`,
+    "--verbose",
+    "--serializable-deferrable",
+  ];
+
+  // Set password in environment since it's not allowed in the CLI
+  const env = {
+    ...process.env,
+    PGPASSWORD: details.password,
+  };
+
+  return await utils.execCommand(args, { env });
+}

app/client/packages/rts/src/ctl/backup/links/index.ts

@@ -16,3 +16,4 @@ export * from "./EnvFileLink";
 export * from "./GitStorageLink";
 export * from "./ManifestLink";
 export * from "./MongoDumpLink";
+export * from "./PostgresDumpLink";