postgresql: add backup and restore

checks/postgresql/default.nix

@@ -1,22 +1,51 @@
-(import ../lib/container-test.nix) (
+(import ../lib/container-test.nix) ({
-  {
+  name = "postgresql";
-    name = "postgresql";
 
-    nodes.machine = { self, ... }:
+  nodes.machine =
-      {
+    { self, config, ... }:
-        imports = [
+    {
-          self.nixosModules.clanCore
+      imports = [
-          self.clanModules.postgresql
+        self.nixosModules.clanCore
-          self.clanModules.localbackup
+        self.clanModules.postgresql
-        ];
+        self.clanModules.localbackup
-        clan.postgresl.databases.test = {};
+      ];
-        clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+      clan.postgresql.users.test = { };
-      };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
-    testScript = ''
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
       start_all()
-      machine.succeed("systemctl status postgresql")
       machine.wait_for_unit("postgresql")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
       machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
-      machine.succeed("ls -la /var/backups/postgresql")
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/dump.sql.zstd || { echo 'dump.sql.zstd not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
     '';
 })

clanModules/borgbackup/default.nix

@@ -63,12 +63,15 @@ in
       preHook = ''
         set -x
         declare -A preCommandErrors
-        ${lib.concatMapStringsSep "\n" (state: ''
+        ${lib.concatMapStringsSep "\n" (
-          echo "Running pre-backup command for ${state.name}"
+          state:
-          if ! ${state.preBackupCommand} then
+          lib.optionalString (state.preBackupCommand != null) ''
-            preCommandErrors["${state.name}"]=1
+            echo "Running pre-backup command for ${state.name}"
-          fi
+            if ! ( ${state.preBackupCommand} ) then
-        '') (lib.attrValues config.clanCore.state)}
+              preCommandErrors["${state.name}"]=1
+            fi
+          ''
+        ) (lib.attrValues config.clanCore.state)}
       '';
       postPrune = ''
         # report any preBackupCommand errors

clanModules/localbackup/default.nix

@@ -6,7 +6,9 @@
 }:
 let
   cfg = config.clan.localbackup;
-  uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state));
+  uniqueFolders = lib.unique (
+    lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
+  );
   rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
@@ -18,19 +20,6 @@ let
     cmd_logger	${pkgs.inetutils}/bin/logger
     cmd_du	${pkgs.coreutils}/bin/du
     cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-    ${lib.optionalString (target.preBackupHook != null) ''
-      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
-        set -efu -o pipefail
-        ${target.preBackupHook}
-
-        # FIXME: we currently fail the backup if the pre-backup command fails
-        # This is not ideal, but at least most of the time we run backup commands in foreground.
-        ${lib.concatMapStringsSep "\n" (state: ''
-          echo "Running pre-backup command for ${state.name}"
-          ${state.preBackupCommand}
-        '') (lib.attrValues config.clanCore.state)}
-      ''}
-    ''}
 
     ${lib.optionalString (target.postBackupHook != null) ''
       cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
@@ -40,7 +29,7 @@ let
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
     ${lib.concatMapStringsSep "\n" (folder: ''
-        backup	${folder}	${config.networking.hostName}/
+      backup	${folder}	${config.networking.hostName}/
     '') uniqueFolders}
   '';
 in
@@ -135,14 +124,30 @@ in
               ]
             }
             ${lib.concatMapStringsSep "\n" (target: ''
-              (
+              ${mountHook target}
-                ${mountHook target}
+              set -x
-                echo "Creating backup '${target.name}'"
+              echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+              ${lib.optionalString (target.preBackupHook != null) ''
-              )
+                (
-            '') (builtins.attrValues cfg.targets)}
+                  ${target.preBackupHook}
-          '')
+                )
+              ''}
+
+              declare -A preCommandErrors
+              ${lib.concatMapStringsSep "\n" (
+                state:
+                lib.optionalString (state.preBackupCommand != null) ''
+                  echo "Running pre-backup command for ${state.name}"
+                  if ! ( ${state.preBackupCommand} ) then
+                    preCommandErrors["${state.name}"]=1
+                  fi
+                ''
+              ) (builtins.attrValues config.clanCore.state)}
+
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+            '') (builtins.attrValues cfg.targets)}'')
           (pkgs.writeShellScriptBin "localbackup-list" ''
             set -efu -o pipefail
             export PATH=${
@@ -173,6 +178,14 @@ in
                 pkgs.gawk
               ]
             }
+            if [[ "''${NAME:-}" == "" ]]; then
+              echo "No backup name given via NAME environment variable"
+              exit 1
+            fi
+            if [[ "''${FOLDERS:-}" == "" ]]; then
+              echo "No folders given via FOLDERS environment variable"
+              exit 1
+            fi
             name=$(awk -F'::' '{print $1}' <<< $NAME)
             backupname=''${NAME#$name::}
 
@@ -188,8 +201,9 @@ in
               exit 1
             fi
 
-            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            IFS=':' read -ra FOLDER <<< "''$FOLDERS"
             for folder in "''${FOLDER[@]}"; do
+              mkdir -p "$folder"
               rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
             done
           '')

clanModules/postgresql/default.nix

@@ -8,32 +8,56 @@ let
   createDatatbaseState =
     db:
     let
-      folder = "/var/backup/postgresql/${db.name}";
+      folder = "/var/backup/postgres/${db.name}";
-      curFile = "${folder}/dump.sql.zstd";
+      current = "${folder}/dump.sql.zstd";
-      prevFile = "${folder}/dump.sql.prev.zstd";
-      inProgressFile = "${folder}/dump.sql.in-progress.zstd";
     in
     {
       folders = [ folder ];
       preBackupCommand = ''
-        (
+        export PATH=${
-          umask 0077 # ensure backup is only readable by postgres user
+          lib.makeBinPath [
-          if [ -e ${curFile} ]; then
+            config.services.postgresql.package
-            mv ${curFile} ${prevFile}
+            config.systemd.package
-          fi
+            pkgs.coreutils
-          while [[ "$(systemctl is-active postgres)" == activating ]]; then
+            pkgs.util-linux
-            sleep 1
+            pkgs.zstd
-          done
+          ]
-          systemctl is-active postgres
+        }
-          pg_dump -C ${db.name} | \
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-            ${pkgs.zstd}/bin/zstd --rsyncable | \
+          sleep 1
-            > ${inProgressFile}
+        done
-          mv ${inProgressFile} ${curFile}
+
-        )
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump -d ${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
       '';
+
       postRestoreCommand = ''
-        if [[ -f ${prevFile} ]]; then
+        export PATH=${
-          zstd --decompress --stdout ${prevFile} | psql -d ${db.name}
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          umask 077 # only root can read the backup
+          mkdir -p "${folder}"
+          runuser -u postgres -- dropdb "${db.name}"
+          runuser -u postgres -- pg_restore -C -d postgres "${current}"
+        else
+          echo No database backup found, skipping restore
         fi
       '';
     };
@@ -41,7 +65,7 @@ let
   createDatabase = db: ''
     CREATE DATABASE ${db.name} ${
       lib.concatStringsSep " " (
-        lib.mapAttrsToList (name: value: "${name} = ':${value}'") db.createOptions
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
       )
     }
   '';
@@ -54,16 +78,17 @@ let
   cfg = config.clan.postgresql;
 
   userClauses = lib.mapAttrsToList (
-    _: user: ""
+    _: user:
     ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
   ) cfg.users;
   databaseClauses = lib.mapAttrsToList (
     name: db:
-    lib.optionalString (db.create) ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ${createDatabaseArgs db}''
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c "${createDatabase db}" ''
   ) cfg.databases;
 in
 {
-  options.clan.postgresl = {
+  options.clan.postgresql = {
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
     databases = lib.mkOption {
       default = { };
       type = lib.types.attrsOf (
@@ -76,11 +101,12 @@ in
                 default = name;
               };
               # set to false, in case the upstream module uses ensureDatabase option
-              create = lib.mkOption {
+              create.enable = lib.mkOption {
                 type = lib.types.bool;
                 default = true;
+                description = "Create the database if it does not exist.";
               };
-              createOptions = lib.mkOption {
+              create.options = lib.mkOption {
                 type = lib.types.lazyAttrsOf lib.types.str;
                 default = { };
                 example = {
@@ -91,6 +117,11 @@ in
                   OWNER = "foo";
                 };
               };
+              backup.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Backup the database.";
+              };
             };
           }
         )
@@ -112,18 +143,52 @@ in
     };
   };
   config = {
+    environment.etc."pgbackrest/pgbackrest.conf".text = lib.generators.toINI { } {
+      "main" = {
+        pg1-path = config.services.postgresql.dataDir;
+      };
+
+      global = {
+        repo1-retention-full = 3;
+        repo1-type = "cifs";
+        repo1-path = "/var/lib/pgbackrest";
+        repo1-host-user = "postgres";
+        pg1-host-user = "postgres";
+        pg1-user = "postgres";
+
+        # Force a checkpoint to start backup immediately.
+        start-fast = "y";
+        # Use delta restore.
+        delta = "y";
+
+        # Enable ZSTD compression.
+        compress-type = "zst";
+        compress-level = 6;
+
+        log-level-console = "info";
+        log-level-file = "debug";
+      };
+    };
+
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+    services.postgresql.package = pkgs.postgresql_14;
+
     services.postgresql.enable = true;
     # We are duplicating a bit the upstream module but allow to create databases with options
     systemd.services.postgresql.postStart = ''
       PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
 
       while ! $PSQL -d postgres -c "" 2> /dev/null; do
-          if ! kill -0 "$MAINPID"; then exit 1; fi
+        if ! kill -0 "$MAINPID"; then exit 1; fi
-          sleep 0.1
+        sleep 0.1
       done
       ${lib.concatStringsSep "\n" userClauses}
       ${lib.concatStringsSep "\n" databaseClauses}
     '';
+
     clanCore.state = lib.mapAttrs' (
       _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
     ) config.clan.postgresql.databases;

nixosModules/clanCore/state.nix

@@ -8,9 +8,16 @@
     default = { };
     type = lib.types.attrsOf (
       lib.types.submodule (
-        { ... }:
+        { name, ... }:
         {
           options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              default = name;
+              description = ''
+                Name of the state
+              '';
+            };
             folders = lib.mkOption {
               type = lib.types.listOf lib.types.str;
               description = ''

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -14,9 +14,15 @@ from ..machines.machines import Machine
 def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
     backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
+
+    if service not in backup_folders:
+        msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
+        raise ClanError(msg)
+
     folders = backup_folders[service]["folders"]
     env = {}
     env["NAME"] = name
+    # FIXME: If we have too many folder this might overflow the stack.
     env["FOLDERS"] = ":".join(set(folders))
 
     if pre_restore := backup_folders[service]["preRestoreCommand"]: