add postgresql backup hooks

checks/flake-module.nix

@@ -44,6 +44,7 @@
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
+            postgresql = import ./postgresql nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
 

checks/postgresql/default.nix

@@ -0,0 +1,23 @@
+(import ../lib/container-test.nix) (
+  {
+    name = "postgresql";
+
+    nodes.machine = { self, ... }:
+      {
+        imports = [
+          self.nixosModules.clanCore
+          self.clanModules.postgresql
+          self.clanModules.localbackup
+        ];
+        clan.postgresl.databases = [ "test" ];
+        clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+      };
+    testScript = ''
+      start_all()
+      machine.succeed("systemctl status postgresql")
+      machine.wait_for_unit("postgresql")
+      machine.succeed("localbackup-create")
+      machine.succeed("ls -la /var/backups/postgresql")
+    '';
+  }
+)

clanModules/borgbackup/default.nix

@@ -51,7 +51,7 @@ in
 
   config = lib.mkIf (cfg.destinations != { }) {
     services.borgbackup.jobs = lib.mapAttrs (_: dest: {
-      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
+      paths = lib.unique (lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state)));
       exclude = [ "*.pyc" ];
       repo = dest.repo;
       environment.BORG_RSH = dest.rsh;
@@ -60,6 +60,23 @@ in
       persistentTimer = true;
       preHook = ''
         set -x
+        declare -A preCommandErrors
+        ${lib.concatMapStringsSep "\n" (state: ''
+          echo "Running pre-backup command for ${state.name}"
+          if ! ${state.preBackupCommand} then
+            preCommandErrors["${state.name}"]=1
+          fi
+        '') config.clanCore.state}
+      '';
+      postPrune = ''
+        # report any preBackupCommand errors
+        if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+          echo "PreBackupCommand failed for the following services:"
+          for state in "''${!preCommandErrors[@]}"; do
+            echo "  $state"
+          done
+          exit 1
+        fi
       '';
 
       encryption = {

clanModules/flake-module.nix

@@ -12,6 +12,7 @@
     localsend = ./localsend;
     matrix-synapse = ./matrix-synapse;
     moonlight = ./moonlight;
+    postgresql = ./postgresql;
     root-password = ./root-password;
     sshd = ./sshd;
     sunshine = ./sunshine;

clanModules/localbackup/default.nix

@@ -6,7 +6,8 @@
 }:
 let
   cfg = config.clan.localbackup;
-  rsnapshotConfig = target: states: ''
+  uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state));
+  rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
     sync_first	1
@@ -21,6 +22,13 @@ let
       cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
         set -efu -o pipefail
         ${target.preBackupHook}
+
+        # FIXME: we currently fail the backup if the pre-backup command fails
+        # This is not ideal, but at least most of the time we run backup commands in foreground.
+        ${lib.concatMapStringsSep "\n" (state: ''
+          echo "Running pre-backup command for ${state.name}"
+          ${state.preBackupCommand}
+        '') (lib.attrValues config.clanCore.state)}
       ''}
     ''}
 
@@ -31,11 +39,9 @@ let
       ''}
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
-    ${lib.concatMapStringsSep "\n" (state: ''
+    ${lib.concatMapStringsSep "\n" (folder: ''
-      ${lib.concatMapStringsSep "\n" (folder: ''
         backup	${folder}	${config.networking.hostName}/
-      '') state.folders}
+    '') uniqueFolders}
-    '') states}
   '';
 in
 {
@@ -132,8 +138,8 @@ in
               (
                 ${mountHook target}
                 echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
+                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
+                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
               )
             '') (builtins.attrValues cfg.targets)}
           '')

clanModules/matrix-synapse/default.nix

@@ -8,13 +8,14 @@ let
   cfg = config.clan.matrix-synapse;
   nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
   element-web =
-    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; } ''
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
-      cp -r ${pkgs.element-web} $out
+      ''
-      chmod -R u+w $out
+        cp -r ${pkgs.element-web} $out
-      jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
+        chmod -R u+w $out
-        > $out/config.json < ${pkgs.element-web}/config.json
+        jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
-      ln -s $out/config.json $out/config.${nginx-vhost}.json
+          > $out/config.json < ${pkgs.element-web}/config.json
-    '';
+        ln -s $out/config.json $out/config.${nginx-vhost}.json
+      '';
 in
 {
   options.clan.matrix-synapse = {
@@ -69,7 +70,8 @@ in
     };
     systemd.tmpfiles.settings."synapse" = {
       "/run/synapse-registration-shared-secret.yaml" = {
-        C.argument = config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
+        C.argument =
+          config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
         z = {
           mode = "0400";
           user = "matrix-synapse";

clanModules/postgresql/README.md

@@ -0,0 +1,2 @@
+A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
+---

clanModules/postgresql/default.nix

@@ -0,0 +1,52 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  createDatatbaseState =
+    db:
+    let
+      folder = "/var/backup/postgresql/${db}";
+      curFile = "${folder}/dump.sql.zstd";
+      prevFile = "${folder}/dump.sql.prev.zstd";
+      inProgressFile = "${folder}/dump.sql.in-progress.zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupCommand = ''
+        (
+          umask 0077 # ensure backup is only readable by postgres user
+          if [ -e ${curFile} ]; then
+            mv ${curFile} ${prevFile}
+          fi
+          pg_dump -C ${db} | \
+            ${pkgs.zstd}/bin/zstd --rsyncable | \
+            > ${inProgressFile}
+          mv ${inProgressFile} ${curFile}
+        )
+      '';
+      postRestoreCommand = ''
+        if [[ -f ${prevFile} ]]; then
+          zstd --decompress --stdout ${prevFile} | psql -d ${db}
+        fi
+      '';
+    };
+in
+{
+  options.clan.postgresl = {
+    databases = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ "clan" ];
+    };
+  };
+  config = {
+    services.postgresql.enable = true;
+    clanCore.state = lib.listToAttrs (
+      builtins.map (
+        db: lib.nameValuePair "postgresql-${db}" (createDatatbaseState db)
+      ) config.clan.postgresl.databases
+    );
+  };
+}

nixosModules/clanCore/state.nix

@@ -17,6 +17,15 @@
                 Folder where state resides in
               '';
             };
+            preBackupCommand = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              description = ''
+                script to run before backing up the state dir
+                This is for example useful for services that require an export of their state
+                e.g. a database dump
+              '';
+            };
             preRestoreCommand = lib.mkOption {
               type = lib.types.nullOr lib.types.str;
               default = null;

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -17,7 +17,7 @@ def restore_service(machine: Machine, name: str, provider: str, service: str) ->
     folders = backup_folders[service]["folders"]
     env = {}
     env["NAME"] = name
-    env["FOLDERS"] = ":".join(folders)
+    env["FOLDERS"] = ":".join(set(folders))
 
     if pre_restore := backup_folders[service]["preRestoreCommand"]:
         proc = machine.target_host.run(
@@ -58,12 +58,20 @@ def restore_backup(
     name: str,
     service: str | None = None,
 ) -> None:
+    errors = []
     if service is None:
         backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
         for _service in backup_folders:
-            restore_service(machine, name, provider, _service)
+            try:
+                restore_service(machine, name, provider, _service)
+            except ClanError as e:
+                errors.append(f"{_service}: {e}")
     else:
-        restore_service(machine, name, provider, service)
+        try:
+            restore_service(machine, name, provider, service)
+        except ClanError as e:
+            errors.append(f"{service}: {e}")
+    raise ClanError("Restore failed for the following services:\n" + "\n".join(errors))
 
 
 def restore_command(args: argparse.Namespace) -> None: