From ba6840d978c4a83de723d24510f21e24a5fa40ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 29 May 2024 14:24:36 +0200
Subject: [PATCH 01/10] matrix-synapse: create database with right collation
 also when postgresql already exists #1108

---
 clanModules/matrix-synapse/default.nix | 77 ++++++++++++++++----------
 1 file changed, 47 insertions(+), 30 deletions(-)

diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix
index f3611bb9..0da0ef9b 100644
--- a/clanModules/matrix-synapse/default.nix
+++ b/clanModules/matrix-synapse/default.nix
@@ -6,6 +6,15 @@
 }:
 let
   cfg = config.clan.matrix-synapse;
+  nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
+  element-web =
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; } ''
+      cp -r ${pkgs.element-web} $out
+      chmod -R u+w $out
+      jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
+        > $out/config.json < ${pkgs.element-web}/config.json
+      ln -s $out/config.json $out/config.${nginx-vhost}.json
+    '';
 in
 {
   options.clan.matrix-synapse = {
@@ -13,6 +22,7 @@ in
     domain = lib.mkOption {
       type = lib.types.str;
       description = "The domain name of the matrix server";
+      example = "example.com";
     };
   };
   config = lib.mkIf cfg.enable {
@@ -49,14 +59,40 @@ in
           }
         ];
       };
-      extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
+      extraConfigFiles = [ "/run/synapse-registration-shared-secret.yaml" ];
     };
+    systemd.tmpfiles.settings."synapse" = {
+      "/run/synapse-registration-shared-secret.yaml" = {
+        C.argument = config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
+        z = {
+          mode = "0400";
+          user = "matrix-synapse";
+        };
+      };
+    };
+
     systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
-      "+${pkgs.writeScript "copy_registration_shared_secret" ''
-        #!/bin/sh
-        cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
+      "+${pkgs.writeShellScript "create-matrix-synapse-db" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            pkgs.util-linux
+            pkgs.gnugrep
+          ]
+        }
+        psql() { runuser -u postgres -- psql "$@"; }
+        # wait for postgres to be ready
+        while ! runuser -u postgres pg_isready; do
+          sleep 1
+        done
+        if ! psql -tAc "SELECT 1 FROM pg_database WHERE datname = 'matrix-synapse'" | grep -q 1; then
+          psql -c "CREATE DATABASE \"matrix-synapse\" TEMPLATE template0 LC_COLLATE = 'C' LC_CTYPE = 'C'"
+        fi
+        # create user if it doesn't exist and make it owner of the database
+        if ! psql -tAc "SELECT 1 FROM pg_user WHERE usename = 'matrix-synapse'" | grep -q 1; then
+          psql -c "CREATE USER \"matrix-synapse\""
+          psql -c "ALTER DATABASE \"matrix-synapse\" OWNER TO \"matrix-synapse\""
+        fi
       ''}"
     ];
 
@@ -72,22 +108,6 @@ in
     };
 
     services.postgresql.enable = true;
-    # we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
-    services.postgresql = {
-      ensureDatabases = [ "matrix-synapse" ];
-      ensureUsers = [
-        {
-          name = "matrix-synapse";
-          ensureDBOwnership = true;
-        }
-      ];
-      initialScript = pkgs.writeText "synapse-init.sql" ''
-        CREATE DATABASE "matrix-synapse"
-          TEMPLATE template0
-          LC_COLLATE = "C"
-          LC_CTYPE = "C";
-      '';
-    };
     services.nginx = {
       enable = true;
       virtualHosts = {
@@ -102,7 +122,7 @@ in
             return 200 '${
               builtins.toJSON {
                 "m.homeserver" = {
-                  "base_url" = "https://matrix.${cfg.domain}";
+                  "base_url" = "https://${nginx-vhost}";
                 };
                 "m.identity_server" = {
                   "base_url" = "https://vector.im";
@@ -111,15 +131,12 @@ in
             }';
           '';
         };
-        "matrix.${cfg.domain}" = {
+        ${nginx-vhost} = {
           forceSSL = true;
           enableACME = true;
-          locations."/_matrix" = {
-            proxyPass = "http://localhost:8008";
-          };
-          locations."/test".extraConfig = ''
-            return 200 "Hello, world!";
-          '';
+          locations."/_matrix".proxyPass = "http://localhost:8008";
+          locations."/_synapse".proxyPass = "http://localhost:8008";
+          locations."/".root = element-web;
         };
       };
     };

From d6a9f6d3f972a1a770566259bb86237072fb44a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 29 May 2024 14:26:35 +0200
Subject: [PATCH 02/10] change clan url to gitea archive url

---
 templates/new-clan/flake.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix
index 3cda5782..7fb1361e 100644
--- a/templates/new-clan/flake.nix
+++ b/templates/new-clan/flake.nix
@@ -1,7 +1,7 @@
 {
   description = "<Put your description here>";
 
-  inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core";
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
 
   outputs =
     { self, clan-core, ... }:

From c1aedc5bb8e0b9b2945cce53b556e664b89af19c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 29 May 2024 16:12:06 +0200
Subject: [PATCH 03/10] matrix-enable: drop enable option

---
 checks/matrix-synapse/default.nix      |  1 -
 clanModules/matrix-synapse/default.nix | 12 ++++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/checks/matrix-synapse/default.nix b/checks/matrix-synapse/default.nix
index cb3c9482..0e879e3d 100644
--- a/checks/matrix-synapse/default.nix
+++ b/checks/matrix-synapse/default.nix
@@ -13,7 +13,6 @@
             clanCore.machineName = "machine";
             clanCore.clanDir = ./.;
             clan.matrix-synapse = {
-              enable = true;
               domain = "clan.test";
             };
           }
diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix
index 0da0ef9b..2f642c35 100644
--- a/clanModules/matrix-synapse/default.nix
+++ b/clanModules/matrix-synapse/default.nix
@@ -18,14 +18,22 @@ let
 in
 {
   options.clan.matrix-synapse = {
-    enable = lib.mkEnableOption "Enable matrix-synapse";
     domain = lib.mkOption {
       type = lib.types.str;
       description = "The domain name of the matrix server";
       example = "example.com";
     };
   };
-  config = lib.mkIf cfg.enable {
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "matrix-synapse"
+      "enable"
+    ] "Importing the module will already enable the service.")
+
+    ../postgresql
+  ];
+  config = {
     services.matrix-synapse = {
       enable = true;
       settings = {

From f71295e640273baed80ae64c44adfd6a4e6aa1d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 4 Jun 2024 13:22:32 +0200
Subject: [PATCH 04/10] fix running cli without arguments

---
 pkgs/clan-cli/clan_cli/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py
index 76b45591..9506f79f 100644
--- a/pkgs/clan-cli/clan_cli/__init__.py
+++ b/pkgs/clan-cli/clan_cli/__init__.py
@@ -313,7 +313,7 @@ def main() -> None:
     if len(sys.argv) == 1:
         parser.print_help()
 
-    if args.debug:
+    if getattr(args, "debug", False):
         setup_logging(logging.DEBUG, root_log_name=__name__.split(".")[0])
         log.debug("Debug log activated")
         if flatpak.is_flatpak():

From 6dec2a9222bcb8a044bc5a272e1475815aa46dd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Fri, 31 May 2024 16:36:37 +0200
Subject: [PATCH 05/10] add postgresql backup hooks

---
 checks/flake-module.nix                   |  1 +
 checks/postgresql/default.nix             | 22 ++++++++++
 clanModules/borgbackup/default.nix        | 21 ++++++++-
 clanModules/flake-module.nix              |  1 +
 clanModules/localbackup/default.nix       | 20 ++++++---
 clanModules/matrix-synapse/default.nix    | 18 ++++----
 clanModules/postgresql/README.md          |  2 +
 clanModules/postgresql/default.nix        | 52 +++++++++++++++++++++++
 docs/mkdocs.yml                           |  1 +
 nixosModules/clanCore/state.nix           |  9 ++++
 pkgs/clan-cli/clan_cli/backups/restore.py | 17 ++++++--
 11 files changed, 145 insertions(+), 19 deletions(-)
 create mode 100644 checks/postgresql/default.nix
 create mode 100644 clanModules/postgresql/README.md
 create mode 100644 clanModules/postgresql/default.nix

diff --git a/checks/flake-module.nix b/checks/flake-module.nix
index efb3d1fe..cd8281f3 100644
--- a/checks/flake-module.nix
+++ b/checks/flake-module.nix
@@ -44,6 +44,7 @@
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
+            postgresql = import ./postgresql nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
 
diff --git a/checks/postgresql/default.nix b/checks/postgresql/default.nix
new file mode 100644
index 00000000..d334c03c
--- /dev/null
+++ b/checks/postgresql/default.nix
@@ -0,0 +1,22 @@
+(import ../lib/container-test.nix) (
+  {
+    name = "postgresql";
+
+    nodes.machine = { self, ... }:
+      {
+        imports = [
+          self.nixosModules.clanCore
+          self.clanModules.postgresql
+          self.clanModules.localbackup
+        ];
+        clan.postgresl.databases = [ "test" ];
+        clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+      };
+    testScript = ''
+      start_all()
+      machine.succeed("systemctl status postgresql")
+      machine.wait_for_unit("postgresql")
+      machine.succeed("localbackup-create")
+      machine.succeed("ls -la /var/backups/postgresql")
+    '';
+})
diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix
index fec4707b..aef8dd6d 100644
--- a/clanModules/borgbackup/default.nix
+++ b/clanModules/borgbackup/default.nix
@@ -51,7 +51,9 @@ in
 
   config = lib.mkIf (cfg.destinations != { }) {
     services.borgbackup.jobs = lib.mapAttrs (_: dest: {
-      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
+      paths = lib.unique (
+        lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state))
+      );
       exclude = [ "*.pyc" ];
       repo = dest.repo;
       environment.BORG_RSH = dest.rsh;
@@ -60,6 +62,23 @@ in
       persistentTimer = true;
       preHook = ''
         set -x
+        declare -A preCommandErrors
+        ${lib.concatMapStringsSep "\n" (state: ''
+          echo "Running pre-backup command for ${state.name}"
+          if ! ${state.preBackupCommand} then
+            preCommandErrors["${state.name}"]=1
+          fi
+        '') (lib.attrValues config.clanCore.state)}
+      '';
+      postPrune = ''
+        # report any preBackupCommand errors
+        if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+          echo "PreBackupCommand failed for the following services:"
+          for state in "''${!preCommandErrors[@]}"; do
+            echo "  $state"
+          done
+          exit 1
+        fi
       '';
 
       encryption = {
diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix
index 7e1b46e5..928b8e5b 100644
--- a/clanModules/flake-module.nix
+++ b/clanModules/flake-module.nix
@@ -12,6 +12,7 @@
     localsend = ./localsend;
     matrix-synapse = ./matrix-synapse;
     moonlight = ./moonlight;
+    postgresql = ./postgresql;
     root-password = ./root-password;
     sshd = ./sshd;
     sunshine = ./sunshine;
diff --git a/clanModules/localbackup/default.nix b/clanModules/localbackup/default.nix
index 74c81d0d..c957b88d 100644
--- a/clanModules/localbackup/default.nix
+++ b/clanModules/localbackup/default.nix
@@ -6,7 +6,8 @@
 }:
 let
   cfg = config.clan.localbackup;
-  rsnapshotConfig = target: states: ''
+  uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state));
+  rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
     sync_first	1
@@ -21,6 +22,13 @@ let
       cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
         set -efu -o pipefail
         ${target.preBackupHook}
+
+        # FIXME: we currently fail the backup if the pre-backup command fails
+        # This is not ideal, but at least most of the time we run backup commands in foreground.
+        ${lib.concatMapStringsSep "\n" (state: ''
+          echo "Running pre-backup command for ${state.name}"
+          ${state.preBackupCommand}
+        '') (lib.attrValues config.clanCore.state)}
       ''}
     ''}
 
@@ -31,11 +39,9 @@ let
       ''}
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
-    ${lib.concatMapStringsSep "\n" (state: ''
-      ${lib.concatMapStringsSep "\n" (folder: ''
+    ${lib.concatMapStringsSep "\n" (folder: ''
         backup	${folder}	${config.networking.hostName}/
-      '') state.folders}
-    '') states}
+    '') uniqueFolders}
   '';
 in
 {
@@ -132,8 +138,8 @@ in
               (
                 ${mountHook target}
                 echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
+                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
               )
             '') (builtins.attrValues cfg.targets)}
           '')
diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix
index 2f642c35..e0eb7dbe 100644
--- a/clanModules/matrix-synapse/default.nix
+++ b/clanModules/matrix-synapse/default.nix
@@ -8,13 +8,14 @@ let
   cfg = config.clan.matrix-synapse;
   nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
   element-web =
-    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; } ''
-      cp -r ${pkgs.element-web} $out
-      chmod -R u+w $out
-      jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
-        > $out/config.json < ${pkgs.element-web}/config.json
-      ln -s $out/config.json $out/config.${nginx-vhost}.json
-    '';
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
+      ''
+        cp -r ${pkgs.element-web} $out
+        chmod -R u+w $out
+        jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
+          > $out/config.json < ${pkgs.element-web}/config.json
+        ln -s $out/config.json $out/config.${nginx-vhost}.json
+      '';
 in
 {
   options.clan.matrix-synapse = {
@@ -71,7 +72,8 @@ in
     };
     systemd.tmpfiles.settings."synapse" = {
       "/run/synapse-registration-shared-secret.yaml" = {
-        C.argument = config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
+        C.argument =
+          config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
         z = {
           mode = "0400";
           user = "matrix-synapse";
diff --git a/clanModules/postgresql/README.md b/clanModules/postgresql/README.md
new file mode 100644
index 00000000..76f5b0f8
--- /dev/null
+++ b/clanModules/postgresql/README.md
@@ -0,0 +1,2 @@
+A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
+---
diff --git a/clanModules/postgresql/default.nix b/clanModules/postgresql/default.nix
new file mode 100644
index 00000000..5d819477
--- /dev/null
+++ b/clanModules/postgresql/default.nix
@@ -0,0 +1,52 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  createDatatbaseState =
+    db:
+    let
+      folder = "/var/backup/postgresql/${db}";
+      curFile = "${folder}/dump.sql.zstd";
+      prevFile = "${folder}/dump.sql.prev.zstd";
+      inProgressFile = "${folder}/dump.sql.in-progress.zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupCommand = ''
+        (
+          umask 0077 # ensure backup is only readable by postgres user
+          if [ -e ${curFile} ]; then
+            mv ${curFile} ${prevFile}
+          fi
+          pg_dump -C ${db} | \
+            ${pkgs.zstd}/bin/zstd --rsyncable | \
+            > ${inProgressFile}
+          mv ${inProgressFile} ${curFile}
+        )
+      '';
+      postRestoreCommand = ''
+        if [[ -f ${prevFile} ]]; then
+          zstd --decompress --stdout ${prevFile} | psql -d ${db}
+        fi
+      '';
+    };
+in
+{
+  options.clan.postgresl = {
+    databases = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ "clan" ];
+    };
+  };
+  config = {
+    services.postgresql.enable = true;
+    clanCore.state = lib.listToAttrs (
+      builtins.map (
+        db: lib.nameValuePair "postgresql-${db}" (createDatatbaseState db)
+      ) config.clan.postgresl.databases
+    );
+  };
+}
diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
index 56bae1f5..b98240bc 100644
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -60,6 +60,7 @@ nav:
           - reference/clanModules/localsend.md
           - reference/clanModules/matrix-synapse.md
           - reference/clanModules/moonlight.md
+          - reference/clanModules/postgresql.md
           - reference/clanModules/root-password.md
           - reference/clanModules/sshd.md
           - reference/clanModules/static-hosts.md
diff --git a/nixosModules/clanCore/state.nix b/nixosModules/clanCore/state.nix
index 26555c15..7e562d48 100644
--- a/nixosModules/clanCore/state.nix
+++ b/nixosModules/clanCore/state.nix
@@ -17,6 +17,15 @@
                 Folder where state resides in
               '';
             };
+            preBackupCommand = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              description = ''
+                script to run before backing up the state dir
+                This is for example useful for services that require an export of their state
+                e.g. a database dump
+              '';
+            };
             preRestoreCommand = lib.mkOption {
               type = lib.types.nullOr lib.types.str;
               default = null;
diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py
index 31b8cb86..3ff0e3a1 100644
--- a/pkgs/clan-cli/clan_cli/backups/restore.py
+++ b/pkgs/clan-cli/clan_cli/backups/restore.py
@@ -17,7 +17,7 @@ def restore_service(machine: Machine, name: str, provider: str, service: str) ->
     folders = backup_folders[service]["folders"]
     env = {}
     env["NAME"] = name
-    env["FOLDERS"] = ":".join(folders)
+    env["FOLDERS"] = ":".join(set(folders))
 
     if pre_restore := backup_folders[service]["preRestoreCommand"]:
         proc = machine.target_host.run(
@@ -58,12 +58,23 @@ def restore_backup(
     name: str,
     service: str | None = None,
 ) -> None:
+    errors = []
     if service is None:
         backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
         for _service in backup_folders:
-            restore_service(machine, name, provider, _service)
+            try:
+                restore_service(machine, name, provider, _service)
+            except ClanError as e:
+                errors.append(f"{_service}: {e}")
     else:
-        restore_service(machine, name, provider, service)
+        try:
+            restore_service(machine, name, provider, service)
+        except ClanError as e:
+            errors.append(f"{service}: {e}")
+    if errors:
+        raise ClanError(
+            "Restore failed for the following services:\n" + "\n".join(errors)
+        )
 
 
 def restore_command(args: argparse.Namespace) -> None:

From 2a0019457d3c00dc32899ab9486d555a1b5db958 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Fri, 31 May 2024 17:16:52 +0200
Subject: [PATCH 06/10] matrix-synapse: create with utf-8 encoding

---
 clanModules/matrix-synapse/default.nix | 32 +++++++-------------------
 1 file changed, 8 insertions(+), 24 deletions(-)

diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix
index e0eb7dbe..09429a38 100644
--- a/clanModules/matrix-synapse/default.nix
+++ b/clanModules/matrix-synapse/default.nix
@@ -81,30 +81,14 @@ in
       };
     };
 
-    systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
-      "+${pkgs.writeShellScript "create-matrix-synapse-db" ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            pkgs.util-linux
-            pkgs.gnugrep
-          ]
-        }
-        psql() { runuser -u postgres -- psql "$@"; }
-        # wait for postgres to be ready
-        while ! runuser -u postgres pg_isready; do
-          sleep 1
-        done
-        if ! psql -tAc "SELECT 1 FROM pg_database WHERE datname = 'matrix-synapse'" | grep -q 1; then
-          psql -c "CREATE DATABASE \"matrix-synapse\" TEMPLATE template0 LC_COLLATE = 'C' LC_CTYPE = 'C'"
-        fi
-        # create user if it doesn't exist and make it owner of the database
-        if ! psql -tAc "SELECT 1 FROM pg_user WHERE usename = 'matrix-synapse'" | grep -q 1; then
-          psql -c "CREATE USER \"matrix-synapse\""
-          psql -c "ALTER DATABASE \"matrix-synapse\" OWNER TO \"matrix-synapse\""
-        fi
-      ''}"
-    ];
+    clan.postgresql.users.matrix-synapse = { };
+    clan.postgresql.databases.matrix-synapse.create.options = {
+      TEMPLATE = "template0";
+      LC_COLLATE = "C";
+      LC_CTYPE = "C";
+      ENCODING = "UTF8";
+      OWNER = "matrix-synapse";
+    };
 
     clanCore.facts.services."matrix-synapse" = {
       secret."synapse-registration_shared_secret" = { };

From c0d8aaf73a61748c196c905f92b67fdd0ff08a02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 4 Jun 2024 12:56:39 +0200
Subject: [PATCH 07/10] postgresql: add new method to create users and
 databases

---
 checks/postgresql/default.nix          |  4 +-
 clanModules/matrix-synapse/default.nix |  1 -
 clanModules/postgresql/default.nix     | 93 +++++++++++++++++++++++---
 3 files changed, 85 insertions(+), 13 deletions(-)

diff --git a/checks/postgresql/default.nix b/checks/postgresql/default.nix
index d334c03c..42baf9ca 100644
--- a/checks/postgresql/default.nix
+++ b/checks/postgresql/default.nix
@@ -9,14 +9,14 @@
           self.clanModules.postgresql
           self.clanModules.localbackup
         ];
-        clan.postgresl.databases = [ "test" ];
+        clan.postgresl.databases.test = {};
         clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
       };
     testScript = ''
       start_all()
       machine.succeed("systemctl status postgresql")
       machine.wait_for_unit("postgresql")
-      machine.succeed("localbackup-create")
+      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
       machine.succeed("ls -la /var/backups/postgresql")
     '';
 })
diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix
index 09429a38..6a06708d 100644
--- a/clanModules/matrix-synapse/default.nix
+++ b/clanModules/matrix-synapse/default.nix
@@ -101,7 +101,6 @@ in
       '';
     };
 
-    services.postgresql.enable = true;
     services.nginx = {
       enable = true;
       virtualHosts = {
diff --git a/clanModules/postgresql/default.nix b/clanModules/postgresql/default.nix
index 5d819477..edb542af 100644
--- a/clanModules/postgresql/default.nix
+++ b/clanModules/postgresql/default.nix
@@ -8,7 +8,7 @@ let
   createDatatbaseState =
     db:
     let
-      folder = "/var/backup/postgresql/${db}";
+      folder = "/var/backup/postgresql/${db.name}";
       curFile = "${folder}/dump.sql.zstd";
       prevFile = "${folder}/dump.sql.prev.zstd";
       inProgressFile = "${folder}/dump.sql.in-progress.zstd";
@@ -21,7 +21,11 @@ let
           if [ -e ${curFile} ]; then
             mv ${curFile} ${prevFile}
           fi
-          pg_dump -C ${db} | \
+          while [[ "$(systemctl is-active postgres)" == activating ]]; then
+            sleep 1
+          done
+          systemctl is-active postgres
+          pg_dump -C ${db.name} | \
             ${pkgs.zstd}/bin/zstd --rsyncable | \
             > ${inProgressFile}
           mv ${inProgressFile} ${curFile}
@@ -29,24 +33,93 @@ let
       '';
       postRestoreCommand = ''
         if [[ -f ${prevFile} ]]; then
-          zstd --decompress --stdout ${prevFile} | psql -d ${db}
+          zstd --decompress --stdout ${prevFile} | psql -d ${db.name}
         fi
       '';
     };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = ':${value}'") db.createOptions
+      )
+    }
+  '';
+  cfg = config.clan.postgresql;
+
+  userClauses = lib.mapAttrsToList (
+    _: user: ""
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString (db.create) ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ${createDatabaseArgs db}''
+  ) cfg.databases;
 in
 {
   options.clan.postgresl = {
     databases = lib.mkOption {
-      type = lib.types.listOf lib.types.str;
-      default = [ "clan" ];
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+              };
+              createOptions = lib.mkOption {
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
     };
   };
   config = {
     services.postgresql.enable = true;
-    clanCore.state = lib.listToAttrs (
-      builtins.map (
-        db: lib.nameValuePair "postgresql-${db}" (createDatatbaseState db)
-      ) config.clan.postgresl.databases
-    );
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+          if ! kill -0 "$MAINPID"; then exit 1; fi
+          sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+    clanCore.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
+    ) config.clan.postgresql.databases;
   };
 }

From 0dc3b9f05697bfcce32d3e5a756fdcb50989be83 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 5 Jun 2024 18:37:31 +0200
Subject: [PATCH 08/10] postgresql: add backup and restore

---
 checks/postgresql/default.nix             | 61 +++++++++++----
 clanModules/borgbackup/default.nix        | 15 ++--
 clanModules/localbackup/default.nix       | 62 +++++++++------
 clanModules/postgresql/default.nix        | 93 ++++++++++++++++-------
 nixosModules/clanCore/state.nix           |  9 ++-
 pkgs/clan-cli/clan_cli/backups/restore.py |  6 ++
 6 files changed, 171 insertions(+), 75 deletions(-)

diff --git a/checks/postgresql/default.nix b/checks/postgresql/default.nix
index 42baf9ca..2a0eeadc 100644
--- a/checks/postgresql/default.nix
+++ b/checks/postgresql/default.nix
@@ -1,22 +1,51 @@
-(import ../lib/container-test.nix) (
-  {
-    name = "postgresql";
+(import ../lib/container-test.nix) ({
+  name = "postgresql";
 
-    nodes.machine = { self, ... }:
-      {
-        imports = [
-          self.nixosModules.clanCore
-          self.clanModules.postgresql
-          self.clanModules.localbackup
-        ];
-        clan.postgresl.databases.test = {};
-        clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
-      };
-    testScript = ''
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [
+        self.nixosModules.clanCore
+        self.clanModules.postgresql
+        self.clanModules.localbackup
+      ];
+      clan.postgresql.users.test = { };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
       start_all()
-      machine.succeed("systemctl status postgresql")
       machine.wait_for_unit("postgresql")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
       machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
-      machine.succeed("ls -la /var/backups/postgresql")
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/dump.sql.zstd || { echo 'dump.sql.zstd not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
     '';
 })
diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix
index aef8dd6d..2e16b3b9 100644
--- a/clanModules/borgbackup/default.nix
+++ b/clanModules/borgbackup/default.nix
@@ -63,12 +63,15 @@ in
       preHook = ''
         set -x
         declare -A preCommandErrors
-        ${lib.concatMapStringsSep "\n" (state: ''
-          echo "Running pre-backup command for ${state.name}"
-          if ! ${state.preBackupCommand} then
-            preCommandErrors["${state.name}"]=1
-          fi
-        '') (lib.attrValues config.clanCore.state)}
+        ${lib.concatMapStringsSep "\n" (
+          state:
+          lib.optionalString (state.preBackupCommand != null) ''
+            echo "Running pre-backup command for ${state.name}"
+            if ! ( ${state.preBackupCommand} ) then
+              preCommandErrors["${state.name}"]=1
+            fi
+          ''
+        ) (lib.attrValues config.clanCore.state)}
       '';
       postPrune = ''
         # report any preBackupCommand errors
diff --git a/clanModules/localbackup/default.nix b/clanModules/localbackup/default.nix
index c957b88d..917406a8 100644
--- a/clanModules/localbackup/default.nix
+++ b/clanModules/localbackup/default.nix
@@ -6,7 +6,9 @@
 }:
 let
   cfg = config.clan.localbackup;
-  uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state));
+  uniqueFolders = lib.unique (
+    lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
+  );
   rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
@@ -18,19 +20,6 @@ let
     cmd_logger	${pkgs.inetutils}/bin/logger
     cmd_du	${pkgs.coreutils}/bin/du
     cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-    ${lib.optionalString (target.preBackupHook != null) ''
-      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
-        set -efu -o pipefail
-        ${target.preBackupHook}
-
-        # FIXME: we currently fail the backup if the pre-backup command fails
-        # This is not ideal, but at least most of the time we run backup commands in foreground.
-        ${lib.concatMapStringsSep "\n" (state: ''
-          echo "Running pre-backup command for ${state.name}"
-          ${state.preBackupCommand}
-        '') (lib.attrValues config.clanCore.state)}
-      ''}
-    ''}
 
     ${lib.optionalString (target.postBackupHook != null) ''
       cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
@@ -40,7 +29,7 @@ let
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
     ${lib.concatMapStringsSep "\n" (folder: ''
-        backup	${folder}	${config.networking.hostName}/
+      backup	${folder}	${config.networking.hostName}/
     '') uniqueFolders}
   '';
 in
@@ -135,14 +124,30 @@ in
               ]
             }
             ${lib.concatMapStringsSep "\n" (target: ''
-              (
-                ${mountHook target}
-                echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
-              )
-            '') (builtins.attrValues cfg.targets)}
-          '')
+              ${mountHook target}
+              set -x
+              echo "Creating backup '${target.name}'"
+
+              ${lib.optionalString (target.preBackupHook != null) ''
+                (
+                  ${target.preBackupHook}
+                )
+              ''}
+
+              declare -A preCommandErrors
+              ${lib.concatMapStringsSep "\n" (
+                state:
+                lib.optionalString (state.preBackupCommand != null) ''
+                  echo "Running pre-backup command for ${state.name}"
+                  if ! ( ${state.preBackupCommand} ) then
+                    preCommandErrors["${state.name}"]=1
+                  fi
+                ''
+              ) (builtins.attrValues config.clanCore.state)}
+
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+            '') (builtins.attrValues cfg.targets)}'')
           (pkgs.writeShellScriptBin "localbackup-list" ''
             set -efu -o pipefail
             export PATH=${
@@ -173,6 +178,14 @@ in
                 pkgs.gawk
               ]
             }
+            if [[ "''${NAME:-}" == "" ]]; then
+              echo "No backup name given via NAME environment variable"
+              exit 1
+            fi
+            if [[ "''${FOLDERS:-}" == "" ]]; then
+              echo "No folders given via FOLDERS environment variable"
+              exit 1
+            fi
             name=$(awk -F'::' '{print $1}' <<< $NAME)
             backupname=''${NAME#$name::}
 
@@ -188,8 +201,9 @@ in
               exit 1
             fi
 
-            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            IFS=':' read -ra FOLDER <<< "''$FOLDERS"
             for folder in "''${FOLDER[@]}"; do
+              mkdir -p "$folder"
               rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
             done
           '')
diff --git a/clanModules/postgresql/default.nix b/clanModules/postgresql/default.nix
index edb542af..c1b24104 100644
--- a/clanModules/postgresql/default.nix
+++ b/clanModules/postgresql/default.nix
@@ -8,32 +8,56 @@ let
   createDatatbaseState =
     db:
     let
-      folder = "/var/backup/postgresql/${db.name}";
-      curFile = "${folder}/dump.sql.zstd";
-      prevFile = "${folder}/dump.sql.prev.zstd";
-      inProgressFile = "${folder}/dump.sql.in-progress.zstd";
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/dump.sql.zstd";
     in
     {
       folders = [ folder ];
       preBackupCommand = ''
-        (
-          umask 0077 # ensure backup is only readable by postgres user
-          if [ -e ${curFile} ]; then
-            mv ${curFile} ${prevFile}
-          fi
-          while [[ "$(systemctl is-active postgres)" == activating ]]; then
-            sleep 1
-          done
-          systemctl is-active postgres
-          pg_dump -C ${db.name} | \
-            ${pkgs.zstd}/bin/zstd --rsyncable | \
-            > ${inProgressFile}
-          mv ${inProgressFile} ${curFile}
-        )
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump --compress=zstd --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
       '';
+
       postRestoreCommand = ''
-        if [[ -f ${prevFile} ]]; then
-          zstd --decompress --stdout ${prevFile} | psql -d ${db.name}
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          umask 077 # only root can read the backup
+          mkdir -p "${folder}"
+          runuser -u postgres -- dropdb "${db.name}"
+          runuser -u postgres -- pg_restore -C -d postgres "${current}"
+        else
+          echo No database backup found, skipping restore
         fi
       '';
     };
@@ -41,23 +65,24 @@ let
   createDatabase = db: ''
     CREATE DATABASE "${db.name}" ${
       lib.concatStringsSep " " (
-        lib.mapAttrsToList (name: value: "${name} = ':${value}'") db.createOptions
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
       )
     }
   '';
   cfg = config.clan.postgresql;
 
   userClauses = lib.mapAttrsToList (
-    _: user: ""
+    _: user:
     ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
   ) cfg.users;
   databaseClauses = lib.mapAttrsToList (
     name: db:
-    lib.optionalString (db.create) ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ${createDatabaseArgs db}''
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
   ) cfg.databases;
 in
 {
-  options.clan.postgresl = {
+  options.clan.postgresql = {
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
     databases = lib.mkOption {
       default = { };
       type = lib.types.attrsOf (
@@ -70,11 +95,12 @@ in
                 default = name;
               };
               # set to false, in case the upstream module uses ensureDatabase option
-              create = lib.mkOption {
+              create.enable = lib.mkOption {
                 type = lib.types.bool;
                 default = true;
+                description = "Create the database if it does not exist.";
               };
-              createOptions = lib.mkOption {
+              create.options = lib.mkOption {
                 type = lib.types.lazyAttrsOf lib.types.str;
                 default = { };
                 example = {
@@ -85,6 +111,11 @@ in
                   OWNER = "foo";
                 };
               };
+              backup.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Backup the database.";
+              };
             };
           }
         )
@@ -106,18 +137,24 @@ in
     };
   };
   config = {
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
     services.postgresql.enable = true;
     # We are duplicating a bit the upstream module but allow to create databases with options
     systemd.services.postgresql.postStart = ''
       PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
 
       while ! $PSQL -d postgres -c "" 2> /dev/null; do
-          if ! kill -0 "$MAINPID"; then exit 1; fi
-          sleep 0.1
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
       done
       ${lib.concatStringsSep "\n" userClauses}
       ${lib.concatStringsSep "\n" databaseClauses}
     '';
+
     clanCore.state = lib.mapAttrs' (
       _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
     ) config.clan.postgresql.databases;
diff --git a/nixosModules/clanCore/state.nix b/nixosModules/clanCore/state.nix
index 7e562d48..24054cbc 100644
--- a/nixosModules/clanCore/state.nix
+++ b/nixosModules/clanCore/state.nix
@@ -8,9 +8,16 @@
     default = { };
     type = lib.types.attrsOf (
       lib.types.submodule (
-        { ... }:
+        { name, ... }:
         {
           options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              default = name;
+              description = ''
+                Name of the state
+              '';
+            };
             folders = lib.mkOption {
               type = lib.types.listOf lib.types.str;
               description = ''
diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py
index 3ff0e3a1..84b7e30b 100644
--- a/pkgs/clan-cli/clan_cli/backups/restore.py
+++ b/pkgs/clan-cli/clan_cli/backups/restore.py
@@ -14,9 +14,15 @@ from ..machines.machines import Machine
 def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
     backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
+
+    if service not in backup_folders:
+        msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
+        raise ClanError(msg)
+
     folders = backup_folders[service]["folders"]
     env = {}
     env["NAME"] = name
+    # FIXME: If we have too many folder this might overflow the stack.
     env["FOLDERS"] = ":".join(set(folders))
 
     if pre_restore := backup_folders[service]["preRestoreCommand"]:

From 5c11a30b4685d992d955a2849129c4a7ea1a29ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 6 Jun 2024 15:17:47 +0200
Subject: [PATCH 09/10] backup: add a way to stop services before restoring a
 state.

---
 checks/postgresql/default.nix      | 22 +++++++++++++++++++---
 clanModules/postgresql/default.nix | 26 +++++++++++++++-----------
 nixosModules/clanCore/state.nix    | 13 +++++++++++++
 3 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/checks/postgresql/default.nix b/checks/postgresql/default.nix
index 2a0eeadc..91bf2e52 100644
--- a/checks/postgresql/default.nix
+++ b/checks/postgresql/default.nix
@@ -11,8 +11,19 @@
       ];
       clan.postgresql.users.test = { };
       clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
       clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
 
+      systemd.services.sample-service = {
+        wantedBy = [ "multi-user.target" ];
+        script = ''
+          while true; do
+            echo "Hello, world!"
+            sleep 5
+          done
+        '';
+      };
+
       environment.systemPackages = [ config.services.postgresql.package ];
     };
   testScript =
@@ -20,20 +31,22 @@
     ''
       start_all()
       machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("sample-service")
       # Create a test table
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
 
       machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
 
-      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/dump.sql.zstd || { echo 'dump.sql.zstd not found'; exit 1; }")
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
-      machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
 
       machine.succeed("rm -rf /var/backup/postgres")
 
       machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
-      machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
 
       machine.succeed("""
       set -x
@@ -42,6 +55,9 @@
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
 
+      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
       # Check that the table is still there
       machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
       output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
diff --git a/clanModules/postgresql/default.nix b/clanModules/postgresql/default.nix
index c1b24104..b1e7621c 100644
--- a/clanModules/postgresql/default.nix
+++ b/clanModules/postgresql/default.nix
@@ -9,7 +9,8 @@ let
     db:
     let
       folder = "/var/backup/postgres/${db.name}";
-      current = "${folder}/dump.sql.zstd";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
     in
     {
       folders = [ folder ];
@@ -28,10 +29,9 @@ let
         done
 
         mkdir -p "${folder}"
-        runuser -u postgres -- pg_dump --compress=zstd --dbname=${db.name} -Fc -c > "${current}.tmp"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
         mv "${current}.tmp" ${current}
       '';
-
       postRestoreCommand = ''
         export PATH=${
           lib.makeBinPath [
@@ -52,10 +52,14 @@ let
         done
 
         if [[ -e "${current}" ]]; then
-          umask 077 # only root can read the backup
-          mkdir -p "${folder}"
-          runuser -u postgres -- dropdb "${db.name}"
-          runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            runuser -u postgres -- dropdb "${db.name}"
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
         else
           echo No database backup found, skipping restore
         fi
@@ -111,10 +115,10 @@ in
                   OWNER = "foo";
                 };
               };
-              backup.enable = lib.mkOption {
-                type = lib.types.bool;
-                default = true;
-                description = "Backup the database.";
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of services to stop before restoring the database.";
               };
             };
           }
diff --git a/nixosModules/clanCore/state.nix b/nixosModules/clanCore/state.nix
index 24054cbc..1128fdcc 100644
--- a/nixosModules/clanCore/state.nix
+++ b/nixosModules/clanCore/state.nix
@@ -33,6 +33,18 @@
                 e.g. a database dump
               '';
             };
+
+            # TODO: implement this
+            #stopOnRestore = lib.mkOption {
+            #  type = lib.types.listOf lib.types.str;
+            #  default = [];
+            #  description = ''
+            #    List of services to stop before restoring the state dir from a backup
+
+            #    Utilize this to stop services which currently access these folders or or other services affected by the restore
+            #  '';
+            #};
+
             preRestoreCommand = lib.mkOption {
               type = lib.types.nullOr lib.types.str;
               default = null;
@@ -42,6 +54,7 @@
                 Utilize this to stop services which currently access these folders
               '';
             };
+
             postRestoreCommand = lib.mkOption {
               type = lib.types.nullOr lib.types.str;
               default = null;

From 66bdc61e3d8139917b19108e065f75eab804ff9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Mon, 10 Jun 2024 14:54:01 +0200
Subject: [PATCH 10/10] borgbackup: move preBackupScript to a different systemd
 context

---
 clanModules/borgbackup/default.nix | 54 +++++++++++++++++-------------
 1 file changed, 31 insertions(+), 23 deletions(-)

diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix
index 2e16b3b9..1b9bdbbf 100644
--- a/clanModules/borgbackup/default.nix
+++ b/clanModules/borgbackup/default.nix
@@ -6,6 +6,27 @@
 }:
 let
   cfg = config.clan.borgbackup;
+  preBackupScript = ''
+    declare -A preCommandErrors
+
+    ${lib.concatMapStringsSep "\n" (
+      state:
+      lib.optionalString (state.preBackupCommand != null) ''
+        echo "Running pre-backup command for ${state.name}"
+        if ! ( ${state.preBackupCommand} ) then
+          preCommandErrors["${state.name}"]=1
+        fi
+      ''
+    ) (lib.attrValues config.clanCore.state)}
+
+    if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+      echo "PreBackupCommand failed for the following services:"
+      for state in "''${!preCommandErrors[@]}"; do
+        echo "  $state"
+      done
+      exit 1
+    fi
+  '';
 in
 {
   options.clan.borgbackup.destinations = lib.mkOption {
@@ -50,6 +71,16 @@ in
   ];
 
   config = lib.mkIf (cfg.destinations != { }) {
+    systemd.services = lib.mapAttrs' (
+      _: dest:
+      lib.nameValuePair "borgbackup-job-${dest.name}" {
+        # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
+        serviceConfig.ExecStartPre = [
+          (''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'')
+        ];
+      }
+    ) cfg.destinations;
+
     services.borgbackup.jobs = lib.mapAttrs (_: dest: {
       paths = lib.unique (
         lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state))
@@ -60,29 +91,6 @@ in
       compression = "auto,zstd";
       startAt = "*-*-* 01:00:00";
       persistentTimer = true;
-      preHook = ''
-        set -x
-        declare -A preCommandErrors
-        ${lib.concatMapStringsSep "\n" (
-          state:
-          lib.optionalString (state.preBackupCommand != null) ''
-            echo "Running pre-backup command for ${state.name}"
-            if ! ( ${state.preBackupCommand} ) then
-              preCommandErrors["${state.name}"]=1
-            fi
-          ''
-        ) (lib.attrValues config.clanCore.state)}
-      '';
-      postPrune = ''
-        # report any preBackupCommand errors
-        if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
-          echo "PreBackupCommand failed for the following services:"
-          for state in "''${!preCommandErrors[@]}"; do
-            echo "  $state"
-          done
-          exit 1
-        fi
-      '';
 
       encryption = {
         mode = "repokey";