From b19ac7e490ad5a76b0bf11d815b01a8b69bf4ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 5 Jun 2024 18:37:31 +0200 Subject: [PATCH] postgresql: add backup and restore --- checks/postgresql/default.nix | 61 ++++++++--- clanModules/borgbackup/default.nix | 15 +-- clanModules/localbackup/default.nix | 62 ++++++----- clanModules/postgresql/default.nix | 121 +++++++++++++++++----- nixosModules/clanCore/state.nix | 9 +- pkgs/clan-cli/clan_cli/backups/restore.py | 6 ++ 6 files changed, 199 insertions(+), 75 deletions(-) diff --git a/checks/postgresql/default.nix b/checks/postgresql/default.nix index 42baf9ca..2a0eeadc 100644 --- a/checks/postgresql/default.nix +++ b/checks/postgresql/default.nix @@ -1,22 +1,51 @@ -(import ../lib/container-test.nix) ( - { - name = "postgresql"; +(import ../lib/container-test.nix) ({ + name = "postgresql"; - nodes.machine = { self, ... }: - { - imports = [ - self.nixosModules.clanCore - self.clanModules.postgresql - self.clanModules.localbackup - ]; - clan.postgresl.databases.test = {}; - clan.localbackup.targets.hdd.directory = "/mnt/external-disk"; - }; - testScript = '' + nodes.machine = + { self, config, ... }: + { + imports = [ + self.nixosModules.clanCore + self.clanModules.postgresql + self.clanModules.localbackup + ]; + clan.postgresql.users.test = { }; + clan.postgresql.databases.test.create.options.OWNER = "test"; + clan.localbackup.targets.hdd.directory = "/mnt/external-disk"; + + environment.systemPackages = [ config.services.postgresql.package ]; + }; + testScript = + { nodes, ... }: + '' start_all() - machine.succeed("systemctl status postgresql") machine.wait_for_unit("postgresql") + # Create a test table + machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test") + machine.succeed("/run/current-system/sw/bin/localbackup-create >&2") - machine.succeed("ls -la /var/backups/postgresql") + + machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/dump.sql.zstd || { echo 'dump.sql.zstd not found'; exit 1; }") + machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'") + machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'") + machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }") + + machine.succeed("rm -rf /var/backup/postgres") + + machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2") + machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }") + + machine.succeed(""" + set -x + ${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand} + """) + machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2") + machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2") + + # Check that the table is still there + machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'") + output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"") + owner = output.split("\n")[1] + assert owner == "test", f"Expected database owner to be 'test', got '{owner}'" ''; }) diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix index aef8dd6d..2e16b3b9 100644 --- a/clanModules/borgbackup/default.nix +++ b/clanModules/borgbackup/default.nix @@ -63,12 +63,15 @@ in preHook = '' set -x declare -A preCommandErrors - ${lib.concatMapStringsSep "\n" (state: '' - echo "Running pre-backup command for ${state.name}" - if ! ${state.preBackupCommand} then - preCommandErrors["${state.name}"]=1 - fi - '') (lib.attrValues config.clanCore.state)} + ${lib.concatMapStringsSep "\n" ( + state: + lib.optionalString (state.preBackupCommand != null) '' + echo "Running pre-backup command for ${state.name}" + if ! ( ${state.preBackupCommand} ) then + preCommandErrors["${state.name}"]=1 + fi + '' + ) (lib.attrValues config.clanCore.state)} ''; postPrune = '' # report any preBackupCommand errors diff --git a/clanModules/localbackup/default.nix b/clanModules/localbackup/default.nix index c957b88d..917406a8 100644 --- a/clanModules/localbackup/default.nix +++ b/clanModules/localbackup/default.nix @@ -6,7 +6,9 @@ }: let cfg = config.clan.localbackup; - uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state)); + uniqueFolders = lib.unique ( + lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state) + ); rsnapshotConfig = target: '' config_version 1.2 snapshot_root ${target.directory} @@ -18,19 +20,6 @@ let cmd_logger ${pkgs.inetutils}/bin/logger cmd_du ${pkgs.coreutils}/bin/du cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff - ${lib.optionalString (target.preBackupHook != null) '' - cmd_preexec ${pkgs.writeShellScript "preexec.sh" '' - set -efu -o pipefail - ${target.preBackupHook} - - # FIXME: we currently fail the backup if the pre-backup command fails - # This is not ideal, but at least most of the time we run backup commands in foreground. - ${lib.concatMapStringsSep "\n" (state: '' - echo "Running pre-backup command for ${state.name}" - ${state.preBackupCommand} - '') (lib.attrValues config.clanCore.state)} - ''} - ''} ${lib.optionalString (target.postBackupHook != null) '' cmd_postexec ${pkgs.writeShellScript "postexec.sh" '' @@ -40,7 +29,7 @@ let ''} retain snapshot ${builtins.toString config.clan.localbackup.snapshots} ${lib.concatMapStringsSep "\n" (folder: '' - backup ${folder} ${config.networking.hostName}/ + backup ${folder} ${config.networking.hostName}/ '') uniqueFolders} ''; in @@ -135,14 +124,30 @@ in ] } ${lib.concatMapStringsSep "\n" (target: '' - ( - ${mountHook target} - echo "Creating backup '${target.name}'" - rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync - rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot - ) - '') (builtins.attrValues cfg.targets)} - '') + ${mountHook target} + set -x + echo "Creating backup '${target.name}'" + + ${lib.optionalString (target.preBackupHook != null) '' + ( + ${target.preBackupHook} + ) + ''} + + declare -A preCommandErrors + ${lib.concatMapStringsSep "\n" ( + state: + lib.optionalString (state.preBackupCommand != null) '' + echo "Running pre-backup command for ${state.name}" + if ! ( ${state.preBackupCommand} ) then + preCommandErrors["${state.name}"]=1 + fi + '' + ) (builtins.attrValues config.clanCore.state)} + + rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync + rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot + '') (builtins.attrValues cfg.targets)}'') (pkgs.writeShellScriptBin "localbackup-list" '' set -efu -o pipefail export PATH=${ @@ -173,6 +178,14 @@ in pkgs.gawk ] } + if [[ "''${NAME:-}" == "" ]]; then + echo "No backup name given via NAME environment variable" + exit 1 + fi + if [[ "''${FOLDERS:-}" == "" ]]; then + echo "No folders given via FOLDERS environment variable" + exit 1 + fi name=$(awk -F'::' '{print $1}' <<< $NAME) backupname=''${NAME#$name::} @@ -188,8 +201,9 @@ in exit 1 fi - IFS=';' read -ra FOLDER <<< "$FOLDERS" + IFS=':' read -ra FOLDER <<< "''$FOLDERS" for folder in "''${FOLDER[@]}"; do + mkdir -p "$folder" rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder" done '') diff --git a/clanModules/postgresql/default.nix b/clanModules/postgresql/default.nix index 32faba94..c7e4747b 100644 --- a/clanModules/postgresql/default.nix +++ b/clanModules/postgresql/default.nix @@ -8,32 +8,56 @@ let createDatatbaseState = db: let - folder = "/var/backup/postgresql/${db.name}"; - curFile = "${folder}/dump.sql.zstd"; - prevFile = "${folder}/dump.sql.prev.zstd"; - inProgressFile = "${folder}/dump.sql.in-progress.zstd"; + folder = "/var/backup/postgres/${db.name}"; + current = "${folder}/dump.sql.zstd"; in { folders = [ folder ]; preBackupCommand = '' - ( - umask 0077 # ensure backup is only readable by postgres user - if [ -e ${curFile} ]; then - mv ${curFile} ${prevFile} - fi - while [[ "$(systemctl is-active postgres)" == activating ]]; then - sleep 1 - done - systemctl is-active postgres - pg_dump -C ${db.name} | \ - ${pkgs.zstd}/bin/zstd --rsyncable | \ - > ${inProgressFile} - mv ${inProgressFile} ${curFile} - ) + export PATH=${ + lib.makeBinPath [ + config.services.postgresql.package + config.systemd.package + pkgs.coreutils + pkgs.util-linux + pkgs.zstd + ] + } + while [[ "$(systemctl is-active postgresql)" == activating ]]; do + sleep 1 + done + + mkdir -p "${folder}" + runuser -u postgres -- pg_dump -d ${db.name} -Fc -c > "${current}.tmp" + mv "${current}.tmp" ${current} ''; + postRestoreCommand = '' - if [[ -f ${prevFile} ]]; then - zstd --decompress --stdout ${prevFile} | psql -d ${db.name} + export PATH=${ + lib.makeBinPath [ + config.services.postgresql.package + config.systemd.package + pkgs.coreutils + pkgs.util-linux + pkgs.zstd + ] + } + while [[ "$(systemctl is-active postgresql)" == activating ]]; do + sleep 1 + done + echo "Waiting for postgres to be ready..." + while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do + if ! systemctl is-active postgresql; then exit 1; fi + sleep 0.1 + done + + if [[ -e "${current}" ]]; then + umask 077 # only root can read the backup + mkdir -p "${folder}" + runuser -u postgres -- dropdb "${db.name}" + runuser -u postgres -- pg_restore -C -d postgres "${current}" + else + echo No database backup found, skipping restore fi ''; }; @@ -41,23 +65,24 @@ let createDatabase = db: '' CREATE DATABASE ${db.name} ${ lib.concatStringsSep " " ( - lib.mapAttrsToList (name: value: "${name} = ':${value}'") db.createOptions + lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options ) } ''; cfg = config.clan.postgresql; userClauses = lib.mapAttrsToList ( - _: user: "" + _: user: ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' '' ) cfg.users; databaseClauses = lib.mapAttrsToList ( name: db: - lib.optionalString (db.create) ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ${createDatabaseArgs db}'' + lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c "${createDatabase db}" '' ) cfg.databases; in { - options.clan.postgresl = { + options.clan.postgresql = { + # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options databases = lib.mkOption { default = { }; type = lib.types.attrsOf ( @@ -70,11 +95,12 @@ in default = name; }; # set to false, in case the upstream module uses ensureDatabase option - create = lib.mkOption { + create.enable = lib.mkOption { type = lib.types.bool; default = true; + description = "Create the database if it does not exist."; }; - createOptions = lib.mkOption { + create.options = lib.mkOption { type = lib.types.lazyAttrsOf lib.types.str; default = { }; example = { @@ -85,6 +111,11 @@ in OWNER = "foo"; }; }; + backup.enable = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Backup the database."; + }; }; } ) @@ -106,18 +137,52 @@ in }; }; config = { + environment.etc."pgbackrest/pgbackrest.conf".text = lib.generators.toINI { } { + "main" = { + pg1-path = config.services.postgresql.dataDir; + }; + + global = { + repo1-retention-full = 3; + repo1-type = "cifs"; + repo1-path = "/var/lib/pgbackrest"; + repo1-host-user = "postgres"; + pg1-host-user = "postgres"; + pg1-user = "postgres"; + + # Force a checkpoint to start backup immediately. + start-fast = "y"; + # Use delta restore. + delta = "y"; + + # Enable ZSTD compression. + compress-type = "zst"; + compress-level = 6; + + log-level-console = "info"; + log-level-file = "debug"; + }; + }; + + services.postgresql.settings = { + wal_level = "replica"; + max_wal_senders = 3; + }; + services.postgresql.package = pkgs.postgresql_14; + services.postgresql.enable = true; # We are duplicating a bit the upstream module but allow to create databases with options systemd.services.postgresql.postStart = '' PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}" while ! $PSQL -d postgres -c "" 2> /dev/null; do - if ! kill -0 "$MAINPID"; then exit 1; fi - sleep 0.1 + if ! kill -0 "$MAINPID"; then exit 1; fi + sleep 0.1 done ${lib.concatStringsSep "\n" userClauses} ${lib.concatStringsSep "\n" databaseClauses} ''; + clanCore.state = lib.mapAttrs' ( _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db) ) config.clan.postgresql.databases; diff --git a/nixosModules/clanCore/state.nix b/nixosModules/clanCore/state.nix index 7e562d48..24054cbc 100644 --- a/nixosModules/clanCore/state.nix +++ b/nixosModules/clanCore/state.nix @@ -8,9 +8,16 @@ default = { }; type = lib.types.attrsOf ( lib.types.submodule ( - { ... }: + { name, ... }: { options = { + name = lib.mkOption { + type = lib.types.str; + default = name; + description = '' + Name of the state + ''; + }; folders = lib.mkOption { type = lib.types.listOf lib.types.str; description = '' diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py index 3ff0e3a1..84b7e30b 100644 --- a/pkgs/clan-cli/clan_cli/backups/restore.py +++ b/pkgs/clan-cli/clan_cli/backups/restore.py @@ -14,9 +14,15 @@ from ..machines.machines import Machine def restore_service(machine: Machine, name: str, provider: str, service: str) -> None: backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups")) backup_folders = json.loads(machine.eval_nix("config.clanCore.state")) + + if service not in backup_folders: + msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}" + raise ClanError(msg) + folders = backup_folders[service]["folders"] env = {} env["NAME"] = name + # FIXME: If we have too many folder this might overflow the stack. env["FOLDERS"] = ":".join(set(folders)) if pre_restore := backup_folders[service]["preRestoreCommand"]: