postgresql: add backup and restore
Some checks failed
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-fakeroot Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-iso-installer Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.11-mypy" Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.11-qemu" Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
checks / checks-impure (pull_request) Successful in 2m32s
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-eval Build done.
Some checks failed
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-fakeroot Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-iso-installer Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.11-mypy" Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.11-qemu" Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
checks / checks-impure (pull_request) Successful in 2m32s
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-eval Build done.
This commit is contained in:
parent
c0d8aaf73a
commit
0dc3b9f056
|
@ -1,22 +1,51 @@
|
||||||
(import ../lib/container-test.nix) (
|
(import ../lib/container-test.nix) ({
|
||||||
{
|
name = "postgresql";
|
||||||
name = "postgresql";
|
|
||||||
|
|
||||||
nodes.machine = { self, ... }:
|
nodes.machine =
|
||||||
{
|
{ self, config, ... }:
|
||||||
imports = [
|
{
|
||||||
self.nixosModules.clanCore
|
imports = [
|
||||||
self.clanModules.postgresql
|
self.nixosModules.clanCore
|
||||||
self.clanModules.localbackup
|
self.clanModules.postgresql
|
||||||
];
|
self.clanModules.localbackup
|
||||||
clan.postgresl.databases.test = {};
|
];
|
||||||
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
clan.postgresql.users.test = { };
|
||||||
};
|
clan.postgresql.databases.test.create.options.OWNER = "test";
|
||||||
testScript = ''
|
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
||||||
|
|
||||||
|
environment.systemPackages = [ config.services.postgresql.package ];
|
||||||
|
};
|
||||||
|
testScript =
|
||||||
|
{ nodes, ... }:
|
||||||
|
''
|
||||||
start_all()
|
start_all()
|
||||||
machine.succeed("systemctl status postgresql")
|
|
||||||
machine.wait_for_unit("postgresql")
|
machine.wait_for_unit("postgresql")
|
||||||
|
# Create a test table
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
|
||||||
|
|
||||||
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
|
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
|
||||||
machine.succeed("ls -la /var/backups/postgresql")
|
|
||||||
|
machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/dump.sql.zstd || { echo 'dump.sql.zstd not found'; exit 1; }")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
|
||||||
|
machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
|
||||||
|
|
||||||
|
machine.succeed("rm -rf /var/backup/postgres")
|
||||||
|
|
||||||
|
machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
|
||||||
|
machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
|
||||||
|
|
||||||
|
machine.succeed("""
|
||||||
|
set -x
|
||||||
|
${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand}
|
||||||
|
""")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
||||||
|
|
||||||
|
# Check that the table is still there
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
|
||||||
|
output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
|
||||||
|
owner = output.split("\n")[1]
|
||||||
|
assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
|
||||||
'';
|
'';
|
||||||
})
|
})
|
||||||
|
|
|
@ -63,12 +63,15 @@ in
|
||||||
preHook = ''
|
preHook = ''
|
||||||
set -x
|
set -x
|
||||||
declare -A preCommandErrors
|
declare -A preCommandErrors
|
||||||
${lib.concatMapStringsSep "\n" (state: ''
|
${lib.concatMapStringsSep "\n" (
|
||||||
echo "Running pre-backup command for ${state.name}"
|
state:
|
||||||
if ! ${state.preBackupCommand} then
|
lib.optionalString (state.preBackupCommand != null) ''
|
||||||
preCommandErrors["${state.name}"]=1
|
echo "Running pre-backup command for ${state.name}"
|
||||||
fi
|
if ! ( ${state.preBackupCommand} ) then
|
||||||
'') (lib.attrValues config.clanCore.state)}
|
preCommandErrors["${state.name}"]=1
|
||||||
|
fi
|
||||||
|
''
|
||||||
|
) (lib.attrValues config.clanCore.state)}
|
||||||
'';
|
'';
|
||||||
postPrune = ''
|
postPrune = ''
|
||||||
# report any preBackupCommand errors
|
# report any preBackupCommand errors
|
||||||
|
|
|
@ -6,7 +6,9 @@
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
cfg = config.clan.localbackup;
|
cfg = config.clan.localbackup;
|
||||||
uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state));
|
uniqueFolders = lib.unique (
|
||||||
|
lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
|
||||||
|
);
|
||||||
rsnapshotConfig = target: ''
|
rsnapshotConfig = target: ''
|
||||||
config_version 1.2
|
config_version 1.2
|
||||||
snapshot_root ${target.directory}
|
snapshot_root ${target.directory}
|
||||||
|
@ -18,19 +20,6 @@ let
|
||||||
cmd_logger ${pkgs.inetutils}/bin/logger
|
cmd_logger ${pkgs.inetutils}/bin/logger
|
||||||
cmd_du ${pkgs.coreutils}/bin/du
|
cmd_du ${pkgs.coreutils}/bin/du
|
||||||
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
|
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
|
||||||
${lib.optionalString (target.preBackupHook != null) ''
|
|
||||||
cmd_preexec ${pkgs.writeShellScript "preexec.sh" ''
|
|
||||||
set -efu -o pipefail
|
|
||||||
${target.preBackupHook}
|
|
||||||
|
|
||||||
# FIXME: we currently fail the backup if the pre-backup command fails
|
|
||||||
# This is not ideal, but at least most of the time we run backup commands in foreground.
|
|
||||||
${lib.concatMapStringsSep "\n" (state: ''
|
|
||||||
echo "Running pre-backup command for ${state.name}"
|
|
||||||
${state.preBackupCommand}
|
|
||||||
'') (lib.attrValues config.clanCore.state)}
|
|
||||||
''}
|
|
||||||
''}
|
|
||||||
|
|
||||||
${lib.optionalString (target.postBackupHook != null) ''
|
${lib.optionalString (target.postBackupHook != null) ''
|
||||||
cmd_postexec ${pkgs.writeShellScript "postexec.sh" ''
|
cmd_postexec ${pkgs.writeShellScript "postexec.sh" ''
|
||||||
|
@ -40,7 +29,7 @@ let
|
||||||
''}
|
''}
|
||||||
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
|
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
|
||||||
${lib.concatMapStringsSep "\n" (folder: ''
|
${lib.concatMapStringsSep "\n" (folder: ''
|
||||||
backup ${folder} ${config.networking.hostName}/
|
backup ${folder} ${config.networking.hostName}/
|
||||||
'') uniqueFolders}
|
'') uniqueFolders}
|
||||||
'';
|
'';
|
||||||
in
|
in
|
||||||
|
@ -135,14 +124,30 @@ in
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
${lib.concatMapStringsSep "\n" (target: ''
|
${lib.concatMapStringsSep "\n" (target: ''
|
||||||
(
|
${mountHook target}
|
||||||
${mountHook target}
|
set -x
|
||||||
echo "Creating backup '${target.name}'"
|
echo "Creating backup '${target.name}'"
|
||||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
|
|
||||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
|
${lib.optionalString (target.preBackupHook != null) ''
|
||||||
)
|
(
|
||||||
'') (builtins.attrValues cfg.targets)}
|
${target.preBackupHook}
|
||||||
'')
|
)
|
||||||
|
''}
|
||||||
|
|
||||||
|
declare -A preCommandErrors
|
||||||
|
${lib.concatMapStringsSep "\n" (
|
||||||
|
state:
|
||||||
|
lib.optionalString (state.preBackupCommand != null) ''
|
||||||
|
echo "Running pre-backup command for ${state.name}"
|
||||||
|
if ! ( ${state.preBackupCommand} ) then
|
||||||
|
preCommandErrors["${state.name}"]=1
|
||||||
|
fi
|
||||||
|
''
|
||||||
|
) (builtins.attrValues config.clanCore.state)}
|
||||||
|
|
||||||
|
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
|
||||||
|
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
|
||||||
|
'') (builtins.attrValues cfg.targets)}'')
|
||||||
(pkgs.writeShellScriptBin "localbackup-list" ''
|
(pkgs.writeShellScriptBin "localbackup-list" ''
|
||||||
set -efu -o pipefail
|
set -efu -o pipefail
|
||||||
export PATH=${
|
export PATH=${
|
||||||
|
@ -173,6 +178,14 @@ in
|
||||||
pkgs.gawk
|
pkgs.gawk
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
if [[ "''${NAME:-}" == "" ]]; then
|
||||||
|
echo "No backup name given via NAME environment variable"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ "''${FOLDERS:-}" == "" ]]; then
|
||||||
|
echo "No folders given via FOLDERS environment variable"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
name=$(awk -F'::' '{print $1}' <<< $NAME)
|
name=$(awk -F'::' '{print $1}' <<< $NAME)
|
||||||
backupname=''${NAME#$name::}
|
backupname=''${NAME#$name::}
|
||||||
|
|
||||||
|
@ -188,8 +201,9 @@ in
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
IFS=';' read -ra FOLDER <<< "$FOLDERS"
|
IFS=':' read -ra FOLDER <<< "''$FOLDERS"
|
||||||
for folder in "''${FOLDER[@]}"; do
|
for folder in "''${FOLDER[@]}"; do
|
||||||
|
mkdir -p "$folder"
|
||||||
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
|
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
|
||||||
done
|
done
|
||||||
'')
|
'')
|
||||||
|
|
|
@ -8,32 +8,56 @@ let
|
||||||
createDatatbaseState =
|
createDatatbaseState =
|
||||||
db:
|
db:
|
||||||
let
|
let
|
||||||
folder = "/var/backup/postgresql/${db.name}";
|
folder = "/var/backup/postgres/${db.name}";
|
||||||
curFile = "${folder}/dump.sql.zstd";
|
current = "${folder}/dump.sql.zstd";
|
||||||
prevFile = "${folder}/dump.sql.prev.zstd";
|
|
||||||
inProgressFile = "${folder}/dump.sql.in-progress.zstd";
|
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
folders = [ folder ];
|
folders = [ folder ];
|
||||||
preBackupCommand = ''
|
preBackupCommand = ''
|
||||||
(
|
export PATH=${
|
||||||
umask 0077 # ensure backup is only readable by postgres user
|
lib.makeBinPath [
|
||||||
if [ -e ${curFile} ]; then
|
config.services.postgresql.package
|
||||||
mv ${curFile} ${prevFile}
|
config.systemd.package
|
||||||
fi
|
pkgs.coreutils
|
||||||
while [[ "$(systemctl is-active postgres)" == activating ]]; then
|
pkgs.util-linux
|
||||||
sleep 1
|
pkgs.zstd
|
||||||
done
|
]
|
||||||
systemctl is-active postgres
|
}
|
||||||
pg_dump -C ${db.name} | \
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
${pkgs.zstd}/bin/zstd --rsyncable | \
|
sleep 1
|
||||||
> ${inProgressFile}
|
done
|
||||||
mv ${inProgressFile} ${curFile}
|
|
||||||
)
|
mkdir -p "${folder}"
|
||||||
|
runuser -u postgres -- pg_dump --compress=zstd --dbname=${db.name} -Fc -c > "${current}.tmp"
|
||||||
|
mv "${current}.tmp" ${current}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
postRestoreCommand = ''
|
postRestoreCommand = ''
|
||||||
if [[ -f ${prevFile} ]]; then
|
export PATH=${
|
||||||
zstd --decompress --stdout ${prevFile} | psql -d ${db.name}
|
lib.makeBinPath [
|
||||||
|
config.services.postgresql.package
|
||||||
|
config.systemd.package
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.util-linux
|
||||||
|
pkgs.zstd
|
||||||
|
]
|
||||||
|
}
|
||||||
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "Waiting for postgres to be ready..."
|
||||||
|
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
||||||
|
if ! systemctl is-active postgresql; then exit 1; fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -e "${current}" ]]; then
|
||||||
|
umask 077 # only root can read the backup
|
||||||
|
mkdir -p "${folder}"
|
||||||
|
runuser -u postgres -- dropdb "${db.name}"
|
||||||
|
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
||||||
|
else
|
||||||
|
echo No database backup found, skipping restore
|
||||||
fi
|
fi
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
@ -41,23 +65,24 @@ let
|
||||||
createDatabase = db: ''
|
createDatabase = db: ''
|
||||||
CREATE DATABASE "${db.name}" ${
|
CREATE DATABASE "${db.name}" ${
|
||||||
lib.concatStringsSep " " (
|
lib.concatStringsSep " " (
|
||||||
lib.mapAttrsToList (name: value: "${name} = ':${value}'") db.createOptions
|
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
'';
|
'';
|
||||||
cfg = config.clan.postgresql;
|
cfg = config.clan.postgresql;
|
||||||
|
|
||||||
userClauses = lib.mapAttrsToList (
|
userClauses = lib.mapAttrsToList (
|
||||||
_: user: ""
|
_: user:
|
||||||
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
|
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
|
||||||
) cfg.users;
|
) cfg.users;
|
||||||
databaseClauses = lib.mapAttrsToList (
|
databaseClauses = lib.mapAttrsToList (
|
||||||
name: db:
|
name: db:
|
||||||
lib.optionalString (db.create) ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ${createDatabaseArgs db}''
|
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
|
||||||
) cfg.databases;
|
) cfg.databases;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.clan.postgresl = {
|
options.clan.postgresql = {
|
||||||
|
# we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
|
||||||
databases = lib.mkOption {
|
databases = lib.mkOption {
|
||||||
default = { };
|
default = { };
|
||||||
type = lib.types.attrsOf (
|
type = lib.types.attrsOf (
|
||||||
|
@ -70,11 +95,12 @@ in
|
||||||
default = name;
|
default = name;
|
||||||
};
|
};
|
||||||
# set to false, in case the upstream module uses ensureDatabase option
|
# set to false, in case the upstream module uses ensureDatabase option
|
||||||
create = lib.mkOption {
|
create.enable = lib.mkOption {
|
||||||
type = lib.types.bool;
|
type = lib.types.bool;
|
||||||
default = true;
|
default = true;
|
||||||
|
description = "Create the database if it does not exist.";
|
||||||
};
|
};
|
||||||
createOptions = lib.mkOption {
|
create.options = lib.mkOption {
|
||||||
type = lib.types.lazyAttrsOf lib.types.str;
|
type = lib.types.lazyAttrsOf lib.types.str;
|
||||||
default = { };
|
default = { };
|
||||||
example = {
|
example = {
|
||||||
|
@ -85,6 +111,11 @@ in
|
||||||
OWNER = "foo";
|
OWNER = "foo";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
backup.enable = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = "Backup the database.";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
@ -106,18 +137,24 @@ in
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
config = {
|
config = {
|
||||||
|
services.postgresql.settings = {
|
||||||
|
wal_level = "replica";
|
||||||
|
max_wal_senders = 3;
|
||||||
|
};
|
||||||
|
|
||||||
services.postgresql.enable = true;
|
services.postgresql.enable = true;
|
||||||
# We are duplicating a bit the upstream module but allow to create databases with options
|
# We are duplicating a bit the upstream module but allow to create databases with options
|
||||||
systemd.services.postgresql.postStart = ''
|
systemd.services.postgresql.postStart = ''
|
||||||
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
|
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
|
||||||
|
|
||||||
while ! $PSQL -d postgres -c "" 2> /dev/null; do
|
while ! $PSQL -d postgres -c "" 2> /dev/null; do
|
||||||
if ! kill -0 "$MAINPID"; then exit 1; fi
|
if ! kill -0 "$MAINPID"; then exit 1; fi
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
done
|
done
|
||||||
${lib.concatStringsSep "\n" userClauses}
|
${lib.concatStringsSep "\n" userClauses}
|
||||||
${lib.concatStringsSep "\n" databaseClauses}
|
${lib.concatStringsSep "\n" databaseClauses}
|
||||||
'';
|
'';
|
||||||
|
|
||||||
clanCore.state = lib.mapAttrs' (
|
clanCore.state = lib.mapAttrs' (
|
||||||
_: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
|
_: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
|
||||||
) config.clan.postgresql.databases;
|
) config.clan.postgresql.databases;
|
||||||
|
|
|
@ -8,9 +8,16 @@
|
||||||
default = { };
|
default = { };
|
||||||
type = lib.types.attrsOf (
|
type = lib.types.attrsOf (
|
||||||
lib.types.submodule (
|
lib.types.submodule (
|
||||||
{ ... }:
|
{ name, ... }:
|
||||||
{
|
{
|
||||||
options = {
|
options = {
|
||||||
|
name = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
description = ''
|
||||||
|
Name of the state
|
||||||
|
'';
|
||||||
|
};
|
||||||
folders = lib.mkOption {
|
folders = lib.mkOption {
|
||||||
type = lib.types.listOf lib.types.str;
|
type = lib.types.listOf lib.types.str;
|
||||||
description = ''
|
description = ''
|
||||||
|
|
|
@ -14,9 +14,15 @@ from ..machines.machines import Machine
|
||||||
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
|
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
|
||||||
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
|
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
|
||||||
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
||||||
|
|
||||||
|
if service not in backup_folders:
|
||||||
|
msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
|
||||||
|
raise ClanError(msg)
|
||||||
|
|
||||||
folders = backup_folders[service]["folders"]
|
folders = backup_folders[service]["folders"]
|
||||||
env = {}
|
env = {}
|
||||||
env["NAME"] = name
|
env["NAME"] = name
|
||||||
|
# FIXME: If we have too many folder this might overflow the stack.
|
||||||
env["FOLDERS"] = ":".join(set(folders))
|
env["FOLDERS"] = ":".join(set(folders))
|
||||||
|
|
||||||
if pre_restore := backup_folders[service]["preRestoreCommand"]:
|
if pre_restore := backup_folders[service]["preRestoreCommand"]:
|
||||||
|
|
Loading…
Reference in New Issue
Block a user