forked from clan/clan-core
postgresql: add backup and restore
This commit is contained in:
parent
c0d8aaf73a
commit
0dc3b9f056
@ -1,22 +1,51 @@
|
||||
(import ../lib/container-test.nix) (
|
||||
{
|
||||
name = "postgresql";
|
||||
(import ../lib/container-test.nix) ({
|
||||
name = "postgresql";
|
||||
|
||||
nodes.machine = { self, ... }:
|
||||
{
|
||||
imports = [
|
||||
self.nixosModules.clanCore
|
||||
self.clanModules.postgresql
|
||||
self.clanModules.localbackup
|
||||
];
|
||||
clan.postgresl.databases.test = {};
|
||||
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
||||
};
|
||||
testScript = ''
|
||||
nodes.machine =
|
||||
{ self, config, ... }:
|
||||
{
|
||||
imports = [
|
||||
self.nixosModules.clanCore
|
||||
self.clanModules.postgresql
|
||||
self.clanModules.localbackup
|
||||
];
|
||||
clan.postgresql.users.test = { };
|
||||
clan.postgresql.databases.test.create.options.OWNER = "test";
|
||||
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
||||
|
||||
environment.systemPackages = [ config.services.postgresql.package ];
|
||||
};
|
||||
testScript =
|
||||
{ nodes, ... }:
|
||||
''
|
||||
start_all()
|
||||
machine.succeed("systemctl status postgresql")
|
||||
machine.wait_for_unit("postgresql")
|
||||
# Create a test table
|
||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
|
||||
|
||||
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
|
||||
machine.succeed("ls -la /var/backups/postgresql")
|
||||
|
||||
machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/dump.sql.zstd || { echo 'dump.sql.zstd not found'; exit 1; }")
|
||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
|
||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
|
||||
machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
|
||||
|
||||
machine.succeed("rm -rf /var/backup/postgres")
|
||||
|
||||
machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
|
||||
machine.succeed("test -e /var/backup/postgres/test/dump.sql.zstd || { echo 'backup.info not found'; exit 1; }")
|
||||
|
||||
machine.succeed("""
|
||||
set -x
|
||||
${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand}
|
||||
""")
|
||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
|
||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
||||
|
||||
# Check that the table is still there
|
||||
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
|
||||
output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
|
||||
owner = output.split("\n")[1]
|
||||
assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
|
||||
'';
|
||||
})
|
||||
|
@ -63,12 +63,15 @@ in
|
||||
preHook = ''
|
||||
set -x
|
||||
declare -A preCommandErrors
|
||||
${lib.concatMapStringsSep "\n" (state: ''
|
||||
echo "Running pre-backup command for ${state.name}"
|
||||
if ! ${state.preBackupCommand} then
|
||||
preCommandErrors["${state.name}"]=1
|
||||
fi
|
||||
'') (lib.attrValues config.clanCore.state)}
|
||||
${lib.concatMapStringsSep "\n" (
|
||||
state:
|
||||
lib.optionalString (state.preBackupCommand != null) ''
|
||||
echo "Running pre-backup command for ${state.name}"
|
||||
if ! ( ${state.preBackupCommand} ) then
|
||||
preCommandErrors["${state.name}"]=1
|
||||
fi
|
||||
''
|
||||
) (lib.attrValues config.clanCore.state)}
|
||||
'';
|
||||
postPrune = ''
|
||||
# report any preBackupCommand errors
|
||||
|
@ -6,7 +6,9 @@
|
||||
}:
|
||||
let
|
||||
cfg = config.clan.localbackup;
|
||||
uniqueFolders = lib.unique (lib.flatten (lib.mapAttrsToList (name: state: state.folders) config.clanCore.state));
|
||||
uniqueFolders = lib.unique (
|
||||
lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
|
||||
);
|
||||
rsnapshotConfig = target: ''
|
||||
config_version 1.2
|
||||
snapshot_root ${target.directory}
|
||||
@ -18,19 +20,6 @@ let
|
||||
cmd_logger ${pkgs.inetutils}/bin/logger
|
||||
cmd_du ${pkgs.coreutils}/bin/du
|
||||
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
|
||||
${lib.optionalString (target.preBackupHook != null) ''
|
||||
cmd_preexec ${pkgs.writeShellScript "preexec.sh" ''
|
||||
set -efu -o pipefail
|
||||
${target.preBackupHook}
|
||||
|
||||
# FIXME: we currently fail the backup if the pre-backup command fails
|
||||
# This is not ideal, but at least most of the time we run backup commands in foreground.
|
||||
${lib.concatMapStringsSep "\n" (state: ''
|
||||
echo "Running pre-backup command for ${state.name}"
|
||||
${state.preBackupCommand}
|
||||
'') (lib.attrValues config.clanCore.state)}
|
||||
''}
|
||||
''}
|
||||
|
||||
${lib.optionalString (target.postBackupHook != null) ''
|
||||
cmd_postexec ${pkgs.writeShellScript "postexec.sh" ''
|
||||
@ -40,7 +29,7 @@ let
|
||||
''}
|
||||
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
|
||||
${lib.concatMapStringsSep "\n" (folder: ''
|
||||
backup ${folder} ${config.networking.hostName}/
|
||||
backup ${folder} ${config.networking.hostName}/
|
||||
'') uniqueFolders}
|
||||
'';
|
||||
in
|
||||
@ -135,14 +124,30 @@ in
|
||||
]
|
||||
}
|
||||
${lib.concatMapStringsSep "\n" (target: ''
|
||||
(
|
||||
${mountHook target}
|
||||
echo "Creating backup '${target.name}'"
|
||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
|
||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
|
||||
)
|
||||
'') (builtins.attrValues cfg.targets)}
|
||||
'')
|
||||
${mountHook target}
|
||||
set -x
|
||||
echo "Creating backup '${target.name}'"
|
||||
|
||||
${lib.optionalString (target.preBackupHook != null) ''
|
||||
(
|
||||
${target.preBackupHook}
|
||||
)
|
||||
''}
|
||||
|
||||
declare -A preCommandErrors
|
||||
${lib.concatMapStringsSep "\n" (
|
||||
state:
|
||||
lib.optionalString (state.preBackupCommand != null) ''
|
||||
echo "Running pre-backup command for ${state.name}"
|
||||
if ! ( ${state.preBackupCommand} ) then
|
||||
preCommandErrors["${state.name}"]=1
|
||||
fi
|
||||
''
|
||||
) (builtins.attrValues config.clanCore.state)}
|
||||
|
||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
|
||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
|
||||
'') (builtins.attrValues cfg.targets)}'')
|
||||
(pkgs.writeShellScriptBin "localbackup-list" ''
|
||||
set -efu -o pipefail
|
||||
export PATH=${
|
||||
@ -173,6 +178,14 @@ in
|
||||
pkgs.gawk
|
||||
]
|
||||
}
|
||||
if [[ "''${NAME:-}" == "" ]]; then
|
||||
echo "No backup name given via NAME environment variable"
|
||||
exit 1
|
||||
fi
|
||||
if [[ "''${FOLDERS:-}" == "" ]]; then
|
||||
echo "No folders given via FOLDERS environment variable"
|
||||
exit 1
|
||||
fi
|
||||
name=$(awk -F'::' '{print $1}' <<< $NAME)
|
||||
backupname=''${NAME#$name::}
|
||||
|
||||
@ -188,8 +201,9 @@ in
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IFS=';' read -ra FOLDER <<< "$FOLDERS"
|
||||
IFS=':' read -ra FOLDER <<< "''$FOLDERS"
|
||||
for folder in "''${FOLDER[@]}"; do
|
||||
mkdir -p "$folder"
|
||||
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
|
||||
done
|
||||
'')
|
||||
|
@ -8,32 +8,56 @@ let
|
||||
createDatatbaseState =
|
||||
db:
|
||||
let
|
||||
folder = "/var/backup/postgresql/${db.name}";
|
||||
curFile = "${folder}/dump.sql.zstd";
|
||||
prevFile = "${folder}/dump.sql.prev.zstd";
|
||||
inProgressFile = "${folder}/dump.sql.in-progress.zstd";
|
||||
folder = "/var/backup/postgres/${db.name}";
|
||||
current = "${folder}/dump.sql.zstd";
|
||||
in
|
||||
{
|
||||
folders = [ folder ];
|
||||
preBackupCommand = ''
|
||||
(
|
||||
umask 0077 # ensure backup is only readable by postgres user
|
||||
if [ -e ${curFile} ]; then
|
||||
mv ${curFile} ${prevFile}
|
||||
fi
|
||||
while [[ "$(systemctl is-active postgres)" == activating ]]; then
|
||||
sleep 1
|
||||
done
|
||||
systemctl is-active postgres
|
||||
pg_dump -C ${db.name} | \
|
||||
${pkgs.zstd}/bin/zstd --rsyncable | \
|
||||
> ${inProgressFile}
|
||||
mv ${inProgressFile} ${curFile}
|
||||
)
|
||||
export PATH=${
|
||||
lib.makeBinPath [
|
||||
config.services.postgresql.package
|
||||
config.systemd.package
|
||||
pkgs.coreutils
|
||||
pkgs.util-linux
|
||||
pkgs.zstd
|
||||
]
|
||||
}
|
||||
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
mkdir -p "${folder}"
|
||||
runuser -u postgres -- pg_dump --compress=zstd --dbname=${db.name} -Fc -c > "${current}.tmp"
|
||||
mv "${current}.tmp" ${current}
|
||||
'';
|
||||
|
||||
postRestoreCommand = ''
|
||||
if [[ -f ${prevFile} ]]; then
|
||||
zstd --decompress --stdout ${prevFile} | psql -d ${db.name}
|
||||
export PATH=${
|
||||
lib.makeBinPath [
|
||||
config.services.postgresql.package
|
||||
config.systemd.package
|
||||
pkgs.coreutils
|
||||
pkgs.util-linux
|
||||
pkgs.zstd
|
||||
]
|
||||
}
|
||||
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||
sleep 1
|
||||
done
|
||||
echo "Waiting for postgres to be ready..."
|
||||
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
||||
if ! systemctl is-active postgresql; then exit 1; fi
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
if [[ -e "${current}" ]]; then
|
||||
umask 077 # only root can read the backup
|
||||
mkdir -p "${folder}"
|
||||
runuser -u postgres -- dropdb "${db.name}"
|
||||
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
||||
else
|
||||
echo No database backup found, skipping restore
|
||||
fi
|
||||
'';
|
||||
};
|
||||
@ -41,23 +65,24 @@ let
|
||||
createDatabase = db: ''
|
||||
CREATE DATABASE "${db.name}" ${
|
||||
lib.concatStringsSep " " (
|
||||
lib.mapAttrsToList (name: value: "${name} = ':${value}'") db.createOptions
|
||||
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
|
||||
)
|
||||
}
|
||||
'';
|
||||
cfg = config.clan.postgresql;
|
||||
|
||||
userClauses = lib.mapAttrsToList (
|
||||
_: user: ""
|
||||
_: user:
|
||||
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
|
||||
) cfg.users;
|
||||
databaseClauses = lib.mapAttrsToList (
|
||||
name: db:
|
||||
lib.optionalString (db.create) ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ${createDatabaseArgs db}''
|
||||
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
|
||||
) cfg.databases;
|
||||
in
|
||||
{
|
||||
options.clan.postgresl = {
|
||||
options.clan.postgresql = {
|
||||
# we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
|
||||
databases = lib.mkOption {
|
||||
default = { };
|
||||
type = lib.types.attrsOf (
|
||||
@ -70,11 +95,12 @@ in
|
||||
default = name;
|
||||
};
|
||||
# set to false, in case the upstream module uses ensureDatabase option
|
||||
create = lib.mkOption {
|
||||
create.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Create the database if it does not exist.";
|
||||
};
|
||||
createOptions = lib.mkOption {
|
||||
create.options = lib.mkOption {
|
||||
type = lib.types.lazyAttrsOf lib.types.str;
|
||||
default = { };
|
||||
example = {
|
||||
@ -85,6 +111,11 @@ in
|
||||
OWNER = "foo";
|
||||
};
|
||||
};
|
||||
backup.enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = true;
|
||||
description = "Backup the database.";
|
||||
};
|
||||
};
|
||||
}
|
||||
)
|
||||
@ -106,18 +137,24 @@ in
|
||||
};
|
||||
};
|
||||
config = {
|
||||
services.postgresql.settings = {
|
||||
wal_level = "replica";
|
||||
max_wal_senders = 3;
|
||||
};
|
||||
|
||||
services.postgresql.enable = true;
|
||||
# We are duplicating a bit the upstream module but allow to create databases with options
|
||||
systemd.services.postgresql.postStart = ''
|
||||
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
|
||||
|
||||
while ! $PSQL -d postgres -c "" 2> /dev/null; do
|
||||
if ! kill -0 "$MAINPID"; then exit 1; fi
|
||||
sleep 0.1
|
||||
if ! kill -0 "$MAINPID"; then exit 1; fi
|
||||
sleep 0.1
|
||||
done
|
||||
${lib.concatStringsSep "\n" userClauses}
|
||||
${lib.concatStringsSep "\n" databaseClauses}
|
||||
'';
|
||||
|
||||
clanCore.state = lib.mapAttrs' (
|
||||
_: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
|
||||
) config.clan.postgresql.databases;
|
||||
|
@ -8,9 +8,16 @@
|
||||
default = { };
|
||||
type = lib.types.attrsOf (
|
||||
lib.types.submodule (
|
||||
{ ... }:
|
||||
{ name, ... }:
|
||||
{
|
||||
options = {
|
||||
name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = name;
|
||||
description = ''
|
||||
Name of the state
|
||||
'';
|
||||
};
|
||||
folders = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = ''
|
||||
|
@ -14,9 +14,15 @@ from ..machines.machines import Machine
|
||||
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
|
||||
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
|
||||
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
||||
|
||||
if service not in backup_folders:
|
||||
msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
|
||||
raise ClanError(msg)
|
||||
|
||||
folders = backup_folders[service]["folders"]
|
||||
env = {}
|
||||
env["NAME"] = name
|
||||
# FIXME: If we have too many folder this might overflow the stack.
|
||||
env["FOLDERS"] = ":".join(set(folders))
|
||||
|
||||
if pre_restore := backup_folders[service]["preRestoreCommand"]:
|
||||
|
Loading…
Reference in New Issue
Block a user