Expand backup and restore capabilities w.r.t. postgresql. #1582
@ -44,6 +44,7 @@
|
|||||||
zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
|
zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
|
||||||
borgbackup = import ./borgbackup nixosTestArgs;
|
borgbackup = import ./borgbackup nixosTestArgs;
|
||||||
syncthing = import ./syncthing nixosTestArgs;
|
syncthing = import ./syncthing nixosTestArgs;
|
||||||
|
postgresql = import ./postgresql nixosTestArgs;
|
||||||
wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
|
wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -13,7 +13,6 @@
|
|||||||
clanCore.machineName = "machine";
|
clanCore.machineName = "machine";
|
||||||
clanCore.clanDir = ./.;
|
clanCore.clanDir = ./.;
|
||||||
clan.matrix-synapse = {
|
clan.matrix-synapse = {
|
||||||
enable = true;
|
|
||||||
domain = "clan.test";
|
domain = "clan.test";
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
67
checks/postgresql/default.nix
Normal file
67
checks/postgresql/default.nix
Normal file
@ -0,0 +1,67 @@
|
|||||||
|
(import ../lib/container-test.nix) ({
|
||||||
|
name = "postgresql";
|
||||||
|
|
||||||
|
nodes.machine =
|
||||||
|
{ self, config, ... }:
|
||||||
|
{
|
||||||
|
imports = [
|
||||||
|
self.nixosModules.clanCore
|
||||||
|
self.clanModules.postgresql
|
||||||
|
self.clanModules.localbackup
|
||||||
|
];
|
||||||
|
clan.postgresql.users.test = { };
|
||||||
|
clan.postgresql.databases.test.create.options.OWNER = "test";
|
||||||
|
clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
|
||||||
|
clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
|
||||||
|
|
||||||
|
systemd.services.sample-service = {
|
||||||
|
wantedBy = [ "multi-user.target" ];
|
||||||
|
script = ''
|
||||||
|
while true; do
|
||||||
|
echo "Hello, world!"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
environment.systemPackages = [ config.services.postgresql.package ];
|
||||||
|
};
|
||||||
|
testScript =
|
||||||
|
{ nodes, ... }:
|
||||||
|
''
|
||||||
|
start_all()
|
||||||
|
machine.wait_for_unit("postgresql")
|
||||||
|
machine.wait_for_unit("sample-service")
|
||||||
|
# Create a test table
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
|
||||||
|
|
||||||
|
machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
|
||||||
|
timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
|
||||||
|
|
||||||
|
machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
|
||||||
|
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
||||||
|
|
||||||
|
machine.succeed("rm -rf /var/backup/postgres")
|
||||||
|
|
||||||
|
machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
|
||||||
|
machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
|
||||||
|
|
||||||
|
machine.succeed("""
|
||||||
|
set -x
|
||||||
|
${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand}
|
||||||
|
""")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
|
||||||
|
|
||||||
|
timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
|
||||||
|
assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
|
||||||
|
|
||||||
|
# Check that the table is still there
|
||||||
|
machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
|
||||||
|
output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
|
||||||
|
owner = output.split("\n")[1]
|
||||||
|
assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
|
||||||
|
'';
|
||||||
|
})
|
@ -6,6 +6,27 @@
|
|||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
cfg = config.clan.borgbackup;
|
cfg = config.clan.borgbackup;
|
||||||
|
preBackupScript = ''
|
||||||
|
declare -A preCommandErrors
|
||||||
|
|
||||||
|
${lib.concatMapStringsSep "\n" (
|
||||||
|
state:
|
||||||
|
lib.optionalString (state.preBackupCommand != null) ''
|
||||||
|
echo "Running pre-backup command for ${state.name}"
|
||||||
|
if ! ( ${state.preBackupCommand} ) then
|
||||||
|
preCommandErrors["${state.name}"]=1
|
||||||
|
fi
|
||||||
|
''
|
||||||
|
) (lib.attrValues config.clanCore.state)}
|
||||||
|
|
||||||
|
if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
|
||||||
|
echo "PreBackupCommand failed for the following services:"
|
||||||
|
for state in "''${!preCommandErrors[@]}"; do
|
||||||
|
echo " $state"
|
||||||
|
done
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
'';
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.clan.borgbackup.destinations = lib.mkOption {
|
options.clan.borgbackup.destinations = lib.mkOption {
|
||||||
@ -50,17 +71,26 @@ in
|
|||||||
];
|
];
|
||||||
|
|
||||||
config = lib.mkIf (cfg.destinations != { }) {
|
config = lib.mkIf (cfg.destinations != { }) {
|
||||||
|
systemd.services = lib.mapAttrs' (
|
||||||
|
_: dest:
|
||||||
|
lib.nameValuePair "borgbackup-job-${dest.name}" {
|
||||||
|
# since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
|
||||||
|
serviceConfig.ExecStartPre = [
|
||||||
|
(''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'')
|
||||||
|
];
|
||||||
|
}
|
||||||
|
) cfg.destinations;
|
||||||
|
|
||||||
services.borgbackup.jobs = lib.mapAttrs (_: dest: {
|
services.borgbackup.jobs = lib.mapAttrs (_: dest: {
|
||||||
paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
|
paths = lib.unique (
|
||||||
|
lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state))
|
||||||
|
);
|
||||||
exclude = [ "*.pyc" ];
|
exclude = [ "*.pyc" ];
|
||||||
repo = dest.repo;
|
repo = dest.repo;
|
||||||
environment.BORG_RSH = dest.rsh;
|
environment.BORG_RSH = dest.rsh;
|
||||||
compression = "auto,zstd";
|
compression = "auto,zstd";
|
||||||
startAt = "*-*-* 01:00:00";
|
startAt = "*-*-* 01:00:00";
|
||||||
persistentTimer = true;
|
persistentTimer = true;
|
||||||
preHook = ''
|
|
||||||
set -x
|
|
||||||
'';
|
|
||||||
|
|
||||||
encryption = {
|
encryption = {
|
||||||
mode = "repokey";
|
mode = "repokey";
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
localsend = ./localsend;
|
localsend = ./localsend;
|
||||||
matrix-synapse = ./matrix-synapse;
|
matrix-synapse = ./matrix-synapse;
|
||||||
moonlight = ./moonlight;
|
moonlight = ./moonlight;
|
||||||
|
postgresql = ./postgresql;
|
||||||
root-password = ./root-password;
|
root-password = ./root-password;
|
||||||
sshd = ./sshd;
|
sshd = ./sshd;
|
||||||
sunshine = ./sunshine;
|
sunshine = ./sunshine;
|
||||||
|
@ -6,7 +6,10 @@
|
|||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
cfg = config.clan.localbackup;
|
cfg = config.clan.localbackup;
|
||||||
rsnapshotConfig = target: states: ''
|
uniqueFolders = lib.unique (
|
||||||
|
lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
|
||||||
|
);
|
||||||
|
rsnapshotConfig = target: ''
|
||||||
config_version 1.2
|
config_version 1.2
|
||||||
snapshot_root ${target.directory}
|
snapshot_root ${target.directory}
|
||||||
sync_first 1
|
sync_first 1
|
||||||
@ -17,12 +20,6 @@ let
|
|||||||
cmd_logger ${pkgs.inetutils}/bin/logger
|
cmd_logger ${pkgs.inetutils}/bin/logger
|
||||||
cmd_du ${pkgs.coreutils}/bin/du
|
cmd_du ${pkgs.coreutils}/bin/du
|
||||||
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
|
cmd_rsnapshot_diff ${pkgs.rsnapshot}/bin/rsnapshot-diff
|
||||||
${lib.optionalString (target.preBackupHook != null) ''
|
|
||||||
cmd_preexec ${pkgs.writeShellScript "preexec.sh" ''
|
|
||||||
set -efu -o pipefail
|
|
||||||
${target.preBackupHook}
|
|
||||||
''}
|
|
||||||
''}
|
|
||||||
|
|
||||||
${lib.optionalString (target.postBackupHook != null) ''
|
${lib.optionalString (target.postBackupHook != null) ''
|
||||||
cmd_postexec ${pkgs.writeShellScript "postexec.sh" ''
|
cmd_postexec ${pkgs.writeShellScript "postexec.sh" ''
|
||||||
@ -31,11 +28,9 @@ let
|
|||||||
''}
|
''}
|
||||||
''}
|
''}
|
||||||
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
|
retain snapshot ${builtins.toString config.clan.localbackup.snapshots}
|
||||||
${lib.concatMapStringsSep "\n" (state: ''
|
|
||||||
${lib.concatMapStringsSep "\n" (folder: ''
|
${lib.concatMapStringsSep "\n" (folder: ''
|
||||||
backup ${folder} ${config.networking.hostName}/
|
backup ${folder} ${config.networking.hostName}/
|
||||||
'') state.folders}
|
'') uniqueFolders}
|
||||||
'') states}
|
|
||||||
'';
|
'';
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
@ -129,14 +124,30 @@ in
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
${lib.concatMapStringsSep "\n" (target: ''
|
${lib.concatMapStringsSep "\n" (target: ''
|
||||||
(
|
|
||||||
${mountHook target}
|
${mountHook target}
|
||||||
|
set -x
|
||||||
echo "Creating backup '${target.name}'"
|
echo "Creating backup '${target.name}'"
|
||||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
|
|
||||||
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
|
${lib.optionalString (target.preBackupHook != null) ''
|
||||||
|
(
|
||||||
|
${target.preBackupHook}
|
||||||
)
|
)
|
||||||
'') (builtins.attrValues cfg.targets)}
|
''}
|
||||||
'')
|
|
||||||
|
declare -A preCommandErrors
|
||||||
|
${lib.concatMapStringsSep "\n" (
|
||||||
|
state:
|
||||||
|
lib.optionalString (state.preBackupCommand != null) ''
|
||||||
|
echo "Running pre-backup command for ${state.name}"
|
||||||
|
if ! ( ${state.preBackupCommand} ) then
|
||||||
|
preCommandErrors["${state.name}"]=1
|
||||||
|
fi
|
||||||
|
''
|
||||||
|
) (builtins.attrValues config.clanCore.state)}
|
||||||
|
|
||||||
|
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
|
||||||
|
rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
|
||||||
|
'') (builtins.attrValues cfg.targets)}'')
|
||||||
(pkgs.writeShellScriptBin "localbackup-list" ''
|
(pkgs.writeShellScriptBin "localbackup-list" ''
|
||||||
set -efu -o pipefail
|
set -efu -o pipefail
|
||||||
export PATH=${
|
export PATH=${
|
||||||
@ -167,6 +178,14 @@ in
|
|||||||
pkgs.gawk
|
pkgs.gawk
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
if [[ "''${NAME:-}" == "" ]]; then
|
||||||
|
echo "No backup name given via NAME environment variable"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ "''${FOLDERS:-}" == "" ]]; then
|
||||||
|
echo "No folders given via FOLDERS environment variable"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
name=$(awk -F'::' '{print $1}' <<< $NAME)
|
name=$(awk -F'::' '{print $1}' <<< $NAME)
|
||||||
backupname=''${NAME#$name::}
|
backupname=''${NAME#$name::}
|
||||||
|
|
||||||
@ -182,8 +201,9 @@ in
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
IFS=';' read -ra FOLDER <<< "$FOLDERS"
|
IFS=':' read -ra FOLDER <<< "''$FOLDERS"
|
||||||
for folder in "''${FOLDER[@]}"; do
|
for folder in "''${FOLDER[@]}"; do
|
||||||
|
mkdir -p "$folder"
|
||||||
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
|
rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
|
||||||
done
|
done
|
||||||
'')
|
'')
|
||||||
|
@ -6,16 +6,35 @@
|
|||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
cfg = config.clan.matrix-synapse;
|
cfg = config.clan.matrix-synapse;
|
||||||
|
nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
|
||||||
|
element-web =
|
||||||
|
pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
|
||||||
|
''
|
||||||
|
cp -r ${pkgs.element-web} $out
|
||||||
|
chmod -R u+w $out
|
||||||
|
jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
|
||||||
|
> $out/config.json < ${pkgs.element-web}/config.json
|
||||||
|
ln -s $out/config.json $out/config.${nginx-vhost}.json
|
||||||
|
'';
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
options.clan.matrix-synapse = {
|
options.clan.matrix-synapse = {
|
||||||
enable = lib.mkEnableOption "Enable matrix-synapse";
|
|
||||||
domain = lib.mkOption {
|
domain = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
description = "The domain name of the matrix server";
|
description = "The domain name of the matrix server";
|
||||||
|
example = "example.com";
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
config = lib.mkIf cfg.enable {
|
imports = [
|
||||||
|
(lib.mkRemovedOptionModule [
|
||||||
|
"clan"
|
||||||
|
"matrix-synapse"
|
||||||
|
"enable"
|
||||||
|
] "Importing the module will already enable the service.")
|
||||||
|
|
||||||
|
../postgresql
|
||||||
|
];
|
||||||
|
config = {
|
||||||
services.matrix-synapse = {
|
services.matrix-synapse = {
|
||||||
enable = true;
|
enable = true;
|
||||||
settings = {
|
settings = {
|
||||||
@ -49,16 +68,27 @@ in
|
|||||||
}
|
}
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
|
extraConfigFiles = [ "/run/synapse-registration-shared-secret.yaml" ];
|
||||||
|
};
|
||||||
|
systemd.tmpfiles.settings."synapse" = {
|
||||||
|
"/run/synapse-registration-shared-secret.yaml" = {
|
||||||
|
C.argument =
|
||||||
|
config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
|
||||||
|
z = {
|
||||||
|
mode = "0400";
|
||||||
|
user = "matrix-synapse";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
clan.postgresql.users.matrix-synapse = { };
|
||||||
|
clan.postgresql.databases.matrix-synapse.create.options = {
|
||||||
|
TEMPLATE = "template0";
|
||||||
|
LC_COLLATE = "C";
|
||||||
|
LC_CTYPE = "C";
|
||||||
|
ENCODING = "UTF8";
|
||||||
|
OWNER = "matrix-synapse";
|
||||||
};
|
};
|
||||||
systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
|
|
||||||
"+${pkgs.writeScript "copy_registration_shared_secret" ''
|
|
||||||
#!/bin/sh
|
|
||||||
cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
|
|
||||||
chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
|
|
||||||
chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
|
|
||||||
''}"
|
|
||||||
];
|
|
||||||
|
|
||||||
clanCore.facts.services."matrix-synapse" = {
|
clanCore.facts.services."matrix-synapse" = {
|
||||||
secret."synapse-registration_shared_secret" = { };
|
secret."synapse-registration_shared_secret" = { };
|
||||||
@ -71,23 +101,6 @@ in
|
|||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
services.postgresql.enable = true;
|
|
||||||
# we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
|
|
||||||
services.postgresql = {
|
|
||||||
ensureDatabases = [ "matrix-synapse" ];
|
|
||||||
ensureUsers = [
|
|
||||||
{
|
|
||||||
name = "matrix-synapse";
|
|
||||||
ensureDBOwnership = true;
|
|
||||||
}
|
|
||||||
];
|
|
||||||
initialScript = pkgs.writeText "synapse-init.sql" ''
|
|
||||||
CREATE DATABASE "matrix-synapse"
|
|
||||||
TEMPLATE template0
|
|
||||||
LC_COLLATE = "C"
|
|
||||||
LC_CTYPE = "C";
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
services.nginx = {
|
services.nginx = {
|
||||||
enable = true;
|
enable = true;
|
||||||
virtualHosts = {
|
virtualHosts = {
|
||||||
@ -102,7 +115,7 @@ in
|
|||||||
return 200 '${
|
return 200 '${
|
||||||
builtins.toJSON {
|
builtins.toJSON {
|
||||||
"m.homeserver" = {
|
"m.homeserver" = {
|
||||||
"base_url" = "https://matrix.${cfg.domain}";
|
"base_url" = "https://${nginx-vhost}";
|
||||||
};
|
};
|
||||||
"m.identity_server" = {
|
"m.identity_server" = {
|
||||||
"base_url" = "https://vector.im";
|
"base_url" = "https://vector.im";
|
||||||
@ -111,15 +124,12 @@ in
|
|||||||
}';
|
}';
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
"matrix.${cfg.domain}" = {
|
${nginx-vhost} = {
|
||||||
forceSSL = true;
|
forceSSL = true;
|
||||||
enableACME = true;
|
enableACME = true;
|
||||||
locations."/_matrix" = {
|
locations."/_matrix".proxyPass = "http://localhost:8008";
|
||||||
proxyPass = "http://localhost:8008";
|
locations."/_synapse".proxyPass = "http://localhost:8008";
|
||||||
};
|
locations."/".root = element-web;
|
||||||
locations."/test".extraConfig = ''
|
|
||||||
return 200 "Hello, world!";
|
|
||||||
'';
|
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
2
clanModules/postgresql/README.md
Normal file
2
clanModules/postgresql/README.md
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
|
||||||
|
---
|
166
clanModules/postgresql/default.nix
Normal file
166
clanModules/postgresql/default.nix
Normal file
@ -0,0 +1,166 @@
|
|||||||
|
{
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
config,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
let
|
||||||
|
createDatatbaseState =
|
||||||
|
db:
|
||||||
|
let
|
||||||
|
folder = "/var/backup/postgres/${db.name}";
|
||||||
|
current = "${folder}/pg-dump";
|
||||||
|
compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
|
||||||
|
in
|
||||||
|
{
|
||||||
|
folders = [ folder ];
|
||||||
|
preBackupCommand = ''
|
||||||
|
export PATH=${
|
||||||
|
lib.makeBinPath [
|
||||||
|
config.services.postgresql.package
|
||||||
|
config.systemd.package
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.util-linux
|
||||||
|
pkgs.zstd
|
||||||
|
]
|
||||||
|
}
|
||||||
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
mkdir -p "${folder}"
|
||||||
|
runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
|
||||||
|
mv "${current}.tmp" ${current}
|
||||||
|
'';
|
||||||
|
postRestoreCommand = ''
|
||||||
|
export PATH=${
|
||||||
|
lib.makeBinPath [
|
||||||
|
config.services.postgresql.package
|
||||||
|
config.systemd.package
|
||||||
|
pkgs.coreutils
|
||||||
|
pkgs.util-linux
|
||||||
|
pkgs.zstd
|
||||||
|
]
|
||||||
|
}
|
||||||
|
while [[ "$(systemctl is-active postgresql)" == activating ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
echo "Waiting for postgres to be ready..."
|
||||||
|
while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
|
||||||
|
if ! systemctl is-active postgresql; then exit 1; fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -e "${current}" ]]; then
|
||||||
|
(
|
||||||
|
systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
|
||||||
|
trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
|
||||||
|
|
||||||
|
mkdir -p "${folder}"
|
||||||
|
runuser -u postgres -- dropdb "${db.name}"
|
||||||
|
runuser -u postgres -- pg_restore -C -d postgres "${current}"
|
||||||
|
)
|
||||||
|
else
|
||||||
|
echo No database backup found, skipping restore
|
||||||
|
fi
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
createDatabase = db: ''
|
||||||
|
CREATE DATABASE "${db.name}" ${
|
||||||
|
lib.concatStringsSep " " (
|
||||||
|
lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
|
||||||
|
)
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
cfg = config.clan.postgresql;
|
||||||
|
|
||||||
|
userClauses = lib.mapAttrsToList (
|
||||||
|
_: user:
|
||||||
|
''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
|
||||||
|
) cfg.users;
|
||||||
|
databaseClauses = lib.mapAttrsToList (
|
||||||
|
name: db:
|
||||||
|
lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
|
||||||
|
) cfg.databases;
|
||||||
|
in
|
||||||
|
{
|
||||||
|
options.clan.postgresql = {
|
||||||
|
# we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
|
||||||
|
databases = lib.mkOption {
|
||||||
|
default = { };
|
||||||
|
type = lib.types.attrsOf (
|
||||||
|
lib.types.submodule (
|
||||||
|
{ name, ... }:
|
||||||
|
{
|
||||||
|
options = {
|
||||||
|
name = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
};
|
||||||
|
# set to false, in case the upstream module uses ensureDatabase option
|
||||||
|
create.enable = lib.mkOption {
|
||||||
|
type = lib.types.bool;
|
||||||
|
default = true;
|
||||||
|
description = "Create the database if it does not exist.";
|
||||||
|
};
|
||||||
|
create.options = lib.mkOption {
|
||||||
|
type = lib.types.lazyAttrsOf lib.types.str;
|
||||||
|
default = { };
|
||||||
|
example = {
|
||||||
|
TEMPLATE = "template0";
|
||||||
|
LC_COLLATE = "C";
|
||||||
|
LC_CTYPE = "C";
|
||||||
|
ENCODING = "UTF8";
|
||||||
|
OWNER = "foo";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
restore.stopOnRestore = lib.mkOption {
|
||||||
|
type = lib.types.listOf lib.types.str;
|
||||||
|
default = [ ];
|
||||||
|
description = "List of services to stop before restoring the database.";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
users = lib.mkOption {
|
||||||
|
default = { };
|
||||||
|
type = lib.types.attrsOf (
|
||||||
|
lib.types.submodule (
|
||||||
|
{ name, ... }:
|
||||||
|
{
|
||||||
|
options.name = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
};
|
||||||
|
};
|
||||||
|
config = {
|
||||||
|
services.postgresql.settings = {
|
||||||
|
wal_level = "replica";
|
||||||
|
max_wal_senders = 3;
|
||||||
|
};
|
||||||
|
|
||||||
|
services.postgresql.enable = true;
|
||||||
|
# We are duplicating a bit the upstream module but allow to create databases with options
|
||||||
|
systemd.services.postgresql.postStart = ''
|
||||||
|
PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
|
||||||
|
|
||||||
|
while ! $PSQL -d postgres -c "" 2> /dev/null; do
|
||||||
|
if ! kill -0 "$MAINPID"; then exit 1; fi
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
${lib.concatStringsSep "\n" userClauses}
|
||||||
|
${lib.concatStringsSep "\n" databaseClauses}
|
||||||
|
'';
|
||||||
|
|
||||||
|
clanCore.state = lib.mapAttrs' (
|
||||||
|
_: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
|
||||||
|
) config.clan.postgresql.databases;
|
||||||
|
};
|
||||||
|
}
|
@ -60,6 +60,7 @@ nav:
|
|||||||
- reference/clanModules/localsend.md
|
- reference/clanModules/localsend.md
|
||||||
- reference/clanModules/matrix-synapse.md
|
- reference/clanModules/matrix-synapse.md
|
||||||
- reference/clanModules/moonlight.md
|
- reference/clanModules/moonlight.md
|
||||||
|
- reference/clanModules/postgresql.md
|
||||||
- reference/clanModules/root-password.md
|
- reference/clanModules/root-password.md
|
||||||
- reference/clanModules/sshd.md
|
- reference/clanModules/sshd.md
|
||||||
- reference/clanModules/static-hosts.md
|
- reference/clanModules/static-hosts.md
|
||||||
|
@ -8,15 +8,43 @@
|
|||||||
default = { };
|
default = { };
|
||||||
type = lib.types.attrsOf (
|
type = lib.types.attrsOf (
|
||||||
lib.types.submodule (
|
lib.types.submodule (
|
||||||
{ ... }:
|
{ name, ... }:
|
||||||
{
|
{
|
||||||
options = {
|
options = {
|
||||||
|
name = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = name;
|
||||||
|
description = ''
|
||||||
|
Name of the state
|
||||||
|
'';
|
||||||
|
};
|
||||||
folders = lib.mkOption {
|
folders = lib.mkOption {
|
||||||
type = lib.types.listOf lib.types.str;
|
type = lib.types.listOf lib.types.str;
|
||||||
description = ''
|
description = ''
|
||||||
Folder where state resides in
|
Folder where state resides in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
preBackupCommand = lib.mkOption {
|
||||||
|
type = lib.types.nullOr lib.types.str;
|
||||||
|
default = null;
|
||||||
|
description = ''
|
||||||
|
script to run before backing up the state dir
|
||||||
|
This is for example useful for services that require an export of their state
|
||||||
|
e.g. a database dump
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
# TODO: implement this
|
||||||
|
#stopOnRestore = lib.mkOption {
|
||||||
|
# type = lib.types.listOf lib.types.str;
|
||||||
|
# default = [];
|
||||||
|
# description = ''
|
||||||
|
# List of services to stop before restoring the state dir from a backup
|
||||||
|
|
||||||
|
# Utilize this to stop services which currently access these folders or or other services affected by the restore
|
||||||
|
# '';
|
||||||
|
#};
|
||||||
|
|
||||||
preRestoreCommand = lib.mkOption {
|
preRestoreCommand = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.str;
|
type = lib.types.nullOr lib.types.str;
|
||||||
default = null;
|
default = null;
|
||||||
@ -26,6 +54,7 @@
|
|||||||
Utilize this to stop services which currently access these folders
|
Utilize this to stop services which currently access these folders
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
|
||||||
postRestoreCommand = lib.mkOption {
|
postRestoreCommand = lib.mkOption {
|
||||||
type = lib.types.nullOr lib.types.str;
|
type = lib.types.nullOr lib.types.str;
|
||||||
default = null;
|
default = null;
|
||||||
|
@ -313,7 +313,7 @@ def main() -> None:
|
|||||||
if len(sys.argv) == 1:
|
if len(sys.argv) == 1:
|
||||||
parser.print_help()
|
parser.print_help()
|
||||||
|
|
||||||
if args.debug:
|
if getattr(args, "debug", False):
|
||||||
setup_logging(logging.DEBUG, root_log_name=__name__.split(".")[0])
|
setup_logging(logging.DEBUG, root_log_name=__name__.split(".")[0])
|
||||||
log.debug("Debug log activated")
|
log.debug("Debug log activated")
|
||||||
if flatpak.is_flatpak():
|
if flatpak.is_flatpak():
|
||||||
|
@ -14,10 +14,16 @@ from ..machines.machines import Machine
|
|||||||
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
|
def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
|
||||||
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
|
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
|
||||||
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
||||||
|
|
||||||
|
if service not in backup_folders:
|
||||||
|
msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
|
||||||
|
raise ClanError(msg)
|
||||||
|
|
||||||
folders = backup_folders[service]["folders"]
|
folders = backup_folders[service]["folders"]
|
||||||
env = {}
|
env = {}
|
||||||
env["NAME"] = name
|
env["NAME"] = name
|
||||||
env["FOLDERS"] = ":".join(folders)
|
# FIXME: If we have too many folder this might overflow the stack.
|
||||||
|
env["FOLDERS"] = ":".join(set(folders))
|
||||||
|
|
||||||
if pre_restore := backup_folders[service]["preRestoreCommand"]:
|
if pre_restore := backup_folders[service]["preRestoreCommand"]:
|
||||||
proc = machine.target_host.run(
|
proc = machine.target_host.run(
|
||||||
@ -58,12 +64,23 @@ def restore_backup(
|
|||||||
name: str,
|
name: str,
|
||||||
service: str | None = None,
|
service: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
errors = []
|
||||||
if service is None:
|
if service is None:
|
||||||
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
|
||||||
for _service in backup_folders:
|
for _service in backup_folders:
|
||||||
|
try:
|
||||||
restore_service(machine, name, provider, _service)
|
restore_service(machine, name, provider, _service)
|
||||||
|
except ClanError as e:
|
||||||
|
errors.append(f"{_service}: {e}")
|
||||||
else:
|
else:
|
||||||
|
try:
|
||||||
restore_service(machine, name, provider, service)
|
restore_service(machine, name, provider, service)
|
||||||
|
except ClanError as e:
|
||||||
|
errors.append(f"{service}: {e}")
|
||||||
|
if errors:
|
||||||
|
raise ClanError(
|
||||||
|
"Restore failed for the following services:\n" + "\n".join(errors)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def restore_command(args: argparse.Namespace) -> None:
|
def restore_command(args: argparse.Namespace) -> None:
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
description = "<Put your description here>";
|
description = "<Put your description here>";
|
||||||
|
|
||||||
inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core";
|
inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
|
||||||
|
|
||||||
outputs =
|
outputs =
|
||||||
{ self, clan-core, ... }:
|
{ self, clan-core, ... }:
|
||||||
|
Loading…
Reference in New Issue
Block a user