From 5c1cf947ab512d87ce9198cb7792c9b98719fca5 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 16 Dec 2023 20:57:39 +0100 Subject: [PATCH 1/5] checks: add backup test --- checks/backups/flake-module.nix | 125 ++++++++++++++++++++++++++++++++ checks/flake-module.nix | 1 + 2 files changed, 126 insertions(+) create mode 100644 checks/backups/flake-module.nix diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix new file mode 100644 index 00000000..d2964047 --- /dev/null +++ b/checks/backups/flake-module.nix @@ -0,0 +1,125 @@ +{ self, ... }: +let + clan = self.lib.buildClan { + clanName = "testclan"; + directory = ../..; + machines = { + test_backup_client = { + imports = [ self.nixosModules.test_backup_client ]; + fileSystems."/".device = "/dev/null"; + boot.loader.grub.device = "/dev/null"; + }; + }; + }; +in +{ + flake.nixosConfigurations = { inherit (clan.nixosConfigurations) test_backup_client; }; + flake.clanInternals = clan.clanInternals; + flake.nixosModules = { + test_backup_server = { ... }: { + imports = [ + self.clanModules.borgbackup + ]; + services.sshd.enable = true; + services.borgbackup.repos.testrepo = { + authorizedKeys = [ + (builtins.readFile ../borgbackup/borg_test.pub) + ]; + }; + }; + test_backup_client = { pkgs, lib, ... }: + let + dependencies = [ + self + pkgs.stdenv.drvPath + ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs); + closureInfo = pkgs.closureInfo { rootPaths = dependencies; }; + in + { + imports = [ + self.clanModules.borgbackup + ]; + networking.hostName = "client"; + services.sshd.enable = true; + users.users.root.openssh.authorizedKeys.keyFiles = [ + ../borgbackup/borg_test.pub + ]; + environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ]; + environment.etc."install-closure".source = "${closureInfo}/store-paths"; + nix.settings = { + substituters = lib.mkForce [ ]; + hashed-mirrors = null; + connect-timeout = lib.mkForce 3; + flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}''; + }; + system.extraDependencies = dependencies; + clanCore.state.test-backups.folders = [ "/var/test-backups" ]; + clan.borgbackup = { + enable = true; + destinations.test_backup_server = { + repo = "borg@server:."; + rsh = "ssh -i /root/.ssh/id_ed25519 -o StrictHostKeyChecking=no"; + }; + }; + }; + }; + perSystem = { nodes, pkgs, ... }: { + checks.test-backups = + (import ../lib/test-base.nix) + { + name = "test-backups"; + nodes.server = { + imports = [ + self.nixosModules.test_backup_server + self.nixosModules.clanCore + { + clanCore.machineName = "server"; + clanCore.clanDir = ../..; + } + ]; + }; + nodes.client = { + imports = [ + self.nixosModules.test_backup_client + self.nixosModules.clanCore + { + clanCore.machineName = "client"; + clanCore.clanDir = ../..; + } + ]; + }; + + testScript = '' + import json + start_all() + + # setup + client.succeed("mkdir -m 700 /root/.ssh") + client.succeed( + "cat ${../borgbackup/borg_test} > /root/.ssh/id_ed25519" + ) + client.succeed("chmod 600 /root/.ssh/id_ed25519") + client.wait_for_unit("sshd", timeout=30) + print(client.succeed("ssh -o StrictHostKeyChecking=accept-new -v root@client hostname")) + + # dummy data + client.succeed("mkdir /var/test-backups") + client.succeed("echo testing > /var/test-backups/somefile") + + # create + client.succeed("clan --flake ${../..} backups create test_backup_client") + client.wait_until_succeeds("! systemctl is-active borgbackup-job-test_backup_server") + + # list + backup_id = json.loads(client.succeed("borg-job-test_backup_server list --json"))["archives"][0]["archive"] + assert(backup_id in client.succeed("clan --flake ${../..} backups list test_backup_client")) + + # restore + client.succeed("rm -f /var/test-backups/somefile") + client.succeed(f"clan --flake ${../..} backups restore test_backup_client borgbackup {backup_id}") + assert(client.succeed("cat /var/test-backups/somefile").strip() == "testing") + ''; + } + { inherit pkgs self; }; + }; +} diff --git a/checks/flake-module.nix b/checks/flake-module.nix index 4b11ce1b..5e9a3eaf 100644 --- a/checks/flake-module.nix +++ b/checks/flake-module.nix @@ -1,6 +1,7 @@ { self, ... }: { imports = [ ./impure/flake-module.nix + ./backups/flake-module.nix ]; perSystem = { pkgs, lib, self', ... }: { checks = From 902ccbc90b8e3bf47c505df3b8d677742cf56bb0 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 16 Dec 2023 20:57:53 +0100 Subject: [PATCH 2/5] backups borgbackup: remove dry-run from restore --- clanModules/borgbackup.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clanModules/borgbackup.nix b/clanModules/borgbackup.nix index 4ee06a65..6bfbe632 100644 --- a/clanModules/borgbackup.nix +++ b/clanModules/borgbackup.nix @@ -83,7 +83,7 @@ in set -efu cd / IFS=';' read -ra FOLDER <<< "$FOLDERS" - yes y | borg-job-"$JOB" extract --list --dry-run "$LOCATION"::"$ARCHIVE_ID" "''${FOLDER[@]}" + yes y | borg-job-"$JOB" extract --list "$LOCATION"::"$ARCHIVE_ID" "''${FOLDER[@]}" ''; }; }; From 56dafff1093f9df6129688c6cb4ce380296c321f Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 16 Dec 2023 20:58:35 +0100 Subject: [PATCH 3/5] machines build_machine_data: raise ClanError on error --- pkgs/clan-cli/clan_cli/machines/machines.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 37996850..22aef38e 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -4,6 +4,7 @@ import subprocess import sys from pathlib import Path +from ..errors import ClanError from ..nix import nix_build, nix_config, nix_eval from ..ssh import Host, parse_deployment_address @@ -12,17 +13,22 @@ def build_machine_data(machine_name: str, clan_dir: Path) -> dict: config = nix_config() system = config["system"] - outpath = subprocess.run( + proc = subprocess.run( nix_build( [ - f'path:{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file' + f'{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file' ] ), stdout=subprocess.PIPE, check=True, text=True, - ).stdout.strip() - return json.loads(Path(outpath).read_text()) + ) + + if proc.returncode != 0: + ClanError("failed to build machine data") + exit(1) + + return json.loads(Path(proc.stdout.strip()).read_text()) class Machine: From e98f037dd8a6461296f40820537bfb0af4bd0203 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 16 Dec 2023 20:58:58 +0100 Subject: [PATCH 4/5] clan_cli: remove incorrect clan_flake check --- pkgs/clan-cli/clan_cli/__init__.py | 6 +----- pkgs/clan-cli/clan_cli/dirs.py | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 3aea8bc9..20e83461 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -8,7 +8,7 @@ from typing import Any from . import backups, config, flakes, history, machines, secrets, vms from .custom_logger import setup_logging -from .dirs import get_clan_flake_toplevel, is_clan_flake +from .dirs import get_clan_flake_toplevel from .ssh import cli as ssh_cli log = logging.getLogger(__name__) @@ -66,10 +66,6 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser: raise argparse.ArgumentTypeError( f"flake directory {flake_dir} is not a directory" ) - if not is_clan_flake(flake_dir): - raise argparse.ArgumentTypeError( - f"flake directory {flake_dir} is not a clan flake" - ) return flake_dir parser.add_argument( diff --git a/pkgs/clan-cli/clan_cli/dirs.py b/pkgs/clan-cli/clan_cli/dirs.py index fdf6fe6c..98076db3 100644 --- a/pkgs/clan-cli/clan_cli/dirs.py +++ b/pkgs/clan-cli/clan_cli/dirs.py @@ -10,10 +10,6 @@ def get_clan_flake_toplevel() -> Path | None: return find_toplevel([".clan-flake", ".git", ".hg", ".svn", "flake.nix"]) -def is_clan_flake(path: Path) -> bool: - return (path / ".clan-flake").exists() - - def find_git_repo_root() -> Path | None: return find_toplevel([".git"]) From cc3701f22fae6df2af8736f6eea5563d804c8f7d Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 16 Dec 2023 21:57:27 +0100 Subject: [PATCH 5/5] checks backup: limit to linux systems --- checks/backups/flake-module.nix | 104 ++++++++++++++++---------------- 1 file changed, 53 insertions(+), 51 deletions(-) diff --git a/checks/backups/flake-module.nix b/checks/backups/flake-module.nix index d2964047..be4a57e1 100644 --- a/checks/backups/flake-module.nix +++ b/checks/backups/flake-module.nix @@ -64,62 +64,64 @@ in }; }; perSystem = { nodes, pkgs, ... }: { - checks.test-backups = - (import ../lib/test-base.nix) - { - name = "test-backups"; - nodes.server = { - imports = [ - self.nixosModules.test_backup_server - self.nixosModules.clanCore - { - clanCore.machineName = "server"; - clanCore.clanDir = ../..; - } - ]; - }; - nodes.client = { - imports = [ - self.nixosModules.test_backup_client - self.nixosModules.clanCore - { - clanCore.machineName = "client"; - clanCore.clanDir = ../..; - } - ]; - }; + checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) { + test-backups = + (import ../lib/test-base.nix) + { + name = "test-backups"; + nodes.server = { + imports = [ + self.nixosModules.test_backup_server + self.nixosModules.clanCore + { + clanCore.machineName = "server"; + clanCore.clanDir = ../..; + } + ]; + }; + nodes.client = { + imports = [ + self.nixosModules.test_backup_client + self.nixosModules.clanCore + { + clanCore.machineName = "client"; + clanCore.clanDir = ../..; + } + ]; + }; - testScript = '' - import json - start_all() + testScript = '' + import json + start_all() - # setup - client.succeed("mkdir -m 700 /root/.ssh") - client.succeed( - "cat ${../borgbackup/borg_test} > /root/.ssh/id_ed25519" - ) - client.succeed("chmod 600 /root/.ssh/id_ed25519") - client.wait_for_unit("sshd", timeout=30) - print(client.succeed("ssh -o StrictHostKeyChecking=accept-new -v root@client hostname")) + # setup + client.succeed("mkdir -m 700 /root/.ssh") + client.succeed( + "cat ${../borgbackup/borg_test} > /root/.ssh/id_ed25519" + ) + client.succeed("chmod 600 /root/.ssh/id_ed25519") + client.wait_for_unit("sshd", timeout=30) + print(client.succeed("ssh -o StrictHostKeyChecking=accept-new -v root@client hostname")) - # dummy data - client.succeed("mkdir /var/test-backups") - client.succeed("echo testing > /var/test-backups/somefile") + # dummy data + client.succeed("mkdir /var/test-backups") + client.succeed("echo testing > /var/test-backups/somefile") - # create - client.succeed("clan --flake ${../..} backups create test_backup_client") - client.wait_until_succeeds("! systemctl is-active borgbackup-job-test_backup_server") + # create + client.succeed("clan --flake ${../..} backups create test_backup_client") + client.wait_until_succeeds("! systemctl is-active borgbackup-job-test_backup_server") - # list - backup_id = json.loads(client.succeed("borg-job-test_backup_server list --json"))["archives"][0]["archive"] - assert(backup_id in client.succeed("clan --flake ${../..} backups list test_backup_client")) + # list + backup_id = json.loads(client.succeed("borg-job-test_backup_server list --json"))["archives"][0]["archive"] + assert(backup_id in client.succeed("clan --flake ${../..} backups list test_backup_client")) - # restore - client.succeed("rm -f /var/test-backups/somefile") - client.succeed(f"clan --flake ${../..} backups restore test_backup_client borgbackup {backup_id}") - assert(client.succeed("cat /var/test-backups/somefile").strip() == "testing") - ''; - } - { inherit pkgs self; }; + # restore + client.succeed("rm -f /var/test-backups/somefile") + client.succeed(f"clan --flake ${../..} backups restore test_backup_client borgbackup {backup_id}") + assert(client.succeed("cat /var/test-backups/somefile").strip() == "testing") + ''; + } + { inherit pkgs self; }; + }; }; }