forked from clan/clan-core
Merge pull request 'backups: add clanCore backup & clan borgbackup module' (#605) from lassulus-backups into main
This commit is contained in:
commit
0b772668a8
@ -16,7 +16,7 @@
|
|||||||
{
|
{
|
||||||
clanCore.machineName = "machine";
|
clanCore.machineName = "machine";
|
||||||
clanCore.clanDir = ./.;
|
clanCore.clanDir = ./.;
|
||||||
clanCore.state."/etc/state" = { };
|
clanCore.state.testState.folders = [ "/etc/state" ];
|
||||||
environment.etc.state.text = "hello world";
|
environment.etc.state.text = "hello world";
|
||||||
clan.borgbackup = {
|
clan.borgbackup = {
|
||||||
enable = true;
|
enable = true;
|
||||||
|
@ -33,7 +33,7 @@ in
|
|||||||
config = lib.mkIf cfg.enable {
|
config = lib.mkIf cfg.enable {
|
||||||
services.borgbackup.jobs = lib.mapAttrs
|
services.borgbackup.jobs = lib.mapAttrs
|
||||||
(_: dest: {
|
(_: dest: {
|
||||||
paths = map (state: state.folder) (lib.attrValues config.clanCore.state);
|
paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
|
||||||
exclude = [
|
exclude = [
|
||||||
"*.pyc"
|
"*.pyc"
|
||||||
];
|
];
|
||||||
@ -58,16 +58,23 @@ in
|
|||||||
clanCore.backups.providers.borgbackup = {
|
clanCore.backups.providers.borgbackup = {
|
||||||
list = ''
|
list = ''
|
||||||
${lib.concatMapStringsSep "\n" (dest: ''
|
${lib.concatMapStringsSep "\n" (dest: ''
|
||||||
echo listing backups for ${dest}
|
(
|
||||||
borg-job-${dest} list
|
export BORG_REPO=${lib.escapeShellArg dest.repo}
|
||||||
'') cfg.destinations}
|
export BORG_RSH=${lib.escapeShellArg dest.rsh}
|
||||||
|
${lib.getExe config.services.borgbackup.package} list
|
||||||
|
)
|
||||||
|
'') (lib.attrValues cfg.destinations)}
|
||||||
'';
|
'';
|
||||||
start = ''
|
start = ''
|
||||||
${lib.concatMapStringsSep "\n" (dest: ''
|
ssh ${config.clan.networking.deploymentAddress} -- '
|
||||||
systemctl start borgbackup-job-${dest}
|
${lib.concatMapStringsSep "\n" (dest: ''
|
||||||
'') cfg.destinations}
|
systemctl start borgbackup-job-${dest.name}
|
||||||
|
'') (lib.attrValues cfg.destinations)}
|
||||||
|
'
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
restore = ''
|
||||||
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -3,15 +3,22 @@
|
|||||||
options.clanCore.state = lib.mkOption {
|
options.clanCore.state = lib.mkOption {
|
||||||
default = { };
|
default = { };
|
||||||
type = lib.types.attrsOf
|
type = lib.types.attrsOf
|
||||||
(lib.types.submodule ({ name, ... }: {
|
(lib.types.submodule ({ ... }: {
|
||||||
options = {
|
options = {
|
||||||
folder = lib.mkOption {
|
folders = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.listOf lib.types.str;
|
||||||
default = name;
|
|
||||||
description = ''
|
description = ''
|
||||||
Folder where state resides in
|
Folder where state resides in
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
restoreScript = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = ":";
|
||||||
|
description = ''
|
||||||
|
script to restore the service after the state dir was restored from a backup
|
||||||
|
'';
|
||||||
|
|
||||||
|
};
|
||||||
};
|
};
|
||||||
}));
|
}));
|
||||||
};
|
};
|
||||||
@ -32,10 +39,11 @@
|
|||||||
script to list backups
|
script to list backups
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
delete = lib.mkOption {
|
restore = lib.mkOption {
|
||||||
type = lib.types.str;
|
type = lib.types.str;
|
||||||
description = ''
|
description = ''
|
||||||
script to delete a backup
|
script to restore a backup
|
||||||
|
should take an optional service name as argument
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
start = lib.mkOption {
|
start = lib.mkOption {
|
||||||
|
@ -147,6 +147,8 @@ in
|
|||||||
--network-id "$facts/zerotier-network-id"
|
--network-id "$facts/zerotier-network-id"
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
|
clanCore.state.zerotier.folders = [ "/var/lib/zerotier-one" ];
|
||||||
|
|
||||||
environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
|
environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
|
||||||
})
|
})
|
||||||
(lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) {
|
(lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) {
|
||||||
|
@ -6,7 +6,7 @@ from pathlib import Path
|
|||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from . import config, flakes, machines, secrets, vms, webui
|
from . import backups, config, flakes, machines, secrets, vms, webui
|
||||||
from .custom_logger import setup_logging
|
from .custom_logger import setup_logging
|
||||||
from .dirs import get_clan_flake_toplevel, is_clan_flake
|
from .dirs import get_clan_flake_toplevel, is_clan_flake
|
||||||
from .ssh import cli as ssh_cli
|
from .ssh import cli as ssh_cli
|
||||||
@ -81,6 +81,11 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
|
|||||||
|
|
||||||
subparsers = parser.add_subparsers()
|
subparsers = parser.add_subparsers()
|
||||||
|
|
||||||
|
parser_backups = subparsers.add_parser(
|
||||||
|
"backups", help="manage backups of clan machines"
|
||||||
|
)
|
||||||
|
backups.register_parser(parser_backups)
|
||||||
|
|
||||||
parser_flake = subparsers.add_parser(
|
parser_flake = subparsers.add_parser(
|
||||||
"flakes", help="create a clan flake inside the current directory"
|
"flakes", help="create a clan flake inside the current directory"
|
||||||
)
|
)
|
||||||
|
25
pkgs/clan-cli/clan_cli/backups/__init__.py
Normal file
25
pkgs/clan-cli/clan_cli/backups/__init__.py
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
# !/usr/bin/env python3
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
from .create import register_create_parser
|
||||||
|
from .list import register_list_parser
|
||||||
|
from .restore import register_restore_parser
|
||||||
|
|
||||||
|
|
||||||
|
# takes a (sub)parser and configures it
|
||||||
|
def register_parser(parser: argparse.ArgumentParser) -> None:
|
||||||
|
subparser = parser.add_subparsers(
|
||||||
|
title="command",
|
||||||
|
description="the command to run",
|
||||||
|
help="the command to run",
|
||||||
|
required=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
list_parser = subparser.add_parser("list", help="list backups")
|
||||||
|
register_list_parser(list_parser)
|
||||||
|
|
||||||
|
create_parser = subparser.add_parser("create", help="create backups")
|
||||||
|
register_create_parser(create_parser)
|
||||||
|
|
||||||
|
restore_parser = subparser.add_parser("restore", help="restore backups")
|
||||||
|
register_restore_parser(restore_parser)
|
45
pkgs/clan-cli/clan_cli/backups/create.py
Normal file
45
pkgs/clan-cli/clan_cli/backups/create.py
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from ..errors import ClanError
|
||||||
|
from ..machines.machines import Machine
|
||||||
|
|
||||||
|
|
||||||
|
def create_backup(machine: Machine, provider: Optional[str] = None) -> None:
|
||||||
|
backup_scripts = json.loads(
|
||||||
|
machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups")
|
||||||
|
)
|
||||||
|
if provider is None:
|
||||||
|
for provider in backup_scripts["providers"]:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["bash", "-c", backup_scripts["providers"][provider]["start"]],
|
||||||
|
)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise ClanError("failed to start backup")
|
||||||
|
else:
|
||||||
|
print("successfully started backup")
|
||||||
|
else:
|
||||||
|
if provider not in backup_scripts["providers"]:
|
||||||
|
raise ClanError(f"provider {provider} not found")
|
||||||
|
proc = subprocess.run(
|
||||||
|
["bash", "-c", backup_scripts["providers"][provider]["start"]],
|
||||||
|
)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise ClanError("failed to start backup")
|
||||||
|
else:
|
||||||
|
print("successfully started backup")
|
||||||
|
|
||||||
|
|
||||||
|
def create_command(args: argparse.Namespace) -> None:
|
||||||
|
machine = Machine(name=args.machine, flake_dir=args.flake)
|
||||||
|
create_backup(machine=machine, provider=args.provider)
|
||||||
|
|
||||||
|
|
||||||
|
def register_create_parser(parser: argparse.ArgumentParser) -> None:
|
||||||
|
parser.add_argument(
|
||||||
|
"machine", type=str, help="machine in the flake to create backups of"
|
||||||
|
)
|
||||||
|
parser.add_argument("--provider", type=str, help="backup provider to use")
|
||||||
|
parser.set_defaults(func=create_command)
|
63
pkgs/clan-cli/clan_cli/backups/list.py
Normal file
63
pkgs/clan-cli/clan_cli/backups/list.py
Normal file
@ -0,0 +1,63 @@
|
|||||||
|
import argparse
|
||||||
|
import pprint
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from ..errors import ClanError
|
||||||
|
|
||||||
|
|
||||||
|
def list_backups(
|
||||||
|
flake_dir: Path, machine: str, provider: Optional[str] = None
|
||||||
|
) -> dict[str, dict[str, list[dict[str, str]]]]:
|
||||||
|
dummy_data = {
|
||||||
|
"testhostname": {
|
||||||
|
"borgbackup": [
|
||||||
|
{"date": "2021-01-01T00:00:00Z", "id": "1"},
|
||||||
|
{"date": "2022-01-01T00:00:00Z", "id": "2"},
|
||||||
|
{"date": "2023-01-01T00:00:00Z", "id": "3"},
|
||||||
|
],
|
||||||
|
"restic": [
|
||||||
|
{"date": "2021-01-01T00:00:00Z", "id": "1"},
|
||||||
|
{"date": "2022-01-01T00:00:00Z", "id": "2"},
|
||||||
|
{"date": "2023-01-01T00:00:00Z", "id": "3"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"another host": {
|
||||||
|
"borgbackup": [
|
||||||
|
{"date": "2021-01-01T00:00:00Z", "id": "1"},
|
||||||
|
{"date": "2022-01-01T00:00:00Z", "id": "2"},
|
||||||
|
{"date": "2023-01-01T00:00:00Z", "id": "3"},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if provider is not None:
|
||||||
|
new_data = {}
|
||||||
|
for machine_ in dummy_data:
|
||||||
|
if provider in dummy_data[machine_]:
|
||||||
|
new_data[machine_] = {provider: dummy_data[machine_][provider]}
|
||||||
|
dummy_data = new_data
|
||||||
|
|
||||||
|
if machine is None:
|
||||||
|
return dummy_data
|
||||||
|
else:
|
||||||
|
return {machine: dummy_data[machine]}
|
||||||
|
|
||||||
|
|
||||||
|
def list_command(args: argparse.Namespace) -> None:
|
||||||
|
if args.flake is None:
|
||||||
|
raise ClanError("Could not find clan flake toplevel directory")
|
||||||
|
backups = list_backups(
|
||||||
|
Path(args.flake), machine=args.machine, provider=args.provider
|
||||||
|
)
|
||||||
|
if len(backups) > 0:
|
||||||
|
pp = pprint.PrettyPrinter(depth=4)
|
||||||
|
pp.pprint(backups)
|
||||||
|
|
||||||
|
|
||||||
|
def register_list_parser(parser: argparse.ArgumentParser) -> None:
|
||||||
|
parser.add_argument(
|
||||||
|
"machine", type=str, help="machine in the flake to show backups of"
|
||||||
|
)
|
||||||
|
parser.add_argument("--provider", type=str, help="backup provider to filter by")
|
||||||
|
parser.set_defaults(func=list_command)
|
42
pkgs/clan-cli/clan_cli/backups/restore.py
Normal file
42
pkgs/clan-cli/clan_cli/backups/restore.py
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from ..errors import ClanError
|
||||||
|
|
||||||
|
|
||||||
|
def restore_backup(
|
||||||
|
flake_dir: Path,
|
||||||
|
machine: str,
|
||||||
|
provider: str,
|
||||||
|
backup_id: str,
|
||||||
|
service: Optional[str] = None,
|
||||||
|
) -> None:
|
||||||
|
if service is None:
|
||||||
|
print("would restore backup", machine, provider, backup_id)
|
||||||
|
else:
|
||||||
|
print(
|
||||||
|
"would restore backup", machine, provider, backup_id, "of service:", service
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def restore_command(args: argparse.Namespace) -> None:
|
||||||
|
if args.flake is None:
|
||||||
|
raise ClanError("Could not find clan flake toplevel directory")
|
||||||
|
restore_backup(
|
||||||
|
Path(args.flake),
|
||||||
|
machine=args.machine,
|
||||||
|
provider=args.provider,
|
||||||
|
backup_id=args.backup_id,
|
||||||
|
service=args.service,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register_restore_parser(parser: argparse.ArgumentParser) -> None:
|
||||||
|
parser.add_argument(
|
||||||
|
"machine", type=str, help="machine in the flake to create backups of"
|
||||||
|
)
|
||||||
|
parser.add_argument("provider", type=str, help="backup provider to use")
|
||||||
|
parser.add_argument("backup_id", type=str, help="id of the backup to restore")
|
||||||
|
parser.add_argument("--service", type=str, help="name of the service to restore")
|
||||||
|
parser.set_defaults(func=restore_command)
|
22
pkgs/clan-cli/tests/test_backups.py
Normal file
22
pkgs/clan-cli/tests/test_backups.py
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
from cli import Cli
|
||||||
|
from fixtures_flakes import FlakeForTest
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def test_backups(
|
||||||
|
test_flake: FlakeForTest,
|
||||||
|
) -> None:
|
||||||
|
cli = Cli()
|
||||||
|
|
||||||
|
cli.run(
|
||||||
|
[
|
||||||
|
"--flake",
|
||||||
|
str(test_flake.path),
|
||||||
|
"backups",
|
||||||
|
"list",
|
||||||
|
"testhostname",
|
||||||
|
]
|
||||||
|
)
|
Loading…
Reference in New Issue
Block a user