From a60b1bfafcee30c082ba2ed073496af57369124f Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 19 Jun 2024 16:56:26 +0200 Subject: [PATCH] Inventory: implement borgbackup --- clanModules/borgbackup-static/default.nix | 96 +++++---------- inventory/default.nix | 89 ++++++++++++++ inventory/example_flake.nix | 137 ---------------------- inventory/flake-module.nix | 4 +- inventory/src/tests/borgbackup.json | 10 +- inventory/src/tests/syncthing.json | 14 +-- 6 files changed, 130 insertions(+), 220 deletions(-) create mode 100644 inventory/default.nix delete mode 100644 inventory/example_flake.nix diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index 36fa6d6f..ee2deccb 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -2,57 +2,36 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; + + cfg = config.clan.borgbackup-static; + + machine_name = config.clan.core.machineName; in { imports = [ ../borgbackup ]; - options.clan.borgbackup-static = { - excludeMachines = lib.mkOption { - type = lib.types.listOf lib.types.str; - example = [ config.clan.core.machineName ]; - default = [ ]; - description = '' - Machines that should not be backuped. - Mutually exclusive with includeMachines. - If this is not empty, every other machine except the targets in the clan will be backuped by this module. - If includeMachines is set, only the included machines will be backuped. - ''; - }; - includeMachines = lib.mkOption { - type = lib.types.listOf lib.types.str; - example = [ config.clan.core.machineName ]; - default = [ ]; - description = '' - Machines that should be backuped. - Mutually exclusive with excludeMachines. - ''; - }; - targets = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - description = '' - Machines that should act as target machines for backups. - ''; - }; + # Inventory / Interface.nix + # options.clan.inventory.borgbackup-static.description. + options.clan.borgbackup-static.roles = lib.mkOption { + type = lib.types.attrsOf (lib.types.listOf lib.types.str); }; config.services.borgbackup.repos = let - machines = builtins.readDir machineDir; + + filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles); + borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; - filteredMachines = - if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then - lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines - else - lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines; - machinesMaybeKey = lib.mapAttrsToList ( - machine: _: + machinesMaybeKey = builtins.map ( + machine: let fullPath = borgbackupIpMachinePath machine; in if builtins.pathExists fullPath then machine else null ) filteredMachines; + machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; + hosts = builtins.map (machine: { name = machine; value = { @@ -61,41 +40,20 @@ in }; }) machinesWithKey; in - lib.mkIf - (builtins.any ( - target: target == config.clan.core.machineName - ) config.clan.borgbackup-static.targets) - (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }); + lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) ( + if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } + ); config.clan.borgbackup.destinations = let - destinations = builtins.map (d: { - name = d; - value = { - repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}"; - }; - }) config.clan.borgbackup-static.targets; - in - lib.mkIf (builtins.any ( - target: target == config.clan.core.machineName - ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations); + servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles); - config.assertions = [ - { - assertion = - !( - ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0) - && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) - ); - message = '' - The options: - config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}] - and - config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}] - are mutually exclusive. - Use excludeMachines to exclude certain machines and backup the other clan machines. - Use include machines to only backup certain machines. - ''; - } - ]; + destinations = builtins.map (server_name: { + name = server_name; + value = { + repo = "borg@${server_name}:/var/lib/borgbackup/${machine_name}"; + }; + }) servers; + in + lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations); } diff --git a/inventory/default.nix b/inventory/default.nix new file mode 100644 index 00000000..8832a7d7 --- /dev/null +++ b/inventory/default.nix @@ -0,0 +1,89 @@ +{ inputs, self, ... }: +let + clan-core = self; + system = "x86_64-linux"; + pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; + + # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json); + syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); + + machines = machinesFromInventory { + inherit clan-core; + lib = pkgs.lib; + } syncthing_inventory; + + /* + Returns a NixOS configuration for every machine in the inventory. + + machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } + */ + machinesFromInventory = + { lib, clan-core, ... }: + inventory: + # For every machine in the inventory, build a NixOS configuration + # For each machine generate config, forEach service, if the machine is used. + builtins.mapAttrs ( + machine_name: _: + builtins.foldl' ( + acc: service_name: + let + service_config = inventory.services.${service_name}; + isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig); + + machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { }; + global_config = inventory.services.${service_name}.config; + module_name = inventory.services.${service_name}.module; + in + # Possible roles: "server", "client", "peer" + if + builtins.trace '' + isInService ${builtins.toJSON isInService}, + ${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)} + '' isInService + then + acc + ++ [ + { + imports = [ clan-core.clanModules.${module_name} ]; + config.clan.${module_name} = lib.mkMerge [ + global_config + machine_service_config + ]; + } + { + config.clan.${module_name} = { + # TODO: filter, show only the roles that are needed by the machine + roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig; + }; + } + ] + else + acc + ) [ ] (builtins.attrNames inventory.services) + ) inventory.machines; +in +{ + clan = clan-core.lib.buildClan { + meta.name = "vis clans"; + # Should usually point to the directory of flake.nix + directory = self; + + machines = { + "vi_machine" = { + imports = machines.vi_machine; + }; + "vyr_machine" = { + imports = machines.vyr_machine; + }; + "camina_machine" = { + imports = machines.camina_machine; + }; + }; + }; + intern = machines; + # inherit (clan) nixosConfigurations clanInternals; + # add the Clan cli tool to the dev shell + devShells.${system}.default = pkgs.mkShell { + packages = [ clan-core.packages.${system}.clan-cli ]; + }; +} diff --git a/inventory/example_flake.nix b/inventory/example_flake.nix deleted file mode 100644 index 11be13a7..00000000 --- a/inventory/example_flake.nix +++ /dev/null @@ -1,137 +0,0 @@ -{ - description = ""; - - inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; - - outputs = - { clan-core, ... }: - let - pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; - system = "x86_64-linux"; - in - # Usage see: https://docs.clan.lol - # nice_flake_interface -> buildInventory() -> Inventory -> buildClanFromInventory() -> nixosConfigurations - # buildClanFromInventory = inventory: evalModules { - # extraAttrs = { inherit inventory; }; - # # (attrNames inventory.machines) - # }; - # clan = - # clan-core.lib.buildClanFromInventory [ - # # Inventory 0 (loads the json file managed by the Python API) - # (builtins.fromJSON (builtins.readFile ./inventory.json)) - # # -> - # # { - # # services."backups_1".autoIncludeMachines = true; - # # services."backups_1".module = "borgbackup"; - # # ... etc. - # # } - # ] - # ++ (buildInventory { - # clanName = "nice_flake_interface"; - # description = "A nice flake interface"; - # icon = "assets/icon.png"; - # machines = { - # jon = { - # # Just regular nixos/clan configuration ? - # # config = { - # # imports = [ - # # ./modules/shared.nix - # # ./machines/jon/configuration.nix - # # ]; - # # nixpkgs.hostPlatform = system; - # # # Set this for clan commands use ssh i.e. `clan machines update` - # # # If you change the hostname, you need to update this line to root@ - # # # This only works however if you have avahi running on your admin machine else use IP - # # clan.networking.targetHost = pkgs.lib.mkDefault "root@jon"; - # # # ssh root@flash-installer.local lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT - # # disko.devices.disk.main = { - # # device = "/dev/disk/by-id/__CHANGE_ME__"; - # # }; - # # # IMPORTANT! Add your SSH key here - # # # e.g. > cat ~/.ssh/id_ed25519.pub - # # users.users.root.openssh.authorizedKeys.keys = throw '' - # # Don't forget to add your SSH key here! - # # users.users.root.openssh.authorizedKeys.keys = [ "" ] - # # ''; - # # # Zerotier needs one controller to accept new nodes. Once accepted - # # # the controller can be offline and routing still works. - # # clan.networking.zerotier.controller.enable = true; - # # }; - # }; - # }; - # }) - # ++ [ - # # Low level inventory overrides (comes at the end) - # { - # services."backups_2".autoIncludeMachines = true; - # services."backups_2".module = "borgbackup"; - # } - # ]; - # # buildClan :: [ Partial ] -> Inventory - # # foldl' (acc: v: lib.recursiveUpdate acc v) {} [] - # inventory = [ - # # import json - # {...} - # # power user flake - # {...} - # ] - # # With Module system - # # Pros: Easy to understand, - # # Cons: Verbose, hard to maintain - # # buildClan :: { modules = [ { config = Partial; options :: InventoryOptions; } } ]; } -> Inventory - # eval = lib.evalModules { - # modules = [ - # { - # # Inventory Schema - # # Python validation - # options = {...} - # } - # { - # config = map lib.mkDefault - # (builtins.fromJSON (builtins.readFile ./inventory.json)) - # } - # { - # # User provided - # config = {...} - # } - # # Later overrides. - # { - # lib.mkForce ... - # } - # ]; - # } - # nixosConfigurations = lib.evalModules inventory; - # eval.config.inventory - # # - # eval.config.machines.jon#nixosConfig - # eval.config.machines.sara#nixosConfig - # - # {inventory, config, ...}:{ - # hostname = config.machines.sara # Invalid - # hostname = inventory.machines.sara.hostname # Valid - # } - /* - # Type - - buildInventory :: { - clanName :: string - machines :: { - ${name} :: { - config :: { - # NixOS configuration - }; - }; - }; - # ... More mapped inventory options - # i.e. shared config for all machines - } -> Inventory - */ - { - # all machines managed by Clan - inherit (clan) nixosConfigurations clanInternals; - # add the Clan cli tool to the dev shell - devShells.${system}.default = pkgs.mkShell { - packages = [ clan-core.packages.${system}.clan-cli ]; - }; - }; -} diff --git a/inventory/flake-module.nix b/inventory/flake-module.nix index 65c6c66f..aace1add 100644 --- a/inventory/flake-module.nix +++ b/inventory/flake-module.nix @@ -1,5 +1,6 @@ -{ ... }: +{ inputs, self, ... }: { + flake.inventory = import ./default.nix { inherit inputs self; }; perSystem = { pkgs, config, ... }: { @@ -13,6 +14,7 @@ mkdir -p $out ''; }; + devShells.inventory-schema = pkgs.mkShell { inputsFrom = [ config.packages.inventory-schema ]; }; checks.inventory-schema-checks = pkgs.stdenv.mkDerivation { diff --git a/inventory/src/tests/borgbackup.json b/inventory/src/tests/borgbackup.json index 43e04322..11aa844e 100644 --- a/inventory/src/tests/borgbackup.json +++ b/inventory/src/tests/borgbackup.json @@ -18,21 +18,19 @@ "meta": { "name": "My backup" }, - "module": "borbackup-static", + "module": "borgbackup-static", "machineConfig": { - "vyr": { + "vyr_machine": { "roles": ["server"] }, - "vi": { + "vi_machine": { "roles": ["client"] }, "camina_machine": { "roles": ["client"] } }, - "config": { - "folders": ["/home", "/root", "/var", "/etc"] - } + "config": {} } } } diff --git a/inventory/src/tests/syncthing.json b/inventory/src/tests/syncthing.json index 1f5d59cf..6656e5ca 100644 --- a/inventory/src/tests/syncthing.json +++ b/inventory/src/tests/syncthing.json @@ -3,10 +3,10 @@ "camina_machine": { "name": "camina" }, - "vyr": { + "vyr_machine": { "name": "vyr" }, - "vi": { + "vi_machine": { "name": "vi" } }, @@ -20,23 +20,23 @@ }, "module": "syncthing-static-peers", "machineConfig": { - "vyr": {}, - "vi": {}, + "vyr_machine": {}, + "vi_machine": {}, "camina_machine": {} }, "config": { "folders": { "test": { "path": "~/data/docs", - "devices": ["camina", "vyr", "vi"] + "devices": ["camina_machine", "vyr_machine", "vi_machine"] }, "videos": { "path": "~/data/videos", - "devices": ["camina", "vyr", "ezra"] + "devices": ["camina_machine", "vyr_machine"] }, "playlist": { "path": "~/data/playlist", - "devices": ["camina", "vyr", "ezra"] + "devices": ["camina_machine", "vi_machine"] } } }