diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index c65dff61..d65fdb5f 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -2,62 +2,57 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; - - # cfg.roles = config.clan.borgbackup-static; - - # machine < machine_module < inventory - # nixos < borgbackup < borgbackup-static > UI - # metadata - # Developer User field descriptions - - roles = config.clan.borgbackup-static.inventory.roles; - - machine_name = config.clan.core.machineName; in -{ +lib.warn "This module is deprecated use the service via the service interface instead." { imports = [ ../borgbackup ]; - # imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ]; - # Inventory / Interface.nix - # options.clan.inventory.borgbackup-static.description. - # options.clan.borgbackup-static.roles = lib.mkOption { - # type = lib.types.attrsOf (lib.types.listOf lib.types.str); - # }; - # Can be used via inventory.json - # - # .borgbackup-static.inventory.roles - # - options.clan.borgbackup-static.inventory = lib.mkOption { - type = lib.types.submodule { - # imports = [./inventory/interface.nix]; - - # idea - # config.metadata = builtins.fromTOML ... - # config.defaultRoles = ["client"]; - - # -> interface.nix - options = { - roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); }; - }; + options.clan.borgbackup-static = { + excludeMachines = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ config.clan.core.machineName ]; + default = [ ]; + description = '' + Machines that should not be backuped. + Mutually exclusive with includeMachines. + If this is not empty, every other machine except the targets in the clan will be backuped by this module. + If includeMachines is set, only the included machines will be backuped. + ''; + }; + includeMachines = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ config.clan.core.machineName ]; + default = [ ]; + description = '' + Machines that should be backuped. + Mutually exclusive with excludeMachines. + ''; + }; + targets = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Machines that should act as target machines for backups. + ''; }; }; config.services.borgbackup.repos = let - - filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles); - + machines = builtins.readDir machineDir; borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; - machinesMaybeKey = builtins.map ( - machine: + filteredMachines = + if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then + lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines + else + lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines; + machinesMaybeKey = lib.mapAttrsToList ( + machine: _: let fullPath = borgbackupIpMachinePath machine; in if builtins.pathExists fullPath then machine else null ) filteredMachines; - machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; - hosts = builtins.map (machine: { name = machine; value = { @@ -66,20 +61,41 @@ in }; }) machinesWithKey; in - lib.mkIf (builtins.elem "server" roles.${machine_name}) ( - if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } - ); + lib.mkIf + (builtins.any ( + target: target == config.clan.core.machineName + ) config.clan.borgbackup-static.targets) + (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }); config.clan.borgbackup.destinations = let - servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles); - - destinations = builtins.map (server_name: { - name = server_name; + destinations = builtins.map (d: { + name = d; value = { - repo = "borg@${server_name}:/var/lib/borgbackup/${machine_name}"; + repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}"; }; - }) servers; + }) config.clan.borgbackup-static.targets; in - lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations); + lib.mkIf (builtins.any ( + target: target == config.clan.core.machineName + ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations); + + config.assertions = [ + { + assertion = + !( + ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0) + && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) + ); + message = '' + The options: + config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}] + and + config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}] + are mutually exclusive. + Use excludeMachines to exclude certain machines and backup the other clan machines. + Use include machines to only backup certain machines. + ''; + } + ]; } diff --git a/clanModules/borgbackup/README.md b/clanModules/borgbackup/README.md index b639786d..13379d26 100644 --- a/clanModules/borgbackup/README.md +++ b/clanModules/borgbackup/README.md @@ -1,2 +1,15 @@ Efficient, deduplicating backup program with optional compression and secure encryption. ---- \ No newline at end of file +--- + +## Roles + +- Client +- Server + +## Configuration + +Configure target machines where the backups should be sent to through `targets`. + +Configure machines that should be backed up either through `includeMachines` +which will exclusively add the included machines to be backed up, or through +`excludeMachines`, which will add every machine except the excluded machine to the backup. diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix new file mode 100644 index 00000000..661cb4f3 --- /dev/null +++ b/clanModules/borgbackup/roles/client.nix @@ -0,0 +1,30 @@ +{ config, lib, ... }: +let + instances = config.clan.inventory.borgbackup; + # roles = { ${role_name} :: { machines :: [string] } } + allServers = lib.foldlAttrs ( + acc: _instanceName: instanceConfig: + acc + ++ ( + if builtins.elem machineName instanceConfig.roles.client.machines then + instanceConfig.roles.server.machines + else + [ ] + ) + ) [ ] instances; + + inherit (config.clan.core) machineName; +in +{ + config.clan.borgbackup.destinations = + let + + destinations = builtins.map (serverName: { + name = serverName; + value = { + repo = "borg@${serverName}:/var/lib/borgbackup/${machineName}"; + }; + }) allServers; + in + (builtins.listToAttrs destinations); +} diff --git a/clanModules/borgbackup/roles/server.nix b/clanModules/borgbackup/roles/server.nix new file mode 100644 index 00000000..c25230da --- /dev/null +++ b/clanModules/borgbackup/roles/server.nix @@ -0,0 +1,45 @@ +{ config, lib, ... }: +let + clanDir = config.clan.core.clanDir; + machineDir = clanDir + "/machines/"; + inherit (config.clan.core) machineName; + instances = config.clan.inventory.borgbackup; + + # roles = { ${role_name} :: { machines :: [string] } } + allClients = lib.foldlAttrs ( + acc: _instanceName: instanceConfig: + acc + ++ ( + if builtins.elem machineName instanceConfig.roles.server.machines then + instanceConfig.roles.client.machines + else + [ ] + ) + ) [ ] instances; +in +{ + config.services.borgbackup.repos = + let + filteredMachines = allClients; + + borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; + machinesMaybeKey = builtins.map ( + machine: + let + fullPath = borgbackupIpMachinePath machine; + in + if builtins.pathExists fullPath then machine else null + ) filteredMachines; + + machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; + + hosts = builtins.map (machine: { + name = machine; + value = { + path = "/var/lib/borgbackup/${machine}"; + authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ]; + }; + }) machinesWithKey; + in + if (builtins.listToAttrs hosts) != [ ] then builtins.listToAttrs hosts else { }; +} diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index 928b8e5b..e3a95adf 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -5,7 +5,7 @@ imports = [ ./disk-layouts ]; }; borgbackup = ./borgbackup; - borgbackup-static = ./borgbackup-static; + deltachat = ./deltachat; ergochat = ./ergochat; localbackup = ./localbackup; diff --git a/inventory/README.md b/inventory/README.md index 64efedd3..effdfc0e 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -3,19 +3,24 @@ The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort. - The inventory acts as a declarative source of truth for all machine configurations. -- Users can easily add or remove machines and services. +- Users can easily add or remove machines to/from services. - Ensures that all machines and services are configured consistently, across multiple nixosConfigs. - Defaults and predefined roles in our modules minimizes the need for manual configuration. Design questions: +- [ ] Is the service config interface the same as the module config interface ? + +- [ ] As a user i dont want to see borgbackup as the high level category ? + - [x] Must roles be a list ? - -> Yes. In zerotier you can be "moon" and "controller" at the same time. + -> Yes. In zerotier a machine can be "moon" and "controller" at the same time. - [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different? -> There are many roles. And they depend on the service. - [x] Should we use the module name in the path of the service? + -> YES ```json // ${module_name}.${instance_name} services.borgbackup-static.backup1 = { @@ -32,8 +37,10 @@ Design questions: Neutral: Module name is hard to change. Exists anyways. - [x] Should the machine specific service config be part of the service? - -> The config implements the schema of the module, which is declared in the service. - -> If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. + -> NO. because ... + - The config implements the schema of the module, which is declared in the service. + - If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. + - If the config is placed in the machine it is de-located into another top-level field. In the module this complicates access. Architecture diff --git a/inventory/default.nix b/inventory/default.nix index fbee6457..8d8e9c8e 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -7,22 +7,25 @@ let machines = machinesFromInventory syncthing_inventory; - resolveGroups = - inventory: members: - lib.unique ( - builtins.foldl' ( - acc: currMember: - let - groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember; - groupMembers = - if inventory.groups.machines ? ${groupName} then - inventory.groups.machines.${groupName} - else - throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory."; - in - if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ] - ) [ ] members - ); + resolveTags = + # Inventory, { machines :: [string], tags :: [string] } + inventory: members: { + machines = + members.machines or [ ] + ++ (builtins.foldl' ( + acc: tag: + let + tagMembers = builtins.attrNames ( + lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines + ); + in + # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory."; + if tagMembers == [ ] then + throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory." + else + acc ++ tagMembers + ) [ ] members.tags or [ ]); + }; /* Returns a NixOS configuration for every machine in the inventory. @@ -45,29 +48,53 @@ let acc2: instanceName: serviceConfig: let resolvedRoles = builtins.mapAttrs ( - _roleName: members: resolveGroups inventory members + _roleName: members: resolveTags inventory members ) serviceConfig.roles; - isInService = builtins.any (members: builtins.elem machineName members) ( + isInService = builtins.any (members: builtins.elem machineName members.machines) ( builtins.attrValues resolvedRoles ); + # Inverse map of roles. Allows for easy lookup of roles for a given machine. + # { ${machine_name} :: [roles] + inverseRoles = lib.foldlAttrs ( + acc: roleName: + { machines }: + acc + // builtins.foldl' ( + acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; } + ) { } machines + ) { } resolvedRoles; + machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; globalConfig = serviceConfig.config; + + # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy + roleModules = builtins.map ( + role: + let + path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix"; + in + if builtins.pathExists path then + path + else + throw "Role doesnt have a module: ${role}. Path: ${path} not found." + ) inverseRoles.${machineName} or [ ]; in if isInService then acc2 ++ [ { - imports = [ clan-core.clanModules.${moduleName} ]; + imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules; config.clan.${moduleName} = lib.mkMerge [ globalConfig machineServiceConfig ]; } { - config.clan.inventory.${instanceName} = { + config.clan.inventory.${moduleName}.${instanceName} = { roles = resolvedRoles; + # inherit inverseRoles; }; } ] @@ -78,8 +105,64 @@ let ) inventory.machines; in { + inherit clan-core; + + new_clan = clan-core.lib.buildInventory { + # High level services. + # If you need multiple instances of a service configure them via: + # inventory.services.[serviceName].[instanceName] = ... + services = { + borbackup = { + roles.server.machines = [ "vyr" ]; + roles.client.tags = [ "laptop" ]; + machines.vyr = { + config = { + + }; + }; + config = { + + }; + }; + }; + + # Low level inventory i.e. if you need multiple instances of a service + # Or if you want to manipulate the created inventory directly. + inventory.services.borbackup.default = { }; + + # Machines. each machine can be referenced by its attribute name under services. + machines = { + camina = { + # This is added to machine tags + clan.tags = [ "laptop" ]; + # These are the inventory machine fields + clan.meta.description = ""; + clan.meta.name = ""; + clan.meta.icon = ""; + # Config ... + }; + vyr = { + # Config ... + }; + vi = { + clan.networking.targetHost = "root@78.47.164.46"; + # Config ... + }; + aya = { + clan.networking.targetHost = "root@78.47.164.46"; + # Config ... + }; + ezra = { + # Config ... + }; + rianon = { + # Config ... + }; + }; + }; + clan = clan-core.lib.buildClan { - meta.name = "vis clans"; + meta.name = "vi's clans"; # Should usually point to the directory of flake.nix directory = self; diff --git a/inventory/src/root.cue b/inventory/src/root.cue index 661e92dd..da64c1b2 100644 --- a/inventory/src/root.cue +++ b/inventory/src/root.cue @@ -20,6 +20,4 @@ import ( // // A map of machines schema.#machine - - schema.#groups } diff --git a/inventory/src/schema/schema.cue b/inventory/src/schema/schema.cue index 7da9dd27..81d197b6 100644 --- a/inventory/src/schema/schema.cue +++ b/inventory/src/schema/schema.cue @@ -1,18 +1,10 @@ package schema -#groups: groups: { - // Machine groups - machines: { - // Group name mapped to list[machineName] - // "group1": ["machine1", "machine2"] - [string]: [...string] - } -} - #machine: machines: [string]: { name: string, description?: string, icon?: string + tags: [...string] } #role: string @@ -26,7 +18,10 @@ package schema }, // We moved the machine sepcific config to "machines". // It may be moved back depending on what makes more sense in the future. - roles: [#role]: [...string], + roles: [#role]: { + machines: [...string], + tags: [...string], + } machines: { [string]: { config?: { diff --git a/inventory/src/tests/borgbackup.json b/inventory/src/tests/borgbackup.json index 55a8c516..7e02bde5 100644 --- a/inventory/src/tests/borgbackup.json +++ b/inventory/src/tests/borgbackup.json @@ -1,39 +1,51 @@ { "machines": { "camina_machine": { - "name": "camina" + "name": "camina", + "tags": ["laptop"] }, "vyr_machine": { "name": "vyr" }, "vi_machine": { - "name": "vi" - } - }, - "groups": { - "machines": { - "laptops": ["camina_machine", "vi_machine"], - "all": ["camina_machine", "vi_machine", "vyr_machine"] + "name": "vi", + "tags": ["laptop"] } }, "meta": { "name": "kenjis clan" }, "services": { - "borgbackup-static": { + "borgbackup": { "instance_1": { "meta": { "name": "My backup" }, "roles": { - "server": ["vyr_machine"], - "client": ["group:laptops"] + "server": { + "machines": ["vyr_machine"] + }, + "client": { + "machines": ["vyr_machine"], + "tags": ["laptop"] + } }, - "machines": { - "vyr_machine": {}, - "vi_machine": {}, - "camina_machine": {} + "machines": {}, + "config": {} + }, + "instance_2": { + "meta": { + "name": "My backup" }, + "roles": { + "server": { + "machines": ["vi_machine"] + }, + "client": { + "machines": ["camina_machine"] + } + }, + "machines": {}, "config": {} } } diff --git a/inventory/src/tests/syncthing.json b/inventory/src/tests/syncthing.json index 58d0e204..e7ed8af6 100644 --- a/inventory/src/tests/syncthing.json +++ b/inventory/src/tests/syncthing.json @@ -14,13 +14,15 @@ "name": "kenjis clan" }, "services": { - "syncthing-static-peers": { + "syncthing": { "instance_1": { "meta": { "name": "My sync" }, "roles": { - "peer": ["vyr_machine", "vi_machine", "camina_machine"] + "peer": { + "machines": ["vyr_machine", "vi_machine", "camina_machine"] + } }, "machines": {}, "config": { diff --git a/inventory/src/tests/zerotier.json b/inventory/src/tests/zerotier.json index 17b226ab..f35e2ab4 100644 --- a/inventory/src/tests/zerotier.json +++ b/inventory/src/tests/zerotier.json @@ -14,14 +14,15 @@ "name": "kenjis clan" }, "services": { - "zerotier-static": { + "zerotier": { "instance_1": { "meta": { "name": "My Network" }, "roles": { - "server": ["vyr_machine"], - "peer": ["vi_machine", "camina_machine"] + "controller": { "machines": ["vyr_machine"] }, + "moon": { "machines": ["vyr_machine"] }, + "peer": { "machines": ["vi_machine", "camina_machine"] } }, "machines": { "vyr_machine": { diff --git a/nixosModules/clanCore/default.nix b/nixosModules/clanCore/default.nix index 9926155c..e356a74f 100644 --- a/nixosModules/clanCore/default.nix +++ b/nixosModules/clanCore/default.nix @@ -14,5 +14,7 @@ ./vm.nix ./wayland-proxy-virtwl.nix ./zerotier + # Inventory + ./inventory/interface.nix ]; } diff --git a/nixosModules/clanCore/inventory/interface.nix b/nixosModules/clanCore/inventory/interface.nix new file mode 100644 index 00000000..7280b560 --- /dev/null +++ b/nixosModules/clanCore/inventory/interface.nix @@ -0,0 +1,36 @@ +{ lib, ... }: +let + # { + # roles = { + # client = { + # machines = [ + # "camina_machine" + # "vi_machine" + # ]; + # }; + # server = { + # machines = [ "vyr_machine" ]; + # }; + # }; + # } + instanceOptions = lib.types.submodule { + options.roles = lib.mkOption { type = lib.types.attrsOf machinesList; }; + }; + + # { + # machines = [ + # "camina_machine" + # "vi_machine" + # "vyr_machine" + # ]; + # } + machinesList = lib.types.submodule { + options.machines = lib.mkOption { type = lib.types.listOf lib.types.str; }; + }; +in +{ + # clan.inventory.${moduleName}.${instanceName} = { ... } + options.clan.inventory = lib.mkOption { + type = lib.types.attrsOf (lib.types.attrsOf instanceOptions); + }; +}