From d934b67c7245bd5b67e0d2efd18a39acb1d20c61 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 19 Jun 2024 16:56:26 +0200 Subject: [PATCH 01/18] Inventory: implement borgbackup --- clanModules/borgbackup-static/default.nix | 96 +++++---------- inventory/default.nix | 89 ++++++++++++++ inventory/example_flake.nix | 137 ---------------------- inventory/flake-module.nix | 4 +- inventory/src/tests/borgbackup.json | 10 +- inventory/src/tests/syncthing.json | 14 +-- 6 files changed, 130 insertions(+), 220 deletions(-) create mode 100644 inventory/default.nix delete mode 100644 inventory/example_flake.nix diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index 36fa6d6f..ee2deccb 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -2,57 +2,36 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; + + cfg = config.clan.borgbackup-static; + + machine_name = config.clan.core.machineName; in { imports = [ ../borgbackup ]; - options.clan.borgbackup-static = { - excludeMachines = lib.mkOption { - type = lib.types.listOf lib.types.str; - example = [ config.clan.core.machineName ]; - default = [ ]; - description = '' - Machines that should not be backuped. - Mutually exclusive with includeMachines. - If this is not empty, every other machine except the targets in the clan will be backuped by this module. - If includeMachines is set, only the included machines will be backuped. - ''; - }; - includeMachines = lib.mkOption { - type = lib.types.listOf lib.types.str; - example = [ config.clan.core.machineName ]; - default = [ ]; - description = '' - Machines that should be backuped. - Mutually exclusive with excludeMachines. - ''; - }; - targets = lib.mkOption { - type = lib.types.listOf lib.types.str; - default = [ ]; - description = '' - Machines that should act as target machines for backups. - ''; - }; + # Inventory / Interface.nix + # options.clan.inventory.borgbackup-static.description. + options.clan.borgbackup-static.roles = lib.mkOption { + type = lib.types.attrsOf (lib.types.listOf lib.types.str); }; config.services.borgbackup.repos = let - machines = builtins.readDir machineDir; + + filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles); + borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; - filteredMachines = - if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then - lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines - else - lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines; - machinesMaybeKey = lib.mapAttrsToList ( - machine: _: + machinesMaybeKey = builtins.map ( + machine: let fullPath = borgbackupIpMachinePath machine; in if builtins.pathExists fullPath then machine else null ) filteredMachines; + machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; + hosts = builtins.map (machine: { name = machine; value = { @@ -61,41 +40,20 @@ in }; }) machinesWithKey; in - lib.mkIf - (builtins.any ( - target: target == config.clan.core.machineName - ) config.clan.borgbackup-static.targets) - (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }); + lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) ( + if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } + ); config.clan.borgbackup.destinations = let - destinations = builtins.map (d: { - name = d; - value = { - repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}"; - }; - }) config.clan.borgbackup-static.targets; - in - lib.mkIf (builtins.any ( - target: target == config.clan.core.machineName - ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations); + servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles); - config.assertions = [ - { - assertion = - !( - ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0) - && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) - ); - message = '' - The options: - config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}] - and - config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}] - are mutually exclusive. - Use excludeMachines to exclude certain machines and backup the other clan machines. - Use include machines to only backup certain machines. - ''; - } - ]; + destinations = builtins.map (server_name: { + name = server_name; + value = { + repo = "borg@${server_name}:/var/lib/borgbackup/${machine_name}"; + }; + }) servers; + in + lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations); } diff --git a/inventory/default.nix b/inventory/default.nix new file mode 100644 index 00000000..8832a7d7 --- /dev/null +++ b/inventory/default.nix @@ -0,0 +1,89 @@ +{ inputs, self, ... }: +let + clan-core = self; + system = "x86_64-linux"; + pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; + + # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json); + syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); + + machines = machinesFromInventory { + inherit clan-core; + lib = pkgs.lib; + } syncthing_inventory; + + /* + Returns a NixOS configuration for every machine in the inventory. + + machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } + */ + machinesFromInventory = + { lib, clan-core, ... }: + inventory: + # For every machine in the inventory, build a NixOS configuration + # For each machine generate config, forEach service, if the machine is used. + builtins.mapAttrs ( + machine_name: _: + builtins.foldl' ( + acc: service_name: + let + service_config = inventory.services.${service_name}; + isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig); + + machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { }; + global_config = inventory.services.${service_name}.config; + module_name = inventory.services.${service_name}.module; + in + # Possible roles: "server", "client", "peer" + if + builtins.trace '' + isInService ${builtins.toJSON isInService}, + ${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)} + '' isInService + then + acc + ++ [ + { + imports = [ clan-core.clanModules.${module_name} ]; + config.clan.${module_name} = lib.mkMerge [ + global_config + machine_service_config + ]; + } + { + config.clan.${module_name} = { + # TODO: filter, show only the roles that are needed by the machine + roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig; + }; + } + ] + else + acc + ) [ ] (builtins.attrNames inventory.services) + ) inventory.machines; +in +{ + clan = clan-core.lib.buildClan { + meta.name = "vis clans"; + # Should usually point to the directory of flake.nix + directory = self; + + machines = { + "vi_machine" = { + imports = machines.vi_machine; + }; + "vyr_machine" = { + imports = machines.vyr_machine; + }; + "camina_machine" = { + imports = machines.camina_machine; + }; + }; + }; + intern = machines; + # inherit (clan) nixosConfigurations clanInternals; + # add the Clan cli tool to the dev shell + devShells.${system}.default = pkgs.mkShell { + packages = [ clan-core.packages.${system}.clan-cli ]; + }; +} diff --git a/inventory/example_flake.nix b/inventory/example_flake.nix deleted file mode 100644 index 11be13a7..00000000 --- a/inventory/example_flake.nix +++ /dev/null @@ -1,137 +0,0 @@ -{ - description = ""; - - inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz"; - - outputs = - { clan-core, ... }: - let - pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; - system = "x86_64-linux"; - in - # Usage see: https://docs.clan.lol - # nice_flake_interface -> buildInventory() -> Inventory -> buildClanFromInventory() -> nixosConfigurations - # buildClanFromInventory = inventory: evalModules { - # extraAttrs = { inherit inventory; }; - # # (attrNames inventory.machines) - # }; - # clan = - # clan-core.lib.buildClanFromInventory [ - # # Inventory 0 (loads the json file managed by the Python API) - # (builtins.fromJSON (builtins.readFile ./inventory.json)) - # # -> - # # { - # # services."backups_1".autoIncludeMachines = true; - # # services."backups_1".module = "borgbackup"; - # # ... etc. - # # } - # ] - # ++ (buildInventory { - # clanName = "nice_flake_interface"; - # description = "A nice flake interface"; - # icon = "assets/icon.png"; - # machines = { - # jon = { - # # Just regular nixos/clan configuration ? - # # config = { - # # imports = [ - # # ./modules/shared.nix - # # ./machines/jon/configuration.nix - # # ]; - # # nixpkgs.hostPlatform = system; - # # # Set this for clan commands use ssh i.e. `clan machines update` - # # # If you change the hostname, you need to update this line to root@ - # # # This only works however if you have avahi running on your admin machine else use IP - # # clan.networking.targetHost = pkgs.lib.mkDefault "root@jon"; - # # # ssh root@flash-installer.local lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT - # # disko.devices.disk.main = { - # # device = "/dev/disk/by-id/__CHANGE_ME__"; - # # }; - # # # IMPORTANT! Add your SSH key here - # # # e.g. > cat ~/.ssh/id_ed25519.pub - # # users.users.root.openssh.authorizedKeys.keys = throw '' - # # Don't forget to add your SSH key here! - # # users.users.root.openssh.authorizedKeys.keys = [ "" ] - # # ''; - # # # Zerotier needs one controller to accept new nodes. Once accepted - # # # the controller can be offline and routing still works. - # # clan.networking.zerotier.controller.enable = true; - # # }; - # }; - # }; - # }) - # ++ [ - # # Low level inventory overrides (comes at the end) - # { - # services."backups_2".autoIncludeMachines = true; - # services."backups_2".module = "borgbackup"; - # } - # ]; - # # buildClan :: [ Partial ] -> Inventory - # # foldl' (acc: v: lib.recursiveUpdate acc v) {} [] - # inventory = [ - # # import json - # {...} - # # power user flake - # {...} - # ] - # # With Module system - # # Pros: Easy to understand, - # # Cons: Verbose, hard to maintain - # # buildClan :: { modules = [ { config = Partial; options :: InventoryOptions; } } ]; } -> Inventory - # eval = lib.evalModules { - # modules = [ - # { - # # Inventory Schema - # # Python validation - # options = {...} - # } - # { - # config = map lib.mkDefault - # (builtins.fromJSON (builtins.readFile ./inventory.json)) - # } - # { - # # User provided - # config = {...} - # } - # # Later overrides. - # { - # lib.mkForce ... - # } - # ]; - # } - # nixosConfigurations = lib.evalModules inventory; - # eval.config.inventory - # # - # eval.config.machines.jon#nixosConfig - # eval.config.machines.sara#nixosConfig - # - # {inventory, config, ...}:{ - # hostname = config.machines.sara # Invalid - # hostname = inventory.machines.sara.hostname # Valid - # } - /* - # Type - - buildInventory :: { - clanName :: string - machines :: { - ${name} :: { - config :: { - # NixOS configuration - }; - }; - }; - # ... More mapped inventory options - # i.e. shared config for all machines - } -> Inventory - */ - { - # all machines managed by Clan - inherit (clan) nixosConfigurations clanInternals; - # add the Clan cli tool to the dev shell - devShells.${system}.default = pkgs.mkShell { - packages = [ clan-core.packages.${system}.clan-cli ]; - }; - }; -} diff --git a/inventory/flake-module.nix b/inventory/flake-module.nix index 65c6c66f..aace1add 100644 --- a/inventory/flake-module.nix +++ b/inventory/flake-module.nix @@ -1,5 +1,6 @@ -{ ... }: +{ inputs, self, ... }: { + flake.inventory = import ./default.nix { inherit inputs self; }; perSystem = { pkgs, config, ... }: { @@ -13,6 +14,7 @@ mkdir -p $out ''; }; + devShells.inventory-schema = pkgs.mkShell { inputsFrom = [ config.packages.inventory-schema ]; }; checks.inventory-schema-checks = pkgs.stdenv.mkDerivation { diff --git a/inventory/src/tests/borgbackup.json b/inventory/src/tests/borgbackup.json index 43e04322..11aa844e 100644 --- a/inventory/src/tests/borgbackup.json +++ b/inventory/src/tests/borgbackup.json @@ -18,21 +18,19 @@ "meta": { "name": "My backup" }, - "module": "borbackup-static", + "module": "borgbackup-static", "machineConfig": { - "vyr": { + "vyr_machine": { "roles": ["server"] }, - "vi": { + "vi_machine": { "roles": ["client"] }, "camina_machine": { "roles": ["client"] } }, - "config": { - "folders": ["/home", "/root", "/var", "/etc"] - } + "config": {} } } } diff --git a/inventory/src/tests/syncthing.json b/inventory/src/tests/syncthing.json index 1f5d59cf..6656e5ca 100644 --- a/inventory/src/tests/syncthing.json +++ b/inventory/src/tests/syncthing.json @@ -3,10 +3,10 @@ "camina_machine": { "name": "camina" }, - "vyr": { + "vyr_machine": { "name": "vyr" }, - "vi": { + "vi_machine": { "name": "vi" } }, @@ -20,23 +20,23 @@ }, "module": "syncthing-static-peers", "machineConfig": { - "vyr": {}, - "vi": {}, + "vyr_machine": {}, + "vi_machine": {}, "camina_machine": {} }, "config": { "folders": { "test": { "path": "~/data/docs", - "devices": ["camina", "vyr", "vi"] + "devices": ["camina_machine", "vyr_machine", "vi_machine"] }, "videos": { "path": "~/data/videos", - "devices": ["camina", "vyr", "ezra"] + "devices": ["camina_machine", "vyr_machine"] }, "playlist": { "path": "~/data/playlist", - "devices": ["camina", "vyr", "ezra"] + "devices": ["camina_machine", "vi_machine"] } } } From 5f72778adec7e3359c25bad6b6124b47dd2a08bf Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 20 Jun 2024 18:36:26 +0200 Subject: [PATCH 02/18] schema improvements --- clanModules/borgbackup-static/default.nix | 42 +++++-- inventory/README.md | 59 ++++++++++ inventory/default.nix | 108 ++++++++++-------- inventory/flake-module.nix | 9 +- inventory/src/machines/machines.cue | 8 -- inventory/src/root.cue | 13 ++- .../services.cue => schema/schema.cue} | 30 +++-- inventory/src/tests/borgbackup.json | 35 +++--- inventory/src/tests/syncthing.json | 46 ++++---- inventory/src/tests/zerotier.json | 29 +++-- 10 files changed, 244 insertions(+), 135 deletions(-) delete mode 100644 inventory/src/machines/machines.cue rename inventory/src/{services/services.cue => schema/schema.cue} (57%) diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index ee2deccb..c65dff61 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -3,23 +3,49 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; - cfg = config.clan.borgbackup-static; + # cfg.roles = config.clan.borgbackup-static; + + # machine < machine_module < inventory + # nixos < borgbackup < borgbackup-static > UI + # metadata + # Developer User field descriptions + + roles = config.clan.borgbackup-static.inventory.roles; machine_name = config.clan.core.machineName; in { imports = [ ../borgbackup ]; - + # imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ]; # Inventory / Interface.nix # options.clan.inventory.borgbackup-static.description. - options.clan.borgbackup-static.roles = lib.mkOption { - type = lib.types.attrsOf (lib.types.listOf lib.types.str); + # options.clan.borgbackup-static.roles = lib.mkOption { + # type = lib.types.attrsOf (lib.types.listOf lib.types.str); + # }; + + # Can be used via inventory.json + # + # .borgbackup-static.inventory.roles + # + options.clan.borgbackup-static.inventory = lib.mkOption { + type = lib.types.submodule { + # imports = [./inventory/interface.nix]; + + # idea + # config.metadata = builtins.fromTOML ... + # config.defaultRoles = ["client"]; + + # -> interface.nix + options = { + roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); }; + }; + }; }; config.services.borgbackup.repos = let - filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles); + filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles); borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; machinesMaybeKey = builtins.map ( @@ -40,13 +66,13 @@ in }; }) machinesWithKey; in - lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) ( + lib.mkIf (builtins.elem "server" roles.${machine_name}) ( if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } ); config.clan.borgbackup.destinations = let - servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles); + servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles); destinations = builtins.map (server_name: { name = server_name; @@ -55,5 +81,5 @@ in }; }) servers; in - lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations); + lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations); } diff --git a/inventory/README.md b/inventory/README.md index 318d1bf0..2bdbf77b 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -1,5 +1,64 @@ # Inventory +Questions: + +- [x] Must roles be a list ? + -> Yes. In zerotier you can be "moon" and "controller" at the same time. + +- [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different? + -> There are many roles. And they depend on the service. + +- [x] Should we use the module name in the path of the service? + ```json + // ${module_name}.${instance_name} + services.borgbackup-static.backup1 = { + + } + ``` + + Pro: + Easier to handle. + Better groups the module specific instances. + Contra: + More nesting in json + + Neutral: Module name is hard to change. Exists anyways. + +- [x] Should the machine specific service config be part of the service? + -> The config implements the schema of the module, which is declared in the service. + -> If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. + +Architecture + +``` +machine < machine_module < inventory +--------------------------------------------- +nixos < borgbackup < borgbackup-static > UI + + creates the config Maps from high level services to the borgbackup clan module + for ONE machine +``` + +- [ ] Why do we need 2 modules? + -> It is technically possible to have only 1 module. + Pros: + Simple to use/Easy to understand. + Less modules + Cons: + Harder to write a module. Because it must do 2 things. + One module should do only 1 thing. + +```nix +clan.machines.${machine_name} = { + # "borgbackup.ssh.pub" = machineDir + machines + "/facts/borgbackup.ssh.pub"; + facts = ... +}; +clan.services.${instance} = { +# roles.server = [ "jon_machine" ] +# roles.${role_name} = [ ${machine_name} ]; +}; +``` + This part provides a specification for the inventory. It is used for design phase and as validation helper. diff --git a/inventory/default.nix b/inventory/default.nix index 8832a7d7..fbee6457 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -1,16 +1,28 @@ -{ inputs, self, ... }: +{ self, lib, ... }: let clan-core = self; - system = "x86_64-linux"; - pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json); syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); - machines = machinesFromInventory { - inherit clan-core; - lib = pkgs.lib; - } syncthing_inventory; + machines = machinesFromInventory syncthing_inventory; + + resolveGroups = + inventory: members: + lib.unique ( + builtins.foldl' ( + acc: currMember: + let + groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember; + groupMembers = + if inventory.groups.machines ? ${groupName} then + inventory.groups.machines.${groupName} + else + throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory."; + in + if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ] + ) [ ] members + ); /* Returns a NixOS configuration for every machine in the inventory. @@ -18,48 +30,51 @@ let machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } */ machinesFromInventory = - { lib, clan-core, ... }: inventory: # For every machine in the inventory, build a NixOS configuration # For each machine generate config, forEach service, if the machine is used. builtins.mapAttrs ( - machine_name: _: - builtins.foldl' ( - acc: service_name: - let - service_config = inventory.services.${service_name}; - isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig); + machineName: _: + lib.foldlAttrs ( + # [ Modules ], String, { ${instance_name} :: ServiceConfig } + acc: moduleName: serviceConfigs: + acc + # Collect service config + ++ (lib.foldlAttrs ( + # [ Modules ], String, ServiceConfig + acc2: instanceName: serviceConfig: + let + resolvedRoles = builtins.mapAttrs ( + _roleName: members: resolveGroups inventory members + ) serviceConfig.roles; - machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { }; - global_config = inventory.services.${service_name}.config; - module_name = inventory.services.${service_name}.module; - in - # Possible roles: "server", "client", "peer" - if - builtins.trace '' - isInService ${builtins.toJSON isInService}, - ${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)} - '' isInService - then - acc - ++ [ - { - imports = [ clan-core.clanModules.${module_name} ]; - config.clan.${module_name} = lib.mkMerge [ - global_config - machine_service_config - ]; - } - { - config.clan.${module_name} = { - # TODO: filter, show only the roles that are needed by the machine - roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig; - }; - } - ] - else - acc - ) [ ] (builtins.attrNames inventory.services) + isInService = builtins.any (members: builtins.elem machineName members) ( + builtins.attrValues resolvedRoles + ); + + machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; + globalConfig = serviceConfig.config; + in + if isInService then + acc2 + ++ [ + { + imports = [ clan-core.clanModules.${moduleName} ]; + config.clan.${moduleName} = lib.mkMerge [ + globalConfig + machineServiceConfig + ]; + } + { + config.clan.inventory.${instanceName} = { + roles = resolvedRoles; + }; + } + ] + else + acc2 + ) [ ] serviceConfigs) + ) [ ] inventory.services ) inventory.machines; in { @@ -81,9 +96,4 @@ in }; }; intern = machines; - # inherit (clan) nixosConfigurations clanInternals; - # add the Clan cli tool to the dev shell - devShells.${system}.default = pkgs.mkShell { - packages = [ clan-core.packages.${system}.clan-cli ]; - }; } diff --git a/inventory/flake-module.nix b/inventory/flake-module.nix index aace1add..782f2496 100644 --- a/inventory/flake-module.nix +++ b/inventory/flake-module.nix @@ -1,6 +1,11 @@ -{ inputs, self, ... }: { - flake.inventory = import ./default.nix { inherit inputs self; }; + inputs, + self, + lib, + ... +}: +{ + flake.inventory = import ./default.nix { inherit inputs self lib; }; perSystem = { pkgs, config, ... }: { diff --git a/inventory/src/machines/machines.cue b/inventory/src/machines/machines.cue deleted file mode 100644 index a917e1e9..00000000 --- a/inventory/src/machines/machines.cue +++ /dev/null @@ -1,8 +0,0 @@ -package machines - - -#machine: machines: [string]: { - name: string, - description?: string, - icon?: string -} \ No newline at end of file diff --git a/inventory/src/root.cue b/inventory/src/root.cue index 3034dab8..661e92dd 100644 --- a/inventory/src/root.cue +++ b/inventory/src/root.cue @@ -1,8 +1,7 @@ package inventory import ( - "clan.lol/inventory/services" - "clan.lol/inventory/machines" + "clan.lol/inventory/schema" ) @jsonschema(schema="http://json-schema.org/schema#") @@ -16,9 +15,11 @@ import ( icon?: string } - // A map of services - services.#service + // // A map of services + schema.#service - // A map of machines - machines.#machine + // // A map of machines + schema.#machine + + schema.#groups } diff --git a/inventory/src/services/services.cue b/inventory/src/schema/schema.cue similarity index 57% rename from inventory/src/services/services.cue rename to inventory/src/schema/schema.cue index 6949c81b..7da9dd27 100644 --- a/inventory/src/services/services.cue +++ b/inventory/src/schema/schema.cue @@ -1,22 +1,34 @@ -package services +package schema -#ServiceRole: "server" | "client" | "both" +#groups: groups: { + // Machine groups + machines: { + // Group name mapped to list[machineName] + // "group1": ["machine1", "machine2"] + [string]: [...string] + } +} -#service: services: [string]: { +#machine: machines: [string]: { + name: string, + description?: string, + icon?: string +} + +#role: string + +#service: services: [string]: [string]: { // Required meta fields meta: { name: string, icon?: string description?: string, }, - // Required module specifies the behavior of the service. - module: string, - // We moved the machine sepcific config to "machines". // It may be moved back depending on what makes more sense in the future. - machineConfig: { + roles: [#role]: [...string], + machines: { [string]: { - roles?: [ ...#ServiceRole ], config?: { ... } @@ -29,4 +41,4 @@ package services // It declares the interface how the service can be configured. ... } -} +} \ No newline at end of file diff --git a/inventory/src/tests/borgbackup.json b/inventory/src/tests/borgbackup.json index 11aa844e..55a8c516 100644 --- a/inventory/src/tests/borgbackup.json +++ b/inventory/src/tests/borgbackup.json @@ -10,27 +10,32 @@ "name": "vi" } }, + "groups": { + "machines": { + "laptops": ["camina_machine", "vi_machine"], + "all": ["camina_machine", "vi_machine", "vyr_machine"] + } + }, "meta": { "name": "kenjis clan" }, "services": { - "backup": { - "meta": { - "name": "My backup" - }, - "module": "borgbackup-static", - "machineConfig": { - "vyr_machine": { - "roles": ["server"] + "borgbackup-static": { + "instance_1": { + "meta": { + "name": "My backup" }, - "vi_machine": { - "roles": ["client"] + "roles": { + "server": ["vyr_machine"], + "client": ["group:laptops"] }, - "camina_machine": { - "roles": ["client"] - } - }, - "config": {} + "machines": { + "vyr_machine": {}, + "vi_machine": {}, + "camina_machine": {} + }, + "config": {} + } } } } diff --git a/inventory/src/tests/syncthing.json b/inventory/src/tests/syncthing.json index 6656e5ca..58d0e204 100644 --- a/inventory/src/tests/syncthing.json +++ b/inventory/src/tests/syncthing.json @@ -14,29 +14,29 @@ "name": "kenjis clan" }, "services": { - "sync_files": { - "meta": { - "name": "My sync" - }, - "module": "syncthing-static-peers", - "machineConfig": { - "vyr_machine": {}, - "vi_machine": {}, - "camina_machine": {} - }, - "config": { - "folders": { - "test": { - "path": "~/data/docs", - "devices": ["camina_machine", "vyr_machine", "vi_machine"] - }, - "videos": { - "path": "~/data/videos", - "devices": ["camina_machine", "vyr_machine"] - }, - "playlist": { - "path": "~/data/playlist", - "devices": ["camina_machine", "vi_machine"] + "syncthing-static-peers": { + "instance_1": { + "meta": { + "name": "My sync" + }, + "roles": { + "peer": ["vyr_machine", "vi_machine", "camina_machine"] + }, + "machines": {}, + "config": { + "folders": { + "test": { + "path": "~/data/docs", + "devices": ["camina_machine", "vyr_machine", "vi_machine"] + }, + "videos": { + "path": "~/data/videos", + "devices": ["camina_machine", "vyr_machine"] + }, + "playlist": { + "path": "~/data/playlist", + "devices": ["camina_machine", "vi_machine"] + } } } } diff --git a/inventory/src/tests/zerotier.json b/inventory/src/tests/zerotier.json index 21a2b097..17b226ab 100644 --- a/inventory/src/tests/zerotier.json +++ b/inventory/src/tests/zerotier.json @@ -14,23 +14,22 @@ "name": "kenjis clan" }, "services": { - "backup": { - "meta": { - "name": "My backup" - }, - "module": "borbackup-static", - "machineConfig": { - "vyr_machine": { - "roles": ["server"] + "zerotier-static": { + "instance_1": { + "meta": { + "name": "My Network" }, - "vi_machine": { - "roles": ["peer"] + "roles": { + "server": ["vyr_machine"], + "peer": ["vi_machine", "camina_machine"] }, - "camina_machine": { - "roles": ["peer"] - } - }, - "config": {} + "machines": { + "vyr_machine": { + "config": {} + } + }, + "config": {} + } } } } From 3a9c56deb29e8747242e63fdc55ee1aeac027ad3 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Thu, 20 Jun 2024 18:52:34 +0200 Subject: [PATCH 03/18] readme improvements --- inventory/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/inventory/README.md b/inventory/README.md index 2bdbf77b..64efedd3 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -1,6 +1,13 @@ # Inventory -Questions: +The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort. + +- The inventory acts as a declarative source of truth for all machine configurations. +- Users can easily add or remove machines and services. +- Ensures that all machines and services are configured consistently, across multiple nixosConfigs. +- Defaults and predefined roles in our modules minimizes the need for manual configuration. + +Design questions: - [x] Must roles be a list ? -> Yes. In zerotier you can be "moon" and "controller" at the same time. @@ -31,9 +38,9 @@ Questions: Architecture ``` -machine < machine_module < inventory +nixosConfig < machine_module < inventory --------------------------------------------- -nixos < borgbackup < borgbackup-static > UI +nixos < borgbackup + borgbackup-static > UI creates the config Maps from high level services to the borgbackup clan module for ONE machine From afca7ae0cc260391218968d77280d80c8d3b6f5a Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 21 Jun 2024 16:45:38 +0200 Subject: [PATCH 04/18] update inventory implementation --- clanModules/borgbackup-static/default.nix | 118 ++++++++++------- clanModules/borgbackup/README.md | 15 ++- clanModules/borgbackup/roles/client.nix | 30 +++++ clanModules/borgbackup/roles/server.nix | 45 +++++++ inventory/README.md | 15 ++- inventory/default.nix | 125 +++++++++++++++--- inventory/src/root.cue | 2 - inventory/src/schema/schema.cue | 15 +-- inventory/src/tests/borgbackup.json | 42 +++--- inventory/src/tests/syncthing.json | 6 +- inventory/src/tests/zerotier.json | 7 +- nixosModules/clanCore/default.nix | 2 + nixosModules/clanCore/inventory/interface.nix | 36 +++++ 13 files changed, 349 insertions(+), 109 deletions(-) create mode 100644 clanModules/borgbackup/roles/client.nix create mode 100644 clanModules/borgbackup/roles/server.nix create mode 100644 nixosModules/clanCore/inventory/interface.nix diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index c65dff61..d65fdb5f 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -2,62 +2,57 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; - - # cfg.roles = config.clan.borgbackup-static; - - # machine < machine_module < inventory - # nixos < borgbackup < borgbackup-static > UI - # metadata - # Developer User field descriptions - - roles = config.clan.borgbackup-static.inventory.roles; - - machine_name = config.clan.core.machineName; in -{ +lib.warn "This module is deprecated use the service via the service interface instead." { imports = [ ../borgbackup ]; - # imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ]; - # Inventory / Interface.nix - # options.clan.inventory.borgbackup-static.description. - # options.clan.borgbackup-static.roles = lib.mkOption { - # type = lib.types.attrsOf (lib.types.listOf lib.types.str); - # }; - # Can be used via inventory.json - # - # .borgbackup-static.inventory.roles - # - options.clan.borgbackup-static.inventory = lib.mkOption { - type = lib.types.submodule { - # imports = [./inventory/interface.nix]; - - # idea - # config.metadata = builtins.fromTOML ... - # config.defaultRoles = ["client"]; - - # -> interface.nix - options = { - roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); }; - }; + options.clan.borgbackup-static = { + excludeMachines = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ config.clan.core.machineName ]; + default = [ ]; + description = '' + Machines that should not be backuped. + Mutually exclusive with includeMachines. + If this is not empty, every other machine except the targets in the clan will be backuped by this module. + If includeMachines is set, only the included machines will be backuped. + ''; + }; + includeMachines = lib.mkOption { + type = lib.types.listOf lib.types.str; + example = [ config.clan.core.machineName ]; + default = [ ]; + description = '' + Machines that should be backuped. + Mutually exclusive with excludeMachines. + ''; + }; + targets = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Machines that should act as target machines for backups. + ''; }; }; config.services.borgbackup.repos = let - - filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles); - + machines = builtins.readDir machineDir; borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; - machinesMaybeKey = builtins.map ( - machine: + filteredMachines = + if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then + lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines + else + lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines; + machinesMaybeKey = lib.mapAttrsToList ( + machine: _: let fullPath = borgbackupIpMachinePath machine; in if builtins.pathExists fullPath then machine else null ) filteredMachines; - machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; - hosts = builtins.map (machine: { name = machine; value = { @@ -66,20 +61,41 @@ in }; }) machinesWithKey; in - lib.mkIf (builtins.elem "server" roles.${machine_name}) ( - if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } - ); + lib.mkIf + (builtins.any ( + target: target == config.clan.core.machineName + ) config.clan.borgbackup-static.targets) + (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }); config.clan.borgbackup.destinations = let - servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles); - - destinations = builtins.map (server_name: { - name = server_name; + destinations = builtins.map (d: { + name = d; value = { - repo = "borg@${server_name}:/var/lib/borgbackup/${machine_name}"; + repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}"; }; - }) servers; + }) config.clan.borgbackup-static.targets; in - lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations); + lib.mkIf (builtins.any ( + target: target == config.clan.core.machineName + ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations); + + config.assertions = [ + { + assertion = + !( + ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0) + && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) + ); + message = '' + The options: + config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}] + and + config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}] + are mutually exclusive. + Use excludeMachines to exclude certain machines and backup the other clan machines. + Use include machines to only backup certain machines. + ''; + } + ]; } diff --git a/clanModules/borgbackup/README.md b/clanModules/borgbackup/README.md index b639786d..13379d26 100644 --- a/clanModules/borgbackup/README.md +++ b/clanModules/borgbackup/README.md @@ -1,2 +1,15 @@ Efficient, deduplicating backup program with optional compression and secure encryption. ---- \ No newline at end of file +--- + +## Roles + +- Client +- Server + +## Configuration + +Configure target machines where the backups should be sent to through `targets`. + +Configure machines that should be backed up either through `includeMachines` +which will exclusively add the included machines to be backed up, or through +`excludeMachines`, which will add every machine except the excluded machine to the backup. diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix new file mode 100644 index 00000000..661cb4f3 --- /dev/null +++ b/clanModules/borgbackup/roles/client.nix @@ -0,0 +1,30 @@ +{ config, lib, ... }: +let + instances = config.clan.inventory.borgbackup; + # roles = { ${role_name} :: { machines :: [string] } } + allServers = lib.foldlAttrs ( + acc: _instanceName: instanceConfig: + acc + ++ ( + if builtins.elem machineName instanceConfig.roles.client.machines then + instanceConfig.roles.server.machines + else + [ ] + ) + ) [ ] instances; + + inherit (config.clan.core) machineName; +in +{ + config.clan.borgbackup.destinations = + let + + destinations = builtins.map (serverName: { + name = serverName; + value = { + repo = "borg@${serverName}:/var/lib/borgbackup/${machineName}"; + }; + }) allServers; + in + (builtins.listToAttrs destinations); +} diff --git a/clanModules/borgbackup/roles/server.nix b/clanModules/borgbackup/roles/server.nix new file mode 100644 index 00000000..c25230da --- /dev/null +++ b/clanModules/borgbackup/roles/server.nix @@ -0,0 +1,45 @@ +{ config, lib, ... }: +let + clanDir = config.clan.core.clanDir; + machineDir = clanDir + "/machines/"; + inherit (config.clan.core) machineName; + instances = config.clan.inventory.borgbackup; + + # roles = { ${role_name} :: { machines :: [string] } } + allClients = lib.foldlAttrs ( + acc: _instanceName: instanceConfig: + acc + ++ ( + if builtins.elem machineName instanceConfig.roles.server.machines then + instanceConfig.roles.client.machines + else + [ ] + ) + ) [ ] instances; +in +{ + config.services.borgbackup.repos = + let + filteredMachines = allClients; + + borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; + machinesMaybeKey = builtins.map ( + machine: + let + fullPath = borgbackupIpMachinePath machine; + in + if builtins.pathExists fullPath then machine else null + ) filteredMachines; + + machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; + + hosts = builtins.map (machine: { + name = machine; + value = { + path = "/var/lib/borgbackup/${machine}"; + authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ]; + }; + }) machinesWithKey; + in + if (builtins.listToAttrs hosts) != [ ] then builtins.listToAttrs hosts else { }; +} diff --git a/inventory/README.md b/inventory/README.md index 64efedd3..effdfc0e 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -3,19 +3,24 @@ The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort. - The inventory acts as a declarative source of truth for all machine configurations. -- Users can easily add or remove machines and services. +- Users can easily add or remove machines to/from services. - Ensures that all machines and services are configured consistently, across multiple nixosConfigs. - Defaults and predefined roles in our modules minimizes the need for manual configuration. Design questions: +- [ ] Is the service config interface the same as the module config interface ? + +- [ ] As a user i dont want to see borgbackup as the high level category ? + - [x] Must roles be a list ? - -> Yes. In zerotier you can be "moon" and "controller" at the same time. + -> Yes. In zerotier a machine can be "moon" and "controller" at the same time. - [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different? -> There are many roles. And they depend on the service. - [x] Should we use the module name in the path of the service? + -> YES ```json // ${module_name}.${instance_name} services.borgbackup-static.backup1 = { @@ -32,8 +37,10 @@ Design questions: Neutral: Module name is hard to change. Exists anyways. - [x] Should the machine specific service config be part of the service? - -> The config implements the schema of the module, which is declared in the service. - -> If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. + -> NO. because ... + - The config implements the schema of the module, which is declared in the service. + - If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. + - If the config is placed in the machine it is de-located into another top-level field. In the module this complicates access. Architecture diff --git a/inventory/default.nix b/inventory/default.nix index fbee6457..8d8e9c8e 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -7,22 +7,25 @@ let machines = machinesFromInventory syncthing_inventory; - resolveGroups = - inventory: members: - lib.unique ( - builtins.foldl' ( - acc: currMember: - let - groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember; - groupMembers = - if inventory.groups.machines ? ${groupName} then - inventory.groups.machines.${groupName} - else - throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory."; - in - if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ] - ) [ ] members - ); + resolveTags = + # Inventory, { machines :: [string], tags :: [string] } + inventory: members: { + machines = + members.machines or [ ] + ++ (builtins.foldl' ( + acc: tag: + let + tagMembers = builtins.attrNames ( + lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines + ); + in + # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory."; + if tagMembers == [ ] then + throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory." + else + acc ++ tagMembers + ) [ ] members.tags or [ ]); + }; /* Returns a NixOS configuration for every machine in the inventory. @@ -45,29 +48,53 @@ let acc2: instanceName: serviceConfig: let resolvedRoles = builtins.mapAttrs ( - _roleName: members: resolveGroups inventory members + _roleName: members: resolveTags inventory members ) serviceConfig.roles; - isInService = builtins.any (members: builtins.elem machineName members) ( + isInService = builtins.any (members: builtins.elem machineName members.machines) ( builtins.attrValues resolvedRoles ); + # Inverse map of roles. Allows for easy lookup of roles for a given machine. + # { ${machine_name} :: [roles] + inverseRoles = lib.foldlAttrs ( + acc: roleName: + { machines }: + acc + // builtins.foldl' ( + acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; } + ) { } machines + ) { } resolvedRoles; + machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; globalConfig = serviceConfig.config; + + # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy + roleModules = builtins.map ( + role: + let + path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix"; + in + if builtins.pathExists path then + path + else + throw "Role doesnt have a module: ${role}. Path: ${path} not found." + ) inverseRoles.${machineName} or [ ]; in if isInService then acc2 ++ [ { - imports = [ clan-core.clanModules.${moduleName} ]; + imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules; config.clan.${moduleName} = lib.mkMerge [ globalConfig machineServiceConfig ]; } { - config.clan.inventory.${instanceName} = { + config.clan.inventory.${moduleName}.${instanceName} = { roles = resolvedRoles; + # inherit inverseRoles; }; } ] @@ -78,8 +105,64 @@ let ) inventory.machines; in { + inherit clan-core; + + new_clan = clan-core.lib.buildInventory { + # High level services. + # If you need multiple instances of a service configure them via: + # inventory.services.[serviceName].[instanceName] = ... + services = { + borbackup = { + roles.server.machines = [ "vyr" ]; + roles.client.tags = [ "laptop" ]; + machines.vyr = { + config = { + + }; + }; + config = { + + }; + }; + }; + + # Low level inventory i.e. if you need multiple instances of a service + # Or if you want to manipulate the created inventory directly. + inventory.services.borbackup.default = { }; + + # Machines. each machine can be referenced by its attribute name under services. + machines = { + camina = { + # This is added to machine tags + clan.tags = [ "laptop" ]; + # These are the inventory machine fields + clan.meta.description = ""; + clan.meta.name = ""; + clan.meta.icon = ""; + # Config ... + }; + vyr = { + # Config ... + }; + vi = { + clan.networking.targetHost = "root@78.47.164.46"; + # Config ... + }; + aya = { + clan.networking.targetHost = "root@78.47.164.46"; + # Config ... + }; + ezra = { + # Config ... + }; + rianon = { + # Config ... + }; + }; + }; + clan = clan-core.lib.buildClan { - meta.name = "vis clans"; + meta.name = "vi's clans"; # Should usually point to the directory of flake.nix directory = self; diff --git a/inventory/src/root.cue b/inventory/src/root.cue index 661e92dd..da64c1b2 100644 --- a/inventory/src/root.cue +++ b/inventory/src/root.cue @@ -20,6 +20,4 @@ import ( // // A map of machines schema.#machine - - schema.#groups } diff --git a/inventory/src/schema/schema.cue b/inventory/src/schema/schema.cue index 7da9dd27..81d197b6 100644 --- a/inventory/src/schema/schema.cue +++ b/inventory/src/schema/schema.cue @@ -1,18 +1,10 @@ package schema -#groups: groups: { - // Machine groups - machines: { - // Group name mapped to list[machineName] - // "group1": ["machine1", "machine2"] - [string]: [...string] - } -} - #machine: machines: [string]: { name: string, description?: string, icon?: string + tags: [...string] } #role: string @@ -26,7 +18,10 @@ package schema }, // We moved the machine sepcific config to "machines". // It may be moved back depending on what makes more sense in the future. - roles: [#role]: [...string], + roles: [#role]: { + machines: [...string], + tags: [...string], + } machines: { [string]: { config?: { diff --git a/inventory/src/tests/borgbackup.json b/inventory/src/tests/borgbackup.json index 55a8c516..7e02bde5 100644 --- a/inventory/src/tests/borgbackup.json +++ b/inventory/src/tests/borgbackup.json @@ -1,39 +1,51 @@ { "machines": { "camina_machine": { - "name": "camina" + "name": "camina", + "tags": ["laptop"] }, "vyr_machine": { "name": "vyr" }, "vi_machine": { - "name": "vi" - } - }, - "groups": { - "machines": { - "laptops": ["camina_machine", "vi_machine"], - "all": ["camina_machine", "vi_machine", "vyr_machine"] + "name": "vi", + "tags": ["laptop"] } }, "meta": { "name": "kenjis clan" }, "services": { - "borgbackup-static": { + "borgbackup": { "instance_1": { "meta": { "name": "My backup" }, "roles": { - "server": ["vyr_machine"], - "client": ["group:laptops"] + "server": { + "machines": ["vyr_machine"] + }, + "client": { + "machines": ["vyr_machine"], + "tags": ["laptop"] + } }, - "machines": { - "vyr_machine": {}, - "vi_machine": {}, - "camina_machine": {} + "machines": {}, + "config": {} + }, + "instance_2": { + "meta": { + "name": "My backup" }, + "roles": { + "server": { + "machines": ["vi_machine"] + }, + "client": { + "machines": ["camina_machine"] + } + }, + "machines": {}, "config": {} } } diff --git a/inventory/src/tests/syncthing.json b/inventory/src/tests/syncthing.json index 58d0e204..e7ed8af6 100644 --- a/inventory/src/tests/syncthing.json +++ b/inventory/src/tests/syncthing.json @@ -14,13 +14,15 @@ "name": "kenjis clan" }, "services": { - "syncthing-static-peers": { + "syncthing": { "instance_1": { "meta": { "name": "My sync" }, "roles": { - "peer": ["vyr_machine", "vi_machine", "camina_machine"] + "peer": { + "machines": ["vyr_machine", "vi_machine", "camina_machine"] + } }, "machines": {}, "config": { diff --git a/inventory/src/tests/zerotier.json b/inventory/src/tests/zerotier.json index 17b226ab..f35e2ab4 100644 --- a/inventory/src/tests/zerotier.json +++ b/inventory/src/tests/zerotier.json @@ -14,14 +14,15 @@ "name": "kenjis clan" }, "services": { - "zerotier-static": { + "zerotier": { "instance_1": { "meta": { "name": "My Network" }, "roles": { - "server": ["vyr_machine"], - "peer": ["vi_machine", "camina_machine"] + "controller": { "machines": ["vyr_machine"] }, + "moon": { "machines": ["vyr_machine"] }, + "peer": { "machines": ["vi_machine", "camina_machine"] } }, "machines": { "vyr_machine": { diff --git a/nixosModules/clanCore/default.nix b/nixosModules/clanCore/default.nix index 9926155c..e356a74f 100644 --- a/nixosModules/clanCore/default.nix +++ b/nixosModules/clanCore/default.nix @@ -14,5 +14,7 @@ ./vm.nix ./wayland-proxy-virtwl.nix ./zerotier + # Inventory + ./inventory/interface.nix ]; } diff --git a/nixosModules/clanCore/inventory/interface.nix b/nixosModules/clanCore/inventory/interface.nix new file mode 100644 index 00000000..7280b560 --- /dev/null +++ b/nixosModules/clanCore/inventory/interface.nix @@ -0,0 +1,36 @@ +{ lib, ... }: +let + # { + # roles = { + # client = { + # machines = [ + # "camina_machine" + # "vi_machine" + # ]; + # }; + # server = { + # machines = [ "vyr_machine" ]; + # }; + # }; + # } + instanceOptions = lib.types.submodule { + options.roles = lib.mkOption { type = lib.types.attrsOf machinesList; }; + }; + + # { + # machines = [ + # "camina_machine" + # "vi_machine" + # "vyr_machine" + # ]; + # } + machinesList = lib.types.submodule { + options.machines = lib.mkOption { type = lib.types.listOf lib.types.str; }; + }; +in +{ + # clan.inventory.${moduleName}.${instanceName} = { ... } + options.clan.inventory = lib.mkOption { + type = lib.types.attrsOf (lib.types.attrsOf instanceOptions); + }; +} From e44b07df66d3ec04124614a0825182044f505ee1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 21 Jun 2024 17:00:44 +0200 Subject: [PATCH 05/18] inventory: rename clan.services --- clanModules/borgbackup/roles/client.nix | 2 +- clanModules/borgbackup/roles/server.nix | 6 +++--- inventory/README.md | 6 ++++-- inventory/default.nix | 5 +++-- nixosModules/clanCore/inventory/interface.nix | 2 +- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index 661cb4f3..182e0dc0 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -1,6 +1,6 @@ { config, lib, ... }: let - instances = config.clan.inventory.borgbackup; + instances = config.clan.services.borgbackup; # roles = { ${role_name} :: { machines :: [string] } } allServers = lib.foldlAttrs ( acc: _instanceName: instanceConfig: diff --git a/clanModules/borgbackup/roles/server.nix b/clanModules/borgbackup/roles/server.nix index c25230da..24d18047 100644 --- a/clanModules/borgbackup/roles/server.nix +++ b/clanModules/borgbackup/roles/server.nix @@ -3,7 +3,8 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; inherit (config.clan.core) machineName; - instances = config.clan.inventory.borgbackup; + + instances = config.clan.services.borgbackup; # roles = { ${role_name} :: { machines :: [string] } } allClients = lib.foldlAttrs ( @@ -20,7 +21,6 @@ in { config.services.borgbackup.repos = let - filteredMachines = allClients; borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; machinesMaybeKey = builtins.map ( @@ -29,7 +29,7 @@ in fullPath = borgbackupIpMachinePath machine; in if builtins.pathExists fullPath then machine else null - ) filteredMachines; + ) allClients; machinesWithKey = lib.filter (x: x != null) machinesMaybeKey; diff --git a/inventory/README.md b/inventory/README.md index effdfc0e..89a1b1e5 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -47,10 +47,12 @@ Architecture ``` nixosConfig < machine_module < inventory --------------------------------------------- -nixos < borgbackup + borgbackup-static > UI +nixos < borgbackup <- inventory <-> UI creates the config Maps from high level services to the borgbackup clan module - for ONE machine + for ONE machine Inventory is completely serializable. + UI can interact with the inventory to define machines, and services + Defining Users is out of scope for the first prototype. ``` - [ ] Why do we need 2 modules? diff --git a/inventory/default.nix b/inventory/default.nix index 8d8e9c8e..f55d01d1 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -92,7 +92,7 @@ let ]; } { - config.clan.inventory.${moduleName}.${instanceName} = { + config.clan.services.${moduleName}.${instanceName} = { roles = resolvedRoles; # inherit inverseRoles; }; @@ -107,7 +107,8 @@ in { inherit clan-core; - new_clan = clan-core.lib.buildInventory { + # Extension of the build clan interface + new_clan = clan-core.lib.buildClan { # High level services. # If you need multiple instances of a service configure them via: # inventory.services.[serviceName].[instanceName] = ... diff --git a/nixosModules/clanCore/inventory/interface.nix b/nixosModules/clanCore/inventory/interface.nix index 7280b560..0e3b99b0 100644 --- a/nixosModules/clanCore/inventory/interface.nix +++ b/nixosModules/clanCore/inventory/interface.nix @@ -30,7 +30,7 @@ let in { # clan.inventory.${moduleName}.${instanceName} = { ... } - options.clan.inventory = lib.mkOption { + options.clan.services = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf instanceOptions); }; } From c89080deb4755f3ecd9a3188e1dc808a634fba5b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Fri, 21 Jun 2024 22:46:12 +0200 Subject: [PATCH 06/18] Extend build-clan interface --- clanModules/borgbackup/roles/server.nix | 4 +- inventory/default.nix | 224 +++++------------- lib/build-clan/default.nix | 75 +++++- lib/build-clan/inventory.nix | 106 +++++++++ nixosModules/clanCore/default.nix | 1 + nixosModules/clanCore/inventory/interface.nix | 1 + nixosModules/clanCore/meta/interface.nix | 10 + 7 files changed, 257 insertions(+), 164 deletions(-) create mode 100644 lib/build-clan/inventory.nix create mode 100644 nixosModules/clanCore/meta/interface.nix diff --git a/clanModules/borgbackup/roles/server.nix b/clanModules/borgbackup/roles/server.nix index 24d18047..3885aa31 100644 --- a/clanModules/borgbackup/roles/server.nix +++ b/clanModules/borgbackup/roles/server.nix @@ -7,11 +7,12 @@ let instances = config.clan.services.borgbackup; # roles = { ${role_name} :: { machines :: [string] } } + allClients = lib.foldlAttrs ( acc: _instanceName: instanceConfig: acc ++ ( - if builtins.elem machineName instanceConfig.roles.server.machines then + if (builtins.elem machineName instanceConfig.roles.server.machines) then instanceConfig.roles.client.machines else [ ] @@ -21,7 +22,6 @@ in { config.services.borgbackup.repos = let - borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; machinesMaybeKey = builtins.map ( machine: diff --git a/inventory/default.nix b/inventory/default.nix index f55d01d1..4f4f3a9e 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -1,183 +1,87 @@ { self, lib, ... }: let clan-core = self; - - # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json); - syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); - - machines = machinesFromInventory syncthing_inventory; - - resolveTags = - # Inventory, { machines :: [string], tags :: [string] } - inventory: members: { - machines = - members.machines or [ ] - ++ (builtins.foldl' ( - acc: tag: - let - tagMembers = builtins.attrNames ( - lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines - ); - in - # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory."; - if tagMembers == [ ] then - throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory." - else - acc ++ tagMembers - ) [ ] members.tags or [ ]); - }; - - /* - Returns a NixOS configuration for every machine in the inventory. - - machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } - */ - machinesFromInventory = - inventory: - # For every machine in the inventory, build a NixOS configuration - # For each machine generate config, forEach service, if the machine is used. - builtins.mapAttrs ( - machineName: _: - lib.foldlAttrs ( - # [ Modules ], String, { ${instance_name} :: ServiceConfig } - acc: moduleName: serviceConfigs: - acc - # Collect service config - ++ (lib.foldlAttrs ( - # [ Modules ], String, ServiceConfig - acc2: instanceName: serviceConfig: - let - resolvedRoles = builtins.mapAttrs ( - _roleName: members: resolveTags inventory members - ) serviceConfig.roles; - - isInService = builtins.any (members: builtins.elem machineName members.machines) ( - builtins.attrValues resolvedRoles - ); - - # Inverse map of roles. Allows for easy lookup of roles for a given machine. - # { ${machine_name} :: [roles] - inverseRoles = lib.foldlAttrs ( - acc: roleName: - { machines }: - acc - // builtins.foldl' ( - acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; } - ) { } machines - ) { } resolvedRoles; - - machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; - globalConfig = serviceConfig.config; - - # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy - roleModules = builtins.map ( - role: - let - path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix"; - in - if builtins.pathExists path then - path - else - throw "Role doesnt have a module: ${role}. Path: ${path} not found." - ) inverseRoles.${machineName} or [ ]; - in - if isInService then - acc2 - ++ [ - { - imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules; - config.clan.${moduleName} = lib.mkMerge [ - globalConfig - machineServiceConfig - ]; - } - { - config.clan.services.${moduleName}.${instanceName} = { - roles = resolvedRoles; - # inherit inverseRoles; - }; - } - ] - else - acc2 - ) [ ] serviceConfigs) - ) [ ] inventory.services - ) inventory.machines; in { - inherit clan-core; - # Extension of the build clan interface - new_clan = clan-core.lib.buildClan { - # High level services. - # If you need multiple instances of a service configure them via: - # inventory.services.[serviceName].[instanceName] = ... - services = { - borbackup = { - roles.server.machines = [ "vyr" ]; - roles.client.tags = [ "laptop" ]; - machines.vyr = { - config = { + # new_clan = clan-core.lib.buildClan { + # # High level services. + # # If you need multiple instances of a service configure them via: + # # inventory.services.[serviceName].[instanceName] = ... + # services = { + # borbackup = { + # roles.server.machines = [ "vyr" ]; + # roles.client.tags = [ "laptop" ]; + # machines.vyr = { + # config = { - }; - }; - config = { + # }; + # }; + # config = { - }; - }; - }; + # }; + # }; + # }; - # Low level inventory i.e. if you need multiple instances of a service - # Or if you want to manipulate the created inventory directly. - inventory.services.borbackup.default = { }; + # # Low level inventory i.e. if you need multiple instances of a service + # # Or if you want to manipulate the created inventory directly. + # inventory.services.borbackup.default = { }; - # Machines. each machine can be referenced by its attribute name under services. - machines = { - camina = { - # This is added to machine tags - clan.tags = [ "laptop" ]; - # These are the inventory machine fields - clan.meta.description = ""; - clan.meta.name = ""; - clan.meta.icon = ""; - # Config ... - }; - vyr = { - # Config ... - }; - vi = { - clan.networking.targetHost = "root@78.47.164.46"; - # Config ... - }; - aya = { - clan.networking.targetHost = "root@78.47.164.46"; - # Config ... - }; - ezra = { - # Config ... - }; - rianon = { - # Config ... - }; - }; - }; + # # Machines. each machine can be referenced by its attribute name under services. + # machines = { + # camina = { + # # This is added to machine tags + # clan.tags = [ "laptop" ]; + # # These are the inventory machine fields + # clan.meta.description = ""; + # clan.meta.name = ""; + # clan.meta.icon = ""; + # # Config ... + # }; + # vyr = { + # # Config ... + # }; + # vi = { + # clan.networking.targetHost = "root@78.47.164.46"; + # # Config ... + # }; + # aya = { + # clan.networking.targetHost = "root@78.47.164.46"; + # # Config ... + # }; + # ezra = { + # # Config ... + # }; + # rianon = { + # # Config ... + # }; + # }; + # }; clan = clan-core.lib.buildClan { - meta.name = "vi's clans"; + meta.name = "sams's clans"; # Should usually point to the directory of flake.nix directory = self; + # services = { + # borgbackup = { + # roles.server.machines = [ "vyr_machine" ]; + # roles.client.tags = [ "laptop" ]; + # }; + # }; + + # OR + inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); + machines = { + "vyr_machine" = { }; "vi_machine" = { - imports = machines.vi_machine; - }; - "vyr_machine" = { - imports = machines.vyr_machine; + clan.tags = [ "laptop" ]; }; "camina_machine" = { - imports = machines.camina_machine; + clan.tags = [ "laptop" ]; + clan.meta.name = "camina"; }; }; }; - intern = machines; } diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 90bb8584..7907970f 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -12,10 +12,79 @@ # DEPRECATED: use meta.icon instead clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string - pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. -# This improves performance, but all nipxkgs.* options will be ignored. + # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. + # This improves performance, but all nipxkgs.* options will be ignored. + pkgsForSystem ? (_system: null), + /* + Distributed services configuration. + + This configures a default instance in the inventory with the name "default". + + If you need multiple instances of a service configure them via: + inventory.services.[serviceName].[instanceName] = ... + */ + services ? { }, + /* + Low level inventory configuration. + Overrides the services configuration. + */ + inventory ? { }, }: let + _inventory = + ( + if services != { } && inventory == { } then + { services = lib.mapAttrs (_name: value: { default = value; }) services; } + else if services == { } && inventory != { } then + inventory + else if services != { } && inventory != { } then + throw "Either services or inventory should be set, but not both." + else + { } + ) + // { + machines = lib.mapAttrs ( + name: config: + (lib.attrByPath [ + "clan" + "meta" + ] { } config) + // { + name = ( + lib.attrByPath [ + "clan" + "meta" + "name" + ] name config + ); + tags = lib.attrByPath [ + "clan" + "tags" + ] [ ] config; + } + ) machines; + }; + + buildInventory = import ./inventory.nix { inherit lib clan-core; }; + + pkgs = import nixpkgs { }; + + inventoryFile = builtins.toFile "inventory.json" (builtins.toJSON _inventory); + + # a Derivation that can be forced to validate the inventory + # It is not used directly here. + validatedFile = pkgs.stdenv.mkDerivation { + name = "validated-inventory"; + src = ../../inventory/src; + buildInputs = [ pkgs.cue ]; + installPhase = '' + cue vet ${inventoryFile} root.cue -d "#Root" + cp ${inventoryFile} $out + ''; + }; + + serviceConfigs = buildInventory _inventory; + deprecationWarnings = [ (lib.warnIf ( clanName != null @@ -98,6 +167,7 @@ let clan-core.nixosModules.clanCore extraConfig (machines.${name} or { }) + { imports = serviceConfigs.${name} or { }; } ( { # Settings @@ -180,6 +250,7 @@ builtins.deepSeq deprecationWarnings { # Evaluated clan meta # Merged /clan/meta.json with overrides from buildClan meta = mergedMeta; + inherit _inventory validatedFile; # machine specifics machines = configsPerSystem; diff --git a/lib/build-clan/inventory.nix b/lib/build-clan/inventory.nix new file mode 100644 index 00000000..075a84d8 --- /dev/null +++ b/lib/build-clan/inventory.nix @@ -0,0 +1,106 @@ +# Generate partial NixOS configurations for every machine in the inventory +# This function is responsible for generating the module configuration for every machine in the inventory. +{ lib, clan-core }: +inventory: +let + machines = machinesFromInventory inventory; + + resolveTags = + # Inventory, { machines :: [string], tags :: [string] } + inventory: members: { + machines = + members.machines or [ ] + ++ (builtins.foldl' ( + acc: tag: + let + tagMembers = builtins.attrNames ( + lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines + ); + in + # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory."; + if tagMembers == [ ] then + throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory." + else + acc ++ tagMembers + ) [ ] members.tags or [ ]); + }; + + /* + Returns a NixOS configuration for every machine in the inventory. + + machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } + */ + machinesFromInventory = + inventory: + # For every machine in the inventory, build a NixOS configuration + # For each machine generate config, forEach service, if the machine is used. + builtins.mapAttrs ( + machineName: _: + lib.foldlAttrs ( + # [ Modules ], String, { ${instance_name} :: ServiceConfig } + acc: moduleName: serviceConfigs: + acc + # Collect service config + ++ (lib.foldlAttrs ( + # [ Modules ], String, ServiceConfig + acc2: instanceName: serviceConfig: + let + resolvedRoles = builtins.mapAttrs ( + _roleName: members: resolveTags inventory members + ) serviceConfig.roles; + + isInService = builtins.any (members: builtins.elem machineName members.machines) ( + builtins.attrValues resolvedRoles + ); + + # Inverse map of roles. Allows for easy lookup of roles for a given machine. + # { ${machine_name} :: [roles] + inverseRoles = lib.foldlAttrs ( + acc: roleName: + { machines }: + acc + // builtins.foldl' ( + acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; } + ) { } machines + ) { } resolvedRoles; + + machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; + globalConfig = serviceConfig.config or { }; + + # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy + roleModules = builtins.map ( + role: + let + path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix"; + in + if builtins.pathExists path then + path + else + throw "Role doesnt have a module: ${role}. Path: ${path} not found." + ) inverseRoles.${machineName} or [ ]; + in + if isInService then + acc2 + ++ [ + { + imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules; + config.clan.${moduleName} = lib.mkMerge [ + globalConfig + machineServiceConfig + ]; + } + { + config.clan.services.${moduleName}.${instanceName} = { + roles = resolvedRoles; + # TODO: Add inverseRoles to the service config if needed + # inherit inverseRoles; + }; + } + ] + else + acc2 + ) [ ] serviceConfigs) + ) [ ] inventory.services + ) inventory.machines; +in +machines diff --git a/nixosModules/clanCore/default.nix b/nixosModules/clanCore/default.nix index e356a74f..c9b7ff3e 100644 --- a/nixosModules/clanCore/default.nix +++ b/nixosModules/clanCore/default.nix @@ -16,5 +16,6 @@ ./zerotier # Inventory ./inventory/interface.nix + ./meta/interface.nix ]; } diff --git a/nixosModules/clanCore/inventory/interface.nix b/nixosModules/clanCore/inventory/interface.nix index 0e3b99b0..3a8228d5 100644 --- a/nixosModules/clanCore/inventory/interface.nix +++ b/nixosModules/clanCore/inventory/interface.nix @@ -30,6 +30,7 @@ let in { # clan.inventory.${moduleName}.${instanceName} = { ... } + # TODO: resolve clash with clan.services.waypipe options.clan.services = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf instanceOptions); }; diff --git a/nixosModules/clanCore/meta/interface.nix b/nixosModules/clanCore/meta/interface.nix new file mode 100644 index 00000000..3b44046e --- /dev/null +++ b/nixosModules/clanCore/meta/interface.nix @@ -0,0 +1,10 @@ +{ lib, ... }: +let + optStr = lib.types.nullOr lib.types.str; +in +{ + options.clan.meta.name = lib.mkOption { type = lib.types.str; }; + options.clan.meta.description = lib.mkOption { type = optStr; }; + options.clan.meta.icon = lib.mkOption { type = optStr; }; + options.clan.tags = lib.mkOption { type = lib.types.listOf lib.types.str; }; +} From 2f8b782a1ff55dc5fe11827f560e6f90670438a8 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 22 Jun 2024 21:31:01 +0200 Subject: [PATCH 07/18] Inventory: init module merge & validation logic for inventory --- clanModules/borgbackup/roles/client.nix | 2 +- clanModules/borgbackup/roles/server.nix | 2 +- flakeModules/clan.nix | 2 +- inventory/README.md | 60 ++------- inventory/default.nix | 75 ++--------- lib/build-clan/default.nix | 99 ++++++++------- lib/build-clan/interface.nix | 120 ++++++++++++++++++ lib/build-clan/inventory.nix | 2 +- nixosModules/clanCore/inventory/interface.nix | 4 +- 9 files changed, 198 insertions(+), 168 deletions(-) create mode 100644 lib/build-clan/interface.nix diff --git a/clanModules/borgbackup/roles/client.nix b/clanModules/borgbackup/roles/client.nix index 182e0dc0..84ce41ef 100644 --- a/clanModules/borgbackup/roles/client.nix +++ b/clanModules/borgbackup/roles/client.nix @@ -1,6 +1,6 @@ { config, lib, ... }: let - instances = config.clan.services.borgbackup; + instances = config.clan.inventory.services.borgbackup; # roles = { ${role_name} :: { machines :: [string] } } allServers = lib.foldlAttrs ( acc: _instanceName: instanceConfig: diff --git a/clanModules/borgbackup/roles/server.nix b/clanModules/borgbackup/roles/server.nix index 3885aa31..48c7089c 100644 --- a/clanModules/borgbackup/roles/server.nix +++ b/clanModules/borgbackup/roles/server.nix @@ -4,7 +4,7 @@ let machineDir = clanDir + "/machines/"; inherit (config.clan.core) machineName; - instances = config.clan.services.borgbackup; + instances = config.clan.inventory.services.borgbackup; # roles = { ${role_name} :: { machines :: [string] } } diff --git a/flakeModules/clan.nix b/flakeModules/clan.nix index 139ef8d9..dec466aa 100644 --- a/flakeModules/clan.nix +++ b/flakeModules/clan.nix @@ -13,7 +13,6 @@ let inherit lib clan-core; inherit (inputs) nixpkgs; }; - cfg = config.clan; in { @@ -91,6 +90,7 @@ in clanInternals = lib.mkOption { type = lib.types.submodule { options = { + inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; diff --git a/inventory/README.md b/inventory/README.md index 89a1b1e5..4776a388 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -7,40 +7,18 @@ The inventory is our concept for distributed services. Users can configure multi - Ensures that all machines and services are configured consistently, across multiple nixosConfigs. - Defaults and predefined roles in our modules minimizes the need for manual configuration. -Design questions: +Open questions: + +- [ ] How do we set default role, description and other metadata? + - It must be accessible from Python. + - It must set the value in the module system. + +- [ ] Inventory might use assertions. Should each machine inherit the inventory assertions ? - [ ] Is the service config interface the same as the module config interface ? -- [ ] As a user i dont want to see borgbackup as the high level category ? +- [ ] As a user do I want to see borgbackup as the high level category? -- [x] Must roles be a list ? - -> Yes. In zerotier a machine can be "moon" and "controller" at the same time. - -- [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different? - -> There are many roles. And they depend on the service. - -- [x] Should we use the module name in the path of the service? - -> YES - ```json - // ${module_name}.${instance_name} - services.borgbackup-static.backup1 = { - - } - ``` - - Pro: - Easier to handle. - Better groups the module specific instances. - Contra: - More nesting in json - - Neutral: Module name is hard to change. Exists anyways. - -- [x] Should the machine specific service config be part of the service? - -> NO. because ... - - The config implements the schema of the module, which is declared in the service. - - If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. - - If the config is placed in the machine it is de-located into another top-level field. In the module this complicates access. Architecture @@ -55,27 +33,7 @@ nixos < borgbackup <- inventory <-> UI Defining Users is out of scope for the first prototype. ``` -- [ ] Why do we need 2 modules? - -> It is technically possible to have only 1 module. - Pros: - Simple to use/Easy to understand. - Less modules - Cons: - Harder to write a module. Because it must do 2 things. - One module should do only 1 thing. - -```nix -clan.machines.${machine_name} = { - # "borgbackup.ssh.pub" = machineDir + machines + "/facts/borgbackup.ssh.pub"; - facts = ... -}; -clan.services.${instance} = { -# roles.server = [ "jon_machine" ] -# roles.${role_name} = [ ${machine_name} ]; -}; -``` - -This part provides a specification for the inventory. +## Provides a specification for the inventory It is used for design phase and as validation helper. diff --git a/inventory/default.nix b/inventory/default.nix index 4f4f3a9e..a60c392a 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -3,76 +3,25 @@ let clan-core = self; in { - # Extension of the build clan interface - # new_clan = clan-core.lib.buildClan { - # # High level services. - # # If you need multiple instances of a service configure them via: - # # inventory.services.[serviceName].[instanceName] = ... - # services = { - # borbackup = { - # roles.server.machines = [ "vyr" ]; - # roles.client.tags = [ "laptop" ]; - # machines.vyr = { - # config = { - - # }; - # }; - # config = { - - # }; - # }; - # }; - - # # Low level inventory i.e. if you need multiple instances of a service - # # Or if you want to manipulate the created inventory directly. - # inventory.services.borbackup.default = { }; - - # # Machines. each machine can be referenced by its attribute name under services. - # machines = { - # camina = { - # # This is added to machine tags - # clan.tags = [ "laptop" ]; - # # These are the inventory machine fields - # clan.meta.description = ""; - # clan.meta.name = ""; - # clan.meta.icon = ""; - # # Config ... - # }; - # vyr = { - # # Config ... - # }; - # vi = { - # clan.networking.targetHost = "root@78.47.164.46"; - # # Config ... - # }; - # aya = { - # clan.networking.targetHost = "root@78.47.164.46"; - # # Config ... - # }; - # ezra = { - # # Config ... - # }; - # rianon = { - # # Config ... - # }; - # }; - # }; - clan = clan-core.lib.buildClan { - meta.name = "sams's clans"; + + meta.name = "kenjis clan"; # Should usually point to the directory of flake.nix directory = self; - # services = { - # borgbackup = { - # roles.server.machines = [ "vyr_machine" ]; - # roles.client.tags = [ "laptop" ]; - # }; - # }; + # service config + # Useful alias: "inventory.services.borgbackup.default" + services = { + borgbackup = { + roles.server.machines = [ "vyr_machine" ]; + roles.client.tags = [ "laptop" ]; + }; + }; - # OR + # merged with inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); + # merged with machines = { "vyr_machine" = { }; "vi_machine" = { diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 7907970f..e09975d5 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -31,59 +31,62 @@ inventory ? { }, }: let - _inventory = - ( - if services != { } && inventory == { } then - { services = lib.mapAttrs (_name: value: { default = value; }) services; } - else if services == { } && inventory != { } then + # Internal inventory, this is the result of merging all potential inventory sources: + # - Default instances configured via 'services' + # - The inventory overrides + # - Machines that exist in inventory.machines + # - Machines explicitly configured via 'machines' argument + # - Machines that exist in the machines directory + # Checks on the module level: + # - Each service role must reference a valid machine after all machines are merged + mergedInventory = + (lib.evalModules { + modules = [ + ./interface.nix + { inherit meta; } + # Default instances configured via 'services' + { + services = lib.mapAttrs (_name: value: { + default = value // { + meta.name = lib.mkDefault _name; + }; + }) services; + } + # The inventory overrides inventory - else if services != { } && inventory != { } then - throw "Either services or inventory should be set, but not both." - else - { } - ) - // { - machines = lib.mapAttrs ( - name: config: - (lib.attrByPath [ - "clan" - "meta" - ] { } config) - // { - name = ( - lib.attrByPath [ + # Machines explicitly configured via 'machines' argument + { + # { ${name} :: meta // { name, tags } } + machines = lib.mapAttrs ( + name: config: + (lib.attrByPath [ "clan" "meta" - "name" - ] name config - ); - tags = lib.attrByPath [ - "clan" - "tags" - ] [ ] config; + ] { } config) + // { + # meta.name default is the attribute name of the machine + name = lib.mkDefault ( + lib.attrByPath [ + "clan" + "meta" + "name" + ] name config + ); + tags = lib.attrByPath [ + "clan" + "tags" + ] [ ] config; + } + ) machines; } - ) machines; - }; + # Machines that exist in the machines directory + { machines = lib.mapAttrs (name: _: { inherit name; }) machinesDirs; } + ]; + }).config; buildInventory = import ./inventory.nix { inherit lib clan-core; }; - pkgs = import nixpkgs { }; - - inventoryFile = builtins.toFile "inventory.json" (builtins.toJSON _inventory); - - # a Derivation that can be forced to validate the inventory - # It is not used directly here. - validatedFile = pkgs.stdenv.mkDerivation { - name = "validated-inventory"; - src = ../../inventory/src; - buildInputs = [ pkgs.cue ]; - installPhase = '' - cue vet ${inventoryFile} root.cue -d "#Root" - cp ${inventoryFile} $out - ''; - }; - - serviceConfigs = buildInventory _inventory; + serviceConfigs = buildInventory mergedInventory; deprecationWarnings = [ (lib.warnIf ( @@ -167,6 +170,8 @@ let clan-core.nixosModules.clanCore extraConfig (machines.${name} or { }) + # Inherit the inventory assertions ? + { inherit (mergedInventory) assertions; } { imports = serviceConfigs.${name} or { }; } ( { @@ -250,7 +255,7 @@ builtins.deepSeq deprecationWarnings { # Evaluated clan meta # Merged /clan/meta.json with overrides from buildClan meta = mergedMeta; - inherit _inventory validatedFile; + inventory = mergedInventory; # machine specifics machines = configsPerSystem; diff --git a/lib/build-clan/interface.nix b/lib/build-clan/interface.nix new file mode 100644 index 00000000..4a46ae39 --- /dev/null +++ b/lib/build-clan/interface.nix @@ -0,0 +1,120 @@ +{ config, lib, ... }: +let + t = lib.types; + + metaOptions = { + name = lib.mkOption { type = t.str; }; + description = lib.mkOption { + default = null; + type = t.nullOr t.str; + }; + icon = lib.mkOption { + default = null; + type = t.nullOr t.str; + }; + }; + + machineRef = lib.mkOptionType { + name = "machineRef"; + description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]"; + check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines); + merge = lib.mergeEqualOption; + }; + + allTags = lib.unique ( + lib.foldlAttrs ( + tags: _: m: + tags ++ m.tags or [ ] + ) [ ] config.machines + ); + + tagRef = lib.mkOptionType { + name = "tagRef"; + description = "Tags :: [${builtins.concatStringsSep " | " allTags}]"; + check = v: lib.isString v && builtins.elem v allTags; + merge = lib.mergeEqualOption; + }; +in +{ + options.assertions = lib.mkOption { + type = t.listOf t.unspecified; + internal = true; + default = [ ]; + }; + config.assertions = lib.foldlAttrs ( + ass1: serviceName: c: + ass1 + ++ lib.foldlAttrs ( + ass2: instanceName: instanceConfig: + let + serviceMachineNames = lib.attrNames instanceConfig.machines; + topLevelMachines = lib.attrNames config.machines; + # All machines must be defined in the top-level machines + assertions = builtins.map (m: { + assertion = builtins.elem m topLevelMachines; + message = "${serviceName}.${instanceName}.machines.${m}. Should be one of [ ${builtins.concatStringsSep " | " topLevelMachines} ]"; + }) serviceMachineNames; + in + ass2 ++ assertions + ) [ ] c + ) [ ] config.services; + + options.meta = metaOptions; + + options.machines = lib.mkOption { + default = { }; + type = t.attrsOf ( + t.submodule { + options = { + inherit (metaOptions) name description icon; + tags = lib.mkOption { + default = [ ]; + apply = lib.unique; + type = t.listOf t.str; + }; + }; + } + ); + }; + + options.services = lib.mkOption { + type = t.attrsOf ( + t.attrsOf ( + t.submodule { + options.meta = metaOptions; + options.config = lib.mkOption { + default = { }; + type = t.anything; + }; + options.machines = lib.mkOption { + default = { }; + type = t.attrsOf ( + t.submodule { + options.config = lib.mkOption { + default = { }; + type = t.anything; + }; + } + ); + }; + options.roles = lib.mkOption { + default = { }; + type = t.attrsOf ( + t.submodule { + options.machines = lib.mkOption { + default = [ ]; + type = t.listOf machineRef; + }; + options.tags = lib.mkOption { + default = [ ]; + apply = lib.unique; + type = t.listOf tagRef; + }; + } + ); + }; + } + ) + ); + }; +} diff --git a/lib/build-clan/inventory.nix b/lib/build-clan/inventory.nix index 075a84d8..a5ab65bc 100644 --- a/lib/build-clan/inventory.nix +++ b/lib/build-clan/inventory.nix @@ -90,7 +90,7 @@ let ]; } { - config.clan.services.${moduleName}.${instanceName} = { + config.clan.inventory.services.${moduleName}.${instanceName} = { roles = resolvedRoles; # TODO: Add inverseRoles to the service config if needed # inherit inverseRoles; diff --git a/nixosModules/clanCore/inventory/interface.nix b/nixosModules/clanCore/inventory/interface.nix index 3a8228d5..895a1599 100644 --- a/nixosModules/clanCore/inventory/interface.nix +++ b/nixosModules/clanCore/inventory/interface.nix @@ -29,9 +29,7 @@ let }; in { - # clan.inventory.${moduleName}.${instanceName} = { ... } - # TODO: resolve clash with clan.services.waypipe - options.clan.services = lib.mkOption { + options.clan.inventory.services = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf instanceOptions); }; } From b74590f381896c5ee3e3176af6d9f16783bbf325 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 24 Jun 2024 15:14:48 +0200 Subject: [PATCH 08/18] Inventory: simplify build-clan interface --- clanModules/borgbackup/default.nix | 49 ++++++++- inventory/default.nix | 16 ++- lib/build-clan/default.nix | 159 +++++++++-------------------- 3 files changed, 100 insertions(+), 124 deletions(-) diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix index a625b83f..f708fe2e 100644 --- a/clanModules/borgbackup/default.nix +++ b/clanModules/borgbackup/default.nix @@ -27,8 +27,50 @@ let exit 1 fi ''; + + # Each .nix file in the roles directory is a role + # TODO: Helper function to set available roles within module meta. + # roles = lib.pipe ./roles [ + # (p: if builtins.pathExists p then p else throw "Role directory does not exist") + # builtins.readDir + # (lib.filterAttrs (_n: v: v == "regular")) + # lib.attrNames + # (map (fileName: lib.removeSuffix ".nix" fileName)) + # ]; + + # TODO: make this an interface of every module + # Maybe load from readme.md + metaInfoOption = lib.mkOption { + readOnly = true; + default = { + description = "A category for the service. This is used to group services in the clan ui"; + }; + type = lib.types.submodule { + options = { + description = lib.mkOption { + description = "A category for the service. This is used to group services in the clan ui"; + type = lib.types.str; + }; + # icon = lib.mkOption { + # description = "A category for the service. This is used to group services in the clan ui"; + # type = lib.types.str; + # }; + # screenshots = lib.mkOption { + # description = "A category for the service. This is used to group services in the clan ui"; + # type = lib.types.str; + # }; + # category = lib.mkOption { + # description = "A category for the service. This is used to group services in the clan ui"; + # type = lib.types.enum ["backup" "network"]; + # }; + }; + }; + }; in { + + options.clan.borgbackup.meta = metaInfoOption; + options.clan.borgbackup.destinations = lib.mkOption { type = lib.types.attrsOf ( lib.types.submodule ( @@ -63,6 +105,11 @@ in }; imports = [ + { + config.clan.borgbackup.meta = { + description = "backup"; + }; + } (lib.mkRemovedOptionModule [ "clan" "borgbackup" @@ -76,7 +123,7 @@ in lib.nameValuePair "borgbackup-job-${dest.name}" { # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files. serviceConfig.ExecStartPre = [ - (''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'') + ''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'' ]; } ) cfg.destinations; diff --git a/inventory/default.nix b/inventory/default.nix index a60c392a..2ca5ac00 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -4,23 +4,19 @@ let in { clan = clan-core.lib.buildClan { - meta.name = "kenjis clan"; # Should usually point to the directory of flake.nix directory = self; - # service config - # Useful alias: "inventory.services.borgbackup.default" - services = { - borgbackup = { - roles.server.machines = [ "vyr_machine" ]; - roles.client.tags = [ "laptop" ]; + inventory = { + services = { + borgbackup.instance_1 = { + roles.server.machines = [ "vyr_machine" ]; + roles.client.tags = [ "laptop" ]; + }; }; }; - # merged with - inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); - # merged with machines = { "vyr_machine" = { }; diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index e09975d5..271f4254 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -15,15 +15,6 @@ # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system. # This improves performance, but all nipxkgs.* options will be ignored. pkgsForSystem ? (_system: null), - /* - Distributed services configuration. - - This configures a default instance in the inventory with the name "default". - - If you need multiple instances of a service configure them via: - inventory.services.[serviceName].[instanceName] = ... - */ - services ? { }, /* Low level inventory configuration. Overrides the services configuration. @@ -44,15 +35,15 @@ let modules = [ ./interface.nix { inherit meta; } - # Default instances configured via 'services' - { - services = lib.mapAttrs (_name: value: { - default = value // { - meta.name = lib.mkDefault _name; - }; - }) services; - } - # The inventory overrides + ( + if + builtins.pathExists "${directory}/inventory.json" + # Is recursively applied. Any explicit nix will override. + then + lib.mkDefault (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) + else + { } + ) inventory # Machines explicitly configured via 'machines' argument { @@ -79,13 +70,17 @@ let } ) machines; } - # Machines that exist in the machines directory - { machines = lib.mapAttrs (name: _: { inherit name; }) machinesDirs; } + + # Deprecated interface + (if clanName != null then { meta.name = clanName; } else { }) + (if clanIcon != null then { meta.icon = clanIcon; } else { }) ]; }).config; buildInventory = import ./inventory.nix { inherit lib clan-core; }; + # map from machine name to service configuration + # { ${machineName} :: Config } serviceConfigs = buildInventory mergedInventory; deprecationWarnings = [ @@ -95,62 +90,6 @@ let (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null) ]; - machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - builtins.readDir (directory + /machines) - ); - - mergedMeta = - let - metaFromFile = - if (builtins.pathExists "${directory}/clan/meta.json") then - let - settings = builtins.fromJSON (builtins.readFile "${directory}/clan/meta.json"); - in - settings - else - { }; - legacyMeta = lib.filterAttrs (_: v: v != null) { - name = clanName; - icon = clanIcon; - }; - optionsMeta = lib.filterAttrs (_: v: v != null) meta; - - warnings = - builtins.map ( - name: - if - metaFromFile.${name} or null != optionsMeta.${name} or null && optionsMeta.${name} or null != null - then - lib.warn "meta.${name} is set in different places. (exlicit option meta.${name} overrides ${directory}/clan/meta.json)" null - else - null - ) (builtins.attrNames metaFromFile) - ++ [ (if (res.name or null == null) then (throw "meta.name should be set") else null) ]; - res = metaFromFile // legacyMeta // optionsMeta; - in - # Print out warnings before returning the merged result - builtins.deepSeq warnings res; - - machineSettings = - machineName: - # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily - # This is useful for doing a dry-run before writing changes into the settings.json - # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then - builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) - else - lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( - builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)) - ); - - # Read additional imports specified via a config option in settings.json - # This is not an infinite recursion, because the imports are discovered here - # before calling evalModules. - # It is still useful to have the imports as an option, as this allows for type - # checking and easy integration with the config frontend(s) - machineImports = - machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); - # TODO: remove default system once we have a hardware-config mechanism nixosConfiguration = { @@ -160,47 +99,41 @@ let extraConfig ? { }, }: nixpkgs.lib.nixosSystem { - modules = - let - settings = machineSettings name; - in - (machineImports settings) - ++ [ - settings - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - # Inherit the inventory assertions ? - { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } - ( - { - # Settings - clan.core.clanDir = directory; - # Inherited from clan wide settings - clan.core.clanName = meta.name or clanName; - clan.core.clanIcon = meta.icon or clanIcon; + modules = [ + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or { }; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + clan.core.clanName = meta.name or clanName; + clan.core.clanIcon = meta.icon or clanIcon; - # Machine specific settings - clan.core.machineName = name; - networking.hostName = lib.mkDefault name; - nixpkgs.hostPlatform = lib.mkDefault system; + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } - // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } - ) - ]; + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; specialArgs = { inherit clan-core; } // specialArgs; }; - allMachines = machinesDirs // machines; + allMachines = mergedInventory.machines or { }; supportedSystems = [ "x86_64-linux" @@ -252,11 +185,11 @@ builtins.deepSeq deprecationWarnings { inherit nixosConfigurations; clanInternals = { - # Evaluated clan meta - # Merged /clan/meta.json with overrides from buildClan - meta = mergedMeta; + meta = mergedInventory.meta; inventory = mergedInventory; + invFile = "${directory}/inventory.json"; + # machine specifics machines = configsPerSystem; machinesFunc = configsFuncPerSystem; From 6378a96b4d3616699141b5fdce26afc190e3b465 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 24 Jun 2024 15:25:36 +0200 Subject: [PATCH 09/18] Template: use inventory --- templates/new-clan/flake.nix | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix index 7fb1361e..e6c67311 100644 --- a/templates/new-clan/flake.nix +++ b/templates/new-clan/flake.nix @@ -11,7 +11,19 @@ # Usage see: https://docs.clan.lol clan = clan-core.lib.buildClan { directory = self; - clanName = "__CHANGE_ME__"; # Ensure this is internet wide unique. + meta.name = "__CHANGE_ME__"; # Ensure this is internet wide unique. + + # Distributed services + inventory = { + services = { + # This example configures a BorgBackup service + # Check: https://docs.clan.lol/reference/clanModules which ones are available in Inventory + borgbackup.instance_1 = { + roles.server.machines = [ "jon" ]; + roles.client.machines = [ "sara" ]; + }; + }; + }; # Prerequisite: boot into the installer # See: https://docs.clan.lol/getting-started/installer From 3aa7a6ee691ab683d0eabc498e50cd15fd33419e Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 24 Jun 2024 21:28:45 +0200 Subject: [PATCH 10/18] Inventory: move to lib.inventory --- flake.nix | 2 -- lib/build-clan/default.nix | 4 ++-- lib/default.nix | 1 + {inventory => lib/inventory}/.envrc | 0 {inventory => lib/inventory}/README.md | 0 inventory/default.nix => lib/inventory/_old_default.nix | 0 .../build-inventory/default.nix} | 0 .../build-inventory}/interface.nix | 1 + lib/inventory/default.nix | 5 +++++ {inventory => lib/inventory}/flake-module.nix | 9 ++------- {inventory => lib/inventory}/src/cue.mod/module.cue | 0 {inventory => lib/inventory}/src/root.cue | 0 {inventory => lib/inventory}/src/schema/schema.cue | 0 {inventory => lib/inventory}/src/tests/borgbackup.json | 0 {inventory => lib/inventory}/src/tests/syncthing.json | 0 {inventory => lib/inventory}/src/tests/zerotier.json | 0 16 files changed, 11 insertions(+), 11 deletions(-) rename {inventory => lib/inventory}/.envrc (100%) rename {inventory => lib/inventory}/README.md (100%) rename inventory/default.nix => lib/inventory/_old_default.nix (100%) rename lib/{build-clan/inventory.nix => inventory/build-inventory/default.nix} (100%) rename lib/{build-clan => inventory/build-inventory}/interface.nix (99%) create mode 100644 lib/inventory/default.nix rename {inventory => lib/inventory}/flake-module.nix (91%) rename {inventory => lib/inventory}/src/cue.mod/module.cue (100%) rename {inventory => lib/inventory}/src/root.cue (100%) rename {inventory => lib/inventory}/src/schema/schema.cue (100%) rename {inventory => lib/inventory}/src/tests/borgbackup.json (100%) rename {inventory => lib/inventory}/src/tests/syncthing.json (100%) rename {inventory => lib/inventory}/src/tests/zerotier.json (100%) diff --git a/flake.nix b/flake.nix index 341c5b84..be6a1db8 100644 --- a/flake.nix +++ b/flake.nix @@ -53,8 +53,6 @@ ./nixosModules/flake-module.nix ./pkgs/flake-module.nix ./templates/flake-module.nix - - ./inventory/flake-module.nix ]; } ); diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 271f4254..9328b460 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -33,7 +33,7 @@ let mergedInventory = (lib.evalModules { modules = [ - ./interface.nix + clan-core.lib.inventory.interface { inherit meta; } ( if @@ -77,7 +77,7 @@ let ]; }).config; - buildInventory = import ./inventory.nix { inherit lib clan-core; }; + inherit (clan-core.lib.inventory) buildInventory; # map from machine name to service configuration # { ${machineName} :: Config } diff --git a/lib/default.nix b/lib/default.nix index ef5fe7ac..086fcce8 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -6,6 +6,7 @@ }: { evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; }; + inventory = import ./inventory { inherit lib clan-core; }; jsonschema = import ./jsonschema { inherit lib; }; modules = import ./description.nix { inherit clan-core lib; }; buildClan = import ./build-clan { inherit clan-core lib nixpkgs; }; diff --git a/inventory/.envrc b/lib/inventory/.envrc similarity index 100% rename from inventory/.envrc rename to lib/inventory/.envrc diff --git a/inventory/README.md b/lib/inventory/README.md similarity index 100% rename from inventory/README.md rename to lib/inventory/README.md diff --git a/inventory/default.nix b/lib/inventory/_old_default.nix similarity index 100% rename from inventory/default.nix rename to lib/inventory/_old_default.nix diff --git a/lib/build-clan/inventory.nix b/lib/inventory/build-inventory/default.nix similarity index 100% rename from lib/build-clan/inventory.nix rename to lib/inventory/build-inventory/default.nix diff --git a/lib/build-clan/interface.nix b/lib/inventory/build-inventory/interface.nix similarity index 99% rename from lib/build-clan/interface.nix rename to lib/inventory/build-inventory/interface.nix index 4a46ae39..c399d2af 100644 --- a/lib/build-clan/interface.nix +++ b/lib/inventory/build-inventory/interface.nix @@ -78,6 +78,7 @@ in }; options.services = lib.mkOption { + default = { }; type = t.attrsOf ( t.attrsOf ( t.submodule { diff --git a/lib/inventory/default.nix b/lib/inventory/default.nix new file mode 100644 index 00000000..78a4c1ac --- /dev/null +++ b/lib/inventory/default.nix @@ -0,0 +1,5 @@ +{ lib, clan-core }: +{ + buildInventory = import ./build-inventory { inherit lib clan-core; }; + interface = ./build-inventory/interface.nix; +} diff --git a/inventory/flake-module.nix b/lib/inventory/flake-module.nix similarity index 91% rename from inventory/flake-module.nix rename to lib/inventory/flake-module.nix index 782f2496..11a5ce9f 100644 --- a/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -1,11 +1,6 @@ +{ ... }: { - inputs, - self, - lib, - ... -}: -{ - flake.inventory = import ./default.nix { inherit inputs self lib; }; + # flake.inventory = import ./default.nix { inherit inputs self lib; }; perSystem = { pkgs, config, ... }: { diff --git a/inventory/src/cue.mod/module.cue b/lib/inventory/src/cue.mod/module.cue similarity index 100% rename from inventory/src/cue.mod/module.cue rename to lib/inventory/src/cue.mod/module.cue diff --git a/inventory/src/root.cue b/lib/inventory/src/root.cue similarity index 100% rename from inventory/src/root.cue rename to lib/inventory/src/root.cue diff --git a/inventory/src/schema/schema.cue b/lib/inventory/src/schema/schema.cue similarity index 100% rename from inventory/src/schema/schema.cue rename to lib/inventory/src/schema/schema.cue diff --git a/inventory/src/tests/borgbackup.json b/lib/inventory/src/tests/borgbackup.json similarity index 100% rename from inventory/src/tests/borgbackup.json rename to lib/inventory/src/tests/borgbackup.json diff --git a/inventory/src/tests/syncthing.json b/lib/inventory/src/tests/syncthing.json similarity index 100% rename from inventory/src/tests/syncthing.json rename to lib/inventory/src/tests/syncthing.json diff --git a/inventory/src/tests/zerotier.json b/lib/inventory/src/tests/zerotier.json similarity index 100% rename from inventory/src/tests/zerotier.json rename to lib/inventory/src/tests/zerotier.json From 7f674e6f6390197b302d3aa3823f2cea884de02b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 24 Jun 2024 21:42:55 +0200 Subject: [PATCH 11/18] Remove useless settings.json --- flakeModules/clan.nix | 2 ++ lib/build-clan/default.nix | 2 +- templates/flake-module.nix | 23 +---------------------- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/flakeModules/clan.nix b/flakeModules/clan.nix index dec466aa..b1a1247a 100644 --- a/flakeModules/clan.nix +++ b/flakeModules/clan.nix @@ -91,6 +91,8 @@ in type = lib.types.submodule { options = { inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; + inventoryFile = lib.mkOption { type = lib.types.unspecified; }; + meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 9328b460..8861c3f8 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -188,7 +188,7 @@ builtins.deepSeq deprecationWarnings { meta = mergedInventory.meta; inventory = mergedInventory; - invFile = "${directory}/inventory.json"; + inventoryFile = "${directory}/inventory.json"; # machine specifics machines = configsPerSystem; diff --git a/templates/flake-module.nix b/templates/flake-module.nix index aa1eaf39..e1c9f774 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -1,4 +1,4 @@ -{ self, inputs, ... }: +{ self, ... }: { flake.templates = { new-clan = { @@ -15,25 +15,4 @@ path = ./minimal; }; }; - flake.checks.x86_64-linux.template-minimal = - let - path = self.templates.minimal.path; - initialized = inputs.nixpkgs.legacyPackages.x86_64-linux.runCommand "minimal-clan-flake" { } '' - mkdir $out - cp -r ${path}/* $out - mkdir -p $out/machines/foo - echo '{ "nixpkgs": { "hostPlatform": "x86_64-linux" } }' > $out/machines/foo/settings.json - ''; - evaled = (import "${initialized}/flake.nix").outputs { - self = evaled // { - outPath = initialized; - }; - clan-core = self; - }; - in - { - type = "derivation"; - name = "minimal-clan-flake-check"; - inherit (evaled.nixosConfigurations.foo.config.system.build.vm) drvPath outPath; - }; } From e6c78054c49c94ad9b87ff118d7b5db34206e595 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 24 Jun 2024 21:55:14 +0200 Subject: [PATCH 12/18] Template: make inventory disabled by default --- templates/new-clan/flake.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/templates/new-clan/flake.nix b/templates/new-clan/flake.nix index e6c67311..8ae5055d 100644 --- a/templates/new-clan/flake.nix +++ b/templates/new-clan/flake.nix @@ -13,17 +13,17 @@ directory = self; meta.name = "__CHANGE_ME__"; # Ensure this is internet wide unique. - # Distributed services - inventory = { - services = { - # This example configures a BorgBackup service - # Check: https://docs.clan.lol/reference/clanModules which ones are available in Inventory - borgbackup.instance_1 = { - roles.server.machines = [ "jon" ]; - roles.client.machines = [ "sara" ]; - }; - }; - }; + # Distributed services, uncomment to enable. + # inventory = { + # services = { + # # This example configures a BorgBackup service + # # Check: https://docs.clan.lol/reference/clanModules which ones are available in Inventory + # borgbackup.instance_1 = { + # roles.server.machines = [ "jon" ]; + # roles.client.machines = [ "sara" ]; + # }; + # }; + # }; # Prerequisite: boot into the installer # See: https://docs.clan.lol/getting-started/installer From 712ed3f738d69fec364a619785bc598f89bd28e3 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 24 Jun 2024 22:58:00 +0200 Subject: [PATCH 13/18] Docs: add inventory module docs --- clanModules/borgbackup-static/default.nix | 2 +- clanModules/borgbackup/README.md | 15 +------- clanModules/borgbackup/default.nix | 47 ++++++++++------------- docs/mkdocs.yml | 2 +- docs/nix/flake-module.nix | 2 + docs/nix/get-module-docs.nix | 7 +++- docs/nix/scripts/renderOptions.py | 47 +++++++++++++++++++++++ 7 files changed, 78 insertions(+), 44 deletions(-) diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index d65fdb5f..931ea15f 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -3,7 +3,7 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; in -lib.warn "This module is deprecated use the service via the service interface instead." { +lib.warn "This module is deprecated use the service via the inventory interface instead." { imports = [ ../borgbackup ]; options.clan.borgbackup-static = { diff --git a/clanModules/borgbackup/README.md b/clanModules/borgbackup/README.md index 13379d26..b639786d 100644 --- a/clanModules/borgbackup/README.md +++ b/clanModules/borgbackup/README.md @@ -1,15 +1,2 @@ Efficient, deduplicating backup program with optional compression and secure encryption. ---- - -## Roles - -- Client -- Server - -## Configuration - -Configure target machines where the backups should be sent to through `targets`. - -Configure machines that should be backed up either through `includeMachines` -which will exclusively add the included machines to be backed up, or through -`excludeMachines`, which will add every machine except the excluded machine to the backup. +--- \ No newline at end of file diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix index f708fe2e..c5f61b3c 100644 --- a/clanModules/borgbackup/default.nix +++ b/clanModules/borgbackup/default.nix @@ -30,39 +30,37 @@ let # Each .nix file in the roles directory is a role # TODO: Helper function to set available roles within module meta. - # roles = lib.pipe ./roles [ - # (p: if builtins.pathExists p then p else throw "Role directory does not exist") - # builtins.readDir - # (lib.filterAttrs (_n: v: v == "regular")) - # lib.attrNames - # (map (fileName: lib.removeSuffix ".nix" fileName)) - # ]; + roles = + if builtins.pathExists ./roles then + lib.pipe ./roles [ + builtins.readDir + (lib.filterAttrs (_n: v: v == "regular")) + lib.attrNames + (map (fileName: lib.removeSuffix ".nix" fileName)) + ] + else + null; # TODO: make this an interface of every module # Maybe load from readme.md metaInfoOption = lib.mkOption { readOnly = true; default = { - description = "A category for the service. This is used to group services in the clan ui"; + description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption."; + availableRoles = roles; + category = "backup"; }; type = lib.types.submodule { options = { - description = lib.mkOption { + description = lib.mkOption { type = lib.types.str; }; + availableRoles = lib.mkOption { type = lib.types.nullOr (lib.types.listOf lib.types.str); }; + category = lib.mkOption { description = "A category for the service. This is used to group services in the clan ui"; - type = lib.types.str; + type = lib.types.enum [ + "backup" + "network" + ]; }; - # icon = lib.mkOption { - # description = "A category for the service. This is used to group services in the clan ui"; - # type = lib.types.str; - # }; - # screenshots = lib.mkOption { - # description = "A category for the service. This is used to group services in the clan ui"; - # type = lib.types.str; - # }; - # category = lib.mkOption { - # description = "A category for the service. This is used to group services in the clan ui"; - # type = lib.types.enum ["backup" "network"]; - # }; }; }; }; @@ -105,11 +103,6 @@ in }; imports = [ - { - config.clan.borgbackup.meta = { - description = "backup"; - }; - } (lib.mkRemovedOptionModule [ "clan" "borgbackup" diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index d43ee81b..daaf98e9 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -49,7 +49,7 @@ nav: - Mesh VPN: getting-started/mesh-vpn.md - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md - - Modules: + - Reference: - Clan Modules: - reference/clanModules/borgbackup-static.md - reference/clanModules/borgbackup.md diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index b398db44..8370c0a2 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -19,6 +19,7 @@ clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules); clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes); + clanModulesMeta = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesMeta); # Simply evaluated options (JSON) renderOptions = @@ -54,6 +55,7 @@ # A file that contains the links to all clanModule docs export CLAN_MODULES=${clanModulesFileInfo} export CLAN_MODULES_READMES=${clanModulesReadmes} + export CLAN_MODULES_META=${clanModulesMeta} mkdir $out diff --git a/docs/nix/get-module-docs.nix b/docs/nix/get-module-docs.nix index a05fe847..b1d2ed9d 100644 --- a/docs/nix/get-module-docs.nix +++ b/docs/nix/get-module-docs.nix @@ -43,11 +43,16 @@ let module_name: _module: self.lib.modules.getReadme module_name ) clanModules; + clanModulesMeta = builtins.mapAttrs ( + module_name: _module: + (self.lib.evalClanModules [ module_name ]).config.clan.${module_name}.meta or { } + ) clanModules; + # clanCore docs clanCoreDocs = (evalDocs (getOptions [ ]).clan.core).optionsJSON; in { - inherit clanModulesReadmes; + inherit clanModulesReadmes clanModulesMeta; clanCore = clanCoreDocs; clanModules = clanModulesDocs; } diff --git a/docs/nix/scripts/renderOptions.py b/docs/nix/scripts/renderOptions.py index 4480d410..096fbe32 100644 --- a/docs/nix/scripts/renderOptions.py +++ b/docs/nix/scripts/renderOptions.py @@ -32,6 +32,8 @@ from typing import Any CLAN_CORE = os.getenv("CLAN_CORE") CLAN_MODULES = os.environ.get("CLAN_MODULES") CLAN_MODULES_READMES = os.environ.get("CLAN_MODULES_READMES") +CLAN_MODULES_META = os.environ.get("CLAN_MODULES_META") + OUT = os.environ.get("out") @@ -198,6 +200,11 @@ def produce_clan_modules_docs() -> None: f"Environment variables are not set correctly: $CLAN_MODULES_READMES={CLAN_MODULES_READMES}" ) + if not CLAN_MODULES_META: + raise ValueError( + f"Environment variables are not set correctly: $CLAN_MODULES_META={CLAN_MODULES_META}" + ) + if not OUT: raise ValueError(f"Environment variables are not set correctly: $out={OUT}") @@ -207,6 +214,42 @@ def produce_clan_modules_docs() -> None: with open(CLAN_MODULES_READMES) as readme: readme_map: dict[str, str] = json.load(readme) + with open(CLAN_MODULES_META) as f: + meta_map: dict[str, Any] = json.load(f) + print(meta_map) + + def render_meta(meta: dict[str, Any], module_name: str) -> str: + roles = meta.get("availableRoles", None) + + if roles: + roles_list = "\n".join([f" - `{r}`" for r in roles]) + return f""" +???+ tip "Inventory (WIP)" + + Predefined roles: + +{roles_list} + + Usage: + + ```{{.nix hl_lines="4"}} + buildClan {{ + inventory.services = {{ + {module_name}.instance_1 = {{ + roles.{roles[0]}.machines = [ "sara_machine" ]; + # ... + }}; + }}; + }} + ``` + +""" + return """ +???+ example "Inventory (WIP)" + This module does not support the inventory yet. + + """ + # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... } for module_name, options_file in links.items(): with open(Path(options_file) / "share/doc/nixos/options.json") as f: @@ -217,6 +260,10 @@ def produce_clan_modules_docs() -> None: if readme_map.get(module_name, None): output += f"{readme_map[module_name]}\n" + # Add meta information: + # - Inventory implementation status + output += render_meta(meta_map.get(module_name, {}), module_name) + output += module_usage(module_name) output += options_head if len(options.items()) else "" From 26c655ff3c35697a8130464362e6572f5f757e0d Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 25 Jun 2024 12:12:09 +0200 Subject: [PATCH 14/18] Add settings.json back --- clanModules/borgbackup/default.nix | 7 +++ docs/nix/scripts/renderOptions.py | 68 ++++++++++++------------- lib/build-clan/default.nix | 80 ++++++++++++++++++++---------- lib/flake-module.nix | 5 +- lib/inventory/_old_default.nix | 32 ------------ lib/inventory/example.nix | 38 ++++++++++++++ lib/inventory/flake-module.nix | 1 - templates/flake-module.nix | 23 ++++++++- 8 files changed, 158 insertions(+), 96 deletions(-) delete mode 100644 lib/inventory/_old_default.nix create mode 100644 lib/inventory/example.nix diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix index c5f61b3c..b46624be 100644 --- a/clanModules/borgbackup/default.nix +++ b/clanModules/borgbackup/default.nix @@ -45,6 +45,13 @@ let # Maybe load from readme.md metaInfoOption = lib.mkOption { readOnly = true; + description = '' + Meta is used to retrieve information about this module. + + - `availableRoles` is a list of roles that can be assigned via the inventory. + - `category` is used to group services in the clan marketplace. + - `description` is a short description of the service for the clan marketplace. + ''; default = { description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption."; availableRoles = roles; diff --git a/docs/nix/scripts/renderOptions.py b/docs/nix/scripts/renderOptions.py index 096fbe32..909d6108 100644 --- a/docs/nix/scripts/renderOptions.py +++ b/docs/nix/scripts/renderOptions.py @@ -78,7 +78,9 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str: res = f""" {"#" * level} {sanitize(name)} -{"Readonly" if read_only else ""} + +{"**Readonly**" if read_only else ""} + {option.get("description", "No description available.")} **Type**: `{option["type"]}` @@ -190,6 +192,35 @@ def produce_clan_core_docs() -> None: of.write(output) +def render_meta(meta: dict[str, Any], module_name: str) -> str: + roles = meta.get("availableRoles", None) + + if roles: + roles_list = "\n".join([f" - `{r}`" for r in roles]) + return f""" +???+ tip "Inventory (WIP)" + + Predefined roles: + +{roles_list} + + Usage: + + ```{{.nix hl_lines="4"}} + buildClan {{ + inventory.services = {{ + {module_name}.instance_1 = {{ + roles.{roles[0]}.machines = [ "sara_machine" ]; + # ... + }}; + }}; + }} + ``` + +""" + return "" + + def produce_clan_modules_docs() -> None: if not CLAN_MODULES: raise ValueError( @@ -218,38 +249,6 @@ def produce_clan_modules_docs() -> None: meta_map: dict[str, Any] = json.load(f) print(meta_map) - def render_meta(meta: dict[str, Any], module_name: str) -> str: - roles = meta.get("availableRoles", None) - - if roles: - roles_list = "\n".join([f" - `{r}`" for r in roles]) - return f""" -???+ tip "Inventory (WIP)" - - Predefined roles: - -{roles_list} - - Usage: - - ```{{.nix hl_lines="4"}} - buildClan {{ - inventory.services = {{ - {module_name}.instance_1 = {{ - roles.{roles[0]}.machines = [ "sara_machine" ]; - # ... - }}; - }}; - }} - ``` - -""" - return """ -???+ example "Inventory (WIP)" - This module does not support the inventory yet. - - """ - # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... } for module_name, options_file in links.items(): with open(Path(options_file) / "share/doc/nixos/options.json") as f: @@ -262,7 +261,8 @@ def produce_clan_modules_docs() -> None: # Add meta information: # - Inventory implementation status - output += render_meta(meta_map.get(module_name, {}), module_name) + if meta_map.get(module_name, None): + output += render_meta(meta_map.get(module_name, {}), module_name) output += module_usage(module_name) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 8861c3f8..0425930a 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -70,6 +70,8 @@ let } ) machines; } + # Will be deprecated + # {machines = lib.mapAttrs (n: _: {}) machinesDirs;} # Deprecated interface (if clanName != null then { meta.name = clanName; } else { }) @@ -83,6 +85,25 @@ let # { ${machineName} :: Config } serviceConfigs = buildInventory mergedInventory; + # machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + # builtins.readDir (directory + /machines) + # ); + + machineSettings = + machineName: + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then + builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) + else + lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( + builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)) + ); + + machineImports = + machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]); + deprecationWarnings = [ (lib.warnIf ( clanName != null @@ -99,35 +120,40 @@ let extraConfig ? { }, }: nixpkgs.lib.nixosSystem { - modules = [ - clan-core.nixosModules.clanCore - extraConfig - (machines.${name} or { }) - # Inherit the inventory assertions ? - { inherit (mergedInventory) assertions; } - { imports = serviceConfigs.${name} or { }; } - ( - { - # Settings - clan.core.clanDir = directory; - # Inherited from clan wide settings - clan.core.clanName = meta.name or clanName; - clan.core.clanIcon = meta.icon or clanIcon; + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + clan-core.nixosModules.clanCore + extraConfig + (machines.${name} or { }) + # Inherit the inventory assertions ? + { inherit (mergedInventory) assertions; } + { imports = serviceConfigs.${name} or { }; } + ( + { + # Settings + clan.core.clanDir = directory; + # Inherited from clan wide settings + clan.core.clanName = meta.name or clanName; + clan.core.clanIcon = meta.icon or clanIcon; - # Machine specific settings - clan.core.machineName = name; - networking.hostName = lib.mkDefault name; - nixpkgs.hostPlatform = lib.mkDefault system; + # Machine specific settings + clan.core.machineName = name; + networking.hostName = lib.mkDefault name; + nixpkgs.hostPlatform = lib.mkDefault system; - # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) - nix.registry.nixpkgs.to = { - type = "path"; - path = lib.mkDefault nixpkgs; - }; - } - // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } - ) - ]; + # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) + nix.registry.nixpkgs.to = { + type = "path"; + path = lib.mkDefault nixpkgs; + }; + } + // lib.optionalAttrs (pkgs != null) { nixpkgs.pkgs = lib.mkForce pkgs; } + ) + ]; specialArgs = { inherit clan-core; } // specialArgs; diff --git a/lib/flake-module.nix b/lib/flake-module.nix index ef93467b..0785872a 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -5,7 +5,10 @@ ... }: { - imports = [ ./jsonschema/flake-module.nix ]; + imports = [ + ./jsonschema/flake-module.nix + ./inventory/flake-module.nix + ]; flake.lib = import ./default.nix { inherit lib inputs; inherit (inputs) nixpkgs; diff --git a/lib/inventory/_old_default.nix b/lib/inventory/_old_default.nix deleted file mode 100644 index 2ca5ac00..00000000 --- a/lib/inventory/_old_default.nix +++ /dev/null @@ -1,32 +0,0 @@ -{ self, lib, ... }: -let - clan-core = self; -in -{ - clan = clan-core.lib.buildClan { - meta.name = "kenjis clan"; - # Should usually point to the directory of flake.nix - directory = self; - - inventory = { - services = { - borgbackup.instance_1 = { - roles.server.machines = [ "vyr_machine" ]; - roles.client.tags = [ "laptop" ]; - }; - }; - }; - - # merged with - machines = { - "vyr_machine" = { }; - "vi_machine" = { - clan.tags = [ "laptop" ]; - }; - "camina_machine" = { - clan.tags = [ "laptop" ]; - clan.meta.name = "camina"; - }; - }; - }; -} diff --git a/lib/inventory/example.nix b/lib/inventory/example.nix new file mode 100644 index 00000000..50b41792 --- /dev/null +++ b/lib/inventory/example.nix @@ -0,0 +1,38 @@ +{ self, ... }: +self.lib.buildClan { + # Name of the clan in the UI, should be unique + meta.name = "Inventory clan"; + + # Should usually point to the directory of flake.nix + directory = self; + + inventory = { + services = { + borgbackup.instance_1 = { + roles.server.machines = [ "backup_server" ]; + roles.client.tags = [ "backup" ]; + }; + }; + }; + + # merged with + machines = { + "backup_server" = { + clan.tags = [ "all" ]; + }; + "client_1_machine" = { + clan.tags = [ + "all" + "backup" + ]; + }; + "client_2_machine" = { + clan.tags = [ + "all" + "backup" + ]; + # Name of the machine in the UI + clan.meta.name = "camina"; + }; + }; +} diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index 11a5ce9f..8778e465 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -1,6 +1,5 @@ { ... }: { - # flake.inventory = import ./default.nix { inherit inputs self lib; }; perSystem = { pkgs, config, ... }: { diff --git a/templates/flake-module.nix b/templates/flake-module.nix index e1c9f774..aa1eaf39 100644 --- a/templates/flake-module.nix +++ b/templates/flake-module.nix @@ -1,4 +1,4 @@ -{ self, ... }: +{ self, inputs, ... }: { flake.templates = { new-clan = { @@ -15,4 +15,25 @@ path = ./minimal; }; }; + flake.checks.x86_64-linux.template-minimal = + let + path = self.templates.minimal.path; + initialized = inputs.nixpkgs.legacyPackages.x86_64-linux.runCommand "minimal-clan-flake" { } '' + mkdir $out + cp -r ${path}/* $out + mkdir -p $out/machines/foo + echo '{ "nixpkgs": { "hostPlatform": "x86_64-linux" } }' > $out/machines/foo/settings.json + ''; + evaled = (import "${initialized}/flake.nix").outputs { + self = evaled // { + outPath = initialized; + }; + clan-core = self; + }; + in + { + type = "derivation"; + name = "minimal-clan-flake-check"; + inherit (evaled.nixosConfigurations.foo.config.system.build.vm) drvPath outPath; + }; } From c0293b889cb3cbda827ef67a0c32377b989028b1 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 25 Jun 2024 12:35:21 +0200 Subject: [PATCH 15/18] Add machinesDirs with default --- lib/build-clan/default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 0425930a..33dea904 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -71,7 +71,7 @@ let ) machines; } # Will be deprecated - # {machines = lib.mapAttrs (n: _: {}) machinesDirs;} + { machines = lib.mkDefault (lib.mapAttrs (_n: _: { }) machinesDirs); } # Deprecated interface (if clanName != null then { meta.name = clanName; } else { }) @@ -85,9 +85,9 @@ let # { ${machineName} :: Config } serviceConfigs = buildInventory mergedInventory; - # machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( - # builtins.readDir (directory + /machines) - # ); + machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") ( + builtins.readDir (directory + /machines) + ); machineSettings = machineName: From 4fb15d8733cb3960a8f83024e4a8f49bdc17412b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 25 Jun 2024 12:51:03 +0200 Subject: [PATCH 16/18] Add machinesDirs with default --- lib/build-clan/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 33dea904..a81ddd2d 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -71,7 +71,7 @@ let ) machines; } # Will be deprecated - { machines = lib.mkDefault (lib.mapAttrs (_n: _: { }) machinesDirs); } + { machines = lib.mapAttrs (_n: _: lib.mkDefault { }) machinesDirs; } # Deprecated interface (if clanName != null then { meta.name = clanName; } else { }) From d03422d004279f0e0c495674af554c5be80f209c Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 25 Jun 2024 12:59:11 +0200 Subject: [PATCH 17/18] test: increase timeout --- pkgs/clan-cli/tests/test_vms_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/tests/test_vms_cli.py b/pkgs/clan-cli/tests/test_vms_cli.py index 73a614f0..a1dba225 100644 --- a/pkgs/clan-cli/tests/test_vms_cli.py +++ b/pkgs/clan-cli/tests/test_vms_cli.py @@ -40,7 +40,7 @@ def run_vm_in_thread(machine_name: str) -> None: # wait for qmp socket to exist def wait_vm_up(state_dir: Path) -> None: socket_file = state_dir / "qga.sock" - timeout: float = 20 + timeout: float = 120 while True: if timeout <= 0: raise TimeoutError( From db951f1d9e18c7a0d9a13484fed19a8a5a3de8d9 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 25 Jun 2024 13:35:38 +0200 Subject: [PATCH 18/18] buildClan: bugfixing --- lib/build-clan/default.nix | 1 + pkgs/clan-cli/tests/test_vms_cli.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index a81ddd2d..452f3839 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -126,6 +126,7 @@ let in (machineImports settings) ++ [ + settings clan-core.nixosModules.clanCore extraConfig (machines.${name} or { }) diff --git a/pkgs/clan-cli/tests/test_vms_cli.py b/pkgs/clan-cli/tests/test_vms_cli.py index a1dba225..73a614f0 100644 --- a/pkgs/clan-cli/tests/test_vms_cli.py +++ b/pkgs/clan-cli/tests/test_vms_cli.py @@ -40,7 +40,7 @@ def run_vm_in_thread(machine_name: str) -> None: # wait for qmp socket to exist def wait_vm_up(state_dir: Path) -> None: socket_file = state_dir / "qga.sock" - timeout: float = 120 + timeout: float = 20 while True: if timeout <= 0: raise TimeoutError(