diff --git a/clanModules/borgbackup-static/default.nix b/clanModules/borgbackup-static/default.nix index ee2deccb..c65dff61 100644 --- a/clanModules/borgbackup-static/default.nix +++ b/clanModules/borgbackup-static/default.nix @@ -3,23 +3,49 @@ let clanDir = config.clan.core.clanDir; machineDir = clanDir + "/machines/"; - cfg = config.clan.borgbackup-static; + # cfg.roles = config.clan.borgbackup-static; + + # machine < machine_module < inventory + # nixos < borgbackup < borgbackup-static > UI + # metadata + # Developer User field descriptions + + roles = config.clan.borgbackup-static.inventory.roles; machine_name = config.clan.core.machineName; in { imports = [ ../borgbackup ]; - + # imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ]; # Inventory / Interface.nix # options.clan.inventory.borgbackup-static.description. - options.clan.borgbackup-static.roles = lib.mkOption { - type = lib.types.attrsOf (lib.types.listOf lib.types.str); + # options.clan.borgbackup-static.roles = lib.mkOption { + # type = lib.types.attrsOf (lib.types.listOf lib.types.str); + # }; + + # Can be used via inventory.json + # + # .borgbackup-static.inventory.roles + # + options.clan.borgbackup-static.inventory = lib.mkOption { + type = lib.types.submodule { + # imports = [./inventory/interface.nix]; + + # idea + # config.metadata = builtins.fromTOML ... + # config.defaultRoles = ["client"]; + + # -> interface.nix + options = { + roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); }; + }; + }; }; config.services.borgbackup.repos = let - filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles); + filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles); borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; machinesMaybeKey = builtins.map ( @@ -40,13 +66,13 @@ in }; }) machinesWithKey; in - lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) ( + lib.mkIf (builtins.elem "server" roles.${machine_name}) ( if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } ); config.clan.borgbackup.destinations = let - servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles); + servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles); destinations = builtins.map (server_name: { name = server_name; @@ -55,5 +81,5 @@ in }; }) servers; in - lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations); + lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations); } diff --git a/inventory/README.md b/inventory/README.md index 318d1bf0..2bdbf77b 100644 --- a/inventory/README.md +++ b/inventory/README.md @@ -1,5 +1,64 @@ # Inventory +Questions: + +- [x] Must roles be a list ? + -> Yes. In zerotier you can be "moon" and "controller" at the same time. + +- [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different? + -> There are many roles. And they depend on the service. + +- [x] Should we use the module name in the path of the service? + ```json + // ${module_name}.${instance_name} + services.borgbackup-static.backup1 = { + + } + ``` + + Pro: + Easier to handle. + Better groups the module specific instances. + Contra: + More nesting in json + + Neutral: Module name is hard to change. Exists anyways. + +- [x] Should the machine specific service config be part of the service? + -> The config implements the schema of the module, which is declared in the service. + -> If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config. + +Architecture + +``` +machine < machine_module < inventory +--------------------------------------------- +nixos < borgbackup < borgbackup-static > UI + + creates the config Maps from high level services to the borgbackup clan module + for ONE machine +``` + +- [ ] Why do we need 2 modules? + -> It is technically possible to have only 1 module. + Pros: + Simple to use/Easy to understand. + Less modules + Cons: + Harder to write a module. Because it must do 2 things. + One module should do only 1 thing. + +```nix +clan.machines.${machine_name} = { + # "borgbackup.ssh.pub" = machineDir + machines + "/facts/borgbackup.ssh.pub"; + facts = ... +}; +clan.services.${instance} = { +# roles.server = [ "jon_machine" ] +# roles.${role_name} = [ ${machine_name} ]; +}; +``` + This part provides a specification for the inventory. It is used for design phase and as validation helper. diff --git a/inventory/default.nix b/inventory/default.nix index 8832a7d7..fbee6457 100644 --- a/inventory/default.nix +++ b/inventory/default.nix @@ -1,16 +1,28 @@ -{ inputs, self, ... }: +{ self, lib, ... }: let clan-core = self; - system = "x86_64-linux"; - pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system}; # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json); syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); - machines = machinesFromInventory { - inherit clan-core; - lib = pkgs.lib; - } syncthing_inventory; + machines = machinesFromInventory syncthing_inventory; + + resolveGroups = + inventory: members: + lib.unique ( + builtins.foldl' ( + acc: currMember: + let + groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember; + groupMembers = + if inventory.groups.machines ? ${groupName} then + inventory.groups.machines.${groupName} + else + throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory."; + in + if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ] + ) [ ] members + ); /* Returns a NixOS configuration for every machine in the inventory. @@ -18,48 +30,51 @@ let machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } */ machinesFromInventory = - { lib, clan-core, ... }: inventory: # For every machine in the inventory, build a NixOS configuration # For each machine generate config, forEach service, if the machine is used. builtins.mapAttrs ( - machine_name: _: - builtins.foldl' ( - acc: service_name: - let - service_config = inventory.services.${service_name}; - isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig); + machineName: _: + lib.foldlAttrs ( + # [ Modules ], String, { ${instance_name} :: ServiceConfig } + acc: moduleName: serviceConfigs: + acc + # Collect service config + ++ (lib.foldlAttrs ( + # [ Modules ], String, ServiceConfig + acc2: instanceName: serviceConfig: + let + resolvedRoles = builtins.mapAttrs ( + _roleName: members: resolveGroups inventory members + ) serviceConfig.roles; - machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { }; - global_config = inventory.services.${service_name}.config; - module_name = inventory.services.${service_name}.module; - in - # Possible roles: "server", "client", "peer" - if - builtins.trace '' - isInService ${builtins.toJSON isInService}, - ${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)} - '' isInService - then - acc - ++ [ - { - imports = [ clan-core.clanModules.${module_name} ]; - config.clan.${module_name} = lib.mkMerge [ - global_config - machine_service_config - ]; - } - { - config.clan.${module_name} = { - # TODO: filter, show only the roles that are needed by the machine - roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig; - }; - } - ] - else - acc - ) [ ] (builtins.attrNames inventory.services) + isInService = builtins.any (members: builtins.elem machineName members) ( + builtins.attrValues resolvedRoles + ); + + machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { }; + globalConfig = serviceConfig.config; + in + if isInService then + acc2 + ++ [ + { + imports = [ clan-core.clanModules.${moduleName} ]; + config.clan.${moduleName} = lib.mkMerge [ + globalConfig + machineServiceConfig + ]; + } + { + config.clan.inventory.${instanceName} = { + roles = resolvedRoles; + }; + } + ] + else + acc2 + ) [ ] serviceConfigs) + ) [ ] inventory.services ) inventory.machines; in { @@ -81,9 +96,4 @@ in }; }; intern = machines; - # inherit (clan) nixosConfigurations clanInternals; - # add the Clan cli tool to the dev shell - devShells.${system}.default = pkgs.mkShell { - packages = [ clan-core.packages.${system}.clan-cli ]; - }; } diff --git a/inventory/flake-module.nix b/inventory/flake-module.nix index aace1add..782f2496 100644 --- a/inventory/flake-module.nix +++ b/inventory/flake-module.nix @@ -1,6 +1,11 @@ -{ inputs, self, ... }: { - flake.inventory = import ./default.nix { inherit inputs self; }; + inputs, + self, + lib, + ... +}: +{ + flake.inventory = import ./default.nix { inherit inputs self lib; }; perSystem = { pkgs, config, ... }: { diff --git a/inventory/src/machines/machines.cue b/inventory/src/machines/machines.cue deleted file mode 100644 index a917e1e9..00000000 --- a/inventory/src/machines/machines.cue +++ /dev/null @@ -1,8 +0,0 @@ -package machines - - -#machine: machines: [string]: { - name: string, - description?: string, - icon?: string -} \ No newline at end of file diff --git a/inventory/src/root.cue b/inventory/src/root.cue index 3034dab8..661e92dd 100644 --- a/inventory/src/root.cue +++ b/inventory/src/root.cue @@ -1,8 +1,7 @@ package inventory import ( - "clan.lol/inventory/services" - "clan.lol/inventory/machines" + "clan.lol/inventory/schema" ) @jsonschema(schema="http://json-schema.org/schema#") @@ -16,9 +15,11 @@ import ( icon?: string } - // A map of services - services.#service + // // A map of services + schema.#service - // A map of machines - machines.#machine + // // A map of machines + schema.#machine + + schema.#groups } diff --git a/inventory/src/services/services.cue b/inventory/src/schema/schema.cue similarity index 57% rename from inventory/src/services/services.cue rename to inventory/src/schema/schema.cue index 6949c81b..7da9dd27 100644 --- a/inventory/src/services/services.cue +++ b/inventory/src/schema/schema.cue @@ -1,22 +1,34 @@ -package services +package schema -#ServiceRole: "server" | "client" | "both" +#groups: groups: { + // Machine groups + machines: { + // Group name mapped to list[machineName] + // "group1": ["machine1", "machine2"] + [string]: [...string] + } +} -#service: services: [string]: { +#machine: machines: [string]: { + name: string, + description?: string, + icon?: string +} + +#role: string + +#service: services: [string]: [string]: { // Required meta fields meta: { name: string, icon?: string description?: string, }, - // Required module specifies the behavior of the service. - module: string, - // We moved the machine sepcific config to "machines". // It may be moved back depending on what makes more sense in the future. - machineConfig: { + roles: [#role]: [...string], + machines: { [string]: { - roles?: [ ...#ServiceRole ], config?: { ... } @@ -29,4 +41,4 @@ package services // It declares the interface how the service can be configured. ... } -} +} \ No newline at end of file diff --git a/inventory/src/tests/borgbackup.json b/inventory/src/tests/borgbackup.json index 11aa844e..55a8c516 100644 --- a/inventory/src/tests/borgbackup.json +++ b/inventory/src/tests/borgbackup.json @@ -10,27 +10,32 @@ "name": "vi" } }, + "groups": { + "machines": { + "laptops": ["camina_machine", "vi_machine"], + "all": ["camina_machine", "vi_machine", "vyr_machine"] + } + }, "meta": { "name": "kenjis clan" }, "services": { - "backup": { - "meta": { - "name": "My backup" - }, - "module": "borgbackup-static", - "machineConfig": { - "vyr_machine": { - "roles": ["server"] + "borgbackup-static": { + "instance_1": { + "meta": { + "name": "My backup" }, - "vi_machine": { - "roles": ["client"] + "roles": { + "server": ["vyr_machine"], + "client": ["group:laptops"] }, - "camina_machine": { - "roles": ["client"] - } - }, - "config": {} + "machines": { + "vyr_machine": {}, + "vi_machine": {}, + "camina_machine": {} + }, + "config": {} + } } } } diff --git a/inventory/src/tests/syncthing.json b/inventory/src/tests/syncthing.json index 6656e5ca..58d0e204 100644 --- a/inventory/src/tests/syncthing.json +++ b/inventory/src/tests/syncthing.json @@ -14,29 +14,29 @@ "name": "kenjis clan" }, "services": { - "sync_files": { - "meta": { - "name": "My sync" - }, - "module": "syncthing-static-peers", - "machineConfig": { - "vyr_machine": {}, - "vi_machine": {}, - "camina_machine": {} - }, - "config": { - "folders": { - "test": { - "path": "~/data/docs", - "devices": ["camina_machine", "vyr_machine", "vi_machine"] - }, - "videos": { - "path": "~/data/videos", - "devices": ["camina_machine", "vyr_machine"] - }, - "playlist": { - "path": "~/data/playlist", - "devices": ["camina_machine", "vi_machine"] + "syncthing-static-peers": { + "instance_1": { + "meta": { + "name": "My sync" + }, + "roles": { + "peer": ["vyr_machine", "vi_machine", "camina_machine"] + }, + "machines": {}, + "config": { + "folders": { + "test": { + "path": "~/data/docs", + "devices": ["camina_machine", "vyr_machine", "vi_machine"] + }, + "videos": { + "path": "~/data/videos", + "devices": ["camina_machine", "vyr_machine"] + }, + "playlist": { + "path": "~/data/playlist", + "devices": ["camina_machine", "vi_machine"] + } } } } diff --git a/inventory/src/tests/zerotier.json b/inventory/src/tests/zerotier.json index 21a2b097..17b226ab 100644 --- a/inventory/src/tests/zerotier.json +++ b/inventory/src/tests/zerotier.json @@ -14,23 +14,22 @@ "name": "kenjis clan" }, "services": { - "backup": { - "meta": { - "name": "My backup" - }, - "module": "borbackup-static", - "machineConfig": { - "vyr_machine": { - "roles": ["server"] + "zerotier-static": { + "instance_1": { + "meta": { + "name": "My Network" }, - "vi_machine": { - "roles": ["peer"] + "roles": { + "server": ["vyr_machine"], + "peer": ["vi_machine", "camina_machine"] }, - "camina_machine": { - "roles": ["peer"] - } - }, - "config": {} + "machines": { + "vyr_machine": { + "config": {} + } + }, + "config": {} + } } } }