schema improvements

clanModules/borgbackup-static/default.nix

@@ -3,23 +3,49 @@ let
   clanDir = config.clan.core.clanDir;
   machineDir = clanDir + "/machines/";
 
-  cfg = config.clan.borgbackup-static;
+  # cfg.roles = config.clan.borgbackup-static;
+
+  # machine < machine_module  < inventory
+  #  nixos  < borgbackup      < borgbackup-static > UI
+  #                           metadata
+  #              Developer    User field descriptions
+
+  roles = config.clan.borgbackup-static.inventory.roles;
 
   machine_name = config.clan.core.machineName;
 in
 {
   imports = [ ../borgbackup ];
-
+  # imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ];
   # Inventory / Interface.nix
   # options.clan.inventory.borgbackup-static.description.
-  options.clan.borgbackup-static.roles = lib.mkOption {
-    type = lib.types.attrsOf (lib.types.listOf lib.types.str);
+  # options.clan.borgbackup-static.roles = lib.mkOption {
+  #   type = lib.types.attrsOf (lib.types.listOf lib.types.str);
+  # };
+
+  # Can be used via inventory.json
+  #
+  # .borgbackup-static.inventory.roles
+  #
+  options.clan.borgbackup-static.inventory = lib.mkOption {
+    type = lib.types.submodule {
+      # imports = [./inventory/interface.nix];
+
+      # idea
+      # config.metadata = builtins.fromTOML ...
+      # config.defaultRoles = ["client"];
+
+      # -> interface.nix
+      options = {
+        roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); };
+      };
+    };
   };
 
   config.services.borgbackup.repos =
     let
 
-      filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles);
+      filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles);
 
       borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
       machinesMaybeKey = builtins.map (
@@ -40,13 +66,13 @@ in
         };
       }) machinesWithKey;
     in
-    lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) (
+    lib.mkIf (builtins.elem "server" roles.${machine_name}) (
       if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }
     );
 
   config.clan.borgbackup.destinations =
     let
-      servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles);
+      servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles);
 
       destinations = builtins.map (server_name: {
         name = server_name;
@@ -55,5 +81,5 @@ in
         };
       }) servers;
     in
-    lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations);
+    lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations);
 }

inventory/README.md

@@ -1,5 +1,64 @@
 # Inventory
 
+Questions:
+
+- [x] Must roles be a list ?
+    -> Yes. In zerotier you can be "moon" and "controller" at the same time.
+
+- [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different?
+    -> There are many roles. And they depend on the service.
+
+- [x] Should we use the module name in the path of the service?
+    ```json
+    // ${module_name}.${instance_name}
+    services.borgbackup-static.backup1 = {
+
+    }
+    ```
+
+    Pro:
+        Easier to handle.
+        Better groups the module specific instances.
+    Contra:
+        More nesting in json
+
+    Neutral: Module name is hard to change. Exists anyways.
+
+- [x] Should the machine specific service config be part of the service?
+    ->  The config implements the schema of the module, which is declared in the service.
+    ->  If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
+
+Architecture
+
+```
+machine < machine_module        < inventory
+---------------------------------------------
+nixos   < borgbackup            < borgbackup-static > UI
+
+        creates the config      Maps from high level services to the borgbackup clan module
+        for ONE machine
+```
+
+- [ ] Why do we need 2 modules?
+    -> It is technically possible to have only 1 module.
+    Pros:
+        Simple to use/Easy to understand.
+        Less modules
+    Cons:
+        Harder to write a module. Because it must do 2 things.
+        One module should do only 1 thing.
+
+```nix
+clan.machines.${machine_name} = {
+    # "borgbackup.ssh.pub" = machineDir + machines + "/facts/borgbackup.ssh.pub";
+    facts = ...
+};
+clan.services.${instance} = {
+#   roles.server = [ "jon_machine" ]
+#   roles.${role_name} = [ ${machine_name} ];
+};
+```
+
 This part provides a specification for the inventory.
 
 It is used for design phase and as validation helper.

inventory/default.nix

@@ -1,16 +1,28 @@
-{ inputs, self, ... }:
+{ self, lib, ... }:
 let
   clan-core = self;
-  system = "x86_64-linux";
-  pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
 
   # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json);
   syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json);
 
-  machines = machinesFromInventory {
-    inherit clan-core;
-    lib = pkgs.lib;
-  } syncthing_inventory;
+  machines = machinesFromInventory syncthing_inventory;
+
+  resolveGroups =
+    inventory: members:
+    lib.unique (
+      builtins.foldl' (
+        acc: currMember:
+        let
+          groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember;
+          groupMembers =
+            if inventory.groups.machines ? ${groupName} then
+              inventory.groups.machines.${groupName}
+            else
+              throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory.";
+        in
+        if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ]
+      ) [ ] members
+    );
 
   /*
     Returns a NixOS configuration for every machine in the inventory.
@@ -18,48 +30,51 @@ let
     machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
   */
   machinesFromInventory =
-    { lib, clan-core, ... }:
     inventory:
     # For every machine in the inventory, build a NixOS configuration
     # For each machine generate config, forEach service, if the machine is used.
     builtins.mapAttrs (
-      machine_name: _:
-      builtins.foldl' (
-        acc: service_name:
-        let
-          service_config = inventory.services.${service_name};
-          isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig);
+      machineName: _:
+      lib.foldlAttrs (
+        # [ Modules ], String, { ${instance_name} :: ServiceConfig }
+        acc: moduleName: serviceConfigs:
+        acc
+        # Collect service config
+        ++ (lib.foldlAttrs (
+          # [ Modules ], String, ServiceConfig
+          acc2: instanceName: serviceConfig:
+          let
+            resolvedRoles = builtins.mapAttrs (
+              _roleName: members: resolveGroups inventory members
+            ) serviceConfig.roles;
 
-          machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { };
-          global_config = inventory.services.${service_name}.config;
-          module_name = inventory.services.${service_name}.module;
-        in
-        # Possible roles: "server", "client", "peer"
-        if
-          builtins.trace ''
-            isInService ${builtins.toJSON isInService},
-            ${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)}
-          '' isInService
-        then
-          acc
-          ++ [
-            {
-              imports = [ clan-core.clanModules.${module_name} ];
-              config.clan.${module_name} = lib.mkMerge [
-                global_config
-                machine_service_config
-              ];
-            }
-            {
-              config.clan.${module_name} = {
-                # TODO: filter, show only the roles that are needed by the machine
-                roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig;
-              };
-            }
-          ]
-        else
-          acc
-      ) [ ] (builtins.attrNames inventory.services)
+            isInService = builtins.any (members: builtins.elem machineName members) (
+              builtins.attrValues resolvedRoles
+            );
+
+            machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
+            globalConfig = serviceConfig.config;
+          in
+          if isInService then
+            acc2
+            ++ [
+              {
+                imports = [ clan-core.clanModules.${moduleName} ];
+                config.clan.${moduleName} = lib.mkMerge [
+                  globalConfig
+                  machineServiceConfig
+                ];
+              }
+              {
+                config.clan.inventory.${instanceName} = {
+                  roles = resolvedRoles;
+                };
+              }
+            ]
+          else
+            acc2
+        ) [ ] serviceConfigs)
+      ) [ ] inventory.services
     ) inventory.machines;
 in
 {
@@ -81,9 +96,4 @@ in
     };
   };
   intern = machines;
-  # inherit (clan) nixosConfigurations clanInternals;
-  # add the Clan cli tool to the dev shell
-  devShells.${system}.default = pkgs.mkShell {
-    packages = [ clan-core.packages.${system}.clan-cli ];
-  };
 }

inventory/flake-module.nix

@@ -1,6 +1,11 @@
-{ inputs, self, ... }:
 {
-  flake.inventory = import ./default.nix { inherit inputs self; };
+  inputs,
+  self,
+  lib,
+  ...
+}:
+{
+  flake.inventory = import ./default.nix { inherit inputs self lib; };
   perSystem =
     { pkgs, config, ... }:
     {

inventory/src/machines/machines.cue

@@ -1,8 +0,0 @@
-package machines
-
-
-#machine: machines: [string]: {
-    name: string,
-    description?: string,
-    icon?: string
-}

inventory/src/root.cue

@@ -1,8 +1,7 @@
 package inventory
 
 import (
-	"clan.lol/inventory/services"
-	"clan.lol/inventory/machines"
+	"clan.lol/inventory/schema"
 )
 
 @jsonschema(schema="http://json-schema.org/schema#")
@@ -16,9 +15,11 @@ import (
 		icon?: string
 	}
 
-	// A map of services
-	services.#service
+	// // A map of services
+	schema.#service
 
-	// A map of machines
-	machines.#machine
+	// // A map of machines
+	schema.#machine
+
+	schema.#groups
 }

inventory/src/schema/schema.cue

@@ -1,22 +1,34 @@
-package services
+package schema
 
-#ServiceRole: "server" | "client" | "both"
+#groups: groups: {
+    // Machine groups
+    machines: {
+        // Group name mapped to list[machineName]
+        // "group1": ["machine1", "machine2"]
+        [string]: [...string]
+    }
+}
 
-#service: services: [string]: {
+#machine: machines: [string]: {
+    name: string,
+    description?: string,
+    icon?: string
+}
+
+#role: string
+
+#service: services: [string]: [string]: {
     // Required meta fields
     meta: {
         name: string,
         icon?: string
         description?: string,
     },
-    // Required module specifies the behavior of the service.
-    module: string,
-
     // We moved the machine sepcific config to "machines".
     // It may be moved back depending on what makes more sense in the future.
-    machineConfig: {
+    roles: [#role]: [...string],
+    machines: {
         [string]: {
-            roles?: [ ...#ServiceRole ],
             config?: {
                 ...
             }
@@ -29,4 +41,4 @@ package services
         // It declares the interface how the service can be configured.
         ...
     }
-}
+}

inventory/src/tests/borgbackup.json

@@ -10,27 +10,32 @@
       "name": "vi"
     }
   },
+  "groups": {
+    "machines": {
+      "laptops": ["camina_machine", "vi_machine"],
+      "all": ["camina_machine", "vi_machine", "vyr_machine"]
+    }
+  },
   "meta": {
     "name": "kenjis clan"
   },
   "services": {
-    "backup": {
-      "meta": {
-        "name": "My backup"
-      },
-      "module": "borgbackup-static",
-      "machineConfig": {
-        "vyr_machine": {
-          "roles": ["server"]
+    "borgbackup-static": {
+      "instance_1": {
+        "meta": {
+          "name": "My backup"
         },
-        "vi_machine": {
-          "roles": ["client"]
+        "roles": {
+          "server": ["vyr_machine"],
+          "client": ["group:laptops"]
         },
-        "camina_machine": {
-          "roles": ["client"]
-        }
-      },
-      "config": {}
+        "machines": {
+          "vyr_machine": {},
+          "vi_machine": {},
+          "camina_machine": {}
+        },
+        "config": {}
+      }
     }
   }
 }

inventory/src/tests/syncthing.json

@@ -14,29 +14,29 @@
     "name": "kenjis clan"
   },
   "services": {
-    "sync_files": {
-      "meta": {
-        "name": "My sync"
-      },
-      "module": "syncthing-static-peers",
-      "machineConfig": {
-        "vyr_machine": {},
-        "vi_machine": {},
-        "camina_machine": {}
-      },
-      "config": {
-        "folders": {
-          "test": {
-            "path": "~/data/docs",
-            "devices": ["camina_machine", "vyr_machine", "vi_machine"]
-          },
-          "videos": {
-            "path": "~/data/videos",
-            "devices": ["camina_machine", "vyr_machine"]
-          },
-          "playlist": {
-            "path": "~/data/playlist",
-            "devices": ["camina_machine", "vi_machine"]
+    "syncthing-static-peers": {
+      "instance_1": {
+        "meta": {
+          "name": "My sync"
+        },
+        "roles": {
+          "peer": ["vyr_machine", "vi_machine", "camina_machine"]
+        },
+        "machines": {},
+        "config": {
+          "folders": {
+            "test": {
+              "path": "~/data/docs",
+              "devices": ["camina_machine", "vyr_machine", "vi_machine"]
+            },
+            "videos": {
+              "path": "~/data/videos",
+              "devices": ["camina_machine", "vyr_machine"]
+            },
+            "playlist": {
+              "path": "~/data/playlist",
+              "devices": ["camina_machine", "vi_machine"]
+            }
           }
         }
       }

inventory/src/tests/zerotier.json

@@ -14,23 +14,22 @@
     "name": "kenjis clan"
   },
   "services": {
-    "backup": {
-      "meta": {
-        "name": "My backup"
-      },
-      "module": "borbackup-static",
-      "machineConfig": {
-        "vyr_machine": {
-          "roles": ["server"]
+    "zerotier-static": {
+      "instance_1": {
+        "meta": {
+          "name": "My Network"
         },
-        "vi_machine": {
-          "roles": ["peer"]
+        "roles": {
+          "server": ["vyr_machine"],
+          "peer": ["vi_machine", "camina_machine"]
         },
-        "camina_machine": {
-          "roles": ["peer"]
-        }
-      },
-      "config": {}
+        "machines": {
+          "vyr_machine": {
+            "config": {}
+          }
+        },
+        "config": {}
+      }
     }
   }
 }