update inventory implementation

clanModules/borgbackup-static/default.nix

@@ -2,62 +2,57 @@
 let
   clanDir = config.clan.core.clanDir;
   machineDir = clanDir + "/machines/";
-
-  # cfg.roles = config.clan.borgbackup-static;
-
-  # machine < machine_module  < inventory
-  #  nixos  < borgbackup      < borgbackup-static > UI
-  #                           metadata
-  #              Developer    User field descriptions
-
-  roles = config.clan.borgbackup-static.inventory.roles;
-
-  machine_name = config.clan.core.machineName;
 in
-{
+lib.warn "This module is deprecated use the service via the service interface instead." {
   imports = [ ../borgbackup ];
-  # imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ];
-  # Inventory / Interface.nix
-  # options.clan.inventory.borgbackup-static.description.
-  # options.clan.borgbackup-static.roles = lib.mkOption {
-  #   type = lib.types.attrsOf (lib.types.listOf lib.types.str);
-  # };
 
-  # Can be used via inventory.json
-  #
-  # .borgbackup-static.inventory.roles
-  #
-  options.clan.borgbackup-static.inventory = lib.mkOption {
-    type = lib.types.submodule {
-      # imports = [./inventory/interface.nix];
-
-      # idea
-      # config.metadata = builtins.fromTOML ...
-      # config.defaultRoles = ["client"];
-
-      # -> interface.nix
-      options = {
-        roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); };
-      };
+  options.clan.borgbackup-static = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be backuped.
+        Mutually exclusive with includeMachines.
+        If this is not empty, every other machine except the targets in the clan will be backuped by this module.
+        If includeMachines is set, only the included machines will be backuped.
+      '';
+    };
+    includeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should be backuped.
+        Mutually exclusive with excludeMachines.
+      '';
+    };
+    targets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Machines that should act as target machines for backups.
+      '';
     };
   };
 
   config.services.borgbackup.repos =
     let
-
-      filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles);
-
+      machines = builtins.readDir machineDir;
       borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
-      machinesMaybeKey = builtins.map (
-        machine:
+      filteredMachines =
+        if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then
+          lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines
+        else
+          lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines;
+      machinesMaybeKey = lib.mapAttrsToList (
+        machine: _:
         let
           fullPath = borgbackupIpMachinePath machine;
         in
         if builtins.pathExists fullPath then machine else null
       ) filteredMachines;
-
       machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
-
       hosts = builtins.map (machine: {
         name = machine;
         value = {
@@ -66,20 +61,41 @@ in
         };
       }) machinesWithKey;
     in
-    lib.mkIf (builtins.elem "server" roles.${machine_name}) (
-      if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }
-    );
+    lib.mkIf
+      (builtins.any (
+        target: target == config.clan.core.machineName
+      ) config.clan.borgbackup-static.targets)
+      (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { });
 
   config.clan.borgbackup.destinations =
     let
-      servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles);
-
-      destinations = builtins.map (server_name: {
-        name = server_name;
+      destinations = builtins.map (d: {
+        name = d;
         value = {
-          repo = "borg@${server_name}:/var/lib/borgbackup/${machine_name}";
+          repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}";
         };
-      }) servers;
+      }) config.clan.borgbackup-static.targets;
     in
-    lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations);
+    lib.mkIf (builtins.any (
+      target: target == config.clan.core.machineName
+    ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations);
+
+  config.assertions = [
+    {
+      assertion =
+        !(
+          ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0)
+          && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0)
+        );
+      message = ''
+        The options:
+        config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}]
+        and
+        config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}]
+        are mutually exclusive.
+        Use excludeMachines to exclude certain machines and backup the other clan machines.
+        Use include machines to only backup certain machines.
+      '';
+    }
+  ];
 }

clanModules/borgbackup/README.md

@@ -1,2 +1,15 @@
 Efficient, deduplicating backup program with optional compression and secure encryption.
----
+---
+
+## Roles
+
+- Client
+- Server
+
+## Configuration
+
+Configure target machines where the backups should be sent to through `targets`.
+
+Configure machines that should be backed up either through `includeMachines`
+which will exclusively add the included machines to be backed up, or through
+`excludeMachines`, which will add every machine except the excluded machine to the backup.

clanModules/borgbackup/roles/client.nix

@@ -0,0 +1,30 @@
+{ config, lib, ... }:
+let
+  instances = config.clan.inventory.borgbackup;
+  # roles = { ${role_name} :: { machines :: [string] } }
+  allServers = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if builtins.elem machineName instanceConfig.roles.client.machines then
+        instanceConfig.roles.server.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+
+  inherit (config.clan.core) machineName;
+in
+{
+  config.clan.borgbackup.destinations =
+    let
+
+      destinations = builtins.map (serverName: {
+        name = serverName;
+        value = {
+          repo = "borg@${serverName}:/var/lib/borgbackup/${machineName}";
+        };
+      }) allServers;
+    in
+    (builtins.listToAttrs destinations);
+}

clanModules/borgbackup/roles/server.nix

@@ -0,0 +1,45 @@
+{ config, lib, ... }:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+  inherit (config.clan.core) machineName;
+  instances = config.clan.inventory.borgbackup;
+
+  # roles = { ${role_name} :: { machines :: [string] } }
+  allClients = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if builtins.elem machineName instanceConfig.roles.server.machines then
+        instanceConfig.roles.client.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+in
+{
+  config.services.borgbackup.repos =
+    let
+      filteredMachines = allClients;
+
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      machinesMaybeKey = builtins.map (
+        machine:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) filteredMachines;
+
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    if (builtins.listToAttrs hosts) != [ ] then builtins.listToAttrs hosts else { };
+}

clanModules/flake-module.nix

@@ -5,7 +5,7 @@
       imports = [ ./disk-layouts ];
     };
     borgbackup = ./borgbackup;
-    borgbackup-static = ./borgbackup-static;
+
     deltachat = ./deltachat;
     ergochat = ./ergochat;
     localbackup = ./localbackup;

inventory/README.md

@@ -3,19 +3,24 @@
 The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort.
 
 - The inventory acts as a declarative source of truth for all machine configurations.
-- Users can easily add or remove machines and services.
+- Users can easily add or remove machines to/from services.
 - Ensures that all machines and services are configured consistently, across multiple nixosConfigs.
 - Defaults and predefined roles in our modules minimizes the need for manual configuration.
 
 Design questions:
 
+- [ ] Is the service config interface the same as the module config interface ?
+
+- [ ] As a user i dont want to see borgbackup as the high level category ?
+
 - [x] Must roles be a list ?
-    -> Yes. In zerotier you can be "moon" and "controller" at the same time.
+    -> Yes. In zerotier a machine can be "moon" and "controller" at the same time.
 
 - [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different?
     -> There are many roles. And they depend on the service.
 
 - [x] Should we use the module name in the path of the service?
+    -> YES
     ```json
     // ${module_name}.${instance_name}
     services.borgbackup-static.backup1 = {
@@ -32,8 +37,10 @@ Design questions:
     Neutral: Module name is hard to change. Exists anyways.
 
 - [x] Should the machine specific service config be part of the service?
-    ->  The config implements the schema of the module, which is declared in the service.
-    ->  If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
+    -> NO. because ...
+    - The config implements the schema of the module, which is declared in the service.
+    - If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
+    - If the config is placed in the machine it is de-located into another top-level field. In the module this complicates access.
 
 Architecture
 

inventory/default.nix

@@ -7,22 +7,25 @@ let
 
   machines = machinesFromInventory syncthing_inventory;
 
-  resolveGroups =
-    inventory: members:
-    lib.unique (
-      builtins.foldl' (
-        acc: currMember:
-        let
-          groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember;
-          groupMembers =
-            if inventory.groups.machines ? ${groupName} then
-              inventory.groups.machines.${groupName}
-            else
-              throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory.";
-        in
-        if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ]
-      ) [ ] members
-    );
+  resolveTags =
+    # Inventory, { machines :: [string], tags :: [string] }
+    inventory: members: {
+      machines =
+        members.machines or [ ]
+        ++ (builtins.foldl' (
+          acc: tag:
+          let
+            tagMembers = builtins.attrNames (
+              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
+            );
+          in
+          # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory.";
+          if tagMembers == [ ] then
+            throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory."
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
 
   /*
     Returns a NixOS configuration for every machine in the inventory.
@@ -45,29 +48,53 @@ let
           acc2: instanceName: serviceConfig:
           let
             resolvedRoles = builtins.mapAttrs (
-              _roleName: members: resolveGroups inventory members
+              _roleName: members: resolveTags inventory members
             ) serviceConfig.roles;
 
-            isInService = builtins.any (members: builtins.elem machineName members) (
+            isInService = builtins.any (members: builtins.elem machineName members.machines) (
               builtins.attrValues resolvedRoles
             );
 
+            # Inverse map of roles. Allows for easy lookup of roles for a given machine.
+            # { ${machine_name} :: [roles]
+            inverseRoles = lib.foldlAttrs (
+              acc: roleName:
+              { machines }:
+              acc
+              // builtins.foldl' (
+                acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; }
+              ) { } machines
+            ) { } resolvedRoles;
+
             machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
             globalConfig = serviceConfig.config;
+
+            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
+            roleModules = builtins.map (
+              role:
+              let
+                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+              in
+              if builtins.pathExists path then
+                path
+              else
+                throw "Role doesnt have a module: ${role}. Path: ${path} not found."
+            ) inverseRoles.${machineName} or [ ];
           in
           if isInService then
             acc2
             ++ [
               {
-                imports = [ clan-core.clanModules.${moduleName} ];
+                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
                 config.clan.${moduleName} = lib.mkMerge [
                   globalConfig
                   machineServiceConfig
                 ];
               }
               {
-                config.clan.inventory.${instanceName} = {
+                config.clan.inventory.${moduleName}.${instanceName} = {
                   roles = resolvedRoles;
+                  # inherit inverseRoles;
                 };
               }
             ]
@@ -78,8 +105,64 @@ let
     ) inventory.machines;
 in
 {
+  inherit clan-core;
+
+  new_clan = clan-core.lib.buildInventory {
+    # High level services.
+    # If you need multiple instances of a service configure them via:
+    # inventory.services.[serviceName].[instanceName] = ...
+    services = {
+      borbackup = {
+        roles.server.machines = [ "vyr" ];
+        roles.client.tags = [ "laptop" ];
+        machines.vyr = {
+          config = {
+
+          };
+        };
+        config = {
+
+        };
+      };
+    };
+
+    # Low level inventory i.e. if you need multiple instances of a service
+    # Or if you want to manipulate the created inventory directly.
+    inventory.services.borbackup.default = { };
+
+    # Machines. each machine can be referenced by its attribute name under services.
+    machines = {
+      camina = {
+        # This is added to machine tags
+        clan.tags = [ "laptop" ];
+        # These are the inventory machine fields
+        clan.meta.description = "";
+        clan.meta.name = "";
+        clan.meta.icon = "";
+        # Config ...
+      };
+      vyr = {
+        # Config ...
+      };
+      vi = {
+        clan.networking.targetHost = "root@78.47.164.46";
+        # Config ...
+      };
+      aya = {
+        clan.networking.targetHost = "root@78.47.164.46";
+        # Config ...
+      };
+      ezra = {
+        # Config ...
+      };
+      rianon = {
+        # Config ...
+      };
+    };
+  };
+
   clan = clan-core.lib.buildClan {
-    meta.name = "vis clans";
+    meta.name = "vi's clans";
     # Should usually point to the directory of flake.nix
     directory = self;
 

inventory/src/root.cue

@@ -20,6 +20,4 @@ import (
 
 	// // A map of machines
 	schema.#machine
-
-	schema.#groups
 }

inventory/src/schema/schema.cue

@@ -1,18 +1,10 @@
 package schema
 
-#groups: groups: {
-    // Machine groups
-    machines: {
-        // Group name mapped to list[machineName]
-        // "group1": ["machine1", "machine2"]
-        [string]: [...string]
-    }
-}
-
 #machine: machines: [string]: {
     name: string,
     description?: string,
     icon?: string
+    tags: [...string]
 }
 
 #role: string
@@ -26,7 +18,10 @@ package schema
     },
     // We moved the machine sepcific config to "machines".
     // It may be moved back depending on what makes more sense in the future.
-    roles: [#role]: [...string],
+    roles: [#role]: {
+        machines: [...string],
+        tags: [...string],
+    }
     machines: {
         [string]: {
             config?: {

inventory/src/tests/borgbackup.json

@@ -1,39 +1,51 @@
 {
   "machines": {
     "camina_machine": {
-      "name": "camina"
+      "name": "camina",
+      "tags": ["laptop"]
     },
     "vyr_machine": {
       "name": "vyr"
     },
     "vi_machine": {
-      "name": "vi"
-    }
-  },
-  "groups": {
-    "machines": {
-      "laptops": ["camina_machine", "vi_machine"],
-      "all": ["camina_machine", "vi_machine", "vyr_machine"]
+      "name": "vi",
+      "tags": ["laptop"]
     }
   },
   "meta": {
     "name": "kenjis clan"
   },
   "services": {
-    "borgbackup-static": {
+    "borgbackup": {
       "instance_1": {
         "meta": {
           "name": "My backup"
         },
         "roles": {
-          "server": ["vyr_machine"],
-          "client": ["group:laptops"]
+          "server": {
+            "machines": ["vyr_machine"]
+          },
+          "client": {
+            "machines": ["vyr_machine"],
+            "tags": ["laptop"]
+          }
         },
-        "machines": {
-          "vyr_machine": {},
-          "vi_machine": {},
-          "camina_machine": {}
+        "machines": {},
+        "config": {}
+      },
+      "instance_2": {
+        "meta": {
+          "name": "My backup"
         },
+        "roles": {
+          "server": {
+            "machines": ["vi_machine"]
+          },
+          "client": {
+            "machines": ["camina_machine"]
+          }
+        },
+        "machines": {},
         "config": {}
       }
     }

inventory/src/tests/syncthing.json

@@ -14,13 +14,15 @@
     "name": "kenjis clan"
   },
   "services": {
-    "syncthing-static-peers": {
+    "syncthing": {
       "instance_1": {
         "meta": {
           "name": "My sync"
         },
         "roles": {
-          "peer": ["vyr_machine", "vi_machine", "camina_machine"]
+          "peer": {
+            "machines": ["vyr_machine", "vi_machine", "camina_machine"]
+          }
         },
         "machines": {},
         "config": {

inventory/src/tests/zerotier.json

@@ -14,14 +14,15 @@
     "name": "kenjis clan"
   },
   "services": {
-    "zerotier-static": {
+    "zerotier": {
       "instance_1": {
         "meta": {
           "name": "My Network"
         },
         "roles": {
-          "server": ["vyr_machine"],
-          "peer": ["vi_machine", "camina_machine"]
+          "controller": { "machines": ["vyr_machine"] },
+          "moon": { "machines": ["vyr_machine"] },
+          "peer": { "machines": ["vi_machine", "camina_machine"] }
         },
         "machines": {
           "vyr_machine": {

nixosModules/clanCore/default.nix

@@ -14,5 +14,7 @@
     ./vm.nix
     ./wayland-proxy-virtwl.nix
     ./zerotier
+    # Inventory
+    ./inventory/interface.nix
   ];
 }

nixosModules/clanCore/inventory/interface.nix

@@ -0,0 +1,36 @@
+{ lib, ... }:
+let
+  # {
+  #   roles = {
+  #     client = {
+  #       machines = [
+  #         "camina_machine"
+  #         "vi_machine"
+  #       ];
+  #     };
+  #     server = {
+  #       machines = [ "vyr_machine" ];
+  #     };
+  #   };
+  # }
+  instanceOptions = lib.types.submodule {
+    options.roles = lib.mkOption { type = lib.types.attrsOf machinesList; };
+  };
+
+  # {
+  #   machines = [
+  #     "camina_machine"
+  #     "vi_machine"
+  #     "vyr_machine"
+  #   ];
+  # }
+  machinesList = lib.types.submodule {
+    options.machines = lib.mkOption { type = lib.types.listOf lib.types.str; };
+  };
+in
+{
+  # clan.inventory.${moduleName}.${instanceName} = { ... }
+  options.clan.inventory = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.attrsOf instanceOptions);
+  };
+}