update inventory implementation

clanModules/borgbackup/README.md

@@ -1,2 +1,15 @@
 Efficient, deduplicating backup program with optional compression and secure encryption.
----
+---
+
+## Roles
+
+- Client
+- Server
+
+## Configuration
+
+Configure target machines where the backups should be sent to through `targets`.
+
+Configure machines that should be backed up either through `includeMachines`
+which will exclusively add the included machines to be backed up, or through
+`excludeMachines`, which will add every machine except the excluded machine to the backup.

clanModules/borgbackup/default.nix

@@ -28,7 +28,7 @@ let
     fi
   '';
 in
-{
+lib.warn "This module is deprecated use the service via the service interface instead." {
   options.clan.borgbackup.destinations = lib.mkOption {
     type = lib.types.attrsOf (
       lib.types.submodule (

clanModules/borgbackup/roles/client.nix

@@ -0,0 +1,30 @@
+{ config, lib, ... }:
+let
+  instances = config.clan.inventory.borgbackup;
+  # roles = { ${role_name} :: { machines :: [string] } }
+  allServers = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if builtins.elem machineName instanceConfig.roles.client.machines then
+        instanceConfig.roles.server.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+
+  inherit (config.clan.core) machineName;
+in
+{
+  config.clan.borgbackup.destinations =
+    let
+
+      destinations = builtins.map (serverName: {
+        name = serverName;
+        value = {
+          repo = "borg@${serverName}:/var/lib/borgbackup/${machineName}";
+        };
+      }) allServers;
+    in
+    (builtins.listToAttrs destinations);
+}

clanModules/borgbackup/roles/server.nix

@@ -0,0 +1,45 @@
+{ config, lib, ... }:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+  inherit (config.clan.core) machineName;
+  instances = config.clan.inventory.borgbackup;
+
+  # roles = { ${role_name} :: { machines :: [string] } }
+  allClients = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if builtins.elem machineName instanceConfig.roles.server.machines then
+        instanceConfig.roles.client.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+in
+{
+  config.services.borgbackup.repos =
+    let
+      filteredMachines = allClients;
+
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      machinesMaybeKey = builtins.map (
+        machine:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) filteredMachines;
+
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    if (builtins.listToAttrs hosts) != [ ] then builtins.listToAttrs hosts else { };
+}

clanModules/flake-module.nix

@@ -5,7 +5,7 @@
       imports = [ ./disk-layouts ];
     };
     borgbackup = ./borgbackup;
-    borgbackup-static = ./borgbackup-static;
+
     deltachat = ./deltachat;
     ergochat = ./ergochat;
     localbackup = ./localbackup;

inventory/README.md

@@ -3,19 +3,24 @@
 The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort.
 
 - The inventory acts as a declarative source of truth for all machine configurations.
-- Users can easily add or remove machines and services.
+- Users can easily add or remove machines to/from services.
 - Ensures that all machines and services are configured consistently, across multiple nixosConfigs.
 - Defaults and predefined roles in our modules minimizes the need for manual configuration.
 
 Design questions:
 
+- [ ] Is the service config interface the same as the module config interface ?
+
+- [ ] As a user i dont want to see borgbackup as the high level category ?
+
 - [x] Must roles be a list ?
-    -> Yes. In zerotier you can be "moon" and "controller" at the same time.
+    -> Yes. In zerotier a machine can be "moon" and "controller" at the same time.
 
 - [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different?
     -> There are many roles. And they depend on the service.
 
 - [x] Should we use the module name in the path of the service?
+    -> YES
     ```json
     // ${module_name}.${instance_name}
     services.borgbackup-static.backup1 = {
@@ -32,8 +37,10 @@ Design questions:
     Neutral: Module name is hard to change. Exists anyways.
 
 - [x] Should the machine specific service config be part of the service?
-    ->  The config implements the schema of the module, which is declared in the service.
+    -> NO. because ...
-    ->  If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
+    - The config implements the schema of the module, which is declared in the service.
+    - If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
+    - If the config is placed in the machine it is de-located into another top-level field. In the module this complicates access.
 
 Architecture
 

inventory/default.nix

@@ -7,22 +7,25 @@ let
 
   machines = machinesFromInventory syncthing_inventory;
 
-  resolveGroups =
+  resolveTags =
-    inventory: members:
+    # Inventory, { machines :: [string], tags :: [string] }
-    lib.unique (
+    inventory: members: {
-      builtins.foldl' (
+      machines =
-        acc: currMember:
+        members.machines or [ ]
-        let
+        ++ (builtins.foldl' (
-          groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember;
+          acc: tag:
-          groupMembers =
+          let
-            if inventory.groups.machines ? ${groupName} then
+            tagMembers = builtins.attrNames (
-              inventory.groups.machines.${groupName}
+              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
-            else
+            );
-              throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory.";
+          in
-        in
+          # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory.";
-        if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ]
+          if tagMembers == [ ] then
-      ) [ ] members
+            throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory."
-    );
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
 
   /*
     Returns a NixOS configuration for every machine in the inventory.
@@ -45,29 +48,53 @@ let
           acc2: instanceName: serviceConfig:
           let
             resolvedRoles = builtins.mapAttrs (
-              _roleName: members: resolveGroups inventory members
+              _roleName: members: resolveTags inventory members
             ) serviceConfig.roles;
 
-            isInService = builtins.any (members: builtins.elem machineName members) (
+            isInService = builtins.any (members: builtins.elem machineName members.machines) (
               builtins.attrValues resolvedRoles
             );
 
+            # Inverse map of roles. Allows for easy lookup of roles for a given machine.
+            # { ${machine_name} :: [roles]
+            inverseRoles = lib.foldlAttrs (
+              acc: roleName:
+              { machines }:
+              acc
+              // builtins.foldl' (
+                acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; }
+              ) { } machines
+            ) { } resolvedRoles;
+
             machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
             globalConfig = serviceConfig.config;
+
+            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
+            roleModules = builtins.map (
+              role:
+              let
+                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+              in
+              if builtins.pathExists path then
+                path
+              else
+                throw "Role doesnt have a module: ${role}. Path: ${path} not found."
+            ) inverseRoles.${machineName} or [ ];
           in
           if isInService then
             acc2
             ++ [
               {
-                imports = [ clan-core.clanModules.${moduleName} ];
+                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
                 config.clan.${moduleName} = lib.mkMerge [
                   globalConfig
                   machineServiceConfig
                 ];
               }
               {
-                config.clan.inventory.${instanceName} = {
+                config.clan.inventory.${moduleName}.${instanceName} = {
                   roles = resolvedRoles;
+                  # inherit inverseRoles;
                 };
               }
             ]
@@ -78,8 +105,64 @@ let
     ) inventory.machines;
 in
 {
+  inherit clan-core;
+
+  new_clan = clan-core.lib.buildInventory {
+    # High level services.
+    # If you need multiple instances of a service configure them via:
+    # inventory.services.[serviceName].[instanceName] = ...
+    services = {
+      borbackup = {
+        roles.server.machines = [ "vyr" ];
+        roles.client.tags = [ "laptop" ];
+        machines.vyr = {
+          config = {
+
+          };
+        };
+        config = {
+
+        };
+      };
+    };
+
+    # Low level inventory i.e. if you need multiple instances of a service
+    # Or if you want to manipulate the created inventory directly.
+    inventory.services.borbackup.default = { };
+
+    # Machines. each machine can be referenced by its attribute name under services.
+    machines = {
+      camina = {
+        # This is added to machine tags
+        clan.tags = [ "laptop" ];
+        # These are the inventory machine fields
+        clan.meta.description = "";
+        clan.meta.name = "";
+        clan.meta.icon = "";
+        # Config ...
+      };
+      vyr = {
+        # Config ...
+      };
+      vi = {
+        clan.networking.targetHost = "root@78.47.164.46";
+        # Config ...
+      };
+      aya = {
+        clan.networking.targetHost = "root@78.47.164.46";
+        # Config ...
+      };
+      ezra = {
+        # Config ...
+      };
+      rianon = {
+        # Config ...
+      };
+    };
+  };
+
   clan = clan-core.lib.buildClan {
-    meta.name = "vis clans";
+    meta.name = "vi's clans";
     # Should usually point to the directory of flake.nix
     directory = self;
 

inventory/src/root.cue

@@ -20,6 +20,4 @@ import (
 
 	// // A map of machines
 	schema.#machine
-
-	schema.#groups
 }

inventory/src/schema/schema.cue

@@ -1,18 +1,10 @@
 package schema
 
-#groups: groups: {
-    // Machine groups
-    machines: {
-        // Group name mapped to list[machineName]
-        // "group1": ["machine1", "machine2"]
-        [string]: [...string]
-    }
-}
-
 #machine: machines: [string]: {
     name: string,
     description?: string,
     icon?: string
+    tags: [...string]
 }
 
 #role: string
@@ -26,7 +18,10 @@ package schema
     },
     // We moved the machine sepcific config to "machines".
     // It may be moved back depending on what makes more sense in the future.
-    roles: [#role]: [...string],
+    roles: [#role]: {
+        machines: [...string],
+        tags: [...string],
+    }
     machines: {
         [string]: {
             config?: {

inventory/src/tests/borgbackup.json

@@ -1,39 +1,51 @@
 {
   "machines": {
     "camina_machine": {
-      "name": "camina"
+      "name": "camina",
+      "tags": ["laptop"]
     },
     "vyr_machine": {
       "name": "vyr"
     },
     "vi_machine": {
-      "name": "vi"
+      "name": "vi",
-    }
+      "tags": ["laptop"]
-  },
-  "groups": {
-    "machines": {
-      "laptops": ["camina_machine", "vi_machine"],
-      "all": ["camina_machine", "vi_machine", "vyr_machine"]
     }
   },
   "meta": {
     "name": "kenjis clan"
   },
   "services": {
-    "borgbackup-static": {
+    "borgbackup": {
       "instance_1": {
         "meta": {
           "name": "My backup"
         },
         "roles": {
-          "server": ["vyr_machine"],
+          "server": {
-          "client": ["group:laptops"]
+            "machines": ["vyr_machine"]
+          },
+          "client": {
+            "machines": ["vyr_machine"],
+            "tags": ["laptop"]
+          }
         },
-        "machines": {
+        "machines": {},
-          "vyr_machine": {},
+        "config": {}
-          "vi_machine": {},
+      },
-          "camina_machine": {}
+      "instance_2": {
+        "meta": {
+          "name": "My backup"
         },
+        "roles": {
+          "server": {
+            "machines": ["vi_machine"]
+          },
+          "client": {
+            "machines": ["camina_machine"]
+          }
+        },
+        "machines": {},
         "config": {}
       }
     }

inventory/src/tests/syncthing.json

@@ -14,13 +14,15 @@
     "name": "kenjis clan"
   },
   "services": {
-    "syncthing-static-peers": {
+    "syncthing": {
       "instance_1": {
         "meta": {
           "name": "My sync"
         },
         "roles": {
-          "peer": ["vyr_machine", "vi_machine", "camina_machine"]
+          "peer": {
+            "machines": ["vyr_machine", "vi_machine", "camina_machine"]
+          }
         },
         "machines": {},
         "config": {

inventory/src/tests/zerotier.json

@@ -14,14 +14,15 @@
     "name": "kenjis clan"
   },
   "services": {
-    "zerotier-static": {
+    "zerotier": {
       "instance_1": {
         "meta": {
           "name": "My Network"
         },
         "roles": {
-          "server": ["vyr_machine"],
+          "controller": { "machines": ["vyr_machine"] },
-          "peer": ["vi_machine", "camina_machine"]
+          "moon": { "machines": ["vyr_machine"] },
+          "peer": { "machines": ["vi_machine", "camina_machine"] }
         },
         "machines": {
           "vyr_machine": {

nixosModules/clanCore/default.nix

@@ -14,5 +14,7 @@
     ./vm.nix
     ./wayland-proxy-virtwl.nix
     ./zerotier
+    # Inventory
+    ./inventory/interface.nix
   ];
 }

nixosModules/clanCore/inventory/interface.nix

@@ -0,0 +1,36 @@
+{ lib, ... }:
+let
+  # {
+  #   roles = {
+  #     client = {
+  #       machines = [
+  #         "camina_machine"
+  #         "vi_machine"
+  #       ];
+  #     };
+  #     server = {
+  #       machines = [ "vyr_machine" ];
+  #     };
+  #   };
+  # }
+  instanceOptions = lib.types.submodule {
+    options.roles = lib.mkOption { type = lib.types.attrsOf machinesList; };
+  };
+
+  # {
+  #   machines = [
+  #     "camina_machine"
+  #     "vi_machine"
+  #     "vyr_machine"
+  #   ];
+  # }
+  machinesList = lib.types.submodule {
+    options.machines = lib.mkOption { type = lib.types.listOf lib.types.str; };
+  };
+in
+{
+  # clan.inventory.${moduleName}.${instanceName} = { ... }
+  options.clan.inventory = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.attrsOf instanceOptions);
+  };
+}