Extend build-clan interface

clanModules/borgbackup/roles/server.nix

@@ -7,11 +7,12 @@ let
   instances = config.clan.services.borgbackup;
 
   # roles = { ${role_name} :: { machines :: [string] } }
+
   allClients = lib.foldlAttrs (
     acc: _instanceName: instanceConfig:
     acc
     ++ (
-      if builtins.elem machineName instanceConfig.roles.server.machines then
+      if (builtins.elem machineName instanceConfig.roles.server.machines) then
         instanceConfig.roles.client.machines
       else
         [ ]
@@ -21,7 +22,6 @@ in
 {
   config.services.borgbackup.repos =
     let
-
       borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
       machinesMaybeKey = builtins.map (
         machine:

inventory/default.nix

@@ -1,183 +1,87 @@
 { self, lib, ... }:
 let
   clan-core = self;
-
-  # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json);
-  syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json);
-
-  machines = machinesFromInventory syncthing_inventory;
-
-  resolveTags =
-    # Inventory, { machines :: [string], tags :: [string] }
-    inventory: members: {
-      machines =
-        members.machines or [ ]
-        ++ (builtins.foldl' (
-          acc: tag:
-          let
-            tagMembers = builtins.attrNames (
-              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
-            );
-          in
-          # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory.";
-          if tagMembers == [ ] then
-            throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory."
-          else
-            acc ++ tagMembers
-        ) [ ] members.tags or [ ]);
-    };
-
-  /*
-    Returns a NixOS configuration for every machine in the inventory.
-
-    machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
-  */
-  machinesFromInventory =
-    inventory:
-    # For every machine in the inventory, build a NixOS configuration
-    # For each machine generate config, forEach service, if the machine is used.
-    builtins.mapAttrs (
-      machineName: _:
-      lib.foldlAttrs (
-        # [ Modules ], String, { ${instance_name} :: ServiceConfig }
-        acc: moduleName: serviceConfigs:
-        acc
-        # Collect service config
-        ++ (lib.foldlAttrs (
-          # [ Modules ], String, ServiceConfig
-          acc2: instanceName: serviceConfig:
-          let
-            resolvedRoles = builtins.mapAttrs (
-              _roleName: members: resolveTags inventory members
-            ) serviceConfig.roles;
-
-            isInService = builtins.any (members: builtins.elem machineName members.machines) (
-              builtins.attrValues resolvedRoles
-            );
-
-            # Inverse map of roles. Allows for easy lookup of roles for a given machine.
-            # { ${machine_name} :: [roles]
-            inverseRoles = lib.foldlAttrs (
-              acc: roleName:
-              { machines }:
-              acc
-              // builtins.foldl' (
-                acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; }
-              ) { } machines
-            ) { } resolvedRoles;
-
-            machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
-            globalConfig = serviceConfig.config;
-
-            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
-            roleModules = builtins.map (
-              role:
-              let
-                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
-              in
-              if builtins.pathExists path then
-                path
-              else
-                throw "Role doesnt have a module: ${role}. Path: ${path} not found."
-            ) inverseRoles.${machineName} or [ ];
-          in
-          if isInService then
-            acc2
-            ++ [
-              {
-                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
-                config.clan.${moduleName} = lib.mkMerge [
-                  globalConfig
-                  machineServiceConfig
-                ];
-              }
-              {
-                config.clan.services.${moduleName}.${instanceName} = {
-                  roles = resolvedRoles;
-                  # inherit inverseRoles;
-                };
-              }
-            ]
-          else
-            acc2
-        ) [ ] serviceConfigs)
-      ) [ ] inventory.services
-    ) inventory.machines;
 in
 {
-  inherit clan-core;
-
   # Extension of the build clan interface
-  new_clan = clan-core.lib.buildClan {
+  # new_clan = clan-core.lib.buildClan {
-    # High level services.
+  #   # High level services.
-    # If you need multiple instances of a service configure them via:
+  #   # If you need multiple instances of a service configure them via:
-    # inventory.services.[serviceName].[instanceName] = ...
+  #   # inventory.services.[serviceName].[instanceName] = ...
-    services = {
+  #   services = {
-      borbackup = {
+  #     borbackup = {
-        roles.server.machines = [ "vyr" ];
+  #       roles.server.machines = [ "vyr" ];
-        roles.client.tags = [ "laptop" ];
+  #       roles.client.tags = [ "laptop" ];
-        machines.vyr = {
+  #       machines.vyr = {
-          config = {
+  #         config = {
 
-          };
+  #         };
-        };
+  #       };
-        config = {
+  #       config = {
 
-        };
+  #       };
-      };
+  #     };
-    };
+  #   };
 
-    # Low level inventory i.e. if you need multiple instances of a service
+  #   # Low level inventory i.e. if you need multiple instances of a service
-    # Or if you want to manipulate the created inventory directly.
+  #   # Or if you want to manipulate the created inventory directly.
-    inventory.services.borbackup.default = { };
+  #   inventory.services.borbackup.default = { };
 
-    # Machines. each machine can be referenced by its attribute name under services.
+  #   # Machines. each machine can be referenced by its attribute name under services.
-    machines = {
+  #   machines = {
-      camina = {
+  #     camina = {
-        # This is added to machine tags
+  #       # This is added to machine tags
-        clan.tags = [ "laptop" ];
+  #       clan.tags = [ "laptop" ];
-        # These are the inventory machine fields
+  #       # These are the inventory machine fields
-        clan.meta.description = "";
+  #       clan.meta.description = "";
-        clan.meta.name = "";
+  #       clan.meta.name = "";
-        clan.meta.icon = "";
+  #       clan.meta.icon = "";
-        # Config ...
+  #       # Config ...
-      };
+  #     };
-      vyr = {
+  #     vyr = {
-        # Config ...
+  #       # Config ...
-      };
+  #     };
-      vi = {
+  #     vi = {
-        clan.networking.targetHost = "root@78.47.164.46";
+  #       clan.networking.targetHost = "root@78.47.164.46";
-        # Config ...
+  #       # Config ...
-      };
+  #     };
-      aya = {
+  #     aya = {
-        clan.networking.targetHost = "root@78.47.164.46";
+  #       clan.networking.targetHost = "root@78.47.164.46";
-        # Config ...
+  #       # Config ...
-      };
+  #     };
-      ezra = {
+  #     ezra = {
-        # Config ...
+  #       # Config ...
-      };
+  #     };
-      rianon = {
+  #     rianon = {
-        # Config ...
+  #       # Config ...
-      };
+  #     };
-    };
+  #   };
-  };
+  # };
 
   clan = clan-core.lib.buildClan {
-    meta.name = "vi's clans";
+    meta.name = "sams's clans";
     # Should usually point to the directory of flake.nix
     directory = self;
 
+    # services = {
+    #   borgbackup = {
+    #     roles.server.machines = [ "vyr_machine" ];
+    #     roles.client.tags = [ "laptop" ];
+    #   };
+    # };
+
+    # OR
+    inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json);
+
     machines = {
+      "vyr_machine" = { };
       "vi_machine" = {
-        imports = machines.vi_machine;
+        clan.tags = [ "laptop" ];
-      };
-      "vyr_machine" = {
-        imports = machines.vyr_machine;
       };
       "camina_machine" = {
-        imports = machines.camina_machine;
+        clan.tags = [ "laptop" ];
+        clan.meta.name = "camina";
       };
     };
   };
-  intern = machines;
 }

lib/build-clan/default.nix

@@ -12,10 +12,79 @@
   # DEPRECATED: use meta.icon instead
   clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
   meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string
-  pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
+  # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
-# This improves performance, but all nipxkgs.* options will be ignored.
+  # This improves performance, but all nipxkgs.* options will be ignored.
+  pkgsForSystem ? (_system: null),
+  /*
+    Distributed services configuration.
+
+    This configures a default instance in the inventory with the name "default".
+
+    If you need multiple instances of a service configure them via:
+    inventory.services.[serviceName].[instanceName] = ...
+  */
+  services ? { },
+  /*
+    Low level inventory configuration.
+    Overrides the services configuration.
+  */
+  inventory ? { },
 }:
 let
+  _inventory =
+    (
+      if services != { } && inventory == { } then
+        { services = lib.mapAttrs (_name: value: { default = value; }) services; }
+      else if services == { } && inventory != { } then
+        inventory
+      else if services != { } && inventory != { } then
+        throw "Either services or inventory should be set, but not both."
+      else
+        { }
+    )
+    // {
+      machines = lib.mapAttrs (
+        name: config:
+        (lib.attrByPath [
+          "clan"
+          "meta"
+        ] { } config)
+        // {
+          name = (
+            lib.attrByPath [
+              "clan"
+              "meta"
+              "name"
+            ] name config
+          );
+          tags = lib.attrByPath [
+            "clan"
+            "tags"
+          ] [ ] config;
+        }
+      ) machines;
+    };
+
+  buildInventory = import ./inventory.nix { inherit lib clan-core; };
+
+  pkgs = import nixpkgs { };
+
+  inventoryFile = builtins.toFile "inventory.json" (builtins.toJSON _inventory);
+
+  # a Derivation that can be forced to validate the inventory
+  # It is not used directly here.
+  validatedFile = pkgs.stdenv.mkDerivation {
+    name = "validated-inventory";
+    src = ../../inventory/src;
+    buildInputs = [ pkgs.cue ];
+    installPhase = ''
+      cue vet ${inventoryFile} root.cue -d "#Root"
+      cp ${inventoryFile} $out
+    '';
+  };
+
+  serviceConfigs = buildInventory _inventory;
+
   deprecationWarnings = [
     (lib.warnIf (
       clanName != null
@@ -98,6 +167,7 @@ let
           clan-core.nixosModules.clanCore
           extraConfig
           (machines.${name} or { })
+          { imports = serviceConfigs.${name} or { }; }
           (
             {
               # Settings
@@ -180,6 +250,7 @@ builtins.deepSeq deprecationWarnings {
     # Evaluated clan meta
     # Merged /clan/meta.json with overrides from buildClan
     meta = mergedMeta;
+    inherit _inventory validatedFile;
 
     # machine specifics
     machines = configsPerSystem;

lib/build-clan/inventory.nix

@@ -0,0 +1,106 @@
+# Generate partial NixOS configurations for every machine in the inventory
+# This function is responsible for generating the module configuration for every machine in the inventory.
+{ lib, clan-core }:
+inventory:
+let
+  machines = machinesFromInventory inventory;
+
+  resolveTags =
+    # Inventory, { machines :: [string], tags :: [string] }
+    inventory: members: {
+      machines =
+        members.machines or [ ]
+        ++ (builtins.foldl' (
+          acc: tag:
+          let
+            tagMembers = builtins.attrNames (
+              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
+            );
+          in
+          # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory.";
+          if tagMembers == [ ] then
+            throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory."
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
+
+  /*
+    Returns a NixOS configuration for every machine in the inventory.
+
+    machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
+  */
+  machinesFromInventory =
+    inventory:
+    # For every machine in the inventory, build a NixOS configuration
+    # For each machine generate config, forEach service, if the machine is used.
+    builtins.mapAttrs (
+      machineName: _:
+      lib.foldlAttrs (
+        # [ Modules ], String, { ${instance_name} :: ServiceConfig }
+        acc: moduleName: serviceConfigs:
+        acc
+        # Collect service config
+        ++ (lib.foldlAttrs (
+          # [ Modules ], String, ServiceConfig
+          acc2: instanceName: serviceConfig:
+          let
+            resolvedRoles = builtins.mapAttrs (
+              _roleName: members: resolveTags inventory members
+            ) serviceConfig.roles;
+
+            isInService = builtins.any (members: builtins.elem machineName members.machines) (
+              builtins.attrValues resolvedRoles
+            );
+
+            # Inverse map of roles. Allows for easy lookup of roles for a given machine.
+            # { ${machine_name} :: [roles]
+            inverseRoles = lib.foldlAttrs (
+              acc: roleName:
+              { machines }:
+              acc
+              // builtins.foldl' (
+                acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; }
+              ) { } machines
+            ) { } resolvedRoles;
+
+            machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
+            globalConfig = serviceConfig.config or { };
+
+            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
+            roleModules = builtins.map (
+              role:
+              let
+                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+              in
+              if builtins.pathExists path then
+                path
+              else
+                throw "Role doesnt have a module: ${role}. Path: ${path} not found."
+            ) inverseRoles.${machineName} or [ ];
+          in
+          if isInService then
+            acc2
+            ++ [
+              {
+                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
+                config.clan.${moduleName} = lib.mkMerge [
+                  globalConfig
+                  machineServiceConfig
+                ];
+              }
+              {
+                config.clan.services.${moduleName}.${instanceName} = {
+                  roles = resolvedRoles;
+                  # TODO: Add inverseRoles to the service config if needed
+                  # inherit inverseRoles;
+                };
+              }
+            ]
+          else
+            acc2
+        ) [ ] serviceConfigs)
+      ) [ ] inventory.services
+    ) inventory.machines;
+in
+machines

nixosModules/clanCore/default.nix

@@ -16,5 +16,6 @@
     ./zerotier
     # Inventory
     ./inventory/interface.nix
+    ./meta/interface.nix
   ];
 }

nixosModules/clanCore/inventory/interface.nix

@@ -30,6 +30,7 @@ let
 in
 {
   # clan.inventory.${moduleName}.${instanceName} = { ... }
+  # TODO: resolve clash with clan.services.waypipe
   options.clan.services = lib.mkOption {
     type = lib.types.attrsOf (lib.types.attrsOf instanceOptions);
   };

nixosModules/clanCore/meta/interface.nix

@@ -0,0 +1,10 @@
+{ lib, ... }:
+let
+  optStr = lib.types.nullOr lib.types.str;
+in
+{
+  options.clan.meta.name = lib.mkOption { type = lib.types.str; };
+  options.clan.meta.description = lib.mkOption { type = optStr; };
+  options.clan.meta.icon = lib.mkOption { type = optStr; };
+  options.clan.tags = lib.mkOption { type = lib.types.listOf lib.types.str; };
+}