Merge pull request 'Inventory: add global imports' (#1749) from inventory-config into main

flakeModules/clan.nix

@@ -60,7 +60,8 @@ in
       description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
     };
     inventory = mkOption {
-      type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; };
+      #type = types.submodule { imports = [ ../lib/inventory/build-inventory/interface.nix ]; };
+      type = types.attrsOf types.raw;
       default = { };
       description = ''
         An abstract service layer for consistently configuring distributed services across machine boundaries.
@@ -117,10 +118,10 @@ in
         directory
         specialArgs
         machines
-        inventory
         pkgsForSystem
         meta
         ;
+      inventory = (lib.traceValSeq cfg.inventory);
     };
   };
   _file = __curPos.file;

lib/build-clan/default.nix

@@ -106,7 +106,10 @@ let
 
   # map from machine name to service configuration
   # { ${machineName} :: Config }
-  serviceConfigs = buildInventory mergedInventory;
+  serviceConfigs = buildInventory {
+    inventory = mergedInventory;
+    inherit directory;
+  };
 
   machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
     builtins.readDir (directory + /machines)

lib/inventory/build-inventory/default.nix

@@ -1,13 +1,20 @@
 # Generate partial NixOS configurations for every machine in the inventory
 # This function is responsible for generating the module configuration for every machine in the inventory.
 { lib, clan-core }:
-inventory:
+{ inventory, directory }:
 let
   machines = machinesFromInventory inventory;
 
   resolveTags =
     # Inventory, { machines :: [string], tags :: [string] }
-    inventory: members: {
+    {
+      serviceName,
+      instanceName,
+      roleName,
+      inventory,
+      members,
+    }:
+    {
       machines =
         members.machines or [ ]
         ++ (builtins.foldl' (
@@ -17,14 +24,17 @@ let
             availableTags = lib.foldlAttrs (
               acc: _: v:
               v.tags or [ ] ++ acc
-            ) [ ] inventory.machines;
+            ) [ ] (lib.traceValSeq inventory.machines);
 
             tagMembers = builtins.attrNames (
               lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
             );
           in
           if tagMembers == [ ] then
-            throw "Tag: '${tag}' not found. Available tags: ${builtins.toJSON (lib.unique availableTags)}"
+            throw ''
+              inventory.services.${serviceName}.${instanceName}: - ${roleName} tags: no machine with tag '${tag}' found.
+              Available tags: ${builtins.toJSON (lib.unique availableTags)}
+            ''
           else
             acc ++ tagMembers
         ) [ ] members.tags or [ ]);
@@ -43,7 +53,7 @@ let
       machineName: machineConfig:
       lib.foldlAttrs (
         # [ Modules ], String, { ${instance_name} :: ServiceConfig }
-        acc: moduleName: serviceConfigs:
+        acc: serviceName: serviceConfigs:
         acc
         # Collect service config
         ++ (lib.foldlAttrs (
@@ -51,7 +61,16 @@ let
           acc2: instanceName: serviceConfig:
           let
             resolvedRoles = builtins.mapAttrs (
-              _roleName: members: resolveTags inventory members
+              roleName: members:
+              resolveTags {
+                inherit
+                  serviceName
+                  instanceName
+                  roleName
+                  inventory
+                  members
+                  ;
+              }
             ) serviceConfig.roles;
 
             isInService = builtins.any (members: builtins.elem machineName members.machines) (
@@ -72,11 +91,17 @@ let
             machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
             globalConfig = serviceConfig.config or { };
 
+            globalImports = serviceConfig.imports or [ ];
+            machineImports = serviceConfig.machines.${machineName}.imports or [ ];
+            roleServiceImports = builtins.foldl' (
+              acc: role: acc ++ serviceConfig.roles.${role}.imports or [ ]
+            ) [ ] inverseRoles.${machineName} or [ ];
+
             # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
             roleModules = builtins.map (
               role:
               let
-                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+                path = "${clan-core.clanModules.${serviceName}}/roles/${role}.nix";
               in
               if builtins.pathExists path then
                 path
@@ -87,13 +112,18 @@ let
             roleServiceConfigs = builtins.map (
               role: serviceConfig.roles.${role}.config or { }
             ) inverseRoles.${machineName} or [ ];
+
+            customImports = map (s: "${directory}/${s}") (
+              globalImports ++ machineImports ++ roleServiceImports
+            );
           in
+
           if isInService then
             acc2
             ++ [
               {
-                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
-                config.clan.${moduleName} = lib.mkMerge (
+                imports = [ clan-core.clanModules.${serviceName} ] ++ roleModules ++ customImports;
+                config.clan.${serviceName} = lib.mkMerge (
                   [
                     globalConfig
                     machineServiceConfig
@@ -102,7 +132,7 @@ let
                 );
               }
               {
-                config.clan.inventory.services.${moduleName}.${instanceName} = {
+                config.clan.inventory.services.${serviceName}.${instanceName} = {
                   roles = resolvedRoles;
                   # TODO: Add inverseRoles to the service config if needed
                   # inherit inverseRoles;

lib/inventory/build-inventory/interface.nix

@@ -1,49 +1,33 @@
 { config, lib, ... }:
 let
-  t = lib.types;
+  types = lib.types;
 
   metaOptions = {
-    name = lib.mkOption { type = t.str; };
+    name = lib.mkOption { type = types.str; };
     description = lib.mkOption {
       default = null;
-      type = t.nullOr t.str;
+      type = types.nullOr types.str;
     };
     icon = lib.mkOption {
       default = null;
-      type = t.nullOr t.str;
+      type = types.nullOr types.str;
     };
   };
 
-  machineRef = lib.mkOptionType {
-    name = "str";
-    description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
-    check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
-    merge = lib.mergeEqualOption;
-  };
-
-  allTags = lib.unique (
-    lib.foldlAttrs (
-      tags: _: m:
-      tags ++ m.tags or [ ]
-    ) [ ] config.machines
-  );
-
-  tagRef = lib.mkOptionType {
-    name = "str";
-    description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
-    check = v: lib.isString v && builtins.elem v allTags;
-    merge = lib.mergeEqualOption;
-  };
-
   moduleConfig = lib.mkOption {
     default = { };
-    type = t.attrsOf t.anything;
+    type = types.attrsOf types.anything;
+  };
+
+  importsOption = lib.mkOption {
+    default = [ ];
+    type = types.listOf types.str;
   };
 in
 {
   options = {
     assertions = lib.mkOption {
-      type = t.listOf t.unspecified;
+      type = types.listOf types.unspecified;
       internal = true;
       visible = false;
       default = [ ];
@@ -52,18 +36,18 @@ in
 
     machines = lib.mkOption {
       default = { };
-      type = t.attrsOf (
-        t.submodule {
+      type = types.attrsOf (
+        types.submodule {
           options = {
             inherit (metaOptions) name description icon;
             tags = lib.mkOption {
               default = [ ];
               apply = lib.unique;
-              type = t.listOf t.str;
+              type = types.listOf types.str;
             };
             system = lib.mkOption {
               default = null;
-              type = t.nullOr t.str;
+              type = types.nullOr types.str;
             };
           };
         }
@@ -72,29 +56,36 @@ in
 
     services = lib.mkOption {
       default = { };
-      type = t.attrsOf (
-        t.attrsOf (
-          t.submodule {
+      type = types.attrsOf (
+        types.attrsOf (
+          types.submodule {
             options.meta = metaOptions;
+            options.imports = importsOption;
             options.config = moduleConfig;
             options.machines = lib.mkOption {
               default = { };
-              type = t.attrsOf (t.submodule { options.config = moduleConfig; });
+              type = types.attrsOf (
+                types.submodule {
+                  options.imports = importsOption;
+                  options.config = moduleConfig;
+                }
+              );
             };
             options.roles = lib.mkOption {
               default = { };
-              type = t.attrsOf (
-                t.submodule {
+              type = types.attrsOf (
+                types.submodule {
                   options.machines = lib.mkOption {
                     default = [ ];
-                    type = t.listOf machineRef;
+                    type = types.listOf types.str;
                   };
                   options.tags = lib.mkOption {
                     default = [ ];
                     apply = lib.unique;
-                    type = t.listOf tagRef;
+                    type = types.listOf types.str;
                   };
                   options.config = moduleConfig;
+                  options.imports = importsOption;
                 }
               );
             };

lib/inventory/tests/default.nix

@@ -2,25 +2,31 @@
 {
   test_inventory_empty = {
     # Empty inventory should return an empty module
-    expr = buildInventory { };
+    expr = buildInventory {
+      inventory = { };
+      directory = ./.;
+    };
     expected = { };
   };
   test_inventory_role_imports =
     let
       configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.server.machines = [ "backup_server" ];
-            roles.client.machines = [
-              "client_1_machine"
-              "client_2_machine"
-            ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.server.machines = [ "backup_server" ];
+              roles.client.machines = [
+                "client_1_machine"
+                "client_2_machine"
+              ];
+            };
+          };
+          machines = {
+            "backup_server" = { };
+            "client_1_machine" = { };
+            "client_2_machine" = { };
           };
-        };
-        machines = {
-          "backup_server" = { };
-          "client_1_machine" = { };
-          "client_2_machine" = { };
         };
       };
     in
@@ -49,18 +55,21 @@
   test_inventory_tag_resolve =
     let
       configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.client.tags = [ "backup" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.client.tags = [ "backup" ];
+            };
           };
-        };
-        machines = {
-          "not_used_machine" = { };
-          "client_1_machine" = {
-            tags = [ "backup" ];
-          };
-          "client_2_machine" = {
-            tags = [ "backup" ];
+          machines = {
+            "not_used_machine" = { };
+            "client_1_machine" = {
+              tags = [ "backup" ];
+            };
+            "client_2_machine" = {
+              tags = [ "backup" ];
+            };
           };
         };
       };
@@ -85,14 +94,17 @@
   test_inventory_multiple_roles =
     let
       configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.client.machines = [ "machine_1" ];
-            roles.server.machines = [ "machine_1" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.client.machines = [ "machine_1" ];
+              roles.server.machines = [ "machine_1" ];
+            };
+          };
+          machines = {
+            "machine_1" = { };
           };
-        };
-        machines = {
-          "machine_1" = { };
         };
       };
     in
@@ -112,13 +124,16 @@
   test_inventory_role_doesnt_exist =
     let
       configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.roleXYZ.machines = [ "machine_1" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.roleXYZ.machines = [ "machine_1" ];
+            };
+          };
+          machines = {
+            "machine_1" = { };
           };
-        };
-        machines = {
-          "machine_1" = { };
         };
       };
     in
@@ -132,15 +147,18 @@
   test_inventory_tag_doesnt_exist =
     let
       configs = buildInventory {
-        services = {
-          borgbackup.instance_1 = {
-            roles.client.machines = [ "machine_1" ];
-            roles.client.tags = [ "tagXYZ" ];
+        directory = ./.;
+        inventory = {
+          services = {
+            borgbackup.instance_1 = {
+              roles.client.machines = [ "machine_1" ];
+              roles.client.tags = [ "tagXYZ" ];
+            };
           };
-        };
-        machines = {
-          "machine_1" = {
-            tags = [ "tagABC" ];
+          machines = {
+            "machine_1" = {
+              tags = [ "tagABC" ];
+            };
           };
         };
       };
@@ -149,7 +167,7 @@
       expr = configs;
       expectedError = {
         type = "ThrownError";
-        msg = "Tag: '\\w+' not found";
+        msg = "no machine with tag '\\w+' found";
       };
     };
 }

nixosModules/clanCore/vars/default.nix

@@ -45,7 +45,7 @@ in
       );
       inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;
     };
-    inherit (config.clan.networking) targetHost buildHost;
-    inherit (config.clan.deployment) requireExplicitUpdate;
+    inherit (config.clan.core.networking) targetHost buildHost;
+    inherit (config.clan.core.deployment) requireExplicitUpdate;
   };
 }