Merge pull request 'Add toml frontmatter description to jsonschema' (#1664) from hsjobeki/clan-core:hsjobeki-main into main

lib/default.nix

@@ -8,7 +8,6 @@
   evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; };
   inventory = import ./inventory { inherit lib clan-core; };
   jsonschema = import ./jsonschema { inherit lib; };
-  # TODO: migrate to also use toml frontmatter
+  modules = import ./description.nix { inherit clan-core lib; };
-  # modules = import ./description.nix { inherit clan-core lib; };
   buildClan = import ./build-clan { inherit clan-core lib nixpkgs; };
 }

lib/description.nix

@@ -1,33 +1,34 @@
-{ ... }:
+{ clan-core, lib }:
-
 rec {
-  # getReadme =
+  getReadme =
-  #   modulename:
+    modulename:
-  #   let
+    let
-  #     readme = "${clan-core}/clanModules/${modulename}/README.md";
+      readme = "${clan-core}/clanModules/${modulename}/README.md";
-  #     readmeContents =
+      readmeContents =
-  #       if (builtins.pathExists readme) then
+        if (builtins.pathExists readme) then
-  #         (builtins.readFile readme)
+          (builtins.readFile readme)
-  #       else
+        else
-  #         throw "No README.md found for module ${modulename}";
+          throw "No README.md found for module ${modulename}";
-  #   in
+    in
-  #   readmeContents;
+    readmeContents;
 
-  # getShortDescription =
+  getShortDescription =
-  #   modulename:
+    modulename:
-  #   let
+    let
-  #     content = (getReadme modulename);
+      content = getReadme modulename;
-  #     parts = lib.splitString "---" content;
+      parts = lib.splitString "---" content;
-  #     description = builtins.head parts;
+      # Partition the parts into the first part (the readme content) and the rest (the metadata)
-  #     number_of_newlines = builtins.length (lib.splitString "\n" description);
+      parsed = builtins.partition ({ index, ... }: if index >= 2 then false else true) (
-  #   in
+        lib.filter ({ index, ... }: index != 0) (lib.imap0 (index: part: { inherit index part; }) parts)
-  #   if (builtins.length parts) > 1 then
+      );
-  #     if number_of_newlines > 4 then
+
-  #       throw "Short description in README.md for module ${modulename} is too long. Max 3 newlines."
+      # Use this if the content is needed
-  #     else if number_of_newlines <= 1 then
+      # readmeContent = lib.concatMapStrings (v: "---" + v.part) parsed.wrong;
-  #       throw "Missing short description in README.md for module ${modulename}."
+
-  #     else
+      meta = builtins.fromTOML (builtins.head parsed.right).part;
-  #       description
+    in
-  #   else
+    if (builtins.length parts >= 3) then
-  #     throw "Short description delimiter `---` not found in README.md for module ${modulename}";
+      meta.description
+    else
+      throw "Short description delimiter `---` not found in README.md for module ${modulename}";
 }

lib/inventory/build-inventory/default.nix

@@ -13,13 +13,18 @@ let
         ++ (builtins.foldl' (
           acc: tag:
           let
+            # For error printing
+            availableTags = lib.foldlAttrs (
+              acc: _: v:
+              v.tags or [ ] ++ acc
+            ) [ ] inventory.machines;
+
             tagMembers = builtins.attrNames (
               lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
             );
           in
-          # throw "Machine tag ${tag} not found. Not machine with: tag ${tagName} not in inventory.";
           if tagMembers == [ ] then
-            throw "Machine tag ${tag} not found. Not machine with: tag ${tag} not in inventory."
+            throw "Tag: '${tag}' not found. Available tags: ${builtins.toJSON (lib.unique availableTags)}"
           else
             acc ++ tagMembers
         ) [ ] members.tags or [ ]);
@@ -76,7 +81,7 @@ let
               if builtins.pathExists path then
                 path
               else
-                throw "Role doesnt have a module: ${role}. Path: ${path} not found."
+                throw "Module doesn't have role: '${role}'. Path: ${path} not found."
             ) inverseRoles.${machineName} or [ ];
           in
           if isInService then
@@ -101,6 +106,6 @@ let
             acc2
         ) [ ] serviceConfigs)
       ) [ ] inventory.services
-    ) inventory.machines;
+    ) inventory.machines or { };
 in
 machines

lib/inventory/example.nix

@@ -19,6 +19,7 @@ self.lib.buildClan {
   machines = {
     "backup_server" = {
       clan.tags = [ "all" ];
+      # ... rest of the machine config
     };
     "client_1_machine" = {
       clan.tags = [

lib/inventory/flake-module.nix

@@ -1,47 +1,81 @@
-{ ... }:
+{ self, inputs, ... }:
+let
+  inputOverrides = builtins.concatStringsSep " " (
+    builtins.map (input: " --override-input ${input} ${inputs.${input}}") (builtins.attrNames inputs)
+  );
+in
 {
+  flake.inventory = import ./example.nix { inherit self; };
+
   perSystem =
-    { pkgs, config, ... }:
     {
-      packages.inventory-schema = pkgs.stdenv.mkDerivation {
+      pkgs,
-        name = "inventory-schema";
+      lib,
-        src = ./src;
+      config,
-
+      system,
-        buildInputs = [ pkgs.cue ];
+      ...
-
+    }:
-        installPhase = ''
+    let
-          mkdir -p $out
+      buildInventory = import ./build-inventory {
-        '';
+        clan-core = self;
+        inherit lib;
+      };
+    in
+    {
+      devShells.inventory-schema = pkgs.mkShell {
+        inputsFrom = with config.checks; [
+          lib-inventory-schema
+          lib-inventory-eval
+        ];
       };
 
-      devShells.inventory-schema = pkgs.mkShell { inputsFrom = [ config.packages.inventory-schema ]; };
+      # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
+      legacyPackages.evalTests = import ./tests {
+        inherit buildInventory;
+        clan-core = self;
+      };
 
-      checks.inventory-schema-checks = pkgs.stdenv.mkDerivation {
+      checks = {
-        name = "inventory-schema-checks";
+        lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
-        src = ./src;
+          export HOME="$(realpath .)"
-        buildInputs = [ pkgs.cue ];
-        buildPhase = ''
-          echo "Running inventory tests..."
 
-          echo "Export cue as json-schema..."
+          nix-unit --eval-store "$HOME" \
-          cue export --out openapi root.cue
+            --extra-experimental-features flakes \
-
+            ${inputOverrides} \
-          echo "Validate test/*.json against inventory-schema..."
+            --flake ${self}#legacyPackages.${system}.evalTests
-
-          test_dir="test"
-          for file in "$test_dir"/*; do
-            # Check if the item is a file
-            if [ -f "$file" ]; then
-              # Print the filename
-              echo "Running test on: $file"
-
-              # Run the cue vet command
-              cue vet "$file" root.cue -d "#Root"
-            fi
-          done
 
           touch $out
         '';
+
+        lib-inventory-schema = pkgs.stdenv.mkDerivation {
+          name = "inventory-schema-checks";
+          src = ./.;
+          buildInputs = [ pkgs.cue ];
+          buildPhase = ''
+            echo "Running inventory tests..."
+            # Cue is easier to run in the same directory as the schema
+            cd spec
+
+            echo "Export cue as json-schema..."
+            cue export --out openapi root.cue
+
+            echo "Validate test/*.json against inventory-schema..."
+
+            test_dir="../examples"
+            for file in "$test_dir"/*; do
+              # Check if the item is a file
+              if [ -f "$file" ]; then
+                # Print the filename
+                echo "Running test on: $file"
+
+                # Run the cue vet command
+                cue vet "$file" root.cue -d "#Root"
+              fi
+            done
+
+            touch $out
+          '';
+        };
       };
     };
 }

lib/inventory/tests/default.nix

@@ -0,0 +1,151 @@
+{ buildInventory, clan-core, ... }:
+{
+  test_inventory_empty = {
+    # Empty inventory should return an empty module
+    expr = buildInventory { };
+    expected = { };
+  };
+  test_inventory_role_imports =
+    let
+      configs = buildInventory {
+        services = {
+          borgbackup.instance_1 = {
+            roles.server.machines = [ "backup_server" ];
+            roles.client.machines = [
+              "client_1_machine"
+              "client_2_machine"
+            ];
+          };
+        };
+        machines = {
+          "backup_server" = { };
+          "client_1_machine" = { };
+          "client_2_machine" = { };
+        };
+      };
+    in
+    {
+      expr = {
+        server_imports = (builtins.head configs."backup_server").imports;
+        client_1_imports = (builtins.head configs."client_1_machine").imports;
+        client_2_imports = (builtins.head configs."client_2_machine").imports;
+      };
+
+      expected = {
+        server_imports = [
+          clan-core.clanModules.borgbackup
+          "${clan-core.clanModules.borgbackup}/roles/server.nix"
+        ];
+        client_1_imports = [
+          clan-core.clanModules.borgbackup
+          "${clan-core.clanModules.borgbackup}/roles/client.nix"
+        ];
+        client_2_imports = [
+          clan-core.clanModules.borgbackup
+          "${clan-core.clanModules.borgbackup}/roles/client.nix"
+        ];
+      };
+    };
+  test_inventory_tag_resolve =
+    let
+      configs = buildInventory {
+        services = {
+          borgbackup.instance_1 = {
+            roles.client.tags = [ "backup" ];
+          };
+        };
+        machines = {
+          "not_used_machine" = { };
+          "client_1_machine" = {
+            tags = [ "backup" ];
+          };
+          "client_2_machine" = {
+            tags = [ "backup" ];
+          };
+        };
+      };
+    in
+    {
+      expr = {
+        client_1_machine = builtins.length configs.client_1_machine;
+        client_2_machine = builtins.length configs.client_2_machine;
+        not_used_machine = builtins.length configs.not_used_machine;
+      };
+      expected = {
+        client_1_machine = 2;
+        client_2_machine = 2;
+        not_used_machine = 0;
+      };
+    };
+
+  test_inventory_multiple_roles =
+    let
+      configs = buildInventory {
+        services = {
+          borgbackup.instance_1 = {
+            roles.client.machines = [ "machine_1" ];
+            roles.server.machines = [ "machine_1" ];
+          };
+        };
+        machines = {
+          "machine_1" = { };
+        };
+      };
+    in
+    {
+      expr = {
+        machine_1_imports = (builtins.head configs."machine_1").imports;
+      };
+      expected = {
+        machine_1_imports = [
+          clan-core.clanModules.borgbackup
+          "${clan-core.clanModules.borgbackup}/roles/client.nix"
+          "${clan-core.clanModules.borgbackup}/roles/server.nix"
+        ];
+      };
+    };
+
+  test_inventory_role_doesnt_exist =
+    let
+      configs = buildInventory {
+        services = {
+          borgbackup.instance_1 = {
+            roles.roleXYZ.machines = [ "machine_1" ];
+          };
+        };
+        machines = {
+          "machine_1" = { };
+        };
+      };
+    in
+    {
+      expr = configs;
+      expectedError = {
+        type = "ThrownError";
+        msg = "Module doesn't have role.*";
+      };
+    };
+  test_inventory_tag_doesnt_exist =
+    let
+      configs = buildInventory {
+        services = {
+          borgbackup.instance_1 = {
+            roles.client.machines = [ "machine_1" ];
+            roles.client.tags = [ "tagXYZ" ];
+          };
+        };
+        machines = {
+          "machine_1" = {
+            tags = [ "tagABC" ];
+          };
+        };
+      };
+    in
+    {
+      expr = configs;
+      expectedError = {
+        type = "ThrownError";
+        msg = "Tag: '\\w+' not found";
+      };
+    };
+}

pkgs/schemas/flake-module.nix

@@ -23,8 +23,7 @@
 
       clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: {
         name = modulename;
-        # TODO: migrate to new toml format
+        description = self.lib.modules.getShortDescription modulename;
-        # description = self.lib.modules.getShortDescription modulename;
         parameters = self.lib.jsonschema.parseOptions (optionsFromModule modulename);
       }) clanModules;
     in