Merge pull request 'Inventory: init draft ideas' (#1632) from hsjobeki/clan-core:hsjobeki-inventory into main

flake.nix

@@ -53,6 +53,8 @@
           ./nixosModules/flake-module.nix
           ./pkgs/flake-module.nix
           ./templates/flake-module.nix
+
+          ./inventory/flake-module.nix
         ];
       }
     );

inventory/.envrc

@@ -0,0 +1,5 @@
+source_up
+
+watch_file flake-module.nix
+
+use flake .#inventory-schema --builders ''

inventory/README.md

@@ -0,0 +1,57 @@
+# Inventory
+
+This part provides a specification for the inventory.
+
+It is used for design phase and as validation helper.
+
+> Cue is less verbose and easier to understand and maintain than json-schema.
+> Json-schema, if needed can be easily generated on-the fly.
+
+## Checking validity
+
+Directly check a json against the schema
+
+`cue vet inventory.json root.cue -d '#Root'`
+
+## Json schema
+
+Export the json-schema i.e. for usage in python / javascript / nix
+
+`cue export --out openapi root.cue`
+
+## Usage
+
+Comments are rendered as descriptions in the json schema.
+
+```cue
+// A name of the clan (primarily shown by the UI)
+name: string
+```
+
+Cue open sets. In the following `foo = {...}` means that the key `foo` can contain any arbitrary json object.
+
+```cue
+foo: { ... }
+```
+
+Cue dynamic keys.
+
+```cue
+[string]: {
+    attr: string
+}
+```
+
+This is the schema of
+
+```json
+{
+    "a": {
+        "attr": "foo"
+    },
+    "b": {
+        "attr": "bar"
+    }
+    // ... Indefinitely more dynamic keys of type "string"
+}
+```

inventory/example_flake.nix

@@ -0,0 +1,137 @@
+{
+  description = "<Put your description here>";
+
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+
+  outputs =
+    { clan-core, ... }:
+    let
+      pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
+      system = "x86_64-linux";
+    in
+    # Usage see: https://docs.clan.lol
+    #  nice_flake_interface -> buildInventory() -> Inventory -> buildClanFromInventory() -> nixosConfigurations
+    # buildClanFromInventory = inventory: evalModules {
+    #     extraAttrs = { inherit inventory; };
+    #     # (attrNames inventory.machines)
+    # };
+    # clan =
+    #   clan-core.lib.buildClanFromInventory [
+    #     # Inventory 0 (loads the json file managed by the Python API)
+    #     (builtins.fromJSON (builtins.readFile ./inventory.json))
+    #     # ->
+    #     # {
+    #     #   services."backups_1".autoIncludeMachines = true;
+    #     #   services."backups_1".module = "borgbackup";
+    #     #   ... etc.
+    #     # }
+    #   ]
+    #   ++ (buildInventory {
+    #     clanName = "nice_flake_interface";
+    #     description = "A nice flake interface";
+    #     icon = "assets/icon.png";
+    #     machines = {
+    #       jon = {
+    #         # Just regular nixos/clan configuration ?
+    #         # config = {
+    #         #   imports = [
+    #         #     ./modules/shared.nix
+    #         #     ./machines/jon/configuration.nix
+    #         #   ];
+    #         #   nixpkgs.hostPlatform = system;
+    #         #   # Set this for clan commands use ssh i.e. `clan machines update`
+    #         #   # If you change the hostname, you need to update this line to root@<new-hostname>
+    #         #   # This only works however if you have avahi running on your admin machine else use IP
+    #         #   clan.networking.targetHost = pkgs.lib.mkDefault "root@jon";
+    #         #   # ssh root@flash-installer.local lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
+    #         #   disko.devices.disk.main = {
+    #         #     device = "/dev/disk/by-id/__CHANGE_ME__";
+    #         #   };
+    #         #   # IMPORTANT! Add your SSH key here
+    #         #   # e.g. > cat ~/.ssh/id_ed25519.pub
+    #         #   users.users.root.openssh.authorizedKeys.keys = throw ''
+    #         #     Don't forget to add your SSH key here!
+    #         #     users.users.root.openssh.authorizedKeys.keys = [ "<YOUR SSH_KEY>" ]
+    #         #   '';
+    #         #   # Zerotier needs one controller to accept new nodes. Once accepted
+    #         #   # the controller can be offline and routing still works.
+    #         #   clan.networking.zerotier.controller.enable = true;
+    #         # };
+    #       };
+    #     };
+    #   })
+    #   ++ [
+    #     # Low level inventory overrides (comes at the end)
+    #     {
+    #       services."backups_2".autoIncludeMachines = true;
+    #       services."backups_2".module = "borgbackup";
+    #     }
+    #   ];
+    # # buildClan :: [ Partial<Inventory> ] -> Inventory
+    # # foldl' (acc: v: lib.recursiveUpdate acc v) {} []
+    # inventory = [
+    #   # import json
+    #   {...}
+    #   # power user flake
+    #   {...}
+    # ]
+    # # With Module system
+    # # Pros: Easy to understand,
+    # # Cons: Verbose, hard to maintain
+    # # buildClan :: { modules = [ { config = Partial<Inventory>; options :: InventoryOptions; } } ]; } -> Inventory
+    # eval = lib.evalModules {
+    #   modules = [
+    #     {
+    #       # Inventory Schema
+    #       # Python validation
+    #       options = {...}
+    #     }
+    #     {
+    #       config = map lib.mkDefault
+    #       (builtins.fromJSON (builtins.readFile ./inventory.json))
+    #     }
+    #     {
+    #       #  User provided
+    #       config = {...}
+    #     }
+    #     # Later overrides.
+    #     {
+    #       lib.mkForce ...
+    #     }
+    #   ];
+    # }
+    # nixosConfigurations = lib.evalModules inventory;
+    # eval.config.inventory
+    # #
+    # eval.config.machines.jon#nixosConfig
+    # eval.config.machines.sara#nixosConfig
+    #
+    # {inventory, config, ...}:{
+    #   hostname = config.machines.sara # Invalid
+    #   hostname = inventory.machines.sara.hostname # Valid
+    # }
+    /*
+      # Type
+
+      buildInventory :: {
+        clanName :: string
+        machines :: {
+          ${name} :: {
+            config :: {
+              # NixOS configuration
+            };
+          };
+        };
+        # ... More mapped inventory options
+        # i.e. shared config for all machines
+      } -> Inventory
+    */
+    {
+      # all machines managed by Clan
+      inherit (clan) nixosConfigurations clanInternals;
+      # add the Clan cli tool to the dev shell
+      devShells.${system}.default = pkgs.mkShell {
+        packages = [ clan-core.packages.${system}.clan-cli ];
+      };
+    };
+}

inventory/flake-module.nix

@@ -0,0 +1,46 @@
+{ ... }:
+{
+  perSystem =
+    { pkgs, config, ... }:
+    {
+      packages.inventory-schema = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema";
+        src = ./src;
+
+        buildInputs = [ pkgs.cue ];
+
+        installPhase = ''
+          mkdir -p $out
+        '';
+      };
+      devShells.inventory-schema = pkgs.mkShell { inputsFrom = [ config.packages.inventory-schema ]; };
+
+      checks.inventory-schema-checks = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema-checks";
+        src = ./src;
+        buildInputs = [ pkgs.cue ];
+        buildPhase = ''
+          echo "Running inventory tests..."
+
+          echo "Export cue as json-schema..."
+          cue export --out openapi root.cue
+
+          echo "Validate test/*.json against inventory-schema..."
+
+          test_dir="test"
+          for file in "$test_dir"/*; do
+            # Check if the item is a file
+            if [ -f "$file" ]; then
+              # Print the filename
+              echo "Running test on: $file"
+
+              # Run the cue vet command
+              cue vet "$file" root.cue -d "#Root"
+            fi
+          done
+
+          touch $out
+        '';
+      };
+    };
+}

inventory/src/cue.mod/module.cue

@@ -0,0 +1,2 @@
+module: "clan.lol/inventory"
+language: version: "v0.8.2"

inventory/src/machines/machines.cue

@@ -0,0 +1,20 @@
+package machines
+
+#ServiceRole: "server" | "client" | "both"
+
+#machine: machines: [string]: {
+    name: string,
+    description?: string,
+    icon?: string,
+    // each machines service
+    services?: [string]: {
+        // Roles if specificed must contain one or more roles
+        // If no roles are specified, the service module defines the default roles.
+        roles?: [ ...#ServiceRole ],
+        // The service config to use
+        // This config is scoped to the service.module, only serializable data (strings,numbers, etc) can be assigned here
+        config: {
+            ...
+        }
+    }
+}

inventory/src/root.cue

@@ -0,0 +1,24 @@
+package inventory
+
+import (
+	"clan.lol/inventory/services"
+	"clan.lol/inventory/machines"
+)
+
+@jsonschema(schema="http://json-schema.org/schema#")
+#Root: {
+	meta: {
+		// A name of the clan (primarily shown by the UI)
+		name: string
+		// A description of the clan
+		description?: string
+		// The icon path
+		icon?: string
+	}
+
+	// A map of services
+	services.#service
+
+	// A map of machines
+	machines.#machine
+}

inventory/src/services/services.cue

@@ -0,0 +1,30 @@
+package services
+
+#service: services: [string]: {
+    // Required meta fields
+    meta: {
+        name: string,
+        icon?: string
+        description?: string,
+    },
+    // Required module specifies the behavior of the service.
+    module: string,
+
+    // We moved the machine sepcific config to "machines".
+    // It may be moved back depending on what makes more sense in the future.
+    // machineConfig: {
+    //     [string]: {
+    //         roles: string[],
+    //         config: {
+    //             defaultUser?: string
+    //         }
+    //     }
+    // },
+
+    // Configuration for the service
+    config: {
+        // Schema depends on the module.
+        // It declares the interface how the service can be configured.
+        ...
+    }
+}

inventory/src/tests/1_inventory.json

@@ -0,0 +1,58 @@
+{
+  "machines": {
+    "jon_machine": {
+      "name": "jon",
+      "description": "Jon's machine",
+      "icon": "assets/icon.png",
+      "services": {
+        "matrix": {
+          "roles": ["server"]
+        }
+      }
+    },
+    "anna_machine": {
+      "name": "anna",
+      "description": "anna's machine"
+    }
+  },
+  "meta": {
+    "name": "clan name"
+  },
+  "services": {
+    "sync-home": {
+      "meta": {
+        "name": "My Home Sync"
+      },
+      "module": "syncthing",
+      "config": {
+        "folders": ["/sync/my_f"]
+      }
+    },
+    "matrix": {
+      "meta": {
+        "name": "Our matrix chat",
+        "description": "Matrix chat service for our clan"
+      },
+      "module": "matrix-synapse",
+      "config": {
+        "compression": "zstd"
+      }
+    },
+    "backup": {
+      "meta": {
+        "name": "My daily backup"
+      },
+      "module": "borgbackup",
+      "config": {}
+    },
+    "borgbackup_1": {
+      "meta": {
+        "name": "My weekly backup"
+      },
+      "module": "borgbackup",
+      "config": {
+        "compression": "lz4"
+      }
+    }
+  }
+}