From c67860810598282f682fe128f5f5813f470e24f5 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 1 Jul 2024 21:16:52 +0200 Subject: [PATCH] Inventory: add system and sample machine --- inventory.json | 12 ++ lib/build-clan/default.nix | 34 ++++- lib/inventory/build-inventory/default.nix | 8 +- lib/inventory/build-inventory/interface.nix | 49 ++++--- lib/inventory/spec/schema/schema.cue | 6 +- lib/inventory/tests/default.nix | 10 +- pkgs/clan-cli/clan_cli/inventory/__init__.py | 138 +++++++++++++++++++ 7 files changed, 229 insertions(+), 28 deletions(-) create mode 100644 inventory.json create mode 100644 pkgs/clan-cli/clan_cli/inventory/__init__.py diff --git a/inventory.json b/inventory.json new file mode 100644 index 00000000..e1503adc --- /dev/null +++ b/inventory.json @@ -0,0 +1,12 @@ +{ + "machines": { + "minimal_inventory_machine": { + "name": "foo", + "system": "x86_64-linux", + "description": "A nice thing", + "icon": "./path/to/icon.png", + "tags": ["1", "2", "3"] + } + }, + "services": {} +} diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index d156f2e4..d0bb8e6a 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -40,7 +40,7 @@ let builtins.pathExists "${directory}/inventory.json" # Is recursively applied. Any explicit nix will override. then - lib.mkDefault (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) + (builtins.fromJSON (builtins.readFile "${directory}/inventory.json")) else { } ) @@ -67,11 +67,27 @@ let "clan" "tags" ] [ ] config; + + system = lib.attrByPath [ + "nixpkgs" + "hostSystem" + ] null config; } ) machines; } + # Will be deprecated - { machines = lib.mapAttrs (_n: _: lib.mkDefault { }) machinesDirs; } + { + machines = lib.mapAttrs ( + name: _: + # Use mkForce to make sure users migrate to the inventory system. + # When the settings.json exists the evaluation will print the deprecation warning. + lib.mkForce { + inherit name; + system = (machineSettings name).nixpkgs.hostSystem or null; + } + ) machinesDirs; + } # Deprecated interface (if clanName != null then { meta.name = clanName; } else { }) @@ -91,14 +107,24 @@ let machineSettings = machineName: + let + warn = lib.warn '' + Usage of Settings.json is only supported for test compatibility. + !!! Consider using the inventory system. !!! + + File: ${directory + /machines/${machineName}/settings.json} + + If there are still features missing in the inventory system, please open an issue on the clan-core repository. + ''; + in # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily # This is useful for doing a dry-run before writing changes into the settings.json # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then - builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) + warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))) else lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") ( - builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)) + warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))) ); machineImports = diff --git a/lib/inventory/build-inventory/default.nix b/lib/inventory/build-inventory/default.nix index 4145354f..6347a58c 100644 --- a/lib/inventory/build-inventory/default.nix +++ b/lib/inventory/build-inventory/default.nix @@ -40,7 +40,7 @@ let # For every machine in the inventory, build a NixOS configuration # For each machine generate config, forEach service, if the machine is used. builtins.mapAttrs ( - machineName: _: + machineName: machineConfig: lib.foldlAttrs ( # [ Modules ], String, { ${instance_name} :: ServiceConfig } acc: moduleName: serviceConfigs: @@ -106,6 +106,12 @@ let acc2 ) [ ] serviceConfigs) ) [ ] inventory.services + # Append each machine config + ++ [ + (lib.optionalAttrs (machineConfig.system or null != null) { + config.nixpkgs.hostPlatform = machineConfig.system; + }) + ] ) inventory.machines or { }; in machines diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix index c399d2af..88663ce9 100644 --- a/lib/inventory/build-inventory/interface.nix +++ b/lib/inventory/build-inventory/interface.nix @@ -41,23 +41,34 @@ in internal = true; default = [ ]; }; - config.assertions = lib.foldlAttrs ( - ass1: serviceName: c: - ass1 - ++ lib.foldlAttrs ( - ass2: instanceName: instanceConfig: - let - serviceMachineNames = lib.attrNames instanceConfig.machines; - topLevelMachines = lib.attrNames config.machines; - # All machines must be defined in the top-level machines - assertions = builtins.map (m: { - assertion = builtins.elem m topLevelMachines; - message = "${serviceName}.${instanceName}.machines.${m}. Should be one of [ ${builtins.concatStringsSep " | " topLevelMachines} ]"; - }) serviceMachineNames; - in - ass2 ++ assertions - ) [ ] c - ) [ ] config.services; + config.assertions = + let + serviceAssertions = lib.foldlAttrs ( + ass1: serviceName: c: + ass1 + ++ lib.foldlAttrs ( + ass2: instanceName: instanceConfig: + let + serviceMachineNames = lib.attrNames instanceConfig.machines; + topLevelMachines = lib.attrNames config.machines; + # All machines must be defined in the top-level machines + assertions = builtins.map (m: { + assertion = builtins.elem m topLevelMachines; + message = "${serviceName}.${instanceName}.machines.${m}. Should be one of [ ${builtins.concatStringsSep " | " topLevelMachines} ]"; + }) serviceMachineNames; + in + ass2 ++ assertions + ) [ ] c + ) [ ] config.services; + machineAssertions = map ( + { name, value }: + { + assertion = true; + message = "Machine ${name} should define its host system in the inventory. ()"; + } + ) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines)); + in + machineAssertions ++ serviceAssertions; options.meta = metaOptions; @@ -72,6 +83,10 @@ in apply = lib.unique; type = t.listOf t.str; }; + system = lib.mkOption { + default = null; + type = t.nullOr t.str; + }; }; } ); diff --git a/lib/inventory/spec/schema/schema.cue b/lib/inventory/spec/schema/schema.cue index 81d197b6..428fbe9b 100644 --- a/lib/inventory/spec/schema/schema.cue +++ b/lib/inventory/spec/schema/schema.cue @@ -22,7 +22,7 @@ package schema machines: [...string], tags: [...string], } - machines: { + machines?: { [string]: { config?: { ... @@ -30,8 +30,8 @@ package schema } }, - // Configuration for the service - config: { + // Global Configuration for the service + config?: { // Schema depends on the module. // It declares the interface how the service can be configured. ... diff --git a/lib/inventory/tests/default.nix b/lib/inventory/tests/default.nix index 8c549cea..14eaa8ac 100644 --- a/lib/inventory/tests/default.nix +++ b/lib/inventory/tests/default.nix @@ -67,14 +67,18 @@ in { expr = { + # A machine that includes the backup service should have 3 imports + # - one for some service agnostic properties of the machine itself + # - One for the service itself (default.nix) + # - one for the role (roles/client.nix) client_1_machine = builtins.length configs.client_1_machine; client_2_machine = builtins.length configs.client_2_machine; not_used_machine = builtins.length configs.not_used_machine; }; expected = { - client_1_machine = 2; - client_2_machine = 2; - not_used_machine = 0; + client_1_machine = 3; + client_2_machine = 3; + not_used_machine = 1; }; }; diff --git a/pkgs/clan-cli/clan_cli/inventory/__init__.py b/pkgs/clan-cli/clan_cli/inventory/__init__.py new file mode 100644 index 00000000..98c231a2 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/inventory/__init__.py @@ -0,0 +1,138 @@ +import re +from dataclasses import asdict, dataclass, is_dataclass +from pathlib import Path +from typing import Any, Literal + +from clan_cli.errors import ClanError + + +def sanitize_string(s: str) -> str: + return s.replace("\\", "\\\\").replace('"', '\\"') + + +def dataclass_to_dict(obj: Any) -> Any: + """ + Utility function to convert dataclasses to dictionaries + It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries + + It does NOT convert member functions. + """ + if is_dataclass(obj): + return { + sanitize_string(k): dataclass_to_dict(v) + for k, v in asdict(obj).items() # type: ignore + } + elif isinstance(obj, list | tuple): + return [dataclass_to_dict(item) for item in obj] + elif isinstance(obj, dict): + return {sanitize_string(k): dataclass_to_dict(v) for k, v in obj.items()} + elif isinstance(obj, Path): + return str(obj) + elif isinstance(obj, str): + return sanitize_string(obj) + else: + return obj + + +@dataclass +class Machine: + """ + Inventory machine model. + + DO NOT EDIT THIS CLASS. + Any changes here must be reflected in the inventory interface file and potentially other nix files. + + - Persisted to the inventory.json file + - Source of truth to generate each clan machine. + - For hardware deployment, the machine must declare the host system. + """ + + name: str + system: Literal["x86_64-linux"] | str | None = None + description: str | None = None + icon: str | None = None + tags: list[str] | None = None + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Machine": + if "name" not in d: + raise ClanError("name not found in machine") + + hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(? "Service": + if "meta" not in d: + raise ClanError("meta not found in service") + + if "roles" not in d: + raise ClanError("roles not found in service") + + return Service( + meta=ServiceMeta(**d["meta"]), + roles={name: Role(**role) for name, role in d["roles"].items()}, + machines={ + name: MachineServiceConfig(**machine) + for name, machine in d.get("machines", {}).items() + }, + ) + + +@dataclass +class Inventory: + machines: dict[str, Machine] + services: dict[str, dict[str, Service]] + + @staticmethod + def from_dict(d: dict[str, Any]) -> "Inventory": + if "machines" not in d: + raise ClanError("machines not found in inventory") + + if "services" not in d: + raise ClanError("services not found in inventory") + + return Inventory( + machines={ + name: Machine.from_dict(machine) + for name, machine in d["machines"].items() + }, + services={ + name: { + role: Service.from_dict(service) + for role, service in services.items() + } + for name, services in d["services"].items() + }, + )