From 9eb00df6b7491fc9db2f3d320cc1341aadc755bb Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 3 Jul 2024 13:26:00 +0200 Subject: [PATCH 1/6] buildClan: autoimport configuration.nix & hardware-configuration.nix --- lib/build-clan/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index d0bb8e6a..13169700 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -152,6 +152,13 @@ let in (machineImports settings) ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } settings clan-core.nixosModules.clanCore extraConfig From d8ff8b042fe24b5b2f259f570ab27591aa975dc7 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 3 Jul 2024 14:13:30 +0200 Subject: [PATCH 2/6] Doc: add conceptual documentation --- docs/mkdocs.yml | 3 + docs/site/concepts/configuration.md | 114 +++++++++++++++++++++++++++ inventory.json | 3 + lib/inventory/spec/schema/schema.cue | 1 + 4 files changed, 121 insertions(+) create mode 100644 docs/site/concepts/configuration.md diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 1432872d..eecfbb33 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -14,6 +14,7 @@ markdown_extensions: - attr_list - footnotes - md_in_html + - def_list - meta - plantuml_markdown - pymdownx.emoji: @@ -49,6 +50,8 @@ nav: - Mesh VPN: getting-started/mesh-vpn.md - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md + - Concepts: + - Configuration: concepts/configuration.md - Reference: - Clan Modules: - reference/clanModules/borgbackup-static.md diff --git a/docs/site/concepts/configuration.md b/docs/site/concepts/configuration.md new file mode 100644 index 00000000..b3d97a1b --- /dev/null +++ b/docs/site/concepts/configuration.md @@ -0,0 +1,114 @@ +# Configuration + +## Introduction + +When managing machine configuration this can be done through many possible ways. +Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming). + +clan currently offers the following methods to configure machines: + +!!! Success "Recommended for nix people" + + - flake.nix (i.e. via `buildClan`) + - `machine` argument + - `inventory` argument + + - machines/`machine_name`/configuration.nix (`autoincluded` if it exists) + +???+ Note "Used by CLI & UI" + + - inventory.json + - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists) + + +!!! Warning "Deprecated" + + machines/`machine_name`/settings.json + +## BuildClan + +The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services. + +### Inputs + +`directory` +: The directory containing the machines subdirectory + +`machines` +: Allows to include machine-specific modules i.e. machines.${name} = { ... } + +`meta` +: An optional set + +: `{ name :: string, icon :: string, description :: string }` + +`inventory` +: Service set for easily configuring distributed services, such as backups + +: For more details see [Inventory](#inventory) + +`specialArgs` +: Extra arguments to pass to nixosSystem i.e. useful to make self available + +`pkgsForSystem` +: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system. + This improves performance, but all nipxkgs.* options will be ignored. + `(string -> pkgs )` + +## Inventory + +`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries. + +The following is the specification of the inventory in `cuelang` + +```cue +{ + meta: { + // A name of the clan (primarily shown by the UI) + name: string + // A description of the clan + description?: string + // The icon path + icon?: string + } + + // A map of services + services: [string]: [string]: { + // Required meta fields + meta: { + name: string, + icon?: string + description?: string, + }, + // Machines are added via the avilable roles + // Membership depends only on this field + roles: [string]: { + machines: [...string], + tags: [...string], + } + machines?: { + [string]: { + config?: { + ... + } + } + }, + + // Global Configuration for the service + // Applied to all machines. + config?: { + // Schema depends on the module. + // It declares the interface how the service can be configured. + ... + } + } + // A map of machines, extends the machines of `buildClan` + machines: [string]: { + name: string, + description?: string, + icon?: string + tags: [...string] + system: string + } +} +``` diff --git a/inventory.json b/inventory.json index bd9d2570..207630e5 100644 --- a/inventory.json +++ b/inventory.json @@ -1,4 +1,7 @@ { + "meta": { + "name": "Minimal inventory" + }, "machines": { "minimal-inventory-machine": { "name": "foo", diff --git a/lib/inventory/spec/schema/schema.cue b/lib/inventory/spec/schema/schema.cue index 428fbe9b..09689584 100644 --- a/lib/inventory/spec/schema/schema.cue +++ b/lib/inventory/spec/schema/schema.cue @@ -5,6 +5,7 @@ package schema description?: string, icon?: string tags: [...string] + system: string } #role: string From cb13ddb46437a34aaeb1a7afc59ab1825bf89e72 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 6 Jul 2024 16:17:44 +0200 Subject: [PATCH 3/6] API Types: treat '_*' as private fields and dont inspect them --- pkgs/clan-cli/clan_cli/api/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/clan_cli/api/util.py b/pkgs/clan-cli/clan_cli/api/util.py index c93c0520..4a49a7a6 100644 --- a/pkgs/clan-cli/clan_cli/api/util.py +++ b/pkgs/clan-cli/clan_cli/api/util.py @@ -76,6 +76,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> properties = { f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}", type_map) for f in fields + if not f.name.startswith("_") } required = set() From a7b7cc888b95d83833d46bf15348eb78577f85f0 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 6 Jul 2024 17:21:18 +0200 Subject: [PATCH 4/6] Test: ensure type inference runs on all dataclasses --- .../tests/test_api_dataclass_compat.py | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 pkgs/clan-cli/tests/test_api_dataclass_compat.py diff --git a/pkgs/clan-cli/tests/test_api_dataclass_compat.py b/pkgs/clan-cli/tests/test_api_dataclass_compat.py new file mode 100644 index 00000000..2a568359 --- /dev/null +++ b/pkgs/clan-cli/tests/test_api_dataclass_compat.py @@ -0,0 +1,125 @@ +import ast +import importlib.util +import os +import sys +from dataclasses import is_dataclass +from pathlib import Path + +from clan_cli.api.util import type_to_dict +from clan_cli.errors import ClanError + + +def find_dataclasses_in_directory( + directory: Path, exclude_paths: list[str] = [] +) -> list[tuple[str, str]]: + """ + Find all dataclass classes in all Python files within a nested directory. + + Args: + directory (str): The root directory to start searching from. + + Returns: + List[Tuple[str, str]]: A list of tuples containing the file path and the dataclass name. + """ + dataclass_files = [] + + excludes = [os.path.join(directory, d) for d in exclude_paths] + + for root, _, files in os.walk(directory, topdown=False): + for file in files: + if not file.endswith(".py"): + continue + + file_path = os.path.join(root, file) + + if file_path in excludes: + print(f"Skipping dataclass check for file: {file_path}") + continue + + with open(file_path, encoding="utf-8") as f: + try: + tree = ast.parse(f.read(), filename=file_path) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + for deco in node.decorator_list: + if ( + isinstance(deco, ast.Name) + and deco.id == "dataclass" + ): + dataclass_files.append((file_path, node.name)) + elif ( + isinstance(deco, ast.Call) + and isinstance(deco.func, ast.Name) + and deco.func.id == "dataclass" + ): + dataclass_files.append((file_path, node.name)) + except (SyntaxError, UnicodeDecodeError) as e: + print(f"Error parsing {file_path}: {e}") + + return dataclass_files + + +def load_dataclass_from_file( + file_path: str, class_name: str, root_dir: str +) -> type | None: + """ + Load a dataclass from a given file path. + + Args: + file_path (str): Path to the file. + class_name (str): Name of the class to load. + + Returns: + List[Type]: The dataclass type if found, else an empty list. + """ + module_name = ( + os.path.relpath(file_path, root_dir).replace(os.path.sep, ".").rstrip(".py") + ) + # spec = importlib.util.spec_from_file_location(module_name, file_path) + # module = importlib.util.module_from_spec(spec) + # spec.loader.exec_module(module) + # breakpoint() + try: + sys.path.insert(0, root_dir) + spec = importlib.util.spec_from_file_location(module_name, file_path) + if not spec: + raise ClanError(f"Could not load spec from file: {file_path}") + + module = importlib.util.module_from_spec(spec) + if not module: + raise ClanError(f"Could not create module: {file_path}") + + if not spec.loader: + raise ClanError(f"Could not load loader from spec: {spec}") + + spec.loader.exec_module(module) + + finally: + sys.path.pop(0) + dataclass_type = getattr(module, class_name, None) + + if dataclass_type and is_dataclass(dataclass_type): + return dataclass_type + return None + + +def test_all_dataclasses() -> None: + # Excludes: + # - API includes Type Generic wrappers, that are not known in the init file. + excludes = ["api/__init__.py"] + + cli_path = Path("clan_cli").resolve() + dataclasses = find_dataclasses_in_directory(cli_path, excludes) + + for file, dataclass in dataclasses: + print(f"Found dataclass {dataclass} in {file}") + # The parent directory of the clan_cli is the projects root directory + try: + dclass = load_dataclass_from_file(file, dataclass, str(cli_path.parent)) + json_schema = type_to_dict(dclass, scope=f"FILE {file} {dataclass}") + except Exception as e: + print(f"Error loading dataclass {dataclass} from {file}: {e}") + raise ClanError( + f"Error loading dataclass {dataclass} from {file}: {e}", + location=__file__, + ) From 2ebc0902c166e8f2d7c0d1cb4a02fe3d324d01cc Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 6 Jul 2024 17:51:46 +0200 Subject: [PATCH 5/6] Test: fixup --- pkgs/clan-cli/clan_cli/api/util.py | 12 +++---- pkgs/clan-cli/clan_cli/machines/machines.py | 2 +- .../tests/test_api_dataclass_compat.py | 35 +++++++++++++++---- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/api/util.py b/pkgs/clan-cli/clan_cli/api/util.py index 4a49a7a6..505fab54 100644 --- a/pkgs/clan-cli/clan_cli/api/util.py +++ b/pkgs/clan-cli/clan_cli/api/util.py @@ -128,7 +128,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> if origin is None: # Non-generic user-defined or built-in type # TODO: handle custom types - raise JSchemaTypeError("Unhandled Type: ", origin) + raise JSchemaTypeError(f"{scope} Unhandled Type: ", origin) elif origin is Literal: # Handle Literal values for enums in JSON Schema @@ -173,7 +173,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> new_map.update(inspect_dataclass_fields(t)) return type_to_dict(origin, scope, new_map) - raise JSchemaTypeError(f"Error api type not yet supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error api type not yet supported {t!s}") elif isinstance(t, type): if t is str: @@ -188,7 +188,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> return {"type": "object"} if t is Any: raise JSchemaTypeError( - f"Usage of the Any type is not supported for API functions. In: {scope}" + f"{scope} - Usage of the Any type is not supported for API functions. In: {scope}" ) if t is pathlib.Path: return { @@ -197,13 +197,13 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> } if t is dict: raise JSchemaTypeError( - "Error: generic dict type not supported. Use dict[str, Any] instead" + f"{scope} - Generic 'dict' type not supported. Use dict[str, Any] or any more expressive type." ) # Optional[T] gets internally transformed Union[T,NoneType] if t is NoneType: return {"type": "null"} - raise JSchemaTypeError(f"Error primitive type not supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error primitive type not supported {t!s}") else: - raise JSchemaTypeError(f"Error type not supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error type not supported {t!s}") diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 877ba642..6e197431 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -20,7 +20,7 @@ class Machine: name: str flake: FlakeId nix_options: list[str] = field(default_factory=list) - cached_deployment: None | dict = None + cached_deployment: None | dict[str, Any] = None _eval_cache: dict[str, str] = field(default_factory=dict) _build_cache: dict[str, Path] = field(default_factory=dict) diff --git a/pkgs/clan-cli/tests/test_api_dataclass_compat.py b/pkgs/clan-cli/tests/test_api_dataclass_compat.py index 2a568359..4641da3d 100644 --- a/pkgs/clan-cli/tests/test_api_dataclass_compat.py +++ b/pkgs/clan-cli/tests/test_api_dataclass_compat.py @@ -5,7 +5,7 @@ import sys from dataclasses import is_dataclass from pathlib import Path -from clan_cli.api.util import type_to_dict +from clan_cli.api.util import JSchemaTypeError, type_to_dict from clan_cli.errors import ClanError @@ -100,10 +100,21 @@ def load_dataclass_from_file( if dataclass_type and is_dataclass(dataclass_type): return dataclass_type - return None + + raise ClanError(f"Could not load dataclass {class_name} from file: {file_path}") def test_all_dataclasses() -> None: + """ + This Test ensures that all dataclasses are compatible with the API. + + It will load all dataclasses from the clan_cli directory and + generate a JSON schema for each of them. + + It will fail if any dataclass cannot be converted to JSON schema. + This means the dataclass in its current form is not compatible with the API. + """ + # Excludes: # - API includes Type Generic wrappers, that are not known in the init file. excludes = ["api/__init__.py"] @@ -112,14 +123,24 @@ def test_all_dataclasses() -> None: dataclasses = find_dataclasses_in_directory(cli_path, excludes) for file, dataclass in dataclasses: - print(f"Found dataclass {dataclass} in {file}") - # The parent directory of the clan_cli is the projects root directory + print(f"checking dataclass {dataclass} in file: {file}") try: dclass = load_dataclass_from_file(file, dataclass, str(cli_path.parent)) - json_schema = type_to_dict(dclass, scope=f"FILE {file} {dataclass}") - except Exception as e: + type_to_dict(dclass) + except JSchemaTypeError as e: print(f"Error loading dataclass {dataclass} from {file}: {e}") raise ClanError( - f"Error loading dataclass {dataclass} from {file}: {e}", + f""" +-------------------------------------------------------------------------------- +Error converting dataclass 'class {dataclass}()' from {file} + +Details: + {e} + +Help: +- Converting public fields to PRIVATE by prefixing them with underscore ('_') +- Ensure all private fields are initialized the API wont provide initial values for them. +-------------------------------------------------------------------------------- +""", location=__file__, ) From 04ef8d824ebf6ae99467bcb29166c864be4c7eee Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 6 Jul 2024 17:55:19 +0200 Subject: [PATCH 6/6] Inventory spec: make system optional --- lib/inventory/spec/schema/schema.cue | 2 +- pkgs/clan-cli/tests/test_api_dataclass_compat.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/inventory/spec/schema/schema.cue b/lib/inventory/spec/schema/schema.cue index 09689584..eded84e2 100644 --- a/lib/inventory/spec/schema/schema.cue +++ b/lib/inventory/spec/schema/schema.cue @@ -5,7 +5,7 @@ package schema description?: string, icon?: string tags: [...string] - system: string + system?: string } #role: string diff --git a/pkgs/clan-cli/tests/test_api_dataclass_compat.py b/pkgs/clan-cli/tests/test_api_dataclass_compat.py index 4641da3d..d54f4e54 100644 --- a/pkgs/clan-cli/tests/test_api_dataclass_compat.py +++ b/pkgs/clan-cli/tests/test_api_dataclass_compat.py @@ -75,10 +75,6 @@ def load_dataclass_from_file( module_name = ( os.path.relpath(file_path, root_dir).replace(os.path.sep, ".").rstrip(".py") ) - # spec = importlib.util.spec_from_file_location(module_name, file_path) - # module = importlib.util.module_from_spec(spec) - # spec.loader.exec_module(module) - # breakpoint() try: sys.path.insert(0, root_dir) spec = importlib.util.spec_from_file_location(module_name, file_path)