diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 1432872d..eecfbb33 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -14,6 +14,7 @@ markdown_extensions: - attr_list - footnotes - md_in_html + - def_list - meta - plantuml_markdown - pymdownx.emoji: @@ -49,6 +50,8 @@ nav: - Mesh VPN: getting-started/mesh-vpn.md - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md + - Concepts: + - Configuration: concepts/configuration.md - Reference: - Clan Modules: - reference/clanModules/borgbackup-static.md diff --git a/docs/site/concepts/configuration.md b/docs/site/concepts/configuration.md new file mode 100644 index 00000000..b3d97a1b --- /dev/null +++ b/docs/site/concepts/configuration.md @@ -0,0 +1,114 @@ +# Configuration + +## Introduction + +When managing machine configuration this can be done through many possible ways. +Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming). + +clan currently offers the following methods to configure machines: + +!!! Success "Recommended for nix people" + + - flake.nix (i.e. via `buildClan`) + - `machine` argument + - `inventory` argument + + - machines/`machine_name`/configuration.nix (`autoincluded` if it exists) + +???+ Note "Used by CLI & UI" + + - inventory.json + - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists) + + +!!! Warning "Deprecated" + + machines/`machine_name`/settings.json + +## BuildClan + +The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services. + +### Inputs + +`directory` +: The directory containing the machines subdirectory + +`machines` +: Allows to include machine-specific modules i.e. machines.${name} = { ... } + +`meta` +: An optional set + +: `{ name :: string, icon :: string, description :: string }` + +`inventory` +: Service set for easily configuring distributed services, such as backups + +: For more details see [Inventory](#inventory) + +`specialArgs` +: Extra arguments to pass to nixosSystem i.e. useful to make self available + +`pkgsForSystem` +: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system. + This improves performance, but all nipxkgs.* options will be ignored. + `(string -> pkgs )` + +## Inventory + +`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries. + +The following is the specification of the inventory in `cuelang` + +```cue +{ + meta: { + // A name of the clan (primarily shown by the UI) + name: string + // A description of the clan + description?: string + // The icon path + icon?: string + } + + // A map of services + services: [string]: [string]: { + // Required meta fields + meta: { + name: string, + icon?: string + description?: string, + }, + // Machines are added via the avilable roles + // Membership depends only on this field + roles: [string]: { + machines: [...string], + tags: [...string], + } + machines?: { + [string]: { + config?: { + ... + } + } + }, + + // Global Configuration for the service + // Applied to all machines. + config?: { + // Schema depends on the module. + // It declares the interface how the service can be configured. + ... + } + } + // A map of machines, extends the machines of `buildClan` + machines: [string]: { + name: string, + description?: string, + icon?: string + tags: [...string] + system: string + } +} +``` diff --git a/inventory.json b/inventory.json index bd9d2570..207630e5 100644 --- a/inventory.json +++ b/inventory.json @@ -1,4 +1,7 @@ { + "meta": { + "name": "Minimal inventory" + }, "machines": { "minimal-inventory-machine": { "name": "foo", diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index d0bb8e6a..13169700 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -152,6 +152,13 @@ let in (machineImports settings) ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } settings clan-core.nixosModules.clanCore extraConfig diff --git a/lib/inventory/spec/schema/schema.cue b/lib/inventory/spec/schema/schema.cue index 428fbe9b..eded84e2 100644 --- a/lib/inventory/spec/schema/schema.cue +++ b/lib/inventory/spec/schema/schema.cue @@ -5,6 +5,7 @@ package schema description?: string, icon?: string tags: [...string] + system?: string } #role: string diff --git a/pkgs/clan-cli/clan_cli/api/util.py b/pkgs/clan-cli/clan_cli/api/util.py index c93c0520..505fab54 100644 --- a/pkgs/clan-cli/clan_cli/api/util.py +++ b/pkgs/clan-cli/clan_cli/api/util.py @@ -76,6 +76,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> properties = { f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}", type_map) for f in fields + if not f.name.startswith("_") } required = set() @@ -127,7 +128,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> if origin is None: # Non-generic user-defined or built-in type # TODO: handle custom types - raise JSchemaTypeError("Unhandled Type: ", origin) + raise JSchemaTypeError(f"{scope} Unhandled Type: ", origin) elif origin is Literal: # Handle Literal values for enums in JSON Schema @@ -172,7 +173,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> new_map.update(inspect_dataclass_fields(t)) return type_to_dict(origin, scope, new_map) - raise JSchemaTypeError(f"Error api type not yet supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error api type not yet supported {t!s}") elif isinstance(t, type): if t is str: @@ -187,7 +188,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> return {"type": "object"} if t is Any: raise JSchemaTypeError( - f"Usage of the Any type is not supported for API functions. In: {scope}" + f"{scope} - Usage of the Any type is not supported for API functions. In: {scope}" ) if t is pathlib.Path: return { @@ -196,13 +197,13 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> } if t is dict: raise JSchemaTypeError( - "Error: generic dict type not supported. Use dict[str, Any] instead" + f"{scope} - Generic 'dict' type not supported. Use dict[str, Any] or any more expressive type." ) # Optional[T] gets internally transformed Union[T,NoneType] if t is NoneType: return {"type": "null"} - raise JSchemaTypeError(f"Error primitive type not supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error primitive type not supported {t!s}") else: - raise JSchemaTypeError(f"Error type not supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error type not supported {t!s}") diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 877ba642..6e197431 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -20,7 +20,7 @@ class Machine: name: str flake: FlakeId nix_options: list[str] = field(default_factory=list) - cached_deployment: None | dict = None + cached_deployment: None | dict[str, Any] = None _eval_cache: dict[str, str] = field(default_factory=dict) _build_cache: dict[str, Path] = field(default_factory=dict) diff --git a/pkgs/clan-cli/tests/test_api_dataclass_compat.py b/pkgs/clan-cli/tests/test_api_dataclass_compat.py new file mode 100644 index 00000000..d54f4e54 --- /dev/null +++ b/pkgs/clan-cli/tests/test_api_dataclass_compat.py @@ -0,0 +1,142 @@ +import ast +import importlib.util +import os +import sys +from dataclasses import is_dataclass +from pathlib import Path + +from clan_cli.api.util import JSchemaTypeError, type_to_dict +from clan_cli.errors import ClanError + + +def find_dataclasses_in_directory( + directory: Path, exclude_paths: list[str] = [] +) -> list[tuple[str, str]]: + """ + Find all dataclass classes in all Python files within a nested directory. + + Args: + directory (str): The root directory to start searching from. + + Returns: + List[Tuple[str, str]]: A list of tuples containing the file path and the dataclass name. + """ + dataclass_files = [] + + excludes = [os.path.join(directory, d) for d in exclude_paths] + + for root, _, files in os.walk(directory, topdown=False): + for file in files: + if not file.endswith(".py"): + continue + + file_path = os.path.join(root, file) + + if file_path in excludes: + print(f"Skipping dataclass check for file: {file_path}") + continue + + with open(file_path, encoding="utf-8") as f: + try: + tree = ast.parse(f.read(), filename=file_path) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + for deco in node.decorator_list: + if ( + isinstance(deco, ast.Name) + and deco.id == "dataclass" + ): + dataclass_files.append((file_path, node.name)) + elif ( + isinstance(deco, ast.Call) + and isinstance(deco.func, ast.Name) + and deco.func.id == "dataclass" + ): + dataclass_files.append((file_path, node.name)) + except (SyntaxError, UnicodeDecodeError) as e: + print(f"Error parsing {file_path}: {e}") + + return dataclass_files + + +def load_dataclass_from_file( + file_path: str, class_name: str, root_dir: str +) -> type | None: + """ + Load a dataclass from a given file path. + + Args: + file_path (str): Path to the file. + class_name (str): Name of the class to load. + + Returns: + List[Type]: The dataclass type if found, else an empty list. + """ + module_name = ( + os.path.relpath(file_path, root_dir).replace(os.path.sep, ".").rstrip(".py") + ) + try: + sys.path.insert(0, root_dir) + spec = importlib.util.spec_from_file_location(module_name, file_path) + if not spec: + raise ClanError(f"Could not load spec from file: {file_path}") + + module = importlib.util.module_from_spec(spec) + if not module: + raise ClanError(f"Could not create module: {file_path}") + + if not spec.loader: + raise ClanError(f"Could not load loader from spec: {spec}") + + spec.loader.exec_module(module) + + finally: + sys.path.pop(0) + dataclass_type = getattr(module, class_name, None) + + if dataclass_type and is_dataclass(dataclass_type): + return dataclass_type + + raise ClanError(f"Could not load dataclass {class_name} from file: {file_path}") + + +def test_all_dataclasses() -> None: + """ + This Test ensures that all dataclasses are compatible with the API. + + It will load all dataclasses from the clan_cli directory and + generate a JSON schema for each of them. + + It will fail if any dataclass cannot be converted to JSON schema. + This means the dataclass in its current form is not compatible with the API. + """ + + # Excludes: + # - API includes Type Generic wrappers, that are not known in the init file. + excludes = ["api/__init__.py"] + + cli_path = Path("clan_cli").resolve() + dataclasses = find_dataclasses_in_directory(cli_path, excludes) + + for file, dataclass in dataclasses: + print(f"checking dataclass {dataclass} in file: {file}") + try: + dclass = load_dataclass_from_file(file, dataclass, str(cli_path.parent)) + type_to_dict(dclass) + except JSchemaTypeError as e: + print(f"Error loading dataclass {dataclass} from {file}: {e}") + raise ClanError( + f""" +-------------------------------------------------------------------------------- +Error converting dataclass 'class {dataclass}()' from {file} + +Details: + {e} + +Help: +- Converting public fields to PRIVATE by prefixing them with underscore ('_') +- Ensure all private fields are initialized the API wont provide initial values for them. +-------------------------------------------------------------------------------- +""", + location=__file__, + )