Merge pull request 'API: extensions & tests' (#1699) from hsjobeki/clan-core:hsjobeki-main into main

docs/mkdocs.yml

@@ -14,6 +14,7 @@ markdown_extensions:
   - attr_list
   - footnotes
   - md_in_html
+  - def_list
   - meta
   - plantuml_markdown
   - pymdownx.emoji:
@@ -49,6 +50,8 @@ nav:
       - Mesh VPN: getting-started/mesh-vpn.md
       - Backup & Restore: getting-started/backups.md
       - Flake-parts: getting-started/flake-parts.md
+  - Concepts:
+      - Configuration: concepts/configuration.md
   - Reference:
       - Clan Modules:
           - reference/clanModules/borgbackup-static.md

docs/site/concepts/configuration.md

@@ -0,0 +1,114 @@
+# Configuration
+
+## Introduction
+
+When managing machine configuration this can be done through many possible ways.
+Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming).
+
+clan currently offers the following methods to configure machines:
+
+!!! Success "Recommended for nix people"
+
+    - flake.nix (i.e. via `buildClan`)
+        - `machine` argument
+        - `inventory` argument
+
+    - machines/`machine_name`/configuration.nix (`autoincluded` if it exists)
+
+???+ Note "Used by CLI & UI"
+
+    - inventory.json
+    - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists)
+
+
+!!! Warning "Deprecated"
+
+    machines/`machine_name`/settings.json
+
+## BuildClan
+
+The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services.
+
+### Inputs
+
+`directory`
+: The directory containing the machines subdirectory
+
+`machines`
+: Allows to include machine-specific modules i.e. machines.${name} = { ... }
+
+`meta`
+: An optional set
+
+: `{ name :: string, icon :: string, description :: string }`
+
+`inventory`
+: Service set for easily configuring distributed services, such as backups
+
+: For more details see [Inventory](#inventory)
+
+`specialArgs`
+: Extra arguments to pass to nixosSystem i.e. useful to make self available
+
+`pkgsForSystem`
+: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system.
+  This improves performance, but all nipxkgs.* options will be ignored.
+  `(string -> pkgs )`
+
+## Inventory
+
+`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries.
+
+The following is the specification of the inventory in `cuelang`
+
+```cue
+{
+    meta: {
+        // A name of the clan (primarily shown by the UI)
+        name: string
+        // A description of the clan
+        description?: string
+        // The icon path
+        icon?: string
+    }
+
+    // A map of services
+    services: [string]: [string]: {
+        // Required meta fields
+        meta: {
+            name: string,
+            icon?: string
+            description?: string,
+        },
+        // Machines are added via the avilable roles
+        // Membership depends only on this field
+        roles: [string]: {
+            machines: [...string],
+            tags: [...string],
+        }
+        machines?: {
+            [string]: {
+                config?: {
+                    ...
+                }
+            }
+        },
+
+        // Global Configuration for the service
+        // Applied to all machines.
+        config?: {
+            // Schema depends on the module.
+            // It declares the interface how the service can be configured.
+            ...
+        }
+    }
+    // A map of machines, extends the machines of `buildClan`
+    machines: [string]: {
+        name: string,
+        description?: string,
+        icon?: string
+        tags: [...string]
+        system: string
+    }
+}
+```

inventory.json

@@ -1,4 +1,7 @@
 {
+  "meta": {
+    "name": "Minimal inventory"
+  },
   "machines": {
     "minimal-inventory-machine": {
       "name": "foo",

lib/build-clan/default.nix

@@ -152,6 +152,13 @@ let
         in
         (machineImports settings)
         ++ [
+          {
+            # Autoinclude configuration.nix and hardware-configuration.nix
+            imports = builtins.filter (p: builtins.pathExists p) [
+              "${directory}/machines/${name}/configuration.nix"
+              "${directory}/machines/${name}/hardware-configuration.nix"
+            ];
+          }
           settings
           clan-core.nixosModules.clanCore
           extraConfig

lib/inventory/spec/schema/schema.cue

@@ -5,6 +5,7 @@ package schema
     description?: string,
     icon?: string
     tags: [...string]
+    system?: string
 }
 
 #role: string

pkgs/clan-cli/clan_cli/api/util.py

@@ -76,6 +76,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) ->
         properties = {
             f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}", type_map)
             for f in fields
+            if not f.name.startswith("_")
         }
 
         required = set()
@@ -127,7 +128,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) ->
         if origin is None:
             # Non-generic user-defined or built-in type
             # TODO: handle custom types
-            raise JSchemaTypeError("Unhandled Type: ", origin)
+            raise JSchemaTypeError(f"{scope} Unhandled Type: ", origin)
 
         elif origin is Literal:
             # Handle Literal values for enums in JSON Schema
@@ -172,7 +173,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) ->
             new_map.update(inspect_dataclass_fields(t))
             return type_to_dict(origin, scope, new_map)
 
-        raise JSchemaTypeError(f"Error api type not yet supported {t!s}")
+        raise JSchemaTypeError(f"{scope} - Error api type not yet supported {t!s}")
 
     elif isinstance(t, type):
         if t is str:
@@ -187,7 +188,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) ->
             return {"type": "object"}
         if t is Any:
             raise JSchemaTypeError(
-                f"Usage of the Any type is not supported for API functions. In: {scope}"
+                f"{scope} - Usage of the Any type is not supported for API functions. In: {scope}"
             )
         if t is pathlib.Path:
             return {
@@ -196,13 +197,13 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) ->
             }
         if t is dict:
             raise JSchemaTypeError(
-                "Error: generic dict type not supported. Use dict[str, Any] instead"
+                f"{scope} - Generic 'dict' type not supported. Use dict[str, Any] or any more expressive type."
             )
 
         # Optional[T] gets internally transformed Union[T,NoneType]
         if t is NoneType:
             return {"type": "null"}
 
-        raise JSchemaTypeError(f"Error primitive type not supported {t!s}")
+        raise JSchemaTypeError(f"{scope} - Error primitive type not supported {t!s}")
     else:
-        raise JSchemaTypeError(f"Error type not supported {t!s}")
+        raise JSchemaTypeError(f"{scope} - Error type not supported {t!s}")

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -20,7 +20,7 @@ class Machine:
     name: str
     flake: FlakeId
     nix_options: list[str] = field(default_factory=list)
-    cached_deployment: None | dict = None
+    cached_deployment: None | dict[str, Any] = None
 
     _eval_cache: dict[str, str] = field(default_factory=dict)
     _build_cache: dict[str, Path] = field(default_factory=dict)

pkgs/clan-cli/tests/test_api_dataclass_compat.py

@@ -0,0 +1,142 @@
+import ast
+import importlib.util
+import os
+import sys
+from dataclasses import is_dataclass
+from pathlib import Path
+
+from clan_cli.api.util import JSchemaTypeError, type_to_dict
+from clan_cli.errors import ClanError
+
+
+def find_dataclasses_in_directory(
+    directory: Path, exclude_paths: list[str] = []
+) -> list[tuple[str, str]]:
+    """
+    Find all dataclass classes in all Python files within a nested directory.
+
+    Args:
+        directory (str): The root directory to start searching from.
+
+    Returns:
+        List[Tuple[str, str]]: A list of tuples containing the file path and the dataclass name.
+    """
+    dataclass_files = []
+
+    excludes = [os.path.join(directory, d) for d in exclude_paths]
+
+    for root, _, files in os.walk(directory, topdown=False):
+        for file in files:
+            if not file.endswith(".py"):
+                continue
+
+            file_path = os.path.join(root, file)
+
+            if file_path in excludes:
+                print(f"Skipping dataclass check for file: {file_path}")
+                continue
+
+            with open(file_path, encoding="utf-8") as f:
+                try:
+                    tree = ast.parse(f.read(), filename=file_path)
+                    for node in ast.walk(tree):
+                        if isinstance(node, ast.ClassDef):
+                            for deco in node.decorator_list:
+                                if (
+                                    isinstance(deco, ast.Name)
+                                    and deco.id == "dataclass"
+                                ):
+                                    dataclass_files.append((file_path, node.name))
+                                elif (
+                                    isinstance(deco, ast.Call)
+                                    and isinstance(deco.func, ast.Name)
+                                    and deco.func.id == "dataclass"
+                                ):
+                                    dataclass_files.append((file_path, node.name))
+                except (SyntaxError, UnicodeDecodeError) as e:
+                    print(f"Error parsing {file_path}: {e}")
+
+    return dataclass_files
+
+
+def load_dataclass_from_file(
+    file_path: str, class_name: str, root_dir: str
+) -> type | None:
+    """
+    Load a dataclass from a given file path.
+
+    Args:
+        file_path (str): Path to the file.
+        class_name (str): Name of the class to load.
+
+    Returns:
+        List[Type]: The dataclass type if found, else an empty list.
+    """
+    module_name = (
+        os.path.relpath(file_path, root_dir).replace(os.path.sep, ".").rstrip(".py")
+    )
+    try:
+        sys.path.insert(0, root_dir)
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        if not spec:
+            raise ClanError(f"Could not load spec from file: {file_path}")
+
+        module = importlib.util.module_from_spec(spec)
+        if not module:
+            raise ClanError(f"Could not create module: {file_path}")
+
+        if not spec.loader:
+            raise ClanError(f"Could not load loader from spec: {spec}")
+
+        spec.loader.exec_module(module)
+
+    finally:
+        sys.path.pop(0)
+    dataclass_type = getattr(module, class_name, None)
+
+    if dataclass_type and is_dataclass(dataclass_type):
+        return dataclass_type
+
+    raise ClanError(f"Could not load dataclass {class_name} from file: {file_path}")
+
+
+def test_all_dataclasses() -> None:
+    """
+    This Test ensures that all dataclasses are compatible with the API.
+
+    It will load all dataclasses from the clan_cli directory and
+    generate a JSON schema for each of them.
+
+    It will fail if any dataclass cannot be converted to JSON schema.
+    This means the dataclass in its current form is not compatible with the API.
+    """
+
+    # Excludes:
+    # - API includes Type Generic wrappers, that are not known in the init file.
+    excludes = ["api/__init__.py"]
+
+    cli_path = Path("clan_cli").resolve()
+    dataclasses = find_dataclasses_in_directory(cli_path, excludes)
+
+    for file, dataclass in dataclasses:
+        print(f"checking dataclass {dataclass} in file: {file}")
+        try:
+            dclass = load_dataclass_from_file(file, dataclass, str(cli_path.parent))
+            type_to_dict(dclass)
+        except JSchemaTypeError as e:
+            print(f"Error loading dataclass {dataclass} from {file}: {e}")
+            raise ClanError(
+                f"""
+--------------------------------------------------------------------------------
+Error converting dataclass 'class {dataclass}()' from {file}
+
+Details:
+ {e}
+
+Help:
+- Converting public fields to PRIVATE by prefixing them with underscore ('_')
+- Ensure all private fields are initialized the API wont provide initial values for them.
+--------------------------------------------------------------------------------
+""",
+                location=__file__,
+            )