Merge pull request 'Inventory improvements' (#1795) from hsjobeki/clan-core:hsjobeki-main into main

lib/build-clan/default.nix

@@ -58,11 +58,11 @@ let
         {
           # { ${name} :: meta // { name, tags } }
           machines = lib.mapAttrs (
-            name: config:
+            name: machineConfig:
             (lib.attrByPath [
               "clan"
               "meta"
-            ] { } config)
+            ] { } machineConfig)
             // {
               # meta.name default is the attribute name of the machine
               name = lib.mkDefault (
@@ -70,11 +70,11 @@ let
                   "clan"
                   "meta"
                   "name"
-                ] name config
+                ] name machineConfig
               );
             }
             # tags
-            // (clanToInventory config {
+            // (clanToInventory machineConfig {
               clanPath = [
                 "clan"
                 "tags"
@@ -82,15 +82,15 @@ let
               inventoryPath = [ "tags" ];
             })
             # system
-            // (clanToInventory config {
+            // (clanToInventory machineConfig {
               clanPath = [
                 "nixpkgs"
-                "hostSystem"
+                "hostPlatform"
               ];
               inventoryPath = [ "system" ];
             })
             # deploy.targetHost
-            // (clanToInventory config {
+            // (clanToInventory machineConfig {
               clanPath = [
                 "clan"
                 "core"

pkgs/clan-cli/clan_cli/api/modules.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 from clan_cli.cmd import run_no_stdout
 from clan_cli.errors import ClanCmdError, ClanError
-from clan_cli.inventory import Inventory, load_inventory
+from clan_cli.inventory import Inventory, load_inventory_json
 from clan_cli.nix import nix_eval
 
 from . import API
@@ -152,4 +152,4 @@ def get_module_info(
 
 @API.register
 def get_inventory(base_path: str) -> Inventory:
-    return load_inventory(base_path)
+    return load_inventory_json(base_path)

pkgs/clan-cli/clan_cli/clan/create.py

@@ -1,12 +1,11 @@
 # !/usr/bin/env python3
 import argparse
 import os
-from dataclasses import dataclass, fields
+from dataclasses import dataclass
 from pathlib import Path
 
 from clan_cli.api import API
-from clan_cli.arg_actions import AppendOptionAction
-from clan_cli.inventory import Meta, load_inventory, save_inventory
+from clan_cli.inventory import Inventory, init_inventory
 
 from ..cmd import CmdOut, run
 from ..errors import ClanError
@@ -29,11 +28,9 @@ class CreateClanResponse:
 @dataclass
 class CreateOptions:
     directory: Path | str
-    # Metadata for the clan
-    # Metadata can be shown with `clan show`
-    meta: Meta | None = None
     # URL to the template to use. Defaults to the "minimal" template
     template_url: str = minimal_template_url
+    initial: Inventory | None = None
 
 
 def git_command(directory: Path, *args: str) -> list[str]:
@@ -88,17 +85,13 @@ def create_clan(options: CreateOptions) -> CreateClanResponse:
             git_command(directory, "config", "user.email", "clan@example.com")
         )
 
-    # Write inventory.json file
-    inventory = load_inventory(directory)
-    if options.meta is not None:
-        inventory.meta = options.meta
-    # Persist creates a commit message for each change
-    save_inventory(inventory, directory, "Init inventory")
-
     flake_update = run(
         nix_shell(["nixpkgs#nix"], ["nix", "flake", "update"]), cwd=directory
     )
 
+    if options.initial:
+        init_inventory(options.directory, init=options.initial)
+
     response = CreateClanResponse(
         flake_init=flake_init,
         git_init=git_init,
@@ -118,15 +111,6 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
         default=default_template_url,
     )
 
-    parser.add_argument(
-        "--meta",
-        help=f"""Metadata to set for the clan. Available options are: {", ".join([f.name for f in fields(Meta)]) }""",
-        nargs=2,
-        metavar=("name", "value"),
-        action=AppendOptionAction,
-        default=[],
-    )
-
     parser.add_argument(
         "path", type=Path, help="Path to the clan directory", default=Path(".")
     )

pkgs/clan-cli/clan_cli/clan/update.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
 from clan_cli.api import API
-from clan_cli.inventory import Meta, load_inventory, save_inventory
+from clan_cli.inventory import Meta, load_inventory_json, save_inventory
 
 
 @dataclass
@@ -12,7 +12,7 @@ class UpdateOptions:
 
 @API.register
 def update_clan_meta(options: UpdateOptions) -> Meta:
-    inventory = load_inventory(options.directory)
+    inventory = load_inventory_json(options.directory)
     inventory.meta = options.meta
 
     save_inventory(inventory, options.directory, "Update clan metadata")

pkgs/clan-cli/clan_cli/clan_uri.py

@@ -9,32 +9,49 @@ from .errors import ClanError
 
 @dataclass
 class FlakeId:
-    # FIXME: this is such a footgun if you accidnetally pass a string
-    _value: str | Path
+    loc: str | Path
 
     def __post_init__(self) -> None:
         assert isinstance(
-            self._value, str | Path
-        ), f"Flake {self._value} has an invalid type: {type(self._value)}"
+            self.loc, str | Path
+        ), f"Flake {self.loc} has an invalid format: {type(self.loc)}"
 
     def __str__(self) -> str:
-        return str(self._value)
+        return str(self.loc)
 
     @property
     def path(self) -> Path:
-        assert isinstance(self._value, Path), f"Flake {self._value} is not a local path"
-        return self._value
+        assert self.is_local(), f"Flake {self.loc} is not a local path"
+        return Path(self.loc)
 
     @property
     def url(self) -> str:
-        assert isinstance(self._value, str), f"Flake {self._value} is not a remote url"
-        return self._value
+        assert self.is_remote(), f"Flake {self.loc} is not a remote url"
+        return str(self.loc)
 
     def is_local(self) -> bool:
-        return isinstance(self._value, Path)
+        """
+        https://nix.dev/manual/nix/2.22/language/builtins.html?highlight=urlS#source-types
+
+        Examples:
+
+        - file:///home/eelco/nix/README.md          file        LOCAL
+        - git+file://git:github.com:NixOS/nixpkgs   git+file    LOCAL
+        - https://example.com/index.html            https       REMOTE
+        - github:nixos/nixpkgs                      github      REMOTE
+        - ftp://serv.file                           ftp         REMOTE
+        - ./.                                       ''          LOCAL
+
+        """
+        x = urllib.parse.urlparse(str(self.loc))
+        if x.scheme == "" or "file" in x.scheme:
+            # See above *file* or empty are the only local schemas
+            return True
+
+        return False
 
     def is_remote(self) -> bool:
-        return isinstance(self._value, str)
+        return not self.is_local()
 
 
 # Define the ClanURI class

pkgs/clan-cli/clan_cli/git.py

@@ -93,6 +93,7 @@ def _commit_file_to_git(
                 "commit",
                 "-m",
                 commit_message,
+                "--no-verify",  # dont run pre-commit hooks
             ]
             + [str(file_path) for file_path in file_paths],
         )

pkgs/clan-cli/clan_cli/inventory/__init__.py

@@ -1,3 +1,17 @@
+"""
+All read/write operations MUST use the inventory.
+
+Machine data, clan data or service data can be accessed in a performant way.
+
+This file exports stable classnames for static & dynamic type safety.
+
+Utilize:
+
+- load_inventory_eval: To load the actual inventory with nix declarations merged.
+Operate on the returned inventory to make changes
+- save_inventory: To persist changes.
+"""
+
 import dataclasses
 import json
 from dataclasses import fields, is_dataclass
@@ -5,9 +19,12 @@ from pathlib import Path
 from types import UnionType
 from typing import Any, get_args, get_origin
 
-from clan_cli.errors import ClanError
+from clan_cli.api import API
+from clan_cli.errors import ClanCmdError, ClanError
 from clan_cli.git import commit_file
 
+from ..cmd import run_no_stdout
+from ..nix import nix_eval
 from .classes import (
     Inventory,
     Machine,
@@ -165,14 +182,42 @@ default_inventory = Inventory(
 )
 
 
-def load_inventory(
+def load_inventory_eval(flake_dir: str | Path) -> Inventory:
+    """
+    Loads the actual inventory.
+    After all merge operations with eventual nix code in buildClan.
+
+    Evaluates clanInternals.inventory with nix. Which is performant.
+
+    - Contains all clan metadata
+    - Contains all machines
+    - and more
+    """
+    cmd = nix_eval(
+        [
+            f"{flake_dir}#clanInternals.inventory",
+            "--json",
+        ]
+    )
+    proc = run_no_stdout(cmd)
+
+    try:
+        res = proc.stdout.strip()
+        data = json.loads(res)
+        inventory = from_dict(Inventory, data)
+        return inventory
+    except json.JSONDecodeError as e:
+        raise ClanError(f"Error decoding inventory from flake: {e}")
+
+
+def load_inventory_json(
     flake_dir: str | Path, default: Inventory = default_inventory
 ) -> Inventory:
     """
     Load the inventory file from the flake directory
     If not file is found, returns the default inventory
     """
-    inventory = default_inventory
+    inventory = default
 
     inventory_file = get_path(flake_dir)
     if inventory_file.exists():
@@ -184,6 +229,10 @@ def load_inventory(
                 # Error decoding the inventory file
                 raise ClanError(f"Error decoding inventory file: {e}")
 
+    if not inventory_file.exists():
+        # Copy over the meta from the flake if the inventory is not initialized
+        inventory.meta = load_inventory_eval(flake_dir).meta
+
     return inventory
 
 
@@ -198,3 +247,22 @@ def save_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
         json.dump(dataclass_to_dict(inventory), f, indent=2)
 
     commit_file(inventory_file, Path(flake_dir), commit_message=message)
+
+
+@API.register
+def init_inventory(directory: str, init: Inventory | None = None) -> None:
+    inventory = None
+    # Try reading the current flake
+    if init is None:
+        try:
+            inventory = load_inventory_eval(directory)
+        except ClanCmdError:
+            pass
+
+    if init is not None:
+        inventory = init
+
+    # Write inventory.json file
+    if inventory is not None:
+        # Persist creates a commit message for each change
+        save_inventory(inventory, directory, "Init inventory")

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,13 +1,17 @@
 import argparse
 import logging
 import re
-from pathlib import Path
 
 from ..api import API
 from ..clan_uri import FlakeId
 from ..errors import ClanError
-from ..git import commit_file
-from ..inventory import Machine, MachineDeploy, get_path, load_inventory, save_inventory
+from ..inventory import (
+    Machine,
+    MachineDeploy,
+    load_inventory_eval,
+    load_inventory_json,
+    save_inventory,
+)
 
 log = logging.getLogger(__name__)
 
@@ -20,12 +24,16 @@ def create_machine(flake: FlakeId, machine: Machine) -> None:
             "Machine name must be a valid hostname", location="Create Machine"
         )
 
-    inventory = load_inventory(flake.path)
+    inventory = load_inventory_json(flake.path)
+
+    full_inventory = load_inventory_eval(flake.path)
+
+    if machine.name in full_inventory.machines.keys():
+        raise ClanError(f"Machine with the name {machine.name} already exists")
+
     inventory.machines.update({machine.name: machine})
     save_inventory(inventory, flake.path, f"Create machine {machine.name}")
 
-    commit_file(get_path(flake.path), Path(flake.path))
-
 
 def create_command(args: argparse.Namespace) -> None:
     create_machine(

pkgs/clan-cli/clan_cli/machines/delete.py

@@ -6,12 +6,12 @@ from ..clan_uri import FlakeId
 from ..completions import add_dynamic_completer, complete_machines
 from ..dirs import specific_machine_dir
 from ..errors import ClanError
-from ..inventory import load_inventory, save_inventory
+from ..inventory import load_inventory_json, save_inventory
 
 
 @API.register
 def delete_machine(flake: FlakeId, name: str) -> None:
-    inventory = load_inventory(flake.path)
+    inventory = load_inventory_json(flake.path)
 
     machine = inventory.machines.pop(name, None)
     if machine is None:

pkgs/clan-cli/clan_cli/machines/list.py

@@ -1,31 +1,17 @@
 import argparse
-import json
 import logging
 from pathlib import Path
 
 from clan_cli.api import API
-from clan_cli.inventory import Machine, from_dict
-
-from ..cmd import run_no_stdout
-from ..nix import nix_eval
+from clan_cli.inventory import Machine, load_inventory_eval
 
 log = logging.getLogger(__name__)
 
 
 @API.register
 def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]:
-    cmd = nix_eval(
-        [
-            f"{flake_url}#clanInternals.inventory.machines",
-            "--json",
-        ]
-    )
-
-    proc = run_no_stdout(cmd)
-
-    res = proc.stdout.strip()
-    data = {name: from_dict(Machine, v) for name, v in json.loads(res).items()}
-    return data
+    inventory = load_inventory_eval(flake_url)
+    return inventory.machines
 
 
 def list_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/tests/test_create_flake.py

@@ -20,12 +20,12 @@ def test_create_flake(
     cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
 
     assert (flake_dir / ".clan-flake").exists()
-
     # Replace the inputs.clan.url in the template flake.nix
     substitute(
         flake_dir / "flake.nix",
         clan_core,
     )
+    # Dont evaluate the inventory before the substitute call
 
     monkeypatch.chdir(flake_dir)
     cli.run(["machines", "create", "machine1"])

pkgs/clan-cli/tests/test_modules.py

@@ -14,7 +14,7 @@ from clan_cli.inventory import (
     ServiceBorgbackupRoleClient,
     ServiceBorgbackupRoleServer,
     ServiceMeta,
-    load_inventory,
+    load_inventory_json,
     save_inventory,
 )
 from clan_cli.machines.create import create_machine
@@ -67,7 +67,7 @@ def test_add_module_to_inventory(
         ),
     )
 
-    inventory = load_inventory(base_path)
+    inventory = load_inventory_json(base_path)
 
     inventory.services.borgbackup = {
         "borg1": ServiceBorgbackup(

pkgs/installer/flake-module.nix

@@ -32,7 +32,7 @@ let
     }
     // flashDiskoConfig;
 
-  # Important: The partition names need to be different to the clan install 
+  # Important: The partition names need to be different to the clan install
   flashDiskoConfig = {
     boot.loader.grub.efiSupport = lib.mkDefault true;
     boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;

pkgs/webview-ui/.gitignore

@@ -1 +1,3 @@
-api
+api
+
+.vite

pkgs/webview-ui/app/src/Routes.tsx

@@ -8,6 +8,7 @@ import { Flash } from "./routes/flash/view";
 import { Settings } from "./routes/settings";
 import { Welcome } from "./routes/welcome";
 import { Deploy } from "./routes/deploy";
+import { CreateMachine } from "./routes/machines/create";
 
 export type Route = keyof typeof routes;
 
@@ -22,6 +23,11 @@ export const routes = {
     label: "Machines",
     icon: "devices_other",
   },
+  "machines/add": {
+    child: CreateMachine,
+    label: "create Machine",
+    icon: "add",
+  },
   hosts: {
     child: HostList,
     label: "hosts",

pkgs/webview-ui/app/src/api.ts

@@ -67,6 +67,17 @@ function createFunctions<K extends OperationNames>(
   dispatch: (args: OperationArgs<K>) => void;
   receive: (fn: (response: OperationResponse<K>) => void, id: string) => void;
 } {
+  window.clan[operationName] = (s: string) => {
+    const f = (response: OperationResponse<K>) => {
+      // Get the correct receiver function for the op_key
+      const receiver = registry[operationName][response.op_key];
+      if (receiver) {
+        receiver(response);
+      }
+    };
+    deserialize(f)(s);
+  };
+
   return {
     dispatch: (args: OperationArgs<K>) => {
       // Send the data to the gtk app
@@ -78,15 +89,6 @@ function createFunctions<K extends OperationNames>(
     receive: (fn: (response: OperationResponse<K>) => void, id: string) => {
       // @ts-expect-error: This should work although typescript doesn't let us write
       registry[operationName][id] = fn;
-
-      window.clan[operationName] = (s: string) => {
-        const f = (response: OperationResponse<K>) => {
-          if (response.op_key === id) {
-            registry[operationName][id](response);
-          }
-        };
-        deserialize(f)(s);
-      };
     },
   };
 }

pkgs/webview-ui/app/src/index.css

@@ -4,7 +4,6 @@
 @tailwind components;
 @tailwind utilities;
 
-
 html {
     overflow-x: hidden;
     overflow-y: scroll;

pkgs/webview-ui/app/src/routes/clan/clanDetails.tsx

@@ -42,7 +42,15 @@ export const ClanForm = () => {
     await toast.promise(
       (async () => {
         await callApi("create_clan", {
-          options: { directory: target_dir, meta, template_url },
+          options: {
+            directory: target_dir,
+            template_url,
+            initial: {
+              meta,
+              services: {},
+              machines: {},
+            },
+          },
         });
         setActiveURI(target_dir);
         setRoute("machines");

pkgs/webview-ui/app/src/routes/machines/create.tsx

@@ -0,0 +1,124 @@
+import { callApi, OperationArgs, pyApi } from "@/src/api";
+import { activeURI } from "@/src/App";
+import { createForm, required } from "@modular-forms/solid";
+import toast from "solid-toast";
+
+type CreateMachineForm = OperationArgs<"create_machine">;
+
+export function CreateMachine() {
+  const [formStore, { Form, Field }] = createForm<CreateMachineForm>({});
+
+  const handleSubmit = async (values: CreateMachineForm) => {
+    const active_dir = activeURI();
+    if (!active_dir) {
+      toast.error("Open a clan to create the machine in");
+      return;
+    }
+
+    callApi("create_machine", {
+      flake: {
+        loc: active_dir,
+      },
+      machine: {
+        name: "jon",
+        deploy: {
+          targetHost: null,
+        },
+      },
+    });
+    console.log("submit", values);
+  };
+  return (
+    <div class="px-1">
+      Create new Machine
+      <Form onSubmit={handleSubmit}>
+        <Field
+          name="machine.name"
+          validate={[required("This field is required")]}
+        >
+          {(field, props) => (
+            <>
+              <label class="input input-bordered flex items-center gap-2">
+                <input
+                  type="text"
+                  class="grow"
+                  placeholder="name"
+                  required
+                  {...props}
+                />
+              </label>
+              <div class="label">
+                {field.error && (
+                  <span class="label-text-alt font-bold text-error">
+                    {field.error}
+                  </span>
+                )}
+              </div>
+            </>
+          )}
+        </Field>
+        <Field name="machine.description">
+          {(field, props) => (
+            <>
+              <label class="input input-bordered flex items-center gap-2">
+                <input
+                  type="text"
+                  class="grow"
+                  placeholder="description"
+                  required
+                  {...props}
+                />
+              </label>
+              <div class="label">
+                {field.error && (
+                  <span class="label-text-alt font-bold text-error">
+                    {field.error}
+                  </span>
+                )}
+              </div>
+            </>
+          )}
+        </Field>
+        <Field name="machine.deploy.targetHost">
+          {(field, props) => (
+            <>
+              <label class="input input-bordered flex items-center gap-2">
+                <input
+                  type="text"
+                  class="grow"
+                  placeholder="root@flash-installer.local"
+                  required
+                  {...props}
+                />
+              </label>
+              <div class="label">
+                <span class="label-text-alt text-neutral">
+                  Must be set before deployment for the following tasks:
+                  <ul>
+                    <li>
+                      <span>Detect hardware config</span>
+                    </li>
+                    <li>
+                      <span>Detect disk layout</span>
+                    </li>
+                    <li>
+                      <span>Remote installation</span>
+                    </li>
+                  </ul>
+                </span>
+                {field.error && (
+                  <span class="label-text-alt font-bold text-error">
+                    {field.error}
+                  </span>
+                )}
+              </div>
+            </>
+          )}
+        </Field>
+        <button class="btn btn-error float-right" type="submit">
+          <span class="material-icons">add</span>Create
+        </button>
+      </Form>
+    </div>
+  );
+}

pkgs/webview-ui/app/src/routes/machines/view.tsx

@@ -7,7 +7,7 @@ import {
   createSignal,
   type Component,
 } from "solid-js";
-import { activeURI, route, setActiveURI } from "@/src/App";
+import { activeURI, route, setActiveURI, setRoute } from "@/src/App";
 import { OperationResponse, callApi, pyApi } from "@/src/api";
 import toast from "solid-toast";
 import { MachineListItem } from "@/src/components/MachineListItem";
@@ -86,6 +86,11 @@ export const MachineListView: Component = () => {
           <span class="material-icons ">refresh</span>
         </button>
       </div>
+      <div class="tooltip tooltip-bottom" data-tip="Create machine">
+        <button class="btn btn-ghost" onClick={() => setRoute("machines/add")}>
+          <span class="material-icons ">add</span>
+        </button>
+      </div>
       {/* <Show when={services()}>
         {(services) => (
           <For each={Object.values(services())}>

templates/minimal/inventory.json

@@ -0,0 +1,5 @@
+{
+  "meta": { "name": "__CHANGE_ME__" },
+  "machines": {},
+  "services": {}
+}