From df934334a207d396e8b5422f7e37314e8fb0a345 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Mon, 1 Jul 2024 21:55:42 +0200 Subject: [PATCH 1/4] API: migrate add machine to inventory --- pkgs/clan-cli/clan_cli/machines/create.py | 79 +++++++++++++++++---- pkgs/clan-cli/tests/test_machines_config.py | 14 +++- 2 files changed, 78 insertions(+), 15 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index ae04690f..5166331d 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -1,30 +1,85 @@ import argparse +import json import logging -from dataclasses import dataclass +import re from pathlib import Path -from typing import Any from clan_cli.api import API -from clan_cli.config.machine import set_config_for_machine +from clan_cli.errors import ClanError +from clan_cli.git import commit_file +from clan_cli.inventory import Inventory, Machine, dataclass_to_dict log = logging.getLogger(__name__) -@dataclass -class MachineCreateRequest: - name: str - config: dict[str, Any] - - @API.register -def create_machine(flake_dir: str | Path, machine: MachineCreateRequest) -> None: - set_config_for_machine(Path(flake_dir), machine.name, machine.config) +def create_machine(flake_dir: str | Path, machine: Machine) -> None: + hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(? None: - create_machine(args.flake, MachineCreateRequest(args.machine, dict())) + create_machine( + args.flake, + Machine( + name=args.machine, + system=args.system, + description=args.description, + tags=args.tags, + icon=args.icon, + ), + ) def register_create_parser(parser: argparse.ArgumentParser) -> None: parser.add_argument("machine", type=str) parser.set_defaults(func=create_command) + + parser.add_argument( + "--system", + type=str, + default=None, + help="Host platform to use. i.e. 'x86_64-linux' or 'aarch64-darwin' etc.", + metavar="PLATFORM", + ) + parser.add_argument( + "--description", + type=str, + default=None, + help="A description of the machine.", + ) + parser.add_argument( + "--icon", + type=str, + default=None, + help="Path to an icon to use for the machine. - Must be a path to icon file relative to the flake directory, or a public url.", + metavar="PATH", + ) + parser.add_argument( + "--tags", + nargs="+", + default=[], + help="Tags to associate with the machine. Can be used to assign multiple machines to services.", + ) diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index 105f2a0a..43adb896 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -7,7 +7,8 @@ from clan_cli.config.machine import ( verify_machine_config, ) from clan_cli.config.schema import machine_schema -from clan_cli.machines.create import MachineCreateRequest, create_machine +from clan_cli.inventory import Machine +from clan_cli.machines.create import create_machine from clan_cli.machines.list import list_machines @@ -22,14 +23,21 @@ def test_create_machine_on_minimal_clan(test_flake_minimal: FlakeForTest) -> Non assert list_machines(test_flake_minimal.path) == [] create_machine( test_flake_minimal.path, - MachineCreateRequest( - name="foo", config=dict(nixpkgs=dict(hostSystem="x86_64-linux")) + Machine( + name="foo", + system="x86_64-linux", + description="A test machine", + tags=["test"], + icon=None, ), ) assert list_machines(test_flake_minimal.path) == ["foo"] + + # Writes into settings.json set_config_for_machine( test_flake_minimal.path, "foo", dict(services=dict(openssh=dict(enable=True))) ) + config = config_for_machine(test_flake_minimal.path, "foo") assert config["services"]["openssh"]["enable"] assert verify_machine_config(test_flake_minimal.path, "foo") is None From 9f484c1d39067a901bdfb8ffceacc16d0bcaecbc Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 2 Jul 2024 11:07:11 +0200 Subject: [PATCH 2/4] API: migrate machines delete and list to inventory --- inventory.json | 2 +- .../README.md | 0 .../settings.json | 0 pkgs/clan-cli/clan_cli/inventory/__init__.py | 24 +++++++++++++ pkgs/clan-cli/clan_cli/machines/create.py | 23 +++--------- pkgs/clan-cli/clan_cli/machines/delete.py | 25 ++++++++++--- pkgs/clan-cli/clan_cli/machines/list.py | 16 ++++----- .../app/src/components/MachineListItem.tsx | 36 +++++-------------- .../app/src/routes/machines/view.tsx | 17 ++++++--- pkgs/webview-ui/app/tests/types.test.ts | 5 +-- 10 files changed, 79 insertions(+), 69 deletions(-) rename machines/{minimal_inventory_machine => minimal-inventory-machine}/README.md (100%) rename machines/{minimal_inventory_machine => minimal-inventory-machine}/settings.json (100%) diff --git a/inventory.json b/inventory.json index e1503adc..7300779f 100644 --- a/inventory.json +++ b/inventory.json @@ -1,6 +1,6 @@ { "machines": { - "minimal_inventory_machine": { + "minimal-inventory-machine": { "name": "foo", "system": "x86_64-linux", "description": "A nice thing", diff --git a/machines/minimal_inventory_machine/README.md b/machines/minimal-inventory-machine/README.md similarity index 100% rename from machines/minimal_inventory_machine/README.md rename to machines/minimal-inventory-machine/README.md diff --git a/machines/minimal_inventory_machine/settings.json b/machines/minimal-inventory-machine/settings.json similarity index 100% rename from machines/minimal_inventory_machine/settings.json rename to machines/minimal-inventory-machine/settings.json diff --git a/pkgs/clan-cli/clan_cli/inventory/__init__.py b/pkgs/clan-cli/clan_cli/inventory/__init__.py index 98c231a2..7215095d 100644 --- a/pkgs/clan-cli/clan_cli/inventory/__init__.py +++ b/pkgs/clan-cli/clan_cli/inventory/__init__.py @@ -1,3 +1,4 @@ +import json import re from dataclasses import asdict, dataclass, is_dataclass from pathlib import Path @@ -136,3 +137,26 @@ class Inventory: for name, services in d["services"].items() }, ) + + @staticmethod + def get_path(flake_dir: str | Path) -> Path: + return Path(flake_dir) / "inventory.json" + + @staticmethod + def load_file(flake_dir: str | Path) -> "Inventory": + inventory = Inventory(machines={}, services={}) + inventory_file = Inventory.get_path(flake_dir) + if inventory_file.exists(): + with open(inventory_file) as f: + try: + res = json.load(f) + inventory = Inventory.from_dict(res) + except json.JSONDecodeError as e: + raise ClanError(f"Error decoding inventory file: {e}") + + return inventory + + def persist(self, flake_dir: str | Path) -> None: + inventory_file = Inventory.get_path(flake_dir) + with open(inventory_file, "w") as f: + json.dump(dataclass_to_dict(self), f, indent=2) diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index 5166331d..5a13ad1a 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -1,5 +1,4 @@ import argparse -import json import logging import re from pathlib import Path @@ -7,7 +6,7 @@ from pathlib import Path from clan_cli.api import API from clan_cli.errors import ClanError from clan_cli.git import commit_file -from clan_cli.inventory import Inventory, Machine, dataclass_to_dict +from clan_cli.inventory import Inventory, Machine log = logging.getLogger(__name__) @@ -18,26 +17,12 @@ def create_machine(flake_dir: str | Path, machine: Machine) -> None: if not re.match(hostname_regex, machine.name): raise ClanError("Machine name must be a valid hostname") - inventory = Inventory(machines={}, services={}) - - inventory_file = Path(flake_dir) / "inventory.json" - if inventory_file.exists(): - with open(inventory_file) as f: - try: - res = json.load(f) - inventory = Inventory.from_dict(res) - - except json.JSONDecodeError as e: - raise ClanError(f"Error decoding inventory file: {e}") - + inventory = Inventory.load_file(flake_dir) inventory.machines.update({machine.name: machine}) - - with open(inventory_file, "w") as g: - d = dataclass_to_dict(inventory) - json.dump(d, g, indent=2) + inventory.persist(flake_dir) if flake_dir is not None: - commit_file(inventory_file, Path(flake_dir)) + commit_file(Inventory.get_path(flake_dir), Path(flake_dir)) def create_command(args: argparse.Namespace) -> None: diff --git a/pkgs/clan-cli/clan_cli/machines/delete.py b/pkgs/clan-cli/clan_cli/machines/delete.py index 9291f957..fcf53a73 100644 --- a/pkgs/clan-cli/clan_cli/machines/delete.py +++ b/pkgs/clan-cli/clan_cli/machines/delete.py @@ -1,21 +1,36 @@ import argparse import shutil +from pathlib import Path + +from clan_cli.api import API +from clan_cli.inventory import Inventory from ..completions import add_dynamic_completer, complete_machines from ..dirs import specific_machine_dir from ..errors import ClanError -def delete_command(args: argparse.Namespace) -> None: - folder = specific_machine_dir(args.flake, args.host) +@API.register +def delete_machine(base_dir: str | Path, name: str) -> None: + inventory = Inventory.load_file(base_dir) + + machine = inventory.machines.pop(name, None) + if machine is None: + raise ClanError(f"Machine {name} does not exist") + + inventory.persist(base_dir) + + folder = specific_machine_dir(Path(base_dir), name) if folder.exists(): shutil.rmtree(folder) - else: - raise ClanError(f"Machine {args.host} does not exist") + + +def delete_command(args: argparse.Namespace) -> None: + delete_machine(args.flake, args.name) def register_delete_parser(parser: argparse.ArgumentParser) -> None: - machines_parser = parser.add_argument("host", type=str) + machines_parser = parser.add_argument("name", type=str) add_dynamic_completer(machines_parser, complete_machines) parser.set_defaults(func=delete_command) diff --git a/pkgs/clan-cli/clan_cli/machines/list.py b/pkgs/clan-cli/clan_cli/machines/list.py index dd71bc7d..83dbf7d7 100644 --- a/pkgs/clan-cli/clan_cli/machines/list.py +++ b/pkgs/clan-cli/clan_cli/machines/list.py @@ -4,22 +4,19 @@ import logging from pathlib import Path from clan_cli.api import API +from clan_cli.inventory import Machine from ..cmd import run_no_stdout -from ..nix import nix_config, nix_eval +from ..nix import nix_eval log = logging.getLogger(__name__) @API.register -def list_machines(flake_url: str | Path, debug: bool = False) -> list[str]: - config = nix_config() - system = config["system"] +def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]: cmd = nix_eval( [ - f"{flake_url}#clanInternals.machines.{system}", - "--apply", - "builtins.attrNames", + f"{flake_url}#clanInternals.inventory.machines", "--json", ] ) @@ -27,12 +24,13 @@ def list_machines(flake_url: str | Path, debug: bool = False) -> list[str]: proc = run_no_stdout(cmd) res = proc.stdout.strip() - return json.loads(res) + data = {name: Machine.from_dict(v) for name, v in json.loads(res).items()} + return data def list_command(args: argparse.Namespace) -> None: flake_path = Path(args.flake).resolve() - for name in list_machines(flake_path, args.debug): + for name in list_machines(flake_path, args.debug).keys(): print(name) diff --git a/pkgs/webview-ui/app/src/components/MachineListItem.tsx b/pkgs/webview-ui/app/src/components/MachineListItem.tsx index 78bc4bf0..68503fe9 100644 --- a/pkgs/webview-ui/app/src/components/MachineListItem.tsx +++ b/pkgs/webview-ui/app/src/components/MachineListItem.tsx @@ -1,12 +1,14 @@ -import { For, Match, Show, Switch, createSignal } from "solid-js"; +import { Match, Show, Switch, createSignal } from "solid-js"; import { ErrorData, SuccessData, pyApi } from "../api"; import { currClanURI } from "../App"; +type MachineDetails = SuccessData<"list_machines">["data"][string]; + interface MachineListItemProps { name: string; + info: MachineDetails; } -type MachineDetails = Record["data"]>; type HWInfo = Record["data"]>; type DeploymentInfo = Record< string, @@ -15,26 +17,12 @@ type DeploymentInfo = Record< type MachineErrors = Record["errors"]>; -const [details, setDetails] = createSignal({}); const [hwInfo, setHwInfo] = createSignal({}); const [deploymentInfo, setDeploymentInfo] = createSignal({}); const [errors, setErrors] = createSignal({}); -pyApi.show_machine.receive((r) => { - if (r.status === "error") { - const { op_key } = r; - if (op_key) { - setErrors((e) => ({ ...e, [op_key]: r.errors })); - } - console.error(r.errors); - } - if (r.status === "success") { - setDetails((d) => ({ ...d, [r.data.machine_name]: r.data })); - } -}); - pyApi.show_machine_hardware_info.receive((r) => { const { op_key } = r; if (r.status === "error") { @@ -64,13 +52,7 @@ pyApi.show_machine_deployment_target.receive((r) => { }); export const MachineListItem = (props: MachineListItemProps) => { - const { name } = props; - - pyApi.show_machine.dispatch({ - op_key: name, - machine_name: name, - flake_url: currClanURI(), - }); + const { name, info } = props; pyApi.show_machine_hardware_info.dispatch({ op_key: name, @@ -97,16 +79,14 @@ export const MachineListItem = (props: MachineListItemProps) => {

{name}

}> - - No description - + No description } > - {(d) => d()?.machine_description} + {(d) => d()?.description}
diff --git a/pkgs/webview-ui/app/src/routes/machines/view.tsx b/pkgs/webview-ui/app/src/routes/machines/view.tsx index 2e6e5b62..1394f5f5 100644 --- a/pkgs/webview-ui/app/src/routes/machines/view.tsx +++ b/pkgs/webview-ui/app/src/routes/machines/view.tsx @@ -23,6 +23,11 @@ type ServiceModel = Extract< { status: "success" } >["data"]["services"]; +type MachinesModel = Extract< + OperationResponse<"list_machines">, + { status: "success" } +>["data"]; + export const MachineListView: Component = () => { const [{ machines, loading }, { getMachines }] = useMachineContext(); @@ -44,11 +49,13 @@ export const MachineListView: Component = () => { console.log(files()); }); - const [data, setData] = createSignal([]); + const [data, setData] = createSignal({}); createEffect(() => { if (route() === "machines") getMachines(); }); + const unpackedMachines = () => Object.entries(data()); + createEffect(() => { const response = machines(); if (response?.status === "success") { @@ -57,7 +64,7 @@ export const MachineListView: Component = () => { toast.success("Machines loaded"); } if (response?.status === "error") { - setData([]); + setData({}); console.error(response.errors); toast.error("Error loading machines"); response.errors.forEach((error) => @@ -162,13 +169,13 @@ export const MachineListView: Component = () => {
- + No machines found
    - - {(entry) => } + + {([name, info]) => }
diff --git a/pkgs/webview-ui/app/tests/types.test.ts b/pkgs/webview-ui/app/tests/types.test.ts index 242d0c6a..f25455e3 100644 --- a/pkgs/webview-ui/app/tests/types.test.ts +++ b/pkgs/webview-ui/app/tests/types.test.ts @@ -28,12 +28,13 @@ describe.concurrent("API types work properly", () => { >(); }); - it("Machine list receives a list of names/id string", async () => { + it("Machine list receives a records of names and machine info.", async () => { expectTypeOf(pyApi.list_machines.receive) .parameter(0) .parameter(0) .toMatchTypeOf< - { status: "success"; data: string[] } | { status: "error"; errors: any } + | { status: "success"; data: Record } + | { status: "error"; errors: any } >(); }); From f7c80834cb14248616bae0b1255f3a7838cb3ff2 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 2 Jul 2024 11:16:54 +0200 Subject: [PATCH 3/4] Inventory persistence improves error resistance --- pkgs/clan-cli/clan_cli/inventory/__init__.py | 31 +++----------------- pkgs/clan-cli/clan_cli/machines/create.py | 7 +++-- 2 files changed, 8 insertions(+), 30 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/inventory/__init__.py b/pkgs/clan-cli/clan_cli/inventory/__init__.py index 7215095d..8141e21f 100644 --- a/pkgs/clan-cli/clan_cli/inventory/__init__.py +++ b/pkgs/clan-cli/clan_cli/inventory/__init__.py @@ -1,5 +1,4 @@ import json -import re from dataclasses import asdict, dataclass, is_dataclass from pathlib import Path from typing import Any, Literal @@ -56,16 +55,6 @@ class Machine: @staticmethod def from_dict(d: dict[str, Any]) -> "Machine": - if "name" not in d: - raise ClanError("name not found in machine") - - hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(? "Service": - if "meta" not in d: - raise ClanError("meta not found in service") - - if "roles" not in d: - raise ClanError("roles not found in service") - return Service( - meta=ServiceMeta(**d["meta"]), - roles={name: Role(**role) for name, role in d["roles"].items()}, + meta=ServiceMeta(**d.get("meta", {})), + roles={name: Role(**role) for name, role in d.get("roles", {}).items()}, machines={ name: MachineServiceConfig(**machine) for name, machine in d.get("machines", {}).items() @@ -118,23 +101,17 @@ class Inventory: @staticmethod def from_dict(d: dict[str, Any]) -> "Inventory": - if "machines" not in d: - raise ClanError("machines not found in inventory") - - if "services" not in d: - raise ClanError("services not found in inventory") - return Inventory( machines={ name: Machine.from_dict(machine) - for name, machine in d["machines"].items() + for name, machine in d.get("machines", {}).items() }, services={ name: { role: Service.from_dict(service) for role, service in services.items() } - for name, services in d["services"].items() + for name, services in d.get("services", {}).items() }, ) diff --git a/pkgs/clan-cli/clan_cli/machines/create.py b/pkgs/clan-cli/clan_cli/machines/create.py index 5a13ad1a..adcf6382 100644 --- a/pkgs/clan-cli/clan_cli/machines/create.py +++ b/pkgs/clan-cli/clan_cli/machines/create.py @@ -15,14 +15,15 @@ log = logging.getLogger(__name__) def create_machine(flake_dir: str | Path, machine: Machine) -> None: hostname_regex = r"^(?!-)[A-Za-z0-9-]{1,63}(? None: From 1b7369cf0df1b1e2b71a7c48c79f37bb117445d5 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 2 Jul 2024 11:21:52 +0200 Subject: [PATCH 4/4] Fix test --- pkgs/clan-cli/tests/test_machines_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index 43adb896..3548a65a 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -20,7 +20,7 @@ def test_schema_for_machine(test_flake_with_core: FlakeForTest) -> None: @pytest.mark.with_core def test_create_machine_on_minimal_clan(test_flake_minimal: FlakeForTest) -> None: - assert list_machines(test_flake_minimal.path) == [] + assert list_machines(test_flake_minimal.path) == {} create_machine( test_flake_minimal.path, Machine( @@ -31,7 +31,7 @@ def test_create_machine_on_minimal_clan(test_flake_minimal: FlakeForTest) -> Non icon=None, ), ) - assert list_machines(test_flake_minimal.path) == ["foo"] + assert list(list_machines(test_flake_minimal.path).keys()) == ["foo"] # Writes into settings.json set_config_for_machine(