DavHau/clan-core
1
0
Fork 0
forked from clan/clan-core
Code Pull Requests Activity

machines.Machine: refactor flake_dir -> flake; use Machine class in vm

Browse Source
This commit is contained in:
lassulus 2024-01-19 14:37:30 +01:00
parent de885c3010
commit 6b004fca6f
13 changed files with 144 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pkgs/clan-cli/clan_cli/__init__.py
View File

@ -57,21 +57,15 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
default=[],
)
def flake_path(arg: str) -> Path:
def flake_path(arg: str) -> str | Path:
flake_dir = Path(arg).resolve()
if not flake_dir.exists():
raise argparse.ArgumentTypeError(
f"flake directory {flake_dir} does not exist"
)
if not flake_dir.is_dir():
raise argparse.ArgumentTypeError(
f"flake directory {flake_dir} is not a directory"
)
return flake_dir
if flake_dir.exists() and flake_dir.is_dir():
return flake_dir
return arg
parser.add_argument(
"--flake",
help="path to the flake where the clan resides in",
help="path to the flake where the clan resides in, can be a remote flake or local",
default=get_clan_flake_toplevel(),
type=flake_path,
)

2
pkgs/clan-cli/clan_cli/backups/create.py
View File

@ -31,7 +31,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
def create_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake_dir=args.flake)
machine = Machine(name=args.machine, flake=args.flake)
create_backup(machine=machine, provider=args.provider)

6
pkgs/clan-cli/clan_cli/backups/list.py
View File

@ -45,9 +45,7 @@ def list_provider(machine: Machine, provider: str) -> list[Backup]:
def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]:
backup_metadata = json.loads(
machine.eval_nix(f"nixosConfigurations.{machine.name}.config.clanCore.backups")
)
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
results = []
if provider is None:
for _provider in backup_metadata["providers"]:
@ -60,7 +58,7 @@ def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]:
def list_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake_dir=args.flake)
machine = Machine(name=args.machine, flake=args.flake)
backups = list_backups(machine=machine, provider=args.provider)
print(backups)

2
pkgs/clan-cli/clan_cli/backups/restore.py
View File

@ -91,7 +91,7 @@ def restore_backup(
def restore_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake_dir=args.flake)
machine = Machine(name=args.machine, flake=args.flake)
backups = list_backups(machine=machine, provider=args.provider)
restore_backup(
machine=machine,

2
pkgs/clan-cli/clan_cli/dirs.py
View File

@ -47,7 +47,7 @@ def user_gcroot_dir() -> Path:
return p
def specific_groot_dir(*, clan_name: str, flake_url: str) -> Path:
def machine_gcroot(*, clan_name: str, flake_url: str) -> Path:
# Always build icon so that we can symlink it to the gcroot
gcroot_dir = user_gcroot_dir()
clan_gcroot = gcroot_dir / clan_key_safe(clan_name, flake_url)

25
pkgs/clan-cli/clan_cli/flakes/inspect.py
View File

@ -3,9 +3,10 @@ from dataclasses import dataclass
from pathlib import Path
from ..cmd import run
from ..dirs import specific_groot_dir
from ..dirs import machine_gcroot
from ..errors import ClanError
from ..machines.list import list_machines
from ..machines.machines import Machine
from ..nix import nix_build, nix_config, nix_eval, nix_metadata
from ..vms.inspect import VmConfig, inspect_vm
@ -29,23 +30,24 @@ def run_cmd(cmd: list[str]) -> str:
return proc.stdout.strip()
def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig:
def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
config = nix_config()
system = config["system"]
# Check if the machine exists
machines = list_machines(flake_url)
if flake_attr not in machines:
if machine_name not in machines:
raise ClanError(
f"Machine {flake_attr} not found in {flake_url}. Available machines: {', '.join(machines)}"
f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}"
)
vm = inspect_vm(flake_url, flake_attr)
machine = Machine(machine_name, flake_url)
vm = inspect_vm(machine)
# Get the cLAN name
cmd = nix_eval(
[
f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.clanName'
f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanName'
]
)
res = run_cmd(cmd)
@ -54,7 +56,7 @@ def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig:
# Get the clan icon path
cmd = nix_eval(
[
f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.clanIcon'
f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanIcon'
]
)
res = run_cmd(cmd)
@ -67,10 +69,9 @@ def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig:
cmd = nix_build(
[
f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.clanIcon'
f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanIcon'
],
specific_groot_dir(clan_name=clan_name, flake_url=str(flake_url))
/ "clanIcon",
machine_gcroot(clan_name=clan_name, flake_url=str(flake_url)) / "clanIcon",
)
run_cmd(cmd)
@ -81,7 +82,7 @@ def inspect_flake(flake_url: str | Path, flake_attr: str) -> FlakeConfig:
vm=vm,
flake_url=flake_url,
clan_name=clan_name,
flake_attr=flake_attr,
flake_attr=machine_name,
nar_hash=meta["locked"]["narHash"],
icon=icon_path,
description=meta.get("description"),
@ -102,7 +103,7 @@ def inspect_command(args: argparse.Namespace) -> None:
flake=args.flake or Path.cwd(),
)
res = inspect_flake(
flake_url=inspect_options.flake, flake_attr=inspect_options.machine
flake_url=inspect_options.flake, machine_name=inspect_options.machine
)
print("cLAN name:", res.clan_name)
print("Icon:", res.icon)

4
pkgs/clan-cli/clan_cli/machines/install.py
View File

@ -34,7 +34,7 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None:
cmd = [
"nixos-anywhere",
"-f",
f"{machine.flake_dir}#{flake_attr}",
f"{machine.flake}#{flake_attr}",
"-t",
"--no-reboot",
"--extra-files",
@ -68,7 +68,7 @@ def install_command(args: argparse.Namespace) -> None:
target_host=args.target_host,
kexec=args.kexec,
)
machine = Machine(opts.machine, flake_dir=opts.flake)
machine = Machine(opts.machine, flake=opts.flake)
machine.deployment_address = opts.target_host
install_nixos(machine, kexec=opts.kexec)

101
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@ -1,33 +1,16 @@
import json
from pathlib import Path
from ..cmd import Log, run
from ..cmd import run
from ..nix import nix_build, nix_config, nix_eval
from ..ssh import Host, parse_deployment_address
def build_machine_data(machine_name: str, clan_dir: Path) -> dict:
config = nix_config()
system = config["system"]
proc = run(
nix_build(
[
f'{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file'
]
),
log=Log.BOTH,
error_msg="failed to build machine data",
)
return json.loads(Path(proc.stdout.strip()).read_text())
class Machine:
def __init__(
self,
name: str,
flake_dir: Path,
flake: Path | str,
machine_data: dict | None = None,
) -> None:
"""
@ -36,22 +19,39 @@ class Machine:
@clan_dir: the directory of the clan, optional, if not set it will be determined from the current working directory
@machine_json: can be optionally used to skip evaluation of the machine, location of the json file with machine data
"""
self.name = name
self.flake_dir = flake_dir
self.name: str = name
self.flake: str | Path = flake
self.eval_cache: dict[str, str] = {}
self.build_cache: dict[str, Path] = {}
# TODO do this lazily
if machine_data is None:
self.machine_data = build_machine_data(name, self.flake_dir)
self.machine_data = json.loads(
self.build_nix("config.system.clan.deployment.file").read_text()
)
else:
self.machine_data = machine_data
self.deployment_address = self.machine_data["deploymentAddress"]
self.secrets_module = self.machine_data["secretsModule"]
self.secrets_data = json.loads(
Path(self.machine_data["secretsData"]).read_text()
)
self.secrets_module = self.machine_data.get("secretsModule", None)
if "secretsData" in self.machine_data:
self.secrets_data = json.loads(
Path(self.machine_data["secretsData"]).read_text()
)
self.secrets_upload_directory = self.machine_data["secretsUploadDirectory"]
self.eval_cache: dict[str, str] = {}
self.build_cache: dict[str, Path] = {}
@property
def flake_dir(self) -> Path:
if isinstance(self.flake, Path):
return self.flake
if hasattr(self, "flake_path"):
return Path(self.flake_path)
print(nix_eval([f"{self.flake}"]))
self.flake_path = run(nix_eval([f"{self.flake}"])).stdout.strip()
return Path(self.flake_path)
@property
def host(self) -> Host:
@ -67,9 +67,25 @@ class Machine:
if attr in self.eval_cache and not refresh:
return self.eval_cache[attr]
output = run(
nix_eval([f"path:{self.flake_dir}#{attr}"]),
).stdout.strip()
config = nix_config()
system = config["system"]
if isinstance(self.flake, Path):
output = run(
nix_eval(
[
f'path:{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}'
]
),
).stdout.strip()
else:
output = run(
nix_eval(
[
f'{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}'
]
),
).stdout.strip()
self.eval_cache[attr] = output
return output
@ -80,8 +96,25 @@ class Machine:
"""
if attr in self.build_cache and not refresh:
return self.build_cache[attr]
outpath = run(
nix_build([f"path:{self.flake_dir}#{attr}"]),
).stdout.strip()
config = nix_config()
system = config["system"]
if isinstance(self.flake, Path):
outpath = run(
nix_build(
[
f'path:{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}'
]
),
).stdout.strip()
else:
outpath = run(
nix_build(
[
f'{self.flake}#clanInternals.machines."{system}"."{self.name}".{attr}'
]
),
).stdout.strip()
self.build_cache[attr] = Path(outpath)
return Path(outpath)

8
pkgs/clan-cli/clan_cli/machines/update.py
View File

@ -93,9 +93,7 @@ def get_all_machines(clan_dir: Path) -> HostGroup:
name,
machine_data["deploymentAddress"],
meta={
"machine": Machine(
name=name, flake_dir=clan_dir, machine_data=machine_data
)
"machine": Machine(name=name, flake=clan_dir, machine_data=machine_data)
},
)
hosts.append(host)
@ -105,7 +103,7 @@ def get_all_machines(clan_dir: Path) -> HostGroup:
def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGroup:
hosts = []
for name in machine_names:
machine = Machine(name=name, flake_dir=flake_dir)
machine = Machine(name=name, flake=flake_dir)
hosts.append(machine.host)
return HostGroup(hosts)
@ -115,7 +113,7 @@ def update(args: argparse.Namespace) -> None:
if args.flake is None:
raise ClanError("Could not find clan flake toplevel directory")
if len(args.machines) == 1 and args.target_host is not None:
machine = Machine(name=args.machines[0], flake_dir=args.flake)
machine = Machine(name=args.machines[0], flake=args.flake)
machine.deployment_address = args.target_host
host = parse_deployment_address(
args.machines[0],

8
pkgs/clan-cli/clan_cli/secrets/generate.py
View File

@ -28,11 +28,11 @@ def generate_secrets(machine: Machine) -> None:
not secret_store.exists(service, secret)
for secret in machine.secrets_data[service]["secrets"]
) or any(
not (machine.flake_dir / fact).exists()
not (machine.flake / fact).exists()
for fact in machine.secrets_data[service]["facts"].values()
)
for fact in machine.secrets_data[service]["facts"].values():
if not (machine.flake_dir / fact).exists():
if not (machine.flake / fact).exists():
print(f"fact {fact} is missing")
if needs_regeneration:
env = os.environ.copy()
@ -66,7 +66,7 @@ def generate_secrets(machine: Machine) -> None:
msg = f"did not generate a file for '{name}' when running the following command:\n"
msg += machine.secrets_data[service]["generator"]
raise ClanError(msg)
fact_path = machine.flake_dir / fact_path
fact_path = machine.flake / fact_path
fact_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(fact_file, fact_path)
@ -74,7 +74,7 @@ def generate_secrets(machine: Machine) -> None:
def generate_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake_dir=args.flake)
machine = Machine(name=args.machine, flake=args.flake)
generate_secrets(machine)

2
pkgs/clan-cli/clan_cli/secrets/upload.py
View File

@ -43,7 +43,7 @@ def upload_secrets(machine: Machine) -> None:
def upload_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake_dir=args.flake)
machine = Machine(name=args.machine, flake=args.flake)
upload_secrets(machine)

31
pkgs/clan-cli/clan_cli/vms/inspect.py
View File

@ -3,15 +3,14 @@ import json
from dataclasses import dataclass
from pathlib import Path
from ..cmd import run
from ..nix import nix_config, nix_eval
from ..machines.machines import Machine
@dataclass
class VmConfig:
clan_name: str
machine_name: str
flake_url: str | Path
flake_attr: str
clan_name: str
cores: int
memory_size: int
@ -19,21 +18,9 @@ class VmConfig:
wayland: bool = False
def inspect_vm(flake_url: str | Path, flake_attr: str) -> VmConfig:
config = nix_config()
system = config["system"]
cmd = nix_eval(
[
f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.vm.inspect',
"--refresh",
]
)
proc = run(cmd)
data = json.loads(proc.stdout)
return VmConfig(flake_url=flake_url, flake_attr=flake_attr, **data)
def inspect_vm(machine: Machine) -> VmConfig:
data = json.loads(machine.eval_nix("config.clanCore.vm.inspect"))
return VmConfig(machine_name=machine.name, flake_url=machine.flake, **data)
@dataclass
@ -47,9 +34,9 @@ def inspect_command(args: argparse.Namespace) -> None:
machine=args.machine,
flake=args.flake or Path.cwd(),
)
res = inspect_vm(
flake_url=inspect_options.flake, flake_attr=inspect_options.machine
)
machine = Machine(inspect_options.machine, inspect_options.flake)
res = inspect_vm(machine)
print("Cores:", res.cores)
print("Memory size:", res.memory_size)
print("Graphics:", res.graphics)

74
pkgs/clan-cli/clan_cli/vms/run.py
View File

@ -1,20 +1,20 @@
import argparse
import importlib
import json
import logging
import os
import sys
import tempfile
import importlib
from dataclasses import dataclass, field
from pathlib import Path
from typing import IO
from ..cmd import Log, run
from ..dirs import module_root, specific_groot_dir, vm_state_dir
from ..dirs import machine_gcroot, module_root, vm_state_dir
from ..errors import ClanError
from ..nix import nix_build, nix_config, nix_shell
from .inspect import VmConfig, inspect_vm
from ..machines.machines import Machine
from ..nix import nix_build, nix_config, nix_shell
from ..secrets.generate import generate_secrets
from .inspect import VmConfig, inspect_vm
log = logging.getLogger(__name__)
@ -75,7 +75,7 @@ def qemu_command(
# fmt: off
command = [
"qemu-kvm",
"-name", vm.flake_attr,
"-name", vm.machine_name,
"-m", f'{nixos_config["memorySize"]}M',
"-smp", str(nixos_config["cores"]),
"-cpu", "max",
@ -104,32 +104,34 @@ def qemu_command(
return command
def get_vm_create_info(vm: VmConfig, nix_options: list[str]) -> dict[str, str]:
# TODO move this to the Machines class
def get_vm_create_info(
machine: Machine, vm: VmConfig, nix_options: list[str]
) -> dict[str, str]:
config = nix_config()
system = config["system"]
clan_dir = vm.flake_url
machine = vm.flake_attr
clan_dir = machine.flake
cmd = nix_build(
[
f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.vm.create',
f'{clan_dir}#clanInternals.machines."{system}"."{machine.name}".config.system.clan.vm.create',
*nix_options,
],
specific_groot_dir(clan_name=vm.clan_name, flake_url=str(vm.flake_url))
/ f"vm-{machine}",
machine_gcroot(clan_name=vm.clan_name, flake_url=str(vm.flake_url))
/ f"vm-{machine.name}",
)
proc = run(
cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine.name}"
)
proc = run(cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine}")
try:
return json.loads(Path(proc.stdout.strip()).read_text())
except json.JSONDecodeError as e:
raise ClanError(f"Failed to parse vm config: {e}")
def generate_secrets(
vm: VmConfig,
nixos_config: dict[str, str],
def get_secrets(
machine: Machine,
tmpdir: Path,
log_fd: IO[str] | None,
) -> Path:
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir(exist_ok=True)
@ -138,19 +140,12 @@ def generate_secrets(
secret_store = secrets_module.SecretStore(machine=machine)
# Only generate secrets for local clans
if isinstance(vm.flake_url, Path) and vm.flake_url.is_dir():
if Path(vm.flake_url).is_dir():
run([nixos_config["generateSecrets"], vm.clan_name], env=env)
else:
log.warning("won't generate secrets for non local clan")
if isinstance(machine.flake, Path) and machine.flake.is_dir():
generate_secrets(machine)
else:
log.warning("won't generate secrets for non local clan")
cmd = [nixos_config["uploadSecrets"]]
run(
cmd,
env=env,
log=Log.BOTH,
error_msg=f"Could not upload secrets for {vm.flake_attr}",
)
secret_store.upload(secrets_dir)
return secrets_dir
@ -191,26 +186,28 @@ def prepare_disk(tmpdir: Path, log_fd: IO[str] | None) -> Path:
def run_vm(
vm: VmConfig, nix_options: list[str] = [], log_fd: IO[str] | None = None
vm: VmConfig,
nix_options: list[str] = [],
log_fd: IO[str] | None = None,
) -> None:
"""
log_fd can be used to stream the output of all commands to a UI
"""
machine = vm.flake_attr
machine = Machine(vm.machine_name, vm.flake_url)
log.debug(f"Creating VM for {machine}")
# TODO: We should get this from the vm argument
nixos_config = get_vm_create_info(vm, nix_options)
nixos_config = get_vm_create_info(machine, vm, nix_options)
with tempfile.TemporaryDirectory() as tmpdir_:
tmpdir = Path(tmpdir_)
xchg_dir = tmpdir / "xchg"
xchg_dir.mkdir(exist_ok=True)
secrets_dir = generate_secrets(vm, nixos_config, tmpdir, log_fd)
secrets_dir = get_secrets(machine, tmpdir)
disk_img = prepare_disk(tmpdir, log_fd)
state_dir = vm_state_dir(vm.clan_name, str(vm.flake_url), machine)
state_dir = vm_state_dir(vm.clan_name, str(machine.flake), machine.name)
state_dir.mkdir(parents=True, exist_ok=True)
qemu_cmd = qemu_command(
@ -243,7 +240,6 @@ def run_vm(
@dataclass
class RunOptions:
machine: str
flake_url: str | None
flake: Path
nix_options: list[str] = field(default_factory=list)
wayland: bool = False
@ -252,14 +248,14 @@ class RunOptions:
def run_command(args: argparse.Namespace) -> None:
run_options = RunOptions(
machine=args.machine,
flake_url=args.flake_url,
flake=args.flake or Path.cwd(),
flake=args.flake,
nix_options=args.option,
wayland=args.wayland,
)
flake_url = run_options.flake_url or run_options.flake
vm = inspect_vm(flake_url=flake_url, flake_attr=run_options.machine)
machine = Machine(run_options.machine, run_options.flake)
vm = inspect_vm(machine=machine)
# TODO: allow to set this in the config
vm.wayland = run_options.wayland