From 6871b29d15724d51bf65ed091db08aee71d0c871 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 14 Feb 2024 07:15:59 +0100 Subject: [PATCH] vms: use vm fact/secret-store --- lib/build-clan/default.nix | 2 +- nixosModules/clanCore/secrets/default.nix | 3 +- nixosModules/clanCore/secrets/vm.nix | 10 +++ nixosModules/clanCore/zerotier/default.nix | 2 +- pkgs/clan-cli/clan_cli/__init__.py | 2 +- pkgs/clan-cli/clan_cli/facts/check.py | 3 +- pkgs/clan-cli/clan_cli/facts/list.py | 2 +- .../clan_cli/facts/modules/in_repo.py | 9 ++- pkgs/clan-cli/clan_cli/facts/modules/vm.py | 44 +++++++++++++ pkgs/clan-cli/clan_cli/machines/machines.py | 28 ++++---- pkgs/clan-cli/clan_cli/secrets/check.py | 6 +- pkgs/clan-cli/clan_cli/secrets/generate.py | 10 +-- pkgs/clan-cli/clan_cli/secrets/modules/vm.py | 31 +++++++++ pkgs/clan-cli/clan_cli/vms/run.py | 65 +++++++++++-------- 14 files changed, 161 insertions(+), 56 deletions(-) create mode 100644 nixosModules/clanCore/secrets/vm.nix create mode 100644 pkgs/clan-cli/clan_cli/facts/modules/vm.py create mode 100644 pkgs/clan-cli/clan_cli/secrets/modules/vm.py diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 39e84705..244fca4f 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -37,9 +37,9 @@ let in (machineImports settings) ++ [ - (nixpkgs.lib.mkOverride 51 extraConfig) settings clan-core.nixosModules.clanCore + extraConfig (machines.${name} or { }) ({ clanCore.clanName = clanName; diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 6c477ca8..08b73448 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -1,7 +1,7 @@ { config, lib, pkgs, ... }: { options.clanCore.secretStore = lib.mkOption { - type = lib.types.enum [ "sops" "password-store" "custom" ]; + type = lib.types.enum [ "sops" "password-store" "vm" "custom" ]; default = "sops"; description = '' method to store secrets @@ -150,5 +150,6 @@ imports = [ ./sops.nix ./password-store.nix + ./vm.nix ]; } diff --git a/nixosModules/clanCore/secrets/vm.nix b/nixosModules/clanCore/secrets/vm.nix new file mode 100644 index 00000000..ce071dd2 --- /dev/null +++ b/nixosModules/clanCore/secrets/vm.nix @@ -0,0 +1,10 @@ +{ config, lib, ... }: +{ + config = lib.mkIf (config.clanCore.secretStore == "vm") { + clanCore.secretsDirectory = "/etc/secrets"; + clanCore.secretsUploadDirectory = "/etc/secrets"; + system.clan.secretsModule = "clan_cli.secrets.modules.vm"; + system.clan.factsModule = "clan_cli.facts.modules.vm"; + }; +} + diff --git a/nixosModules/clanCore/zerotier/default.nix b/nixosModules/clanCore/zerotier/default.nix index 768b0b50..90d5f53e 100644 --- a/nixosModules/clanCore/zerotier/default.nix +++ b/nixosModules/clanCore/zerotier/default.nix @@ -190,7 +190,7 @@ in environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ]; }) - (lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) { + (lib.mkIf (!cfg.controller.enable && cfg.networkId != null) { clanCore.secrets.zerotier = { facts.zerotier-ip = { }; facts.zerotier-meshname = { }; diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index b3acf473..2e29a000 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -6,7 +6,7 @@ from pathlib import Path from types import ModuleType from typing import Any -from . import backups, config, flakes, flash, history, machines, secrets, vms, facts +from . import backups, config, facts, flakes, flash, history, machines, secrets, vms from .custom_logger import setup_logging from .dirs import get_clan_flake_toplevel from .errors import ClanCmdError, ClanError diff --git a/pkgs/clan-cli/clan_cli/facts/check.py b/pkgs/clan-cli/clan_cli/facts/check.py index 11a9cc3a..51f4d9ac 100644 --- a/pkgs/clan-cli/clan_cli/facts/check.py +++ b/pkgs/clan-cli/clan_cli/facts/check.py @@ -11,10 +11,11 @@ def check_facts(machine: Machine) -> bool: facts_module = importlib.import_module(machine.facts_module) fact_store = facts_module.FactStore(machine=machine) + existing_facts = fact_store.get_all() missing_facts = [] for service in machine.secrets_data: for fact in machine.secrets_data[service]["facts"]: - if not fact_store.get(service, fact): + if fact not in existing_facts.get(service, {}): log.info(f"Fact {fact} for service {service} is missing") missing_facts.append((service, fact)) diff --git a/pkgs/clan-cli/clan_cli/facts/list.py b/pkgs/clan-cli/clan_cli/facts/list.py index 59eae3fc..342a7375 100644 --- a/pkgs/clan-cli/clan_cli/facts/list.py +++ b/pkgs/clan-cli/clan_cli/facts/list.py @@ -1,6 +1,6 @@ -import json import argparse import importlib +import json import logging from ..machines.machines import Machine diff --git a/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py b/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py index 225d608e..5806b129 100644 --- a/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py +++ b/pkgs/clan-cli/clan_cli/facts/modules/in_repo.py @@ -7,6 +7,7 @@ from clan_cli.machines.machines import Machine class FactStore: def __init__(self, machine: Machine) -> None: self.machine = machine + self.works_remotely = False def set(self, _service: str, name: str, value: bytes) -> Path | None: if isinstance(self.machine.flake, Path): @@ -23,12 +24,16 @@ class FactStore: ) def exists(self, _service: str, name: str) -> bool: - fact_path = self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + fact_path = ( + self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + ) return fact_path.exists() # get a single fact def get(self, _service: str, name: str) -> bytes: - fact_path = self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + fact_path = ( + self.machine.flake_dir / "machines" / self.machine.name / "facts" / name + ) return fact_path.read_bytes() # get all facts diff --git a/pkgs/clan-cli/clan_cli/facts/modules/vm.py b/pkgs/clan-cli/clan_cli/facts/modules/vm.py new file mode 100644 index 00000000..c2d5817a --- /dev/null +++ b/pkgs/clan-cli/clan_cli/facts/modules/vm.py @@ -0,0 +1,44 @@ +import logging +from pathlib import Path + +from clan_cli.dirs import vm_state_dir +from clan_cli.errors import ClanError +from clan_cli.machines.machines import Machine + +log = logging.getLogger(__name__) + + +class FactStore: + def __init__(self, machine: Machine) -> None: + self.machine = machine + self.works_remotely = False + self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts" + log.debug(f"FactStore initialized with dir {self.dir}") + + def exists(self, service: str, name: str) -> bool: + fact_path = self.dir / service / name + return fact_path.exists() + + def set(self, service: str, name: str, value: bytes) -> Path | None: + fact_path = self.dir / service / name + fact_path.parent.mkdir(parents=True, exist_ok=True) + fact_path.write_bytes(value) + return None + + # get a single fact + def get(self, service: str, name: str) -> bytes: + fact_path = self.dir / service / name + if fact_path.exists(): + return fact_path.read_bytes() + raise ClanError(f"Fact {name} for service {service} not found") + + # get all facts + def get_all(self) -> dict[str, dict[str, bytes]]: + facts: dict[str, dict[str, bytes]] = {} + if self.dir.exists(): + for service in self.dir.iterdir(): + facts[service.name] = {} + for fact in service.iterdir(): + facts[service.name][fact.name] = fact.read_bytes() + + return facts diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 17cfc581..0a79fc99 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -155,6 +155,7 @@ class Machine: attr: str, extra_config: None | dict = None, impure: bool = False, + nix_options: list[str] = [], ) -> str | Path: """ Build the machine and return the path to the result @@ -188,17 +189,15 @@ class Machine: if extra_config is not None: metadata = nix_metadata(self.flake_dir) url = metadata["url"] - if "dirtyRev" in metadata: - if not impure: - raise ClanError( - "The machine has a dirty revision, and impure mode is not allowed" - ) - else: - args += ["--impure"] + if "dirtyRevision" in metadata: + # if not impure: + # raise ClanError( + # "The machine has a dirty revision, and impure mode is not allowed" + # ) + # else: + # args += ["--impure"] + args += ["--impure"] - if "dirtyRev" in nix_metadata(self.flake_dir): - dirty_rev = nix_metadata(self.flake_dir)["dirtyRevision"] - url = f"{url}?rev={dirty_rev}" args += [ "--expr", f""" @@ -220,7 +219,8 @@ class Machine: else: flake = self.flake args += [ - f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}' + f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}', + *nix_options, ] if method == "eval": @@ -238,6 +238,7 @@ class Machine: refresh: bool = False, extra_config: None | dict = None, impure: bool = False, + nix_options: list[str] = [], ) -> str: """ eval a nix attribute of the machine @@ -246,7 +247,7 @@ class Machine: if attr in self.eval_cache and not refresh and extra_config is None: return self.eval_cache[attr] - output = self.nix("eval", attr, extra_config, impure) + output = self.nix("eval", attr, extra_config, impure, nix_options) if isinstance(output, str): self.eval_cache[attr] = output return output @@ -259,6 +260,7 @@ class Machine: refresh: bool = False, extra_config: None | dict = None, impure: bool = False, + nix_options: list[str] = [], ) -> Path: """ build a nix attribute of the machine @@ -268,7 +270,7 @@ class Machine: if attr in self.build_cache and not refresh and extra_config is None: return self.build_cache[attr] - output = self.nix("build", attr, extra_config, impure) + output = self.nix("build", attr, extra_config, impure, nix_options) if isinstance(output, Path): self.build_cache[attr] = output return output diff --git a/pkgs/clan-cli/clan_cli/secrets/check.py b/pkgs/clan-cli/clan_cli/secrets/check.py index a79452db..83c92680 100644 --- a/pkgs/clan-cli/clan_cli/secrets/check.py +++ b/pkgs/clan-cli/clan_cli/secrets/check.py @@ -11,7 +11,7 @@ def check_secrets(machine: Machine) -> bool: secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) facts_module = importlib.import_module(machine.facts_module) - fact_store = facts_module.FactsStore(machine=machine) + fact_store = facts_module.FactStore(machine=machine) missing_secrets = [] missing_facts = [] @@ -21,11 +21,13 @@ def check_secrets(machine: Machine) -> bool: log.info(f"Secret {secret} for service {service} is missing") missing_secrets.append((service, secret)) - for fact in machine.secrets_data[service]["facts"].values(): + for fact in machine.secrets_data[service]["facts"]: if not fact_store.exists(service, fact): log.info(f"Fact {fact} for service {service} is missing") missing_facts.append((service, fact)) + log.debug(f"missing_secrets: {missing_secrets}") + log.debug(f"missing_facts: {missing_facts}") if missing_secrets or missing_facts: return False return True diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index b38901f8..2cbf255f 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -28,6 +28,7 @@ def generate_secrets(machine: Machine) -> None: tmpdir = Path(d) / service # check if all secrets exist and generate them if at least one is missing needs_regeneration = not check_secrets(machine) + log.debug(f"{service} needs_regeneration: {needs_regeneration}") if needs_regeneration: if not isinstance(machine.flake, Path): msg = f"flake is not a Path: {machine.flake}" @@ -80,16 +81,15 @@ def generate_secrets(machine: Machine) -> None: files_to_commit.append(secret_path) # store facts - for name, fact_path in machine.secrets_data[service]["facts"].items(): + for name in machine.secrets_data[service]["facts"]: fact_file = facts_dir / name if not fact_file.is_file(): msg = f"did not generate a file for '{name}' when running the following command:\n" msg += machine.secrets_data[service]["generator"] raise ClanError(msg) - fact_file = fact_store.set( - service, fact_path, fact_file.read_bytes() - ) - files_to_commit.append(fact_file) + fact_file = fact_store.set(service, name, fact_file.read_bytes()) + if fact_file: + files_to_commit.append(fact_file) commit_files( files_to_commit, machine.flake_dir, diff --git a/pkgs/clan-cli/clan_cli/secrets/modules/vm.py b/pkgs/clan-cli/clan_cli/secrets/modules/vm.py new file mode 100644 index 00000000..33701c51 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/secrets/modules/vm.py @@ -0,0 +1,31 @@ +import os +import shutil +from pathlib import Path + +from clan_cli.dirs import vm_state_dir +from clan_cli.machines.machines import Machine + + +class SecretStore: + def __init__(self, machine: Machine) -> None: + self.machine = machine + self.dir = vm_state_dir(str(machine.flake), machine.name) / "secrets" + self.dir.mkdir(parents=True, exist_ok=True) + + def set(self, service: str, name: str, value: bytes) -> Path | None: + secret_file = self.dir / service / name + secret_file.parent.mkdir(parents=True, exist_ok=True) + secret_file.write_bytes(value) + return None # we manage the files outside of the git repo + + def get(self, service: str, name: str) -> bytes: + secret_file = self.dir / service / name + return secret_file.read_bytes() + + def exists(self, service: str, name: str) -> bool: + return (self.dir / service / name).exists() + + def upload(self, output_dir: Path) -> None: + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + shutil.copytree(self.dir, output_dir) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 487266fd..1ea1f0f6 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -15,10 +15,10 @@ from pathlib import Path from tempfile import TemporaryDirectory from ..cmd import Log, run -from ..dirs import machine_gcroot, module_root, user_cache_dir, vm_state_dir +from ..dirs import module_root, user_cache_dir, vm_state_dir from ..errors import ClanError from ..machines.machines import Machine -from ..nix import nix_build, nix_config, nix_shell +from ..nix import nix_shell from ..secrets.generate import generate_secrets from .inspect import VmConfig, inspect_vm @@ -153,26 +153,39 @@ def qemu_command( return QemuCommand(command, vsock_cid=vsock_cid) +def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict: + nixos_config: dict = {} + nixos_config["clanCore"] = {} + nixos_config["clanCore"]["secrets"] = {} + for service, service_facts in facts.items(): + nixos_config["clanCore"]["secrets"][service] = {} + nixos_config["clanCore"]["secrets"][service]["facts"] = {} + for fact, value in service_facts.items(): + nixos_config["clanCore"]["secrets"][service]["facts"][fact] = { + "value": value.decode() + } + return nixos_config + + # TODO move this to the Machines class def build_vm( - machine: Machine, vm: VmConfig, nix_options: list[str] = [] + machine: Machine, vm: VmConfig, tmpdir: Path, nix_options: list[str] ) -> dict[str, str]: - config = nix_config() - system = config["system"] + secrets_dir = get_secrets(machine, tmpdir) - clan_dir = machine.flake - cmd = nix_build( - [ - f'{clan_dir}#clanInternals.machines."{system}"."{machine.name}".config.system.clan.vm.create', - *nix_options, - ], - machine_gcroot(flake_url=str(vm.flake_url)) / f"vm-{machine.name}", - ) - proc = run( - cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine.name}" + facts_module = importlib.import_module(machine.facts_module) + fact_store = facts_module.FactStore(machine=machine) + facts = fact_store.get_all() + + nixos_config_file = machine.build_nix( + "config.system.clan.vm.create", + extra_config=facts_to_nixos_config(facts), + nix_options=nix_options, ) try: - return json.loads(Path(proc.stdout.strip()).read_text()) + vm_data = json.loads(Path(nixos_config_file).read_text()) + vm_data["secrets_dir"] = str(secrets_dir) + return vm_data except json.JSONDecodeError as e: raise ClanError(f"Failed to parse vm config: {e}") @@ -182,16 +195,13 @@ def get_secrets( tmpdir: Path, ) -> Path: secrets_dir = tmpdir / "secrets" - secrets_dir.mkdir(exist_ok=True) + secrets_dir.mkdir(parents=True, exist_ok=True) secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) - # Only generate secrets for local clans - if isinstance(machine.flake, Path) and machine.flake.is_dir(): - generate_secrets(machine) - else: - log.warning("won't generate secrets for non local clan") + # TODO Only generate secrets for local clans + generate_secrets(machine) secret_store.upload(secrets_dir) return secrets_dir @@ -302,20 +312,19 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None: machine = Machine(vm.machine_name, vm.flake_url) log.debug(f"Creating VM for {machine}") - # TODO: We should get this from the vm argument - nixos_config = build_vm(machine, vm, nix_options) - # store the temporary rootfs inside XDG_CACHE_HOME on the host # otherwise, when using /tmp, we risk running out of memory cache = user_cache_dir() / "clan" cache.mkdir(exist_ok=True) with TemporaryDirectory(dir=cache) as cachedir, TemporaryDirectory() as sockets: tmpdir = Path(cachedir) + + # TODO: We should get this from the vm argument + nixos_config = build_vm(machine, vm, tmpdir, nix_options) + xchg_dir = tmpdir / "xchg" xchg_dir.mkdir(exist_ok=True) - secrets_dir = get_secrets(machine, tmpdir) - state_dir = vm_state_dir(str(vm.flake_url), machine.name) state_dir.mkdir(parents=True, exist_ok=True) @@ -350,7 +359,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None: vm, nixos_config, xchg_dir=xchg_dir, - secrets_dir=secrets_dir, + secrets_dir=Path(nixos_config["secrets_dir"]), rootfs_img=rootfs_img, state_img=state_img, virtiofsd_socket=virtiofsd_socket,