vms: use vm fact/secret-store

lib/build-clan/default.nix

@@ -37,9 +37,9 @@ let
       in
       (machineImports settings)
       ++ [
-        (nixpkgs.lib.mkOverride 51 extraConfig)
         settings
         clan-core.nixosModules.clanCore
+        extraConfig
         (machines.${name} or { })
         ({
           clanCore.clanName = clanName;

nixosModules/clanCore/secrets/default.nix

@@ -1,7 +1,7 @@
 { config, lib, pkgs, ... }:
 {
   options.clanCore.secretStore = lib.mkOption {
-    type = lib.types.enum [ "sops" "password-store" "custom" ];
+    type = lib.types.enum [ "sops" "password-store" "vm" "custom" ];
     default = "sops";
     description = ''
       method to store secrets
@@ -150,5 +150,6 @@
   imports = [
     ./sops.nix
     ./password-store.nix
+    ./vm.nix
   ];
 }

nixosModules/clanCore/secrets/vm.nix

@@ -0,0 +1,10 @@
+{ config, lib, ... }:
+{
+  config = lib.mkIf (config.clanCore.secretStore == "vm") {
+    clanCore.secretsDirectory = "/etc/secrets";
+    clanCore.secretsUploadDirectory = "/etc/secrets";
+    system.clan.secretsModule = "clan_cli.secrets.modules.vm";
+    system.clan.factsModule = "clan_cli.facts.modules.vm";
+  };
+}
+

nixosModules/clanCore/zerotier/default.nix

@@ -190,7 +190,7 @@ in
 
       environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
     })
-    (lib.mkIf (config.clanCore.secretsUploadDirectory != null && !cfg.controller.enable && cfg.networkId != null) {
+    (lib.mkIf (!cfg.controller.enable && cfg.networkId != null) {
       clanCore.secrets.zerotier = {
         facts.zerotier-ip = { };
         facts.zerotier-meshname = { };

pkgs/clan-cli/clan_cli/__init__.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any
 
-from . import backups, config, flakes, flash, history, machines, secrets, vms, facts
+from . import backups, config, facts, flakes, flash, history, machines, secrets, vms
 from .custom_logger import setup_logging
 from .dirs import get_clan_flake_toplevel
 from .errors import ClanCmdError, ClanError

pkgs/clan-cli/clan_cli/facts/check.py

@@ -11,10 +11,11 @@ def check_facts(machine: Machine) -> bool:
     facts_module = importlib.import_module(machine.facts_module)
     fact_store = facts_module.FactStore(machine=machine)
 
+    existing_facts = fact_store.get_all()
     missing_facts = []
     for service in machine.secrets_data:
         for fact in machine.secrets_data[service]["facts"]:
-            if not fact_store.get(service, fact):
+            if fact not in existing_facts.get(service, {}):
                 log.info(f"Fact {fact} for service {service} is missing")
                 missing_facts.append((service, fact))
 

pkgs/clan-cli/clan_cli/facts/list.py

@@ -1,6 +1,6 @@
-import json
 import argparse
 import importlib
+import json
 import logging
 
 from ..machines.machines import Machine

pkgs/clan-cli/clan_cli/facts/modules/in_repo.py

@@ -7,6 +7,7 @@ from clan_cli.machines.machines import Machine
 class FactStore:
     def __init__(self, machine: Machine) -> None:
         self.machine = machine
+        self.works_remotely = False
 
     def set(self, _service: str, name: str, value: bytes) -> Path | None:
         if isinstance(self.machine.flake, Path):
@@ -23,12 +24,16 @@ class FactStore:
             )
 
     def exists(self, _service: str, name: str) -> bool:
-        fact_path = self.machine.flake_dir / "machines" / self.machine.name / "facts" / name
+        fact_path = (
+            self.machine.flake_dir / "machines" / self.machine.name / "facts" / name
+        )
         return fact_path.exists()
 
     # get a single fact
     def get(self, _service: str, name: str) -> bytes:
-        fact_path = self.machine.flake_dir / "machines" / self.machine.name / "facts" / name
+        fact_path = (
+            self.machine.flake_dir / "machines" / self.machine.name / "facts" / name
+        )
         return fact_path.read_bytes()
 
     # get all facts

pkgs/clan-cli/clan_cli/facts/modules/vm.py

@@ -0,0 +1,44 @@
+import logging
+from pathlib import Path
+
+from clan_cli.dirs import vm_state_dir
+from clan_cli.errors import ClanError
+from clan_cli.machines.machines import Machine
+
+log = logging.getLogger(__name__)
+
+
+class FactStore:
+    def __init__(self, machine: Machine) -> None:
+        self.machine = machine
+        self.works_remotely = False
+        self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts"
+        log.debug(f"FactStore initialized with dir {self.dir}")
+
+    def exists(self, service: str, name: str) -> bool:
+        fact_path = self.dir / service / name
+        return fact_path.exists()
+
+    def set(self, service: str, name: str, value: bytes) -> Path | None:
+        fact_path = self.dir / service / name
+        fact_path.parent.mkdir(parents=True, exist_ok=True)
+        fact_path.write_bytes(value)
+        return None
+
+    # get a single fact
+    def get(self, service: str, name: str) -> bytes:
+        fact_path = self.dir / service / name
+        if fact_path.exists():
+            return fact_path.read_bytes()
+        raise ClanError(f"Fact {name} for service {service} not found")
+
+    # get all facts
+    def get_all(self) -> dict[str, dict[str, bytes]]:
+        facts: dict[str, dict[str, bytes]] = {}
+        if self.dir.exists():
+            for service in self.dir.iterdir():
+                facts[service.name] = {}
+                for fact in service.iterdir():
+                    facts[service.name][fact.name] = fact.read_bytes()
+
+        return facts

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -155,6 +155,7 @@ class Machine:
         attr: str,
         extra_config: None | dict = None,
         impure: bool = False,
+        nix_options: list[str] = [],
     ) -> str | Path:
         """
         Build the machine and return the path to the result
@@ -188,17 +189,15 @@ class Machine:
             if extra_config is not None:
                 metadata = nix_metadata(self.flake_dir)
                 url = metadata["url"]
-                if "dirtyRev" in metadata:
-                    if not impure:
-                        raise ClanError(
-                            "The machine has a dirty revision, and impure mode is not allowed"
-                        )
-                    else:
-                        args += ["--impure"]
+                if "dirtyRevision" in metadata:
+                    # if not impure:
+                    #     raise ClanError(
+                    #         "The machine has a dirty revision, and impure mode is not allowed"
+                    #     )
+                    # else:
+                    #     args += ["--impure"]
+                    args += ["--impure"]
 
-                if "dirtyRev" in nix_metadata(self.flake_dir):
-                    dirty_rev = nix_metadata(self.flake_dir)["dirtyRevision"]
-                    url = f"{url}?rev={dirty_rev}"
                 args += [
                     "--expr",
                     f"""
@@ -220,7 +219,8 @@ class Machine:
                 else:
                     flake = self.flake
                 args += [
-                    f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}'
+                    f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}',
+                    *nix_options,
                 ]
 
             if method == "eval":
@@ -238,6 +238,7 @@ class Machine:
         refresh: bool = False,
         extra_config: None | dict = None,
         impure: bool = False,
+        nix_options: list[str] = [],
     ) -> str:
         """
         eval a nix attribute of the machine
@@ -246,7 +247,7 @@ class Machine:
         if attr in self.eval_cache and not refresh and extra_config is None:
             return self.eval_cache[attr]
 
-        output = self.nix("eval", attr, extra_config, impure)
+        output = self.nix("eval", attr, extra_config, impure, nix_options)
         if isinstance(output, str):
             self.eval_cache[attr] = output
             return output
@@ -259,6 +260,7 @@ class Machine:
         refresh: bool = False,
         extra_config: None | dict = None,
         impure: bool = False,
+        nix_options: list[str] = [],
     ) -> Path:
         """
         build a nix attribute of the machine
@@ -268,7 +270,7 @@ class Machine:
         if attr in self.build_cache and not refresh and extra_config is None:
             return self.build_cache[attr]
 
-        output = self.nix("build", attr, extra_config, impure)
+        output = self.nix("build", attr, extra_config, impure, nix_options)
         if isinstance(output, Path):
             self.build_cache[attr] = output
             return output

pkgs/clan-cli/clan_cli/secrets/check.py

@@ -11,7 +11,7 @@ def check_secrets(machine: Machine) -> bool:
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
     facts_module = importlib.import_module(machine.facts_module)
-    fact_store = facts_module.FactsStore(machine=machine)
+    fact_store = facts_module.FactStore(machine=machine)
 
     missing_secrets = []
     missing_facts = []
@@ -21,11 +21,13 @@ def check_secrets(machine: Machine) -> bool:
                 log.info(f"Secret {secret} for service {service} is missing")
                 missing_secrets.append((service, secret))
 
-        for fact in machine.secrets_data[service]["facts"].values():
+        for fact in machine.secrets_data[service]["facts"]:
             if not fact_store.exists(service, fact):
                 log.info(f"Fact {fact} for service {service} is missing")
                 missing_facts.append((service, fact))
 
+    log.debug(f"missing_secrets: {missing_secrets}")
+    log.debug(f"missing_facts: {missing_facts}")
     if missing_secrets or missing_facts:
         return False
     return True

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -28,6 +28,7 @@ def generate_secrets(machine: Machine) -> None:
             tmpdir = Path(d) / service
             # check if all secrets exist and generate them if at least one is missing
             needs_regeneration = not check_secrets(machine)
+            log.debug(f"{service} needs_regeneration: {needs_regeneration}")
             if needs_regeneration:
                 if not isinstance(machine.flake, Path):
                     msg = f"flake is not a Path: {machine.flake}"
@@ -80,16 +81,15 @@ def generate_secrets(machine: Machine) -> None:
                         files_to_commit.append(secret_path)
 
                 # store facts
-                for name, fact_path in machine.secrets_data[service]["facts"].items():
+                for name in machine.secrets_data[service]["facts"]:
                     fact_file = facts_dir / name
                     if not fact_file.is_file():
                         msg = f"did not generate a file for '{name}' when running the following command:\n"
                         msg += machine.secrets_data[service]["generator"]
                         raise ClanError(msg)
-                    fact_file = fact_store.set(
-                        service, fact_path, fact_file.read_bytes()
-                    )
-                    files_to_commit.append(fact_file)
+                    fact_file = fact_store.set(service, name, fact_file.read_bytes())
+                    if fact_file:
+                        files_to_commit.append(fact_file)
                 commit_files(
                     files_to_commit,
                     machine.flake_dir,

pkgs/clan-cli/clan_cli/secrets/modules/vm.py

@@ -0,0 +1,31 @@
+import os
+import shutil
+from pathlib import Path
+
+from clan_cli.dirs import vm_state_dir
+from clan_cli.machines.machines import Machine
+
+
+class SecretStore:
+    def __init__(self, machine: Machine) -> None:
+        self.machine = machine
+        self.dir = vm_state_dir(str(machine.flake), machine.name) / "secrets"
+        self.dir.mkdir(parents=True, exist_ok=True)
+
+    def set(self, service: str, name: str, value: bytes) -> Path | None:
+        secret_file = self.dir / service / name
+        secret_file.parent.mkdir(parents=True, exist_ok=True)
+        secret_file.write_bytes(value)
+        return None  # we manage the files outside of the git repo
+
+    def get(self, service: str, name: str) -> bytes:
+        secret_file = self.dir / service / name
+        return secret_file.read_bytes()
+
+    def exists(self, service: str, name: str) -> bool:
+        return (self.dir / service / name).exists()
+
+    def upload(self, output_dir: Path) -> None:
+        if os.path.exists(output_dir):
+            shutil.rmtree(output_dir)
+        shutil.copytree(self.dir, output_dir)

pkgs/clan-cli/clan_cli/vms/run.py

@@ -15,10 +15,10 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..cmd import Log, run
-from ..dirs import machine_gcroot, module_root, user_cache_dir, vm_state_dir
+from ..dirs import module_root, user_cache_dir, vm_state_dir
 from ..errors import ClanError
 from ..machines.machines import Machine
-from ..nix import nix_build, nix_config, nix_shell
+from ..nix import nix_shell
 from ..secrets.generate import generate_secrets
 from .inspect import VmConfig, inspect_vm
 
@@ -153,26 +153,39 @@ def qemu_command(
     return QemuCommand(command, vsock_cid=vsock_cid)
 
 
+def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict:
+    nixos_config: dict = {}
+    nixos_config["clanCore"] = {}
+    nixos_config["clanCore"]["secrets"] = {}
+    for service, service_facts in facts.items():
+        nixos_config["clanCore"]["secrets"][service] = {}
+        nixos_config["clanCore"]["secrets"][service]["facts"] = {}
+        for fact, value in service_facts.items():
+            nixos_config["clanCore"]["secrets"][service]["facts"][fact] = {
+                "value": value.decode()
+            }
+    return nixos_config
+
+
 # TODO move this to the Machines class
 def build_vm(
-    machine: Machine, vm: VmConfig, nix_options: list[str] = []
+    machine: Machine, vm: VmConfig, tmpdir: Path, nix_options: list[str]
 ) -> dict[str, str]:
-    config = nix_config()
-    system = config["system"]
+    secrets_dir = get_secrets(machine, tmpdir)
 
-    clan_dir = machine.flake
-    cmd = nix_build(
-        [
-            f'{clan_dir}#clanInternals.machines."{system}"."{machine.name}".config.system.clan.vm.create',
-            *nix_options,
-        ],
-        machine_gcroot(flake_url=str(vm.flake_url)) / f"vm-{machine.name}",
-    )
-    proc = run(
-        cmd, log=Log.BOTH, error_msg=f"Could not build vm config for {machine.name}"
+    facts_module = importlib.import_module(machine.facts_module)
+    fact_store = facts_module.FactStore(machine=machine)
+    facts = fact_store.get_all()
+
+    nixos_config_file = machine.build_nix(
+        "config.system.clan.vm.create",
+        extra_config=facts_to_nixos_config(facts),
+        nix_options=nix_options,
     )
     try:
-        return json.loads(Path(proc.stdout.strip()).read_text())
+        vm_data = json.loads(Path(nixos_config_file).read_text())
+        vm_data["secrets_dir"] = str(secrets_dir)
+        return vm_data
     except json.JSONDecodeError as e:
         raise ClanError(f"Failed to parse vm config: {e}")
 
@@ -182,16 +195,13 @@ def get_secrets(
     tmpdir: Path,
 ) -> Path:
     secrets_dir = tmpdir / "secrets"
-    secrets_dir.mkdir(exist_ok=True)
+    secrets_dir.mkdir(parents=True, exist_ok=True)
 
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
 
-    # Only generate secrets for local clans
-    if isinstance(machine.flake, Path) and machine.flake.is_dir():
-        generate_secrets(machine)
-    else:
-        log.warning("won't generate secrets for non local clan")
+    # TODO Only generate secrets for local clans
+    generate_secrets(machine)
 
     secret_store.upload(secrets_dir)
     return secrets_dir
@@ -302,20 +312,19 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
     machine = Machine(vm.machine_name, vm.flake_url)
     log.debug(f"Creating VM for {machine}")
 
-    # TODO: We should get this from the vm argument
-    nixos_config = build_vm(machine, vm, nix_options)
-
     # store the temporary rootfs inside XDG_CACHE_HOME on the host
     # otherwise, when using /tmp, we risk running out of memory
     cache = user_cache_dir() / "clan"
     cache.mkdir(exist_ok=True)
     with TemporaryDirectory(dir=cache) as cachedir, TemporaryDirectory() as sockets:
         tmpdir = Path(cachedir)
+
+        # TODO: We should get this from the vm argument
+        nixos_config = build_vm(machine, vm, tmpdir, nix_options)
+
         xchg_dir = tmpdir / "xchg"
         xchg_dir.mkdir(exist_ok=True)
 
-        secrets_dir = get_secrets(machine, tmpdir)
-
         state_dir = vm_state_dir(str(vm.flake_url), machine.name)
         state_dir.mkdir(parents=True, exist_ok=True)
 
@@ -350,7 +359,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
             vm,
             nixos_config,
             xchg_dir=xchg_dir,
-            secrets_dir=secrets_dir,
+            secrets_dir=Path(nixos_config["secrets_dir"]),
             rootfs_img=rootfs_img,
             state_img=state_img,
             virtiofsd_socket=virtiofsd_socket,