From c29e0086a452e06f6b777de7c3daa52ac2b5c340 Mon Sep 17 00:00:00 2001 From: DavHau Date: Mon, 8 Jan 2024 18:37:50 +0700 Subject: [PATCH] VMs: persist state folders on host Done: - move vm inspect attrs from system.clan.vm.config to clanCore.vm.inspect. This gives us proper name and type checking. everything in `system` is basically freeform, so the previous option definitions were never enforced - when running VMs, mount state directory from ~/.config/clan/vmstate/{...} from the host to /var/vmstate inside the vm - create bind mount inside the VM from /var/vmstate/{folder} to / for all folders defined in clanCore.state..folders TODOs: - make sure directories in ~/.config/clan/vmstate never collide (include hash of clan-url, etc.) - port impure test to python --- checks/impure/flake-module.nix | 81 +++++++++++++++++- nixosModules/clanCore/outputs.nix | 6 -- nixosModules/clanCore/vm.nix | 113 ++++++++++++++++++++++---- pkgs/clan-cli/clan_cli/dirs.py | 4 + pkgs/clan-cli/clan_cli/vms/inspect.py | 2 +- pkgs/clan-cli/clan_cli/vms/run.py | 8 +- 6 files changed, 186 insertions(+), 28 deletions(-) diff --git a/checks/impure/flake-module.nix b/checks/impure/flake-module.nix index 18f3e7ad..77e08a01 100644 --- a/checks/impure/flake-module.nix +++ b/checks/impure/flake-module.nix @@ -1,5 +1,5 @@ -{ ... }: { - perSystem = { pkgs, lib, ... }: { +{ self, ... }: { + perSystem = { pkgs, lib, self', ... }: { packages = rec { # a script that executes all other checks impure-checks = pkgs.writeShellScriptBin "impure-checks" '' @@ -13,9 +13,86 @@ ]}" ROOT=$(git rev-parse --show-toplevel) cd "$ROOT/pkgs/clan-cli" + ${self'.packages.vm-persistence}/bin/vm-persistence nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure ./tests $@" ''; + # TODO: port this to python and make it pure + vm-persistence = + let + machineConfigFile = builtins.toFile "vm-config.json" (builtins.toJSON { + clanCore.state.my-state = { + folders = [ "/var/my-state" ]; + }; + # powers off the machine after the state is created + systemd.services.poweroff = { + description = "Poweroff the machine"; + wantedBy = [ "multi-user.target" ]; + after = [ "my-state.service" ]; + script = '' + echo "Powering off the machine" + poweroff + ''; + }; + # creates a file in the state folder + systemd.services.my-state = { + description = "Create a file in the state folder"; + wantedBy = [ "multi-user.target" ]; + script = '' + echo "Creating a file in the state folder" + echo "dream2nix" > /var/my-state/test + ''; + serviceConfig.Type = "oneshot"; + }; + clan.virtualisation.graphics = false; + users.users.root.password = "root"; + }); + in + pkgs.writeShellScriptBin "vm-persistence" '' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + export PATH="${lib.makeBinPath [ + pkgs.coreutils + pkgs.gitMinimal + pkgs.jq + pkgs.nix + pkgs.gnused + self'.packages.clan-cli + ]}" + + clanName=_test_vm_persistence + testFile=~/".config/clan/vmstate/$clanName/my-machine/var/my-state/test" + + export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d) + trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT + + # clean up vmstate after test + trap "${pkgs.coreutils}/bin/rm -rf ~/.config/clan/vmstate/$clanName" EXIT + + cd $TMPDIR + mkdir ./clan + cd ./clan + nix flake init -t ${self}#templates.new-clan + nix flake lock --override-input clan-core ${self} + sed -i "s/__CHANGE_ME__/$clanName/g" flake.nix + clan machines create my-machine + + cat ${machineConfigFile} | jq > ./machines/my-machine/settings.json + + # clear state from previous runs + rm -rf "$testFile" + + # machine will automatically shutdown due to the shutdown service above + clan vms run my-machine + + set -x + if ! test -e "$testFile"; then + echo "failed: file "$testFile" was not created" + exit 1 + fi + ''; + runMockApi = pkgs.writeShellScriptBin "run-mock-api" '' #!${pkgs.bash}/bin/bash set -euo pipefail diff --git a/nixosModules/clanCore/outputs.nix b/nixosModules/clanCore/outputs.nix index d60f8778..b077a2f1 100644 --- a/nixosModules/clanCore/outputs.nix +++ b/nixosModules/clanCore/outputs.nix @@ -45,12 +45,6 @@ ''; default = "${pkgs.coreutils}/bin/true"; }; - vm.config = lib.mkOption { - type = lib.types.attrs; - description = '' - the vm config - ''; - }; vm.create = lib.mkOption { type = lib.types.path; description = '' diff --git a/nixosModules/clanCore/vm.nix b/nixosModules/clanCore/vm.nix index 0c171e1c..51fcbdba 100644 --- a/nixosModules/clanCore/vm.nix +++ b/nixosModules/clanCore/vm.nix @@ -1,19 +1,59 @@ { lib, config, pkgs, options, extendModules, modulesPath, ... }: let - vmConfig = extendModules { - modules = [ + # Generates a fileSystems entry for bind mounting a given state folder path + # It binds directories from /var/clanstate/{some-path} to /{some-path}. + # As a result, all state paths will be persisted across reboots, because + # the state folder is mounted from the host system. + mkBindMount = path: { + name = path; + value = { + device = "/var/clanstate/${path}"; + options = [ "bind" ]; + }; + }; + + # Flatten the list of state folders into a single list + stateFolders = lib.flatten ( + lib.mapAttrsToList + (_item: attrs: attrs.folders) + config.clanCore.state + ); + + # A module setting up bind mounts for all state folders + stateMounts = { + virtualisation.fileSystems = + lib.listToAttrs + (map mkBindMount stateFolders); + }; + + vmModule = { + imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ./serial.nix - { - virtualisation.fileSystems.${config.clanCore.secretsUploadDirectory} = lib.mkForce { - device = "secrets"; - fsType = "9p"; - neededForBoot = true; - options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ]; - }; - boot.initrd.systemd.enable = true; - } + stateMounts ]; + virtualisation.fileSystems = { + ${config.clanCore.secretsUploadDirectory} = lib.mkForce { + device = "secrets"; + fsType = "9p"; + neededForBoot = true; + options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ]; + }; + "/var/clanstate" = { + device = "state"; + fsType = "9p"; + options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ]; + }; + }; + boot.initrd.systemd.enable = true; + }; + + # We cannot simply merge the VM config into the current system config, because + # it is not necessarily a VM. + # Instead we use extendModules to create a second instance of the current + # system configuration, and then merge the VM config into that. + vmConfig = extendModules { + modules = [ vmModule stateMounts ]; }; in { @@ -47,17 +87,54 @@ in ''; }; }; + # All important VM config variables needed by the vm runner + # this is really just a remapping of values defined elsewhere + # and therefore not intended to be set by the user + clanCore.vm.inspect = { + clan_name = lib.mkOption { + type = lib.types.str; + internal = true; + readOnly = true; + description = '' + the name of the clan + ''; + }; + memory_size = lib.mkOption { + type = lib.types.int; + internal = true; + readOnly = true; + description = '' + the amount of memory to allocate to the vm + ''; + }; + cores = lib.mkOption { + type = lib.types.int; + internal = true; + readOnly = true; + description = '' + the number of cores to allocate to the vm + ''; + }; + graphics = lib.mkOption { + type = lib.types.bool; + internal = true; + readOnly = true; + description = '' + whether to enable graphics for the vm + ''; + }; + }; }; config = { + # for clan vm inspect + clanCore.vm.inspect = { + clan_name = config.clanCore.clanName; + memory_size = config.clan.virtualisation.memorySize; + inherit (config.clan.virtualisation) cores graphics; + }; + # for clan vm create system.clan.vm = { - # for clan vm inspect - config = { - clan_name = config.clanCore.clanName; - memory_size = config.clan.virtualisation.memorySize; - inherit (config.clan.virtualisation) cores graphics; - }; - # for clan vm create create = pkgs.writeText "vm.json" (builtins.toJSON { initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}"; toplevel = vmConfig.config.system.build.toplevel; diff --git a/pkgs/clan-cli/clan_cli/dirs.py b/pkgs/clan-cli/clan_cli/dirs.py index e8957301..77d32990 100644 --- a/pkgs/clan-cli/clan_cli/dirs.py +++ b/pkgs/clan-cli/clan_cli/dirs.py @@ -56,6 +56,10 @@ def user_history_file() -> Path: return user_config_dir() / "clan" / "history" +def vm_state_dir(clan_name: str, vm_name: str) -> Path: + return user_config_dir() / "clan" / "vmstate" / clan_name / vm_name + + def machines_dir(flake_dir: Path) -> Path: return flake_dir / "machines" diff --git a/pkgs/clan-cli/clan_cli/vms/inspect.py b/pkgs/clan-cli/clan_cli/vms/inspect.py index 14d90119..96eaa0dc 100644 --- a/pkgs/clan-cli/clan_cli/vms/inspect.py +++ b/pkgs/clan-cli/clan_cli/vms/inspect.py @@ -25,7 +25,7 @@ def inspect_vm(flake_url: str | Path, flake_attr: str) -> VmConfig: cmd = nix_eval( [ - f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.system.clan.vm.config' + f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.vm.inspect' ] ) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 73ea7eb5..c8f32639 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import IO from ..cmd import run -from ..dirs import module_root, specific_groot_dir +from ..dirs import module_root, specific_groot_dir, vm_state_dir from ..errors import ClanError from ..nix import nix_build, nix_config, nix_shell from .inspect import VmConfig, inspect_vm @@ -82,6 +82,7 @@ def qemu_command( nixos_config: dict[str, str], xchg_dir: Path, secrets_dir: Path, + state_dir: Path, disk_img: Path, ) -> list[str]: kernel_cmdline = [ @@ -107,6 +108,7 @@ def qemu_command( "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared", "-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg", "-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets", + "-virtfs", f"local,path={state_dir},security_model=none,mount_tag=state", "-drive", f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report", "-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root", "-device", "virtio-keyboard", @@ -253,11 +255,15 @@ def run_vm( secrets_dir = generate_secrets(vm, nixos_config, tmpdir, log_fd) disk_img = prepare_disk(tmpdir, log_fd) + state_dir = vm_state_dir(vm.clan_name, machine) + state_dir.mkdir(parents=True, exist_ok=True) + qemu_cmd = qemu_command( vm, nixos_config, xchg_dir=xchg_dir, secrets_dir=secrets_dir, + state_dir=state_dir, disk_img=disk_img, )