VMs: persist state folders on host
Done: - move vm inspect attrs from system.clan.vm.config to clanCore.vm.inspect. This gives us proper name and type checking. everything in `system` is basically freeform, so the previous option definitions were never enforced - when running VMs, mount state directory from ~/.config/clan/vmstate/{...} from the host to /var/vmstate inside the vm - create bind mount inside the VM from /var/vmstate/{folder} to / for all folders defined in clanCore.state.<name>.folders TODOs: - make sure directories in ~/.config/clan/vmstate never collide (include hash of clan-url, etc.) - port impure test to python
This commit is contained in:
parent
4123944732
commit
c29e0086a4
|
@ -1,5 +1,5 @@
|
|||
{ ... }: {
|
||||
perSystem = { pkgs, lib, ... }: {
|
||||
{ self, ... }: {
|
||||
perSystem = { pkgs, lib, self', ... }: {
|
||||
packages = rec {
|
||||
# a script that executes all other checks
|
||||
impure-checks = pkgs.writeShellScriptBin "impure-checks" ''
|
||||
|
@ -13,9 +13,86 @@
|
|||
]}"
|
||||
ROOT=$(git rev-parse --show-toplevel)
|
||||
cd "$ROOT/pkgs/clan-cli"
|
||||
${self'.packages.vm-persistence}/bin/vm-persistence
|
||||
nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure ./tests $@"
|
||||
'';
|
||||
|
||||
# TODO: port this to python and make it pure
|
||||
vm-persistence =
|
||||
let
|
||||
machineConfigFile = builtins.toFile "vm-config.json" (builtins.toJSON {
|
||||
clanCore.state.my-state = {
|
||||
folders = [ "/var/my-state" ];
|
||||
};
|
||||
# powers off the machine after the state is created
|
||||
systemd.services.poweroff = {
|
||||
description = "Poweroff the machine";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "my-state.service" ];
|
||||
script = ''
|
||||
echo "Powering off the machine"
|
||||
poweroff
|
||||
'';
|
||||
};
|
||||
# creates a file in the state folder
|
||||
systemd.services.my-state = {
|
||||
description = "Create a file in the state folder";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
script = ''
|
||||
echo "Creating a file in the state folder"
|
||||
echo "dream2nix" > /var/my-state/test
|
||||
'';
|
||||
serviceConfig.Type = "oneshot";
|
||||
};
|
||||
clan.virtualisation.graphics = false;
|
||||
users.users.root.password = "root";
|
||||
});
|
||||
in
|
||||
pkgs.writeShellScriptBin "vm-persistence" ''
|
||||
#!${pkgs.bash}/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export PATH="${lib.makeBinPath [
|
||||
pkgs.coreutils
|
||||
pkgs.gitMinimal
|
||||
pkgs.jq
|
||||
pkgs.nix
|
||||
pkgs.gnused
|
||||
self'.packages.clan-cli
|
||||
]}"
|
||||
|
||||
clanName=_test_vm_persistence
|
||||
testFile=~/".config/clan/vmstate/$clanName/my-machine/var/my-state/test"
|
||||
|
||||
export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
|
||||
trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
|
||||
|
||||
# clean up vmstate after test
|
||||
trap "${pkgs.coreutils}/bin/rm -rf ~/.config/clan/vmstate/$clanName" EXIT
|
||||
|
||||
cd $TMPDIR
|
||||
mkdir ./clan
|
||||
cd ./clan
|
||||
nix flake init -t ${self}#templates.new-clan
|
||||
nix flake lock --override-input clan-core ${self}
|
||||
sed -i "s/__CHANGE_ME__/$clanName/g" flake.nix
|
||||
clan machines create my-machine
|
||||
|
||||
cat ${machineConfigFile} | jq > ./machines/my-machine/settings.json
|
||||
|
||||
# clear state from previous runs
|
||||
rm -rf "$testFile"
|
||||
|
||||
# machine will automatically shutdown due to the shutdown service above
|
||||
clan vms run my-machine
|
||||
|
||||
set -x
|
||||
if ! test -e "$testFile"; then
|
||||
echo "failed: file "$testFile" was not created"
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
|
||||
runMockApi = pkgs.writeShellScriptBin "run-mock-api" ''
|
||||
#!${pkgs.bash}/bin/bash
|
||||
set -euo pipefail
|
||||
|
|
|
@ -45,12 +45,6 @@
|
|||
'';
|
||||
default = "${pkgs.coreutils}/bin/true";
|
||||
};
|
||||
vm.config = lib.mkOption {
|
||||
type = lib.types.attrs;
|
||||
description = ''
|
||||
the vm config
|
||||
'';
|
||||
};
|
||||
vm.create = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = ''
|
||||
|
|
|
@ -1,19 +1,59 @@
|
|||
{ lib, config, pkgs, options, extendModules, modulesPath, ... }:
|
||||
let
|
||||
vmConfig = extendModules {
|
||||
modules = [
|
||||
# Generates a fileSystems entry for bind mounting a given state folder path
|
||||
# It binds directories from /var/clanstate/{some-path} to /{some-path}.
|
||||
# As a result, all state paths will be persisted across reboots, because
|
||||
# the state folder is mounted from the host system.
|
||||
mkBindMount = path: {
|
||||
name = path;
|
||||
value = {
|
||||
device = "/var/clanstate/${path}";
|
||||
options = [ "bind" ];
|
||||
};
|
||||
};
|
||||
|
||||
# Flatten the list of state folders into a single list
|
||||
stateFolders = lib.flatten (
|
||||
lib.mapAttrsToList
|
||||
(_item: attrs: attrs.folders)
|
||||
config.clanCore.state
|
||||
);
|
||||
|
||||
# A module setting up bind mounts for all state folders
|
||||
stateMounts = {
|
||||
virtualisation.fileSystems =
|
||||
lib.listToAttrs
|
||||
(map mkBindMount stateFolders);
|
||||
};
|
||||
|
||||
vmModule = {
|
||||
imports = [
|
||||
(modulesPath + "/virtualisation/qemu-vm.nix")
|
||||
./serial.nix
|
||||
{
|
||||
virtualisation.fileSystems.${config.clanCore.secretsUploadDirectory} = lib.mkForce {
|
||||
device = "secrets";
|
||||
fsType = "9p";
|
||||
neededForBoot = true;
|
||||
options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
|
||||
};
|
||||
boot.initrd.systemd.enable = true;
|
||||
}
|
||||
stateMounts
|
||||
];
|
||||
virtualisation.fileSystems = {
|
||||
${config.clanCore.secretsUploadDirectory} = lib.mkForce {
|
||||
device = "secrets";
|
||||
fsType = "9p";
|
||||
neededForBoot = true;
|
||||
options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
|
||||
};
|
||||
"/var/clanstate" = {
|
||||
device = "state";
|
||||
fsType = "9p";
|
||||
options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
|
||||
};
|
||||
};
|
||||
boot.initrd.systemd.enable = true;
|
||||
};
|
||||
|
||||
# We cannot simply merge the VM config into the current system config, because
|
||||
# it is not necessarily a VM.
|
||||
# Instead we use extendModules to create a second instance of the current
|
||||
# system configuration, and then merge the VM config into that.
|
||||
vmConfig = extendModules {
|
||||
modules = [ vmModule stateMounts ];
|
||||
};
|
||||
in
|
||||
{
|
||||
|
@ -47,17 +87,54 @@ in
|
|||
'';
|
||||
};
|
||||
};
|
||||
# All important VM config variables needed by the vm runner
|
||||
# this is really just a remapping of values defined elsewhere
|
||||
# and therefore not intended to be set by the user
|
||||
clanCore.vm.inspect = {
|
||||
clan_name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
the name of the clan
|
||||
'';
|
||||
};
|
||||
memory_size = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
the amount of memory to allocate to the vm
|
||||
'';
|
||||
};
|
||||
cores = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
the number of cores to allocate to the vm
|
||||
'';
|
||||
};
|
||||
graphics = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
whether to enable graphics for the vm
|
||||
'';
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
# for clan vm inspect
|
||||
clanCore.vm.inspect = {
|
||||
clan_name = config.clanCore.clanName;
|
||||
memory_size = config.clan.virtualisation.memorySize;
|
||||
inherit (config.clan.virtualisation) cores graphics;
|
||||
};
|
||||
# for clan vm create
|
||||
system.clan.vm = {
|
||||
# for clan vm inspect
|
||||
config = {
|
||||
clan_name = config.clanCore.clanName;
|
||||
memory_size = config.clan.virtualisation.memorySize;
|
||||
inherit (config.clan.virtualisation) cores graphics;
|
||||
};
|
||||
# for clan vm create
|
||||
create = pkgs.writeText "vm.json" (builtins.toJSON {
|
||||
initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}";
|
||||
toplevel = vmConfig.config.system.build.toplevel;
|
||||
|
|
|
@ -56,6 +56,10 @@ def user_history_file() -> Path:
|
|||
return user_config_dir() / "clan" / "history"
|
||||
|
||||
|
||||
def vm_state_dir(clan_name: str, vm_name: str) -> Path:
|
||||
return user_config_dir() / "clan" / "vmstate" / clan_name / vm_name
|
||||
|
||||
|
||||
def machines_dir(flake_dir: Path) -> Path:
|
||||
return flake_dir / "machines"
|
||||
|
||||
|
|
|
@ -25,7 +25,7 @@ def inspect_vm(flake_url: str | Path, flake_attr: str) -> VmConfig:
|
|||
|
||||
cmd = nix_eval(
|
||||
[
|
||||
f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.system.clan.vm.config'
|
||||
f'{flake_url}#clanInternals.machines."{system}"."{flake_attr}".config.clanCore.vm.inspect'
|
||||
]
|
||||
)
|
||||
|
||||
|
|
|
@ -11,7 +11,7 @@ from pathlib import Path
|
|||
from typing import IO
|
||||
|
||||
from ..cmd import run
|
||||
from ..dirs import module_root, specific_groot_dir
|
||||
from ..dirs import module_root, specific_groot_dir, vm_state_dir
|
||||
from ..errors import ClanError
|
||||
from ..nix import nix_build, nix_config, nix_shell
|
||||
from .inspect import VmConfig, inspect_vm
|
||||
|
@ -82,6 +82,7 @@ def qemu_command(
|
|||
nixos_config: dict[str, str],
|
||||
xchg_dir: Path,
|
||||
secrets_dir: Path,
|
||||
state_dir: Path,
|
||||
disk_img: Path,
|
||||
) -> list[str]:
|
||||
kernel_cmdline = [
|
||||
|
@ -107,6 +108,7 @@ def qemu_command(
|
|||
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
|
||||
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
|
||||
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
|
||||
"-virtfs", f"local,path={state_dir},security_model=none,mount_tag=state",
|
||||
"-drive", f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report",
|
||||
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
|
||||
"-device", "virtio-keyboard",
|
||||
|
@ -253,11 +255,15 @@ def run_vm(
|
|||
secrets_dir = generate_secrets(vm, nixos_config, tmpdir, log_fd)
|
||||
disk_img = prepare_disk(tmpdir, log_fd)
|
||||
|
||||
state_dir = vm_state_dir(vm.clan_name, machine)
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
qemu_cmd = qemu_command(
|
||||
vm,
|
||||
nixos_config,
|
||||
xchg_dir=xchg_dir,
|
||||
secrets_dir=secrets_dir,
|
||||
state_dir=state_dir,
|
||||
disk_img=disk_img,
|
||||
)
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user