forked from clan/clan-core
Compare commits
3 Commits
Author | SHA1 | Date | |
---|---|---|---|
6d135dc299 | |||
281fd2d89c | |||
062a0b151e |
@ -1,8 +1,9 @@
|
|||||||
{ lib, ... }:
|
{ lib, ... }:
|
||||||
{
|
{
|
||||||
# defaults
|
# defaults
|
||||||
# FIXME: currently broken, will be fixed soon
|
config.clanCore.state.HOME.folders = [
|
||||||
#config.clanCore.state.HOME.folders = [ "/home" ];
|
"/home"
|
||||||
|
];
|
||||||
|
|
||||||
# interface
|
# interface
|
||||||
options.clanCore.state = lib.mkOption {
|
options.clanCore.state = lib.mkOption {
|
||||||
|
@ -1,17 +1,5 @@
|
|||||||
{ lib, config, pkgs, options, extendModules, modulesPath, ... }:
|
{ lib, config, pkgs, options, extendModules, modulesPath, ... }:
|
||||||
let
|
let
|
||||||
# Generates a fileSystems entry for bind mounting a given state folder path
|
|
||||||
# It binds directories from /var/clanstate/{some-path} to /{some-path}.
|
|
||||||
# As a result, all state paths will be persisted across reboots, because
|
|
||||||
# the state folder is mounted from the host system.
|
|
||||||
mkBindMount = path: {
|
|
||||||
name = path;
|
|
||||||
value = {
|
|
||||||
device = "/var/clanstate/${path}";
|
|
||||||
options = [ "bind" ];
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
# Flatten the list of state folders into a single list
|
# Flatten the list of state folders into a single list
|
||||||
stateFolders = lib.flatten (
|
stateFolders = lib.flatten (
|
||||||
lib.mapAttrsToList
|
lib.mapAttrsToList
|
||||||
@ -19,19 +7,70 @@ let
|
|||||||
config.clanCore.state
|
config.clanCore.state
|
||||||
);
|
);
|
||||||
|
|
||||||
# A module setting up bind mounts for all state folders
|
# Ensure sane mount order by topo-sorting
|
||||||
stateMounts = {
|
sortedStateFolders =
|
||||||
virtualisation.fileSystems =
|
let
|
||||||
lib.listToAttrs
|
sorted = lib.toposort lib.hasPrefix stateFolders;
|
||||||
(map mkBindMount stateFolders);
|
in
|
||||||
};
|
sorted.result or (
|
||||||
|
throw ''
|
||||||
|
The state folders have a cyclic dependency.
|
||||||
|
This is not allowed.
|
||||||
|
The cyclic dependencies are:
|
||||||
|
- ${lib.concatStringsSep "\n - " sorted.loops}
|
||||||
|
''
|
||||||
|
);
|
||||||
|
|
||||||
vmModule = {
|
vmModule = {
|
||||||
imports = [
|
imports = [
|
||||||
(modulesPath + "/virtualisation/qemu-vm.nix")
|
(modulesPath + "/virtualisation/qemu-vm.nix")
|
||||||
./serial.nix
|
./serial.nix
|
||||||
stateMounts
|
|
||||||
];
|
];
|
||||||
|
|
||||||
|
# required for issuing shell commands via qga
|
||||||
|
services.qemuGuest.enable = true;
|
||||||
|
|
||||||
|
boot.initrd.systemd.enable = true;
|
||||||
|
|
||||||
|
# currently needed for system.etc.overlay.enable
|
||||||
|
boot.kernelPackages = pkgs.linuxPackages_latest;
|
||||||
|
|
||||||
|
boot.initrd.systemd.storePaths = [ pkgs.util-linux pkgs.e2fsprogs ];
|
||||||
|
# Ensures, that all state paths will be persisted across reboots
|
||||||
|
# - Mounts the state.qcow2 disk to /vmstate.
|
||||||
|
# - Binds directories from /vmstate/{some-path} to /{some-path}.
|
||||||
|
boot.initrd.systemd.services.rw-etc-pre = {
|
||||||
|
unitConfig = {
|
||||||
|
DefaultDependencies = false;
|
||||||
|
RequiresMountsFor = "/sysroot /dev";
|
||||||
|
};
|
||||||
|
wantedBy = [ "initrd.target" ];
|
||||||
|
requiredBy = [ "rw-etc.service" ];
|
||||||
|
before = [ "rw-etc.service" ];
|
||||||
|
serviceConfig = {
|
||||||
|
Type = "oneshot";
|
||||||
|
};
|
||||||
|
script = ''
|
||||||
|
set -x
|
||||||
|
mkdir -p -m 0755 \
|
||||||
|
/sysroot/vmstate \
|
||||||
|
/sysroot/.rw-etc \
|
||||||
|
/sysroot/var/lib/nixos
|
||||||
|
|
||||||
|
${pkgs.util-linux}/bin/blkid /dev/vdb || ${pkgs.e2fsprogs}/bin/mkfs.ext4 /dev/vdb
|
||||||
|
sync
|
||||||
|
mount /dev/vdb /sysroot/vmstate
|
||||||
|
|
||||||
|
mkdir -p -m 0755 /sysroot/vmstate/{.rw-etc,var/lib/nixos}
|
||||||
|
mount --bind /sysroot/vmstate/.rw-etc /sysroot/.rw-etc
|
||||||
|
mount --bind /sysroot/vmstate/var/lib/nixos /sysroot/var/lib/nixos
|
||||||
|
|
||||||
|
for folder in "${lib.concatStringsSep ''" "'' sortedStateFolders}"; do
|
||||||
|
mkdir -p -m 0755 "/sysroot/vmstate/$folder" "/sysroot/$folder"
|
||||||
|
mount --bind "/sysroot/vmstate/$folder" "/sysroot/$folder"
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
virtualisation.fileSystems = {
|
virtualisation.fileSystems = {
|
||||||
${config.clanCore.secretsUploadDirectory} = lib.mkForce {
|
${config.clanCore.secretsUploadDirectory} = lib.mkForce {
|
||||||
device = "secrets";
|
device = "secrets";
|
||||||
@ -39,13 +78,7 @@ let
|
|||||||
neededForBoot = true;
|
neededForBoot = true;
|
||||||
options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
|
options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
|
||||||
};
|
};
|
||||||
"/var/clanstate" = {
|
|
||||||
device = "state";
|
|
||||||
fsType = "9p";
|
|
||||||
options = [ "trans=virtio" "version=9p2000.L" "cache=loose" ];
|
|
||||||
};
|
|
||||||
};
|
};
|
||||||
boot.initrd.systemd.enable = true;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
# We cannot simply merge the VM config into the current system config, because
|
# We cannot simply merge the VM config into the current system config, because
|
||||||
@ -53,7 +86,7 @@ let
|
|||||||
# Instead we use extendModules to create a second instance of the current
|
# Instead we use extendModules to create a second instance of the current
|
||||||
# system configuration, and then merge the VM config into that.
|
# system configuration, and then merge the VM config into that.
|
||||||
vmConfig = extendModules {
|
vmConfig = extendModules {
|
||||||
modules = [ vmModule stateMounts ];
|
modules = [ vmModule ];
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
|
@ -220,7 +220,7 @@ in
|
|||||||
--network-id "$facts/zerotier-network-id"
|
--network-id "$facts/zerotier-network-id"
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
# clanCore.state.zerotier.folders = [ "/var/lib/zerotier-one" ];
|
clanCore.state.zerotier.folders = [ "/var/lib/zerotier-one" ];
|
||||||
|
|
||||||
environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
|
environment.systemPackages = [ config.clanCore.clanPkgs.zerotier-members ];
|
||||||
})
|
})
|
||||||
|
@ -33,7 +33,7 @@
|
|||||||
systemd.services.hidden-ssh-announce = {
|
systemd.services.hidden-ssh-announce = {
|
||||||
description = "announce hidden ssh";
|
description = "announce hidden ssh";
|
||||||
after = [ "tor.service" "network-online.target" ];
|
after = [ "tor.service" "network-online.target" ];
|
||||||
wants = [ "tor.service" ];
|
wants = [ "tor.service" "network-online.target" ];
|
||||||
wantedBy = [ "multi-user.target" ];
|
wantedBy = [ "multi-user.target" ];
|
||||||
serviceConfig = {
|
serviceConfig = {
|
||||||
# ${pkgs.tor}/bin/torify
|
# ${pkgs.tor}/bin/torify
|
||||||
|
@ -41,6 +41,28 @@ def user_config_dir() -> Path:
|
|||||||
return Path(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")))
|
return Path(os.getenv("XDG_CONFIG_HOME", os.path.expanduser("~/.config")))
|
||||||
|
|
||||||
|
|
||||||
|
def user_data_dir() -> Path:
|
||||||
|
if sys.platform == "win32":
|
||||||
|
return Path(
|
||||||
|
os.getenv("LOCALAPPDATA", os.path.expanduser("~\\AppData\\Local\\"))
|
||||||
|
)
|
||||||
|
elif sys.platform == "darwin":
|
||||||
|
return Path(os.path.expanduser("~/Library/Application Support/"))
|
||||||
|
else:
|
||||||
|
return Path(os.getenv("XDG_DATA_HOME", os.path.expanduser("~/.local/share")))
|
||||||
|
|
||||||
|
|
||||||
|
def user_cache_dir() -> Path:
|
||||||
|
if sys.platform == "win32":
|
||||||
|
return Path(
|
||||||
|
os.getenv("LOCALAPPDATA", os.path.expanduser("~\\AppData\\Local\\"))
|
||||||
|
)
|
||||||
|
elif sys.platform == "darwin":
|
||||||
|
return Path(os.path.expanduser("~/Library/Caches/"))
|
||||||
|
else:
|
||||||
|
return Path(os.getenv("XDG_CACHE_HOME", os.path.expanduser("~/.cache")))
|
||||||
|
|
||||||
|
|
||||||
def user_gcroot_dir() -> Path:
|
def user_gcroot_dir() -> Path:
|
||||||
p = user_config_dir() / "clan" / "gcroots"
|
p = user_config_dir() / "clan" / "gcroots"
|
||||||
p.mkdir(parents=True, exist_ok=True)
|
p.mkdir(parents=True, exist_ok=True)
|
||||||
@ -61,7 +83,7 @@ def user_history_file() -> Path:
|
|||||||
|
|
||||||
def vm_state_dir(clan_name: str, flake_url: str, vm_name: str) -> Path:
|
def vm_state_dir(clan_name: str, flake_url: str, vm_name: str) -> Path:
|
||||||
clan_key = clan_key_safe(clan_name, flake_url)
|
clan_key = clan_key_safe(clan_name, flake_url)
|
||||||
return user_config_dir() / "clan" / "vmstate" / clan_key / vm_name
|
return user_data_dir() / "clan" / "vmstate" / clan_key / vm_name
|
||||||
|
|
||||||
|
|
||||||
def machines_dir(flake_dir: Path) -> Path:
|
def machines_dir(flake_dir: Path) -> Path:
|
||||||
|
@ -7,15 +7,15 @@ import os
|
|||||||
import random
|
import random
|
||||||
import socket
|
import socket
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
|
||||||
import time
|
import time
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
from typing import IO
|
from typing import IO
|
||||||
|
|
||||||
from ..cmd import Log, run
|
from ..cmd import Log, run
|
||||||
from ..dirs import machine_gcroot, module_root, vm_state_dir
|
from ..dirs import machine_gcroot, module_root, user_cache_dir, vm_state_dir
|
||||||
from ..errors import ClanError
|
from ..errors import ClanError
|
||||||
from ..machines.machines import Machine
|
from ..machines.machines import Machine
|
||||||
from ..nix import nix_build, nix_config, nix_shell
|
from ..nix import nix_build, nix_config, nix_shell
|
||||||
@ -90,8 +90,10 @@ def qemu_command(
|
|||||||
nixos_config: dict[str, str],
|
nixos_config: dict[str, str],
|
||||||
xchg_dir: Path,
|
xchg_dir: Path,
|
||||||
secrets_dir: Path,
|
secrets_dir: Path,
|
||||||
state_dir: Path,
|
rootfs_img: Path,
|
||||||
disk_img: Path,
|
state_img: Path,
|
||||||
|
qmp_socket_file: Path,
|
||||||
|
qga_socket_file: Path,
|
||||||
) -> QemuCommand:
|
) -> QemuCommand:
|
||||||
kernel_cmdline = [
|
kernel_cmdline = [
|
||||||
(Path(nixos_config["toplevel"]) / "kernel-params").read_text(),
|
(Path(nixos_config["toplevel"]) / "kernel-params").read_text(),
|
||||||
@ -118,14 +120,20 @@ def qemu_command(
|
|||||||
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
|
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
|
||||||
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
|
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
|
||||||
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
|
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
|
||||||
"-virtfs", f"local,path={state_dir},security_model=none,mount_tag=state",
|
"-drive", f"cache=writeback,file={rootfs_img},format=raw,id=drive1,if=none,index=1,werror=report",
|
||||||
"-drive", f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report",
|
|
||||||
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
|
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
|
||||||
|
"-drive", f"cache=writeback,file={state_img},format=qcow2,id=state,if=none,index=2,werror=report",
|
||||||
|
"-device", "virtio-blk-pci,drive=state",
|
||||||
"-device", "virtio-keyboard",
|
"-device", "virtio-keyboard",
|
||||||
"-usb", "-device", "usb-tablet,bus=usb-bus.0",
|
"-usb", "-device", "usb-tablet,bus=usb-bus.0",
|
||||||
"-kernel", f'{nixos_config["toplevel"]}/kernel',
|
"-kernel", f'{nixos_config["toplevel"]}/kernel',
|
||||||
"-initrd", nixos_config["initrd"],
|
"-initrd", nixos_config["initrd"],
|
||||||
"-append", " ".join(kernel_cmdline),
|
"-append", " ".join(kernel_cmdline),
|
||||||
|
# qmp & qga setup
|
||||||
|
"-qmp", f"unix:{qmp_socket_file},server,wait=off",
|
||||||
|
"-chardev", f"socket,path={qga_socket_file},server=on,wait=off,id=qga0",
|
||||||
|
"-device", "virtio-serial",
|
||||||
|
"-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
|
||||||
] # fmt: on
|
] # fmt: on
|
||||||
|
|
||||||
vsock_cid = None
|
vsock_cid = None
|
||||||
@ -183,17 +191,23 @@ def get_secrets(
|
|||||||
return secrets_dir
|
return secrets_dir
|
||||||
|
|
||||||
|
|
||||||
def prepare_disk(tmpdir: Path, log_fd: IO[str] | None) -> Path:
|
def prepare_disk(
|
||||||
disk_img = tmpdir / "disk.img"
|
directory: Path,
|
||||||
|
disk_format: str = "raw",
|
||||||
|
size: str = "1024M",
|
||||||
|
label: str = "nixos",
|
||||||
|
file_name: str = "disk.img",
|
||||||
|
) -> Path:
|
||||||
|
disk_img = directory / file_name
|
||||||
cmd = nix_shell(
|
cmd = nix_shell(
|
||||||
["nixpkgs#qemu"],
|
["nixpkgs#qemu"],
|
||||||
[
|
[
|
||||||
"qemu-img",
|
"qemu-img",
|
||||||
"create",
|
"create",
|
||||||
"-f",
|
"-f",
|
||||||
"raw",
|
disk_format,
|
||||||
str(disk_img),
|
str(disk_img),
|
||||||
"1024M",
|
size,
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
run(
|
run(
|
||||||
@ -202,20 +216,21 @@ def prepare_disk(tmpdir: Path, log_fd: IO[str] | None) -> Path:
|
|||||||
error_msg=f"Could not create disk image at {disk_img}",
|
error_msg=f"Could not create disk image at {disk_img}",
|
||||||
)
|
)
|
||||||
|
|
||||||
cmd = nix_shell(
|
if disk_format == "raw":
|
||||||
["nixpkgs#e2fsprogs"],
|
cmd = nix_shell(
|
||||||
[
|
["nixpkgs#e2fsprogs"],
|
||||||
"mkfs.ext4",
|
[
|
||||||
"-L",
|
"mkfs.ext4",
|
||||||
"nixos",
|
"-L",
|
||||||
str(disk_img),
|
label,
|
||||||
],
|
str(disk_img),
|
||||||
)
|
],
|
||||||
run(
|
)
|
||||||
cmd,
|
run(
|
||||||
log=Log.BOTH,
|
cmd,
|
||||||
error_msg=f"Could not create ext4 filesystem at {disk_img}",
|
log=Log.BOTH,
|
||||||
)
|
error_msg=f"Could not create ext4 filesystem at {disk_img}",
|
||||||
|
)
|
||||||
return disk_img
|
return disk_img
|
||||||
|
|
||||||
|
|
||||||
@ -272,24 +287,57 @@ def run_vm(
|
|||||||
# TODO: We should get this from the vm argument
|
# TODO: We should get this from the vm argument
|
||||||
nixos_config = get_vm_create_info(machine, vm, nix_options)
|
nixos_config = get_vm_create_info(machine, vm, nix_options)
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory() as tmpdir_:
|
# store the temporary rootfs inside XDG_CACHE_HOME on the host
|
||||||
tmpdir = Path(tmpdir_)
|
# otherwise, when using /tmp, we risk running out of memory
|
||||||
|
cache = user_cache_dir() / "clan"
|
||||||
|
cache.mkdir(exist_ok=True)
|
||||||
|
with TemporaryDirectory(dir=cache) as cachedir, TemporaryDirectory() as sockets:
|
||||||
|
tmpdir = Path(cachedir)
|
||||||
xchg_dir = tmpdir / "xchg"
|
xchg_dir = tmpdir / "xchg"
|
||||||
xchg_dir.mkdir(exist_ok=True)
|
xchg_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
secrets_dir = get_secrets(machine, tmpdir)
|
secrets_dir = get_secrets(machine, tmpdir)
|
||||||
disk_img = prepare_disk(tmpdir, log_fd)
|
|
||||||
|
|
||||||
state_dir = vm_state_dir(vm.clan_name, str(machine.flake), machine.name)
|
state_dir = vm_state_dir(vm.clan_name, str(vm.flake_url), machine.name)
|
||||||
state_dir.mkdir(parents=True, exist_ok=True)
|
state_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# specify socket files for qmp and qga
|
||||||
|
qmp_socket_file = Path(sockets) / "qmp.sock"
|
||||||
|
qga_socket_file = Path(sockets) / "qga.sock"
|
||||||
|
# Create symlinks to the qmp/qga sockets to be able to find them later.
|
||||||
|
# This indirection is needed because we cannot put the sockets directly
|
||||||
|
# in the state_dir.
|
||||||
|
# The reason is, qemu has a length limit of 108 bytes for the qmp socket
|
||||||
|
# path which is violated easily.
|
||||||
|
qmp_link = state_dir / "qmp.sock"
|
||||||
|
if os.path.lexists(qmp_link):
|
||||||
|
qmp_link.unlink()
|
||||||
|
qmp_link.symlink_to(qmp_socket_file)
|
||||||
|
|
||||||
|
qga_link = state_dir / "qga.sock"
|
||||||
|
if os.path.lexists(qga_link):
|
||||||
|
qga_link.unlink()
|
||||||
|
qga_link.symlink_to(qga_socket_file)
|
||||||
|
|
||||||
|
rootfs_img = prepare_disk(tmpdir)
|
||||||
|
state_img = state_dir / "state.qcow2"
|
||||||
|
if not state_img.exists():
|
||||||
|
state_img = prepare_disk(
|
||||||
|
directory=state_dir,
|
||||||
|
file_name="state.qcow2",
|
||||||
|
disk_format="qcow2",
|
||||||
|
size="50G",
|
||||||
|
label="state",
|
||||||
|
)
|
||||||
qemu_cmd = qemu_command(
|
qemu_cmd = qemu_command(
|
||||||
vm,
|
vm,
|
||||||
nixos_config,
|
nixos_config,
|
||||||
xchg_dir=xchg_dir,
|
xchg_dir=xchg_dir,
|
||||||
secrets_dir=secrets_dir,
|
secrets_dir=secrets_dir,
|
||||||
state_dir=state_dir,
|
rootfs_img=rootfs_img,
|
||||||
disk_img=disk_img,
|
state_img=state_img,
|
||||||
|
qmp_socket_file=qmp_socket_file,
|
||||||
|
qga_socket_file=qga_socket_file,
|
||||||
)
|
)
|
||||||
|
|
||||||
packages = ["nixpkgs#qemu"]
|
packages = ["nixpkgs#qemu"]
|
||||||
|
@ -1,5 +1,12 @@
|
|||||||
|
import base64
|
||||||
|
import json
|
||||||
import os
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import traceback
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from time import sleep
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@ -15,6 +22,79 @@ if TYPE_CHECKING:
|
|||||||
no_kvm = not os.path.exists("/dev/kvm")
|
no_kvm = not os.path.exists("/dev/kvm")
|
||||||
|
|
||||||
|
|
||||||
|
# qga is almost like qmp, but not quite, because:
|
||||||
|
# - server doesn't send initial message
|
||||||
|
# - no need to initialize by asking for capabilities
|
||||||
|
# - results need to be base64 decoded
|
||||||
|
# TODO: move this to an extra file and make it available to other parts like GUI
|
||||||
|
class QgaSession:
|
||||||
|
def __init__(self, socket_file: Path | str) -> None:
|
||||||
|
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||||
|
# try to reconnect a couple of times if connetion refused
|
||||||
|
for _ in range(100):
|
||||||
|
try:
|
||||||
|
self.sock.connect(str(socket_file))
|
||||||
|
return
|
||||||
|
except ConnectionRefusedError:
|
||||||
|
sleep(0.1)
|
||||||
|
self.sock.connect(str(socket_file))
|
||||||
|
|
||||||
|
def get_response(self) -> dict:
|
||||||
|
result = self.sock.recv(9999999)
|
||||||
|
return json.loads(result)
|
||||||
|
|
||||||
|
# only execute, don't wait for response
|
||||||
|
def exec_cmd(self, cmd: str) -> None:
|
||||||
|
self.sock.send(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"execute": "guest-exec",
|
||||||
|
"arguments": {
|
||||||
|
"path": "/bin/sh",
|
||||||
|
"arg": ["-l", "-c", cmd],
|
||||||
|
"capture-output": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
).encode("utf-8")
|
||||||
|
)
|
||||||
|
|
||||||
|
# run, wait for result, return exitcode and output
|
||||||
|
def run(self, cmd: str) -> tuple[int, str, str]:
|
||||||
|
self.exec_cmd(cmd)
|
||||||
|
result_pid = self.get_response()
|
||||||
|
pid = result_pid["return"]["pid"]
|
||||||
|
# loop until exited=true
|
||||||
|
status_payload = json.dumps(
|
||||||
|
{
|
||||||
|
"execute": "guest-exec-status",
|
||||||
|
"arguments": {
|
||||||
|
"pid": pid,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
).encode("utf-8")
|
||||||
|
while True:
|
||||||
|
self.sock.send(status_payload)
|
||||||
|
result = self.get_response()
|
||||||
|
if "error" in result and result["error"]["desc"].startswith("PID"):
|
||||||
|
raise Exception("PID could not be found")
|
||||||
|
if result["return"]["exited"]:
|
||||||
|
break
|
||||||
|
sleep(0.1)
|
||||||
|
|
||||||
|
exitcode = result["return"]["exitcode"]
|
||||||
|
stdout = (
|
||||||
|
""
|
||||||
|
if "out-data" not in result["return"]
|
||||||
|
else base64.b64decode(result["return"]["out-data"]).decode("utf-8")
|
||||||
|
)
|
||||||
|
stderr = (
|
||||||
|
""
|
||||||
|
if "err-data" not in result["return"]
|
||||||
|
else base64.b64decode(result["return"]["err-data"]).decode("utf-8")
|
||||||
|
)
|
||||||
|
return exitcode, stdout, stderr
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.impure
|
@pytest.mark.impure
|
||||||
def test_inspect(
|
def test_inspect(
|
||||||
test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture
|
test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture
|
||||||
@ -64,51 +144,199 @@ def test_vm_persistence(
|
|||||||
},
|
},
|
||||||
machine_configs=dict(
|
machine_configs=dict(
|
||||||
my_machine=dict(
|
my_machine=dict(
|
||||||
clanCore=dict(state=dict(my_state=dict(folders=["/var/my-state"]))),
|
services=dict(getty=dict(autologinUser="root")),
|
||||||
|
clanCore=dict(
|
||||||
|
state=dict(
|
||||||
|
my_state=dict(
|
||||||
|
folders=[
|
||||||
|
# to be owned by root
|
||||||
|
"/var/my-state",
|
||||||
|
# to be owned by user 'test'
|
||||||
|
"/var/user-state",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
# create test user
|
||||||
|
# TODO: test persisting files via that user
|
||||||
|
users=dict(
|
||||||
|
users=dict(
|
||||||
|
test=dict(
|
||||||
|
password="test",
|
||||||
|
isNormalUser=True,
|
||||||
|
),
|
||||||
|
root=dict(password="root"),
|
||||||
|
)
|
||||||
|
),
|
||||||
systemd=dict(
|
systemd=dict(
|
||||||
services=dict(
|
services=dict(
|
||||||
poweroff=dict(
|
create_state=dict(
|
||||||
description="Poweroff the machine",
|
|
||||||
wantedBy=["multi-user.target"],
|
|
||||||
after=["my-state.service"],
|
|
||||||
script="""
|
|
||||||
echo "Powering off the machine"
|
|
||||||
poweroff
|
|
||||||
""",
|
|
||||||
),
|
|
||||||
my_state=dict(
|
|
||||||
description="Create a file in the state folder",
|
description="Create a file in the state folder",
|
||||||
wantedBy=["multi-user.target"],
|
wantedBy=["multi-user.target"],
|
||||||
script="""
|
script="""
|
||||||
echo "Creating a file in the state folder"
|
if [ ! -f /var/my-state/root ]; then
|
||||||
echo "dream2nix" > /var/my-state/test
|
echo "Creating a file in the state folder"
|
||||||
""",
|
echo "dream2nix" > /var/my-state/root
|
||||||
serviceConfig=dict(Type="oneshot"),
|
# create /var/my-state/test owned by user test
|
||||||
|
echo "dream2nix" > /var/my-state/test
|
||||||
|
chown test /var/my-state/test
|
||||||
|
# make sure /var/user-state is owned by test
|
||||||
|
chown test /var/user-state
|
||||||
|
fi
|
||||||
|
""",
|
||||||
|
serviceConfig=dict(
|
||||||
|
Type="oneshot",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
reboot=dict(
|
||||||
|
description="Reboot the machine",
|
||||||
|
wantedBy=["multi-user.target"],
|
||||||
|
after=["my-state.service"],
|
||||||
|
script="""
|
||||||
|
if [ ! -f /var/my-state/rebooting ]; then
|
||||||
|
echo "Rebooting the machine"
|
||||||
|
touch /var/my-state/rebooting
|
||||||
|
poweroff
|
||||||
|
else
|
||||||
|
touch /var/my-state/rebooted
|
||||||
|
fi
|
||||||
|
""",
|
||||||
|
),
|
||||||
|
read_after_reboot=dict(
|
||||||
|
description="Read a file in the state folder",
|
||||||
|
wantedBy=["multi-user.target"],
|
||||||
|
after=["reboot.service"],
|
||||||
|
# TODO: currently state folders itself cannot be owned by users
|
||||||
|
script="""
|
||||||
|
if ! cat /var/my-state/test; then
|
||||||
|
echo "cannot read from state file" > /var/my-state/error
|
||||||
|
# ensure root file is owned by root
|
||||||
|
elif [ "$(stat -c '%U' /var/my-state/root)" != "root" ]; then
|
||||||
|
echo "state file /var/my-state/root is not owned by user root" > /var/my-state/error
|
||||||
|
# ensure test file is owned by test
|
||||||
|
elif [ "$(stat -c '%U' /var/my-state/test)" != "test" ]; then
|
||||||
|
echo "state file /var/my-state/test is not owned by user test" > /var/my-state/error
|
||||||
|
# ensure /var/user-state is owned by test
|
||||||
|
elif [ "$(stat -c '%U' /var/user-state)" != "test" ]; then
|
||||||
|
echo "state folder /var/user-state is not owned by user test" > /var/my-state/error
|
||||||
|
fi
|
||||||
|
|
||||||
|
""",
|
||||||
|
serviceConfig=dict(
|
||||||
|
Type="oneshot",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
# TODO: implement shutdown via qmp instead of this hack
|
||||||
|
poweroff=dict(
|
||||||
|
description="Poweroff the machine",
|
||||||
|
wantedBy=["multi-user.target"],
|
||||||
|
after=["read_after_reboot.service"],
|
||||||
|
script="""
|
||||||
|
while [ ! -f /var/my-state/poweroff ]; do
|
||||||
|
sleep 0.1
|
||||||
|
done
|
||||||
|
sleep 0.1
|
||||||
|
poweroff
|
||||||
|
""",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
clan=dict(virtualisation=dict(graphics=False)),
|
clan=dict(virtualisation=dict(graphics=False)),
|
||||||
users=dict(users=dict(root=dict(password="root"))),
|
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
monkeypatch.chdir(flake.path)
|
monkeypatch.chdir(flake.path)
|
||||||
cli = Cli()
|
|
||||||
cli.run(
|
state_dir = vm_state_dir("_test_vm_persistence", str(flake.path), "my_machine")
|
||||||
[
|
socket_file = state_dir / "qga.sock"
|
||||||
"secrets",
|
|
||||||
"users",
|
# wait until socket file exists
|
||||||
"add",
|
def connect() -> QgaSession:
|
||||||
"user1",
|
while True:
|
||||||
age_keys[0].pubkey,
|
if (state_dir / "qga.sock").exists():
|
||||||
]
|
break
|
||||||
|
sleep(0.1)
|
||||||
|
return QgaSession(os.path.realpath(socket_file))
|
||||||
|
|
||||||
|
# run the machine in a separate thread
|
||||||
|
def run() -> None:
|
||||||
|
try:
|
||||||
|
Cli().run(["vms", "run", "my_machine"])
|
||||||
|
except Exception:
|
||||||
|
# print exception details
|
||||||
|
print(traceback.format_exc())
|
||||||
|
print(sys.exc_info()[2])
|
||||||
|
|
||||||
|
t = threading.Thread(target=run, name="run")
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
# wait for socket to be up
|
||||||
|
Path("/tmp/log").write_text(f"wait for socket to be up: {socket_file!s}")
|
||||||
|
while True:
|
||||||
|
if socket_file.exists():
|
||||||
|
break
|
||||||
|
sleep(0.1)
|
||||||
|
|
||||||
|
# wait for socket to be down
|
||||||
|
Path("/tmp/log").write_text("wait for socket to be down")
|
||||||
|
while socket_file.exists():
|
||||||
|
sleep(0.1)
|
||||||
|
Path("/tmp/log").write_text("socket is down")
|
||||||
|
|
||||||
|
# start vm again
|
||||||
|
t = threading.Thread(target=run, name="run")
|
||||||
|
t.daemon = True
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
# wait for the socket to be up
|
||||||
|
Path("/tmp/log").write_text("wait for socket to be up second time")
|
||||||
|
while True:
|
||||||
|
if socket_file.exists():
|
||||||
|
break
|
||||||
|
sleep(0.1)
|
||||||
|
|
||||||
|
# connect second time
|
||||||
|
Path("/tmp/log").write_text("connecting")
|
||||||
|
qga = connect()
|
||||||
|
|
||||||
|
# Path("/tmp/log").write_text("wait for reboot")
|
||||||
|
def wait_cmd_succeed(cmd: str) -> QgaSession:
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
# this might crash as the operation is not atomic
|
||||||
|
exitcode, out, err = qga.run(cmd)
|
||||||
|
if exitcode == 0:
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
sleep(0.1)
|
||||||
|
return qga
|
||||||
|
|
||||||
|
Path("/tmp/log").write_text("trying echo hello")
|
||||||
|
qga = wait_cmd_succeed("echo hello")
|
||||||
|
|
||||||
|
# sleep(9999)
|
||||||
|
|
||||||
|
# ensure that either /var/lib/nixos or /etc gets persisted
|
||||||
|
# (depending on if system.etc.overlay.enable is set or not)
|
||||||
|
exitcode, out, err = qga.run(
|
||||||
|
"ls /vmstate/var/lib/nixos/gid-map || ls /vmstate/.rw-etc/upper"
|
||||||
)
|
)
|
||||||
cli.run(["vms", "run", "my_machine"])
|
assert exitcode == 0, err
|
||||||
test_file = (
|
|
||||||
vm_state_dir("_test_vm_persistence", str(flake.path), "my_machine")
|
exitcode, out, err = qga.run("cat /var/my-state/test")
|
||||||
/ "var"
|
assert exitcode == 0, err
|
||||||
/ "my-state"
|
assert out == "dream2nix\n", out
|
||||||
/ "test"
|
|
||||||
|
# check for errors
|
||||||
|
exitcode, out, err = qga.run("cat /var/my-state/error")
|
||||||
|
assert exitcode == 1, out
|
||||||
|
|
||||||
|
# check all systemd services are OK
|
||||||
|
exitcode, out, err = qga.run(
|
||||||
|
"systemctl --failed | tee /tmp/yolo | grep -q '0 loaded units listed' || ( cat /tmp/yolo && false )"
|
||||||
)
|
)
|
||||||
assert test_file.exists()
|
print(out)
|
||||||
assert test_file.read_text() == "dream2nix\n"
|
assert exitcode == 0, out
|
||||||
|
Loading…
Reference in New Issue
Block a user