DavHau/clan-core
1
0
Fork 0
forked from clan/clan-core
Code Pull Requests Activity

Merge pull request 'add wayland option' (#628) from Mic92-main into main

Browse Source
This commit is contained in:
clan-bot 2023-12-08 14:16:43 +00:00
commit c22280b864
12 changed files with 187 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkgs/clan-cli/clan_cli/flakes/create.py
View File

@ -26,28 +26,30 @@ async def create_flake(directory: Path, url: str) -> dict[str, CmdOut]:
out = await run(command, cwd=directory)
response["flake init"] = out
command = nix_shell(["git"], ["git", "init"])
command = nix_shell(["nixpkgs#git"], ["git", "init"])
out = await run(command, cwd=directory)
response["git init"] = out
command = nix_shell(["git"], ["git", "add", "."])
command = nix_shell(["nixpkgs#git"], ["git", "add", "."])
out = await run(command, cwd=directory)
response["git add"] = out
# command = nix_shell(["git"], ["git", "config", "init.defaultBranch", "main"])
# command = nix_shell(["nixpkgs#git"], ["git", "config", "init.defaultBranch", "main"])
# out = await run(command, cwd=directory)
# response["git config"] = out
command = nix_shell(["git"], ["git", "config", "user.name", "clan-tool"])
command = nix_shell(["nixpkgs#git"], ["git", "config", "user.name", "clan-tool"])
out = await run(command, cwd=directory)
response["git config"] = out
command = nix_shell(["git"], ["git", "config", "user.email", "clan@example.com"])
command = nix_shell(
["nixpkgs#git"], ["git", "config", "user.email", "clan@example.com"]
)
out = await run(command, cwd=directory)
response["git config"] = out
# TODO: Find out why this fails on Johannes machine
# command = nix_shell(["git"], ["git", "commit", "-a", "-m", "Initial commit"])
# command = nix_shell(["nixpkgs#git"], ["git", "commit", "-a", "-m", "Initial commit"])
# out = await run(command, cwd=directory)
# response["git commit"] = out

6
pkgs/clan-cli/clan_cli/git.py
View File

@ -38,7 +38,7 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) ->
:raises ClanError: If the file is not in the git repository.
"""
cmd = nix_shell(
["git"],
["nixpkgs#git"],
["git", "-C", str(repo_dir), "add", str(file_path)],
)
# add the file to the git index
@ -51,7 +51,7 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) ->
# check if there is a diff
cmd = nix_shell(
["git"],
["nixpkgs#git"],
["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code", str(file_path)],
)
result = subprocess.run(cmd, cwd=repo_dir)
@ -61,7 +61,7 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) ->
# commit only that file
cmd = nix_shell(
["git"],
["nixpkgs#git"],
[
"git",
"-C",

2
pkgs/clan-cli/clan_cli/machines/install.py
View File

@ -28,7 +28,7 @@ def install_nixos(machine: Machine) -> None:
subprocess.run(
nix_shell(
["nixos-anywhere"],
["nixpkgs#nixos-anywhere"],
[
"nixos-anywhere",
"-f",

15
pkgs/clan-cli/clan_cli/nix.py
View File

@ -95,16 +95,15 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
# in our tests we just make sure we have all the packages
if os.environ.get("IN_NIX_SANDBOX"):
return cmd
wrapped_packages = [f"nixpkgs#{p}" for p in packages]
return (
nix_command(
return [
*nix_command(
[
"shell",
"--inputs-from",
f"{nixpkgs_flake()!s}",
]
)
+ wrapped_packages
+ ["-c"]
+ cmd
)
),
*packages,
"-c",
*cmd,
]

2
pkgs/clan-cli/clan_cli/secrets/import_sops.py
View File

@ -22,7 +22,7 @@ def import_sops(args: argparse.Namespace) -> None:
if args.input_type:
cmd += ["--input-type", args.input_type]
cmd += ["--output-type", "json", "--decrypt", args.sops_file]
cmd = nix_shell(["sops"], cmd)
cmd = nix_shell(["nixpkgs#sops"], cmd)
try:
res = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
except subprocess.CalledProcessError as e:

13
pkgs/clan-cli/clan_cli/secrets/sops.py
View File

@ -21,7 +21,7 @@ class SopsKey:
def get_public_key(privkey: str) -> str:
cmd = nix_shell(["age"], ["age-keygen", "-y"])
cmd = nix_shell(["nixpkgs#age"], ["age-keygen", "-y"])
try:
res = subprocess.run(
cmd, input=privkey, stdout=subprocess.PIPE, text=True, check=True
@ -34,7 +34,7 @@ def get_public_key(privkey: str) -> str:
def generate_private_key() -> tuple[str, str]:
cmd = nix_shell(["age"], ["age-keygen"])
cmd = nix_shell(["nixpkgs#age"], ["age-keygen"])
try:
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True)
res = proc.stdout.strip()
@ -119,7 +119,7 @@ def sops_manifest(keys: list[str]) -> Iterator[Path]:
def update_keys(secret_path: Path, keys: list[str]) -> None:
with sops_manifest(keys) as manifest:
cmd = nix_shell(
["sops"],
["nixpkgs#sops"],
[
"sops",
"--config",
@ -146,7 +146,7 @@ def encrypt_file(
if not content:
args = ["sops", "--config", str(manifest)]
args.extend([str(secret_path)])
cmd = nix_shell(["sops"], args)
cmd = nix_shell(["nixpkgs#sops"], args)
p = subprocess.run(cmd)
# returns 200 if the file is changed
if p.returncode != 0 and p.returncode != 200:
@ -166,7 +166,7 @@ def encrypt_file(
# we pass an empty manifest to pick up existing configuration of the user
args = ["sops", "--config", str(manifest)]
args.extend(["-i", "--encrypt", str(f.name)])
cmd = nix_shell(["sops"], args)
cmd = nix_shell(["nixpkgs#sops"], args)
subprocess.run(cmd, check=True)
# atomic copy of the encrypted file
with NamedTemporaryFile(dir=folder, delete=False) as f2:
@ -182,7 +182,8 @@ def encrypt_file(
def decrypt_file(secret_path: Path) -> str:
with sops_manifest([]) as manifest:
cmd = nix_shell(
["sops"], ["sops", "--config", str(manifest), "--decrypt", str(secret_path)]
["nixpkgs#sops"],
["sops", "--config", str(manifest), "--decrypt", str(secret_path)],
)
res = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
if res.returncode != 0:

2
pkgs/clan-cli/clan_cli/secrets/sops_generate.py
View File

@ -61,7 +61,7 @@ export secrets={shlex.quote(str(secrets_dir))}
{generator}
"""
try:
cmd = nix_shell(["bash"], ["bash", "-c", text])
cmd = nix_shell(["nixpkgs#bash"], ["bash", "-c", text])
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError:
msg = "failed to the following command:\n"

2
pkgs/clan-cli/clan_cli/secrets/upload.py
View File

@ -21,7 +21,7 @@ def upload_secrets(machine: Machine) -> None:
ssh_cmd = host.ssh_cmd()
subprocess.run(
nix_shell(
["rsync"],
["nixpkgs#rsync"],
[
"rsync",
"-e",

6
pkgs/clan-cli/clan_cli/ssh/cli.py
View File

@ -11,10 +11,10 @@ def ssh(
password: str | None = None,
ssh_args: list[str] = [],
) -> None:
packages = ["tor", "openssh"]
packages = ["nixpkgs#tor", "nixpkgs#openssh"]
password_args = []
if password:
packages.append("sshpass")
packages.append("nixpkgs#sshpass")
password_args = [
"sshpass",
"-p",
@ -37,7 +37,7 @@ def qrcode_scan(picture_file: str) -> str:
return (
subprocess.run(
nix_shell(
["zbar"],
["nixpkgs#zbar"],
[
"zbarimg",
"--quiet",

264
pkgs/clan-cli/clan_cli/vms/run.py
View File

@ -18,13 +18,47 @@ from .inspect import VmConfig, inspect_vm
log = logging.getLogger(__name__)
def graphics_options(vm: VmConfig) -> list[str]:
if vm.wayland:
# fmt: off
return [
"-nographic",
"-vga", "none",
"-device", "virtio-gpu-rutabaga,gfxstream-vulkan=on,cross-domain=on,hostmem=4G,wsi=headless",
]
# fmt: on
else:
# fmt: off
return [
"-audiodev", "spice,id=audio0",
"-device", "intel-hda",
"-device", "hda-duplex,audiodev=audio0",
"-display", "gtk,gl=on",
"-device", "virtio-gpu-gl",
"-display", "spice-app,gl=on",
"-device", "virtio-serial-pci",
"-chardev", "spicevmc,id=vdagent0,name=vdagent",
"-device", "virtserialport,chardev=vdagent0,name=com.redhat.spice.0",
"-device", "qemu-xhci,id=spicepass",
"-chardev", "spicevmc,id=usbredirchardev1,name=usbredir",
"-device", "usb-redir,chardev=usbredirchardev1,id=usbredirdev1",
"-chardev", "spicevmc,id=usbredirchardev2,name=usbredir",
"-device", "usb-redir,chardev=usbredirchardev2,id=usbredirdev2",
"-chardev", "spicevmc,id=usbredirchardev3,name=usbredir",
"-device", "usb-redir,chardev=usbredirchardev3,id=usbredirdev3",
"-device", "pci-ohci,id=smartpass",
"-device", "usb-ccid",
"-chardev", "spicevmc,id=ccid,name=smartcard",
]
# fmt: on
def qemu_command(
vm: VmConfig,
nixos_config: dict[str, str],
xchg_dir: Path,
secrets_dir: Path,
disk_img: Path,
spice_socket: Path,
) -> list[str]:
kernel_cmdline = [
(Path(nixos_config["toplevel"]) / "kernel-params").read_text(),
@ -40,6 +74,7 @@ def qemu_command(
"-m", f'{nixos_config["memorySize"]}M',
"-smp", str(nixos_config["cores"]),
"-cpu", "max",
"-enable-kvm",
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio",
"-netdev", "user,id=user.0",
@ -50,9 +85,6 @@ def qemu_command(
"-drive", f"cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report",
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
# TODO: we also need to fixup timezone than...
# "-rtc", "base=localtime,clock=host,driftfix=slew",
"-vga", "virtio",
"-usb", "-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{nixos_config["toplevel"]}/kernel',
"-initrd", nixos_config["initrd"],
@ -60,31 +92,7 @@ def qemu_command(
] # fmt: on
if vm.graphics:
# fmt: off
command.extend(
[
"-audiodev", "spice,id=audio0",
"-device", "intel-hda",
"-device", "hda-duplex,audiodev=audio0",
"-vga", "none",
"-device", "virtio-gpu-gl",
"-display", "spice-app,gl=on",
"-device", "virtio-serial-pci",
"-chardev", "spicevmc,id=vdagent0,name=vdagent",
"-device", "virtserialport,chardev=vdagent0,name=com.redhat.spice.0",
"-device", "qemu-xhci,id=spicepass",
"-chardev", "spicevmc,id=usbredirchardev1,name=usbredir",
"-device", "usb-redir,chardev=usbredirchardev1,id=usbredirdev1",
"-chardev", "spicevmc,id=usbredirchardev2,name=usbredir",
"-device", "usb-redir,chardev=usbredirchardev2,id=usbredirdev2",
"-chardev", "spicevmc,id=usbredirchardev3,name=usbredir",
"-device", "usb-redir,chardev=usbredirchardev3,id=usbredirdev3",
"-device", "pci-ohci,id=smartpass",
"-device", "usb-ccid",
"-chardev", "spicevmc,id=ccid,name=smartcard",
]
)
# fmt: on
command.extend(graphics_options(vm))
else:
command.append("-nographic")
return command
@ -134,6 +142,96 @@ def get_clan_name(vm: VmConfig, nix_options: list[str]) -> str:
return proc.stdout.strip().strip('"')
def generate_secrets(
vm: VmConfig,
clan_name: str,
nixos_config: dict[str, str],
tmpdir: Path,
log_fd: IO[str] | None,
) -> Path:
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir(exist_ok=True)
env = os.environ.copy()
env["CLAN_DIR"] = str(vm.flake_url)
env["PYTHONPATH"] = str(":".join(sys.path)) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
# Only generate secrets for local clans
if isinstance(vm.flake_url, Path) and vm.flake_url.is_dir():
if Path(vm.flake_url).is_dir():
subprocess.run(
[nixos_config["generateSecrets"], clan_name],
env=env,
check=False,
stdout=log_fd,
stderr=log_fd,
)
else:
log.warning("won't generate secrets for non local clan")
cmd = [nixos_config["uploadSecrets"]]
res = subprocess.run(
cmd,
env=env,
check=False,
stdout=log_fd,
stderr=log_fd,
)
if res.returncode != 0:
raise ClanError(
f"Failed to upload secrets: {shlex.join(cmd)} failed with {res.returncode}"
)
return secrets_dir
def prepare_disk(tmpdir: Path, log_fd: IO[str] | None) -> Path:
disk_img = tmpdir / "disk.img"
cmd = nix_shell(
["nixpkgs#qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
str(disk_img),
"1024M",
],
)
res = subprocess.run(
cmd,
check=False,
stdout=log_fd,
stderr=log_fd,
)
if res.returncode != 0:
raise ClanError(
f"Failed to create disk image: {shlex.join(cmd)} failed with {res.returncode}"
)
cmd = nix_shell(
["nixpkgs#e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
str(disk_img),
],
)
res = subprocess.run(
cmd,
check=False,
stdout=log_fd,
stderr=log_fd,
)
if res.returncode != 0:
raise ClanError(
f"Failed to create ext4 filesystem: {shlex.join(cmd)} failed with {res.returncode}"
)
return disk_img
def run_vm(
vm: VmConfig, nix_options: list[str] = [], log_fd: IO[str] | None = None
) -> None:
@ -156,86 +254,9 @@ def run_vm(
tmpdir = Path(tmpdir_)
xchg_dir = tmpdir / "xchg"
xchg_dir.mkdir(exist_ok=True)
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir(exist_ok=True)
disk_img = tmpdir / "disk.img"
spice_socket = tmpdir / "spice.sock"
env = os.environ.copy()
env["CLAN_DIR"] = str(vm.flake_url)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
# Only generate secrets for local clans
if isinstance(vm.flake_url, Path) and vm.flake_url.is_dir():
if Path(vm.flake_url).is_dir():
subprocess.run(
[nixos_config["generateSecrets"], clan_name],
env=env,
check=False,
stdout=log_fd,
stderr=log_fd,
)
else:
log.warning("won't generate secrets for non local clan")
cmd = [nixos_config["uploadSecrets"]]
res = subprocess.run(
cmd,
env=env,
check=False,
stdout=log_fd,
stderr=log_fd,
)
if res.returncode != 0:
raise ClanError(
f"Failed to upload secrets: {shlex.join(cmd)} failed with {res.returncode}"
)
cmd = nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
str(disk_img),
"1024M",
],
)
res = subprocess.run(
cmd,
check=False,
stdout=log_fd,
stderr=log_fd,
)
if res.returncode != 0:
raise ClanError(
f"Failed to create disk image: {shlex.join(cmd)} failed with {res.returncode}"
)
cmd = nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
str(disk_img),
],
)
res = subprocess.run(
cmd,
check=False,
stdout=log_fd,
stderr=log_fd,
)
if res.returncode != 0:
raise ClanError(
f"Failed to create ext4 filesystem: {shlex.join(cmd)} failed with {res.returncode}"
)
secrets_dir = generate_secrets(vm, clan_name, nixos_config, tmpdir, log_fd)
disk_img = prepare_disk(tmpdir, log_fd)
qemu_cmd = qemu_command(
vm,
@ -243,20 +264,22 @@ def run_vm(
xchg_dir=xchg_dir,
secrets_dir=secrets_dir,
disk_img=disk_img,
spice_socket=spice_socket,
)
print("$ " + shlex.join(qemu_cmd))
packages = ["qemu"]
if vm.graphics:
packages.append("virt-viewer")
if vm.wayland:
packages = ["git+https://git.clan.lol/clan/clan-core.git#qemu-wayland"]
else:
packages = ["nixpkgs#qemu"]
env = os.environ.copy()
remote_viewer_mimetypes = module_root() / "vms" / "mimetypes"
env[
"XDG_DATA_DIRS"
] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}"
print(env["XDG_DATA_DIRS"])
if vm.graphics and not vm.wayland:
packages.append("nixpkgs#virt-viewer")
remote_viewer_mimetypes = module_root() / "vms" / "mimetypes"
env[
"XDG_DATA_DIRS"
] = f"{remote_viewer_mimetypes}:{env.get('XDG_DATA_DIRS', '')}"
print("$ " + shlex.join(qemu_cmd))
res = subprocess.run(
nix_shell(packages, qemu_cmd),
env=env,
@ -274,6 +297,7 @@ class RunOptions:
flake_url: str | None
flake: Path
nix_options: list[str] = field(default_factory=list)
wayland: bool = False
def run_command(args: argparse.Namespace) -> None:
@ -282,10 +306,13 @@ def run_command(args: argparse.Namespace) -> None:
flake_url=args.flake_url,
flake=args.flake or Path.cwd(),
nix_options=args.option,
wayland=args.wayland,
)
flake_url = run_options.flake_url or run_options.flake
vm = inspect_vm(flake_url=flake_url, flake_attr=run_options.machine)
# TODO: allow to set this in the config
vm.wayland = run_options.wayland
run_vm(vm, run_options.nix_options)
@ -293,4 +320,5 @@ def run_command(args: argparse.Namespace) -> None:
def register_run_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument("machine", type=str, help="machine in the flake to run")
parser.add_argument("--flake-url", type=str, help="flake url")
parser.add_argument("--wayland", action="store_true", help="use wayland")
parser.set_defaults(func=run_command)

6
pkgs/clan-cli/tests/conftest.py
View File

@ -26,12 +26,12 @@ pytest_plugins = [
@pytest.fixture
def git_repo(tmp_path: Path) -> Path:
# initialize a git repository
cmd = nix_shell(["git"], ["git", "init"])
cmd = nix_shell(["nixpkgs#git"], ["git", "init"])
subprocess.run(cmd, cwd=tmp_path, check=True)
# set user.name and user.email
cmd = nix_shell(["git"], ["git", "config", "user.name", "test"])
cmd = nix_shell(["nixpkgs#git"], ["git", "config", "user.name", "test"])
subprocess.run(cmd, cwd=tmp_path, check=True)
cmd = nix_shell(["git"], ["git", "config", "user.email", "test@test.test"])
cmd = nix_shell(["nixpkgs#git"], ["git", "config", "user.email", "test@test.test"])
subprocess.run(cmd, cwd=tmp_path, check=True)
# return the path to the git repository
return tmp_path

8
pkgs/clan-cli/tests/test_secrets_password_store.py
View File

@ -35,10 +35,14 @@ def test_upload_secret(
)
cli = Cli()
subprocess.run(
nix_shell(["gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]),
nix_shell(
["nixpkgs#gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]
),
check=True,
)
subprocess.run(nix_shell(["pass"], ["pass", "init", "test@local"]), check=True)
subprocess.run(
nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True
)
cli.run(["secrets", "generate", "vm1"])
network_id = machine_get_fact(
test_flake_with_core_and_pass.path, "vm1", "zerotier-network-id"