clan/clan-core
11
2
Fork 13
Code Issues 91 Pull Requests 5 Actions Packages Projects 1 Releases Wiki Activity

Added ref to Qubasa-main in template/new-clan/flake.nix

Browse Source
This commit is contained in:
Luis Hebendanz 2023-10-20 01:11:32 +02:00
parent a71584d9d2
commit 26bfb793b1
9 changed files with 153 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkgs/clan-cli/clan_cli/flakes/create.py
View File

@ -12,7 +12,7 @@ from ..errors import ClanError
from ..nix import nix_command, nix_shell
DEFAULT_URL: AnyUrl = parse_obj_as(
AnyUrl, "git+https://git.clan.lol/clan/clan-core#new-clan"
AnyUrl, "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan" # TODO: Change me back to main branch
)

4
pkgs/clan-cli/clan_cli/task_manager.py
View File

@ -115,6 +115,10 @@ class BaseTask:
self.status = TaskStatus.RUNNING
try:
self.run()
# TODO: We need to check, if too many commands have been initialized,
# but not run. This would deadlock the log_lines() function.
# Idea: Run next(cmds) and check if it raises StopIteration if not,
# we have too many commands
except Exception as e:
# FIXME: fix exception handling here
traceback.print_exception(*sys.exc_info())

20
pkgs/clan-cli/clan_cli/types.py
View File

@ -1,3 +1,23 @@
from typing import NewType
from pathlib import Path
import logging
log = logging.getLogger(__name__)
FlakeName = NewType("FlakeName", str)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path

188
pkgs/clan-cli/clan_cli/vms/create.py
View File

@ -9,15 +9,16 @@ from pathlib import Path
from typing import Iterator
from uuid import UUID
from ..dirs import specific_flake_dir
from ..nix import nix_build, nix_config, nix_shell
from ..dirs import specific_flake_dir, clan_flakes_dir
from ..nix import nix_build, nix_config, nix_shell, nix_eval
from ..task_manager import BaseTask, Command, create_task
from .inspect import VmConfig, inspect_vm
from ..flakes.create import create_flake
from ..types import validate_path
class BuildVmTask(BaseTask):
def __init__(self, uuid: UUID, vm: VmConfig) -> None:
super().__init__(uuid, num_cmds=6)
super().__init__(uuid, num_cmds=7)
self.vm = vm
def get_vm_create_info(self, cmds: Iterator[Command]) -> dict:
@ -39,6 +40,19 @@ class BuildVmTask(BaseTask):
with open(vm_json) as f:
return json.load(f)
def get_clan_name(self, cmds: Iterator[Command]) -> str:
clan_dir = self.vm.flake_url
cmd = next(cmds)
cmd.run(
nix_eval(
[
f'{clan_dir}#clanInternals.clanName'
]
)
)
clan_name = "".join(cmd.stdout).strip()
return clan_name
def run(self) -> None:
cmds = self.commands()
@ -47,101 +61,103 @@ class BuildVmTask(BaseTask):
# TODO: We should get this from the vm argument
vm_config = self.get_vm_create_info(cmds)
clan_name = self.get_clan_name(cmds)
# TODO: Don't use a temporary directory, instead create a new flake directory
with tempfile.TemporaryDirectory() as tmpdir_:
tmpdir = Path(tmpdir_)
xchg_dir = tmpdir / "xchg"
xchg_dir.mkdir()
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir()
disk_img = f"{tmpdir_}/disk.img"
env = os.environ.copy()
env["CLAN_DIR"] = str(self.vm.flake_url)
flake_dir = clan_flakes_dir() / clan_name
validate_path(clan_flakes_dir(), flake_dir)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
xchg_dir = flake_dir / "xchg"
xchg_dir.mkdir()
secrets_dir = flake_dir / "secrets"
secrets_dir.mkdir()
disk_img = f"{flake_dir}/disk.img"
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run(
[vm_config["generateSecrets"]],
env=env,
)
else:
self.log.warning("won't generate secrets for non local clan")
env = os.environ.copy()
env["CLAN_DIR"] = str(self.vm.flake_url)
cmd = next(cmds)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run(
[vm_config["uploadSecrets"]],
[vm_config["generateSecrets"]],
env=env,
)
else:
self.log.warning("won't generate secrets for non local clan")
cmd = next(cmds)
cmd.run(
nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
disk_img,
"1024M",
],
)
cmd = next(cmds)
cmd.run(
[vm_config["uploadSecrets"]],
env=env,
)
cmd = next(cmds)
cmd.run(
nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
disk_img,
"1024M",
],
)
)
cmd = next(cmds)
cmd.run(
nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
disk_img,
],
)
cmd = next(cmds)
cmd.run(
nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
disk_img,
],
)
)
cmd = next(cmds)
cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8",
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
]
if not self.vm.graphics:
qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command))
cmd = next(cmds)
cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8",
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
]
if not self.vm.graphics:
qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command))
def create_vm(vm: VmConfig) -> BuildVmTask:

19
pkgs/clan-cli/clan_cli/webui/api_inputs.py
View File

@ -6,26 +6,11 @@ from pydantic import AnyUrl, BaseModel, validator
from ..dirs import clan_data_dir, clan_flakes_dir
from ..flakes.create import DEFAULT_URL
from ..types import validate_path
log = logging.getLogger(__name__)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path
class ClanDataPath(BaseModel):
dest: Path

7
pkgs/clan-cli/tests/temporary_dir.py
View File

@ -10,13 +10,16 @@ log = logging.getLogger(__name__)
@pytest.fixture
def temporary_dir() -> Iterator[Path]:
def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
if os.getenv("TEST_KEEP_TEMPORARY_DIR") is not None:
temp_dir = tempfile.mkdtemp(prefix="pytest-")
path = Path(temp_dir)
log.info("Keeping temporary test directory: ", path)
log.debug("Temp HOME directory: %s", str(path))
monkeypatch.setenv("HOME", str(temp_dir))
yield path
else:
log.debug("TEST_KEEP_TEMPORARY_DIR not set, using TemporaryDirectory")
with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath:
monkeypatch.setenv("HOME", str(dirpath))
log.debug("Temp HOME directory: %s", str(dirpath))
yield Path(dirpath)

2
pkgs/clan-cli/tests/test_config.py
View File

@ -57,7 +57,7 @@ def test_configure_machine(
capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("HOME", str(temporary_dir))
cli = Cli()
cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true"])
# clear the output buffer

30
pkgs/clan-cli/tests/test_create_flake.py
View File

@ -5,7 +5,8 @@ from pathlib import Path
import pytest
from api import TestClient
from cli import Cli
from clan_cli.flakes.create import DEFAULT_URL
from clan_cli.dirs import clan_flakes_dir, clan_data_dir
@pytest.fixture
def cli() -> Cli:
@ -14,15 +15,16 @@ def cli() -> Cli:
@pytest.mark.impure
def test_create_flake_api(
monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_dir: Path
monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_home: Path
) -> None:
flake_dir = temporary_dir / "flake_dir"
flake_dir_str = str(flake_dir.resolve())
monkeypatch.chdir(clan_flakes_dir())
flake_name = "flake_dir"
flake_dir = clan_flakes_dir() / flake_name
response = api.post(
"/api/flake/create",
json=dict(
dest=flake_dir_str,
url="git+https://git.clan.lol/clan/clan-core#new-clan",
dest=str(flake_dir),
url=str(DEFAULT_URL),
),
)
@ -34,19 +36,21 @@ def test_create_flake_api(
@pytest.mark.impure
def test_create_flake(
monkeypatch: pytest.MonkeyPatch,
temporary_dir: Path,
capsys: pytest.CaptureFixture,
temporary_home: Path,
cli: Cli,
) -> None:
monkeypatch.chdir(temporary_dir)
flake_dir = temporary_dir / "flake_dir"
flake_dir_str = str(flake_dir.resolve())
cli.run(["flake", "create", flake_dir_str])
monkeypatch.chdir(clan_flakes_dir())
flake_name = "flake_dir"
flake_dir = clan_flakes_dir() / flake_name
cli.run(["flakes", "create", flake_name])
assert (flake_dir / ".clan-flake").exists()
monkeypatch.chdir(flake_dir)
cli.run(["machines", "create", "machine1"])
cli.run(["machines", "create", "machine1", flake_name])
capsys.readouterr() # flush cache
cli.run(["machines", "list"])
cli.run(["machines", "list", flake_name])
assert "machine1" in capsys.readouterr().out
flake_show = subprocess.run(
["nix", "flake", "show", "--json"],

2
templates/new-clan/flake.nix
View File

@ -1,7 +1,7 @@
{
description = "<Put your description here>";
inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core";
inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main";
outputs = { self, clan-core, ... }:
let