resolve conflicts

nixosModules/clanCore/flake-module.nix

@@ -66,12 +66,14 @@
             description = ''
               script to upload secrets to the deployment server
             '';
+            default = "${pkgs.coreutils}/bin/true";
           };
           generateSecrets = lib.mkOption {
             type = lib.types.path;
             description = ''
               script to generate secrets
             '';
+            default = "${pkgs.coreutils}/bin/true";
           };
           vm.config = lib.mkOption {
             type = lib.types.attrs;

nixosModules/clanCore/secrets/password-store.nix

@@ -13,44 +13,46 @@ in
   config = lib.mkIf (config.clanCore.secretStore == "password-store") {
     clanCore.secretsDirectory = config.clan.password-store.targetDirectory;
     clanCore.secretsUploadDirectory = config.clan.password-store.targetDirectory;
-    system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
-      #!/bin/sh
-      set -efu
+    system.clan.generateSecrets = lib.mkIf (config.clanCore.secrets != { }) (
+      pkgs.writeScript "generate-secrets" ''
+        #!/bin/sh
+        set -efu
 
-      test -d "$CLAN_DIR"
-      PATH=${lib.makeBinPath [
-        pkgs.pass
-      ]}:$PATH
+        test -d "$CLAN_DIR"
+        PATH=${lib.makeBinPath [
+          pkgs.pass
+        ]}:$PATH
 
-      # TODO maybe initialize password store if it doesn't exist yet
+        # TODO maybe initialize password store if it doesn't exist yet
 
-      ${lib.foldlAttrs (acc: n: v: ''
-        ${acc}
-        # ${n}
-        # if any of the secrets are missing, we regenerate all connected facts/secrets
-        (if ! (${lib.concatMapStringsSep " && " (x: "test -e ${passwordstoreDir}/machines/${config.clanCore.machineName}/${x.name}.gpg >/dev/null") (lib.attrValues v.secrets)}); then
+        ${lib.foldlAttrs (acc: n: v: ''
+          ${acc}
+          # ${n}
+          # if any of the secrets are missing, we regenerate all connected facts/secrets
+          (if ! (${lib.concatMapStringsSep " && " (x: "test -e ${passwordstoreDir}/machines/${config.clanCore.machineName}/${x.name}.gpg >/dev/null") (lib.attrValues v.secrets)}); then
 
-          tmpdir=$(mktemp -d)
-          trap "rm -rf $tmpdir" EXIT
-          cd $tmpdir
+            tmpdir=$(mktemp -d)
+            trap "rm -rf $tmpdir" EXIT
+            cd $tmpdir
 
-          facts=$(mktemp -d)
-          trap "rm -rf $facts" EXIT
-          secrets=$(mktemp -d)
-          trap "rm -rf $secrets" EXIT
-          ( ${v.generator} )
+            facts=$(mktemp -d)
+            trap "rm -rf $facts" EXIT
+            secrets=$(mktemp -d)
+            trap "rm -rf $secrets" EXIT
+            ( ${v.generator} )
 
-          ${lib.concatMapStrings (fact: ''
-            mkdir -p "$CLAN_DIR"/"$(dirname ${fact.path})"
-            cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
-          '') (lib.attrValues v.facts)}
+            ${lib.concatMapStrings (fact: ''
+              mkdir -p "$CLAN_DIR"/"$(dirname ${fact.path})"
+              cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
+            '') (lib.attrValues v.facts)}
 
-          ${lib.concatMapStrings (secret: ''
-            cat "$secrets"/${secret.name} | pass insert -m machines/${config.clanCore.machineName}/${secret.name}
-          '') (lib.attrValues v.secrets)}
-        fi)
-      '') "" config.clanCore.secrets}
-    '';
+            ${lib.concatMapStrings (secret: ''
+              cat "$secrets"/${secret.name} | pass insert -m machines/${config.clanCore.machineName}/${secret.name}
+            '') (lib.attrValues v.secrets)}
+          fi)
+        '') "" config.clanCore.secrets}
+      ''
+    );
     system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
       #!/bin/sh
       set -efu

nixosModules/clanCore/secrets/sops.nix

@@ -25,7 +25,7 @@ in
   config = lib.mkIf (config.clanCore.secretStore == "sops") {
     clanCore.secretsDirectory = "/run/secrets";
     clanCore.secretsPrefix = config.clanCore.machineName + "-";
-    system.clan = {
+    system.clan = lib.mkIf (config.clanCore.secrets != { }) {
 
       generateSecrets = pkgs.writeScript "generate-secrets" ''
         #!${pkgs.python3}/bin/python

pkgs/clan-cli/tests/conftest.py

@@ -18,7 +18,7 @@ pytest_plugins = [
     "command",
     "ports",
     "host_group",
-    "test_flake",
+    "fixtures_flakes",
 ]
 
 

pkgs/clan-cli/tests/fixtures_flakes.py

@@ -0,0 +1,90 @@
+import fileinput
+import shutil
+import tempfile
+from pathlib import Path
+from typing import Iterator
+
+import pytest
+from root import CLAN_CORE
+
+from clan_cli.dirs import nixpkgs_source
+
+
+# substitutes string sin a file.
+# This can be used on the flake.nix or default.nix of a machine
+def substitute(
+    file: Path,
+    clan_core_flake: Path | None = None,
+    flake: Path = Path(__file__).parent,
+) -> None:
+    sops_key = str(flake.joinpath("sops.key"))
+    for line in fileinput.input(file, inplace=True):
+        line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
+        if clan_core_flake:
+            line = line.replace("__CLAN_CORE__", str(clan_core_flake))
+        line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
+        line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
+        print(line, end="")
+
+
+def create_flake(
+    monkeypatch: pytest.MonkeyPatch,
+    name: str,
+    clan_core_flake: Path | None = None,
+    machines: list[str] = [],
+    remote: bool = False,
+) -> Iterator[Path]:
+    """
+    Creates a flake with the given name and machines.
+    The machine names map to the machines in ./test_machines
+    """
+    template = Path(__file__).parent / name
+    # copy the template to a new temporary location
+    with tempfile.TemporaryDirectory() as tmpdir_:
+        home = Path(tmpdir_)
+        flake = home / name
+        shutil.copytree(template, flake)
+        # lookup the requested machines in ./test_machines and include them
+        if machines:
+            (flake / "machines").mkdir(parents=True, exist_ok=True)
+        for machine_name in machines:
+            machine_path = Path(__file__).parent / "machines" / machine_name
+            shutil.copytree(machine_path, flake / "machines" / machine_name)
+            substitute(flake / "machines" / machine_name / "default.nix", flake)
+        # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
+        # provided by get_test_flake_toplevel
+        flake_nix = flake / "flake.nix"
+        # this is where we would install the sops key to, when updating
+        substitute(flake_nix, clan_core_flake, flake)
+        if remote:
+            with tempfile.TemporaryDirectory() as workdir:
+                monkeypatch.chdir(workdir)
+                monkeypatch.setenv("HOME", str(home))
+                yield flake
+        else:
+            monkeypatch.chdir(flake)
+            monkeypatch.setenv("HOME", str(home))
+            yield flake
+
+
+@pytest.fixture
+def test_flake(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
+    yield from create_flake(monkeypatch, "test_flake")
+
+
+@pytest.fixture
+def test_flake_with_core(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
+    if not (CLAN_CORE / "flake.nix").exists():
+        raise Exception(
+            "clan-core flake not found. This test requires the clan-core flake to be present"
+        )
+    yield from create_flake(monkeypatch, "test_flake_with_core", CLAN_CORE)
+
+
+@pytest.fixture
+def test_flake_with_core_and_pass(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
+    if not (CLAN_CORE / "flake.nix").exists():
+        raise Exception(
+            "clan-core flake not found. This test requires the clan-core flake to be present"
+        )
+    yield from create_flake(monkeypatch, "test_flake_with_core_and_pass", CLAN_CORE)

pkgs/clan-cli/tests/machines/vm1/default.nix

@@ -0,0 +1,20 @@
+{ lib, ... }: {
+  clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+  system.stateVersion = lib.version;
+  sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+  clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__";
+  clan.virtualisation.graphics = false;
+
+  clan.networking.zerotier.controller.enable = true;
+  networking.useDHCP = false;
+
+  systemd.services.shutdown-after-boot = {
+    enable = true;
+    wantedBy = [ "multi-user.target" ];
+    after = [ "multi-user.target" ];
+    script = ''
+      #!/usr/bin/env bash
+      shutdown -h now
+    '';
+  };
+}

pkgs/clan-cli/tests/machines/vm_with_secrets/default.nix

@@ -0,0 +1,20 @@
+{ lib, ... }: {
+  clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+  system.stateVersion = lib.version;
+  sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+  clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__";
+  clan.virtualisation.graphics = false;
+
+  clan.networking.zerotier.controller.enable = true;
+  networking.useDHCP = false;
+
+  systemd.services.shutdown-after-boot = {
+    enable = true;
+    wantedBy = [ "multi-user.target" ];
+    after = [ "multi-user.target" ];
+    script = ''
+      #!/usr/bin/env bash
+      shutdown -h now
+    '';
+  };
+}

pkgs/clan-cli/tests/machines/vm_without_secrets/default.nix

@@ -0,0 +1,17 @@
+{ lib, ... }: {
+  clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+  system.stateVersion = lib.version;
+  clan.virtualisation.graphics = false;
+
+  networking.useDHCP = false;
+
+  systemd.services.shutdown-after-boot = {
+    enable = true;
+    wantedBy = [ "multi-user.target" ];
+    after = [ "multi-user.target" ];
+    script = ''
+      #!/usr/bin/env bash
+      shutdown -h now
+    '';
+  };
+}

pkgs/clan-cli/tests/test_flake.py

@@ -1,59 +0,0 @@
-import fileinput
-import shutil
-import tempfile
-from pathlib import Path
-from typing import Iterator
-
-import pytest
-from root import CLAN_CORE
-
-from clan_cli.dirs import nixpkgs_source
-
-
-def create_flake(
-    monkeypatch: pytest.MonkeyPatch, name: str, clan_core_flake: Path | None = None
-) -> Iterator[Path]:
-    template = Path(__file__).parent / name
-    # copy the template to a new temporary location
-    with tempfile.TemporaryDirectory() as tmpdir_:
-        home = Path(tmpdir_)
-        flake = home / name
-        shutil.copytree(template, flake)
-        # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
-        # provided by get_test_flake_toplevel
-        flake_nix = flake / "flake.nix"
-        # this is where we would install the sops key to, when updating
-        sops_key = str(flake.joinpath("sops.key"))
-        for line in fileinput.input(flake_nix, inplace=True):
-            line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
-            if clan_core_flake:
-                line = line.replace("__CLAN_CORE__", str(clan_core_flake))
-            line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
-            line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
-            print(line, end="")
-        monkeypatch.chdir(flake)
-        monkeypatch.setenv("HOME", str(home))
-        yield flake
-
-
-@pytest.fixture
-def test_flake(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
-    yield from create_flake(monkeypatch, "test_flake")
-
-
-@pytest.fixture
-def test_flake_with_core(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
-    if not (CLAN_CORE / "flake.nix").exists():
-        raise Exception(
-            "clan-core flake not found. This test requires the clan-core flake to be present"
-        )
-    yield from create_flake(monkeypatch, "test_flake_with_core", CLAN_CORE)
-
-
-@pytest.fixture
-def test_flake_with_core_and_pass(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
-    if not (CLAN_CORE / "flake.nix").exists():
-        raise Exception(
-            "clan-core flake not found. This test requires the clan-core flake to be present"
-        )
-    yield from create_flake(monkeypatch, "test_flake_with_core_and_pass", CLAN_CORE)

pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix

@@ -2,7 +2,7 @@
   # Use this path to our repo root e.g. for UI test
   # inputs.clan-core.url = "../../../../.";
 
-  # this placeholder is replaced by the path to nixpkgs
+  # this placeholder is replaced by the path to clan-core
   inputs.clan-core.url = "__CLAN_CORE__";
 
   outputs = { self, clan-core }:

pkgs/clan-cli/tests/test_flake_with_core_dynamic_machines/flake.nix

@@ -0,0 +1,24 @@
+{
+  # Use this path to our repo root e.g. for UI test
+  # inputs.clan-core.url = "../../../../.";
+
+  # this placeholder is replaced by the path to nixpkgs
+  inputs.clan-core.url = "__CLAN_CORE__";
+
+  outputs = { self, clan-core }:
+    let
+      clan = clan-core.lib.buildClan {
+        directory = self;
+        machines =
+          let
+            machineModules = builtins.readDir (self + "/machines");
+          in
+          builtins.mapAttrs
+            (name: _type: import (self + "/machines/${name}"))
+            machineModules;
+      };
+    in
+    {
+      inherit (clan) nixosConfigurations clanInternals;
+    };
+}

pkgs/clan-cli/tests/test_vms_api.py

@@ -1,14 +1,7 @@
-import os
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import pytest
 from api import TestClient
-from cli import Cli
-from httpx import SyncByteStream
-
-if TYPE_CHECKING:
-    from age_keys import KeyPair
 
 
 @pytest.mark.impure
@@ -34,51 +27,3 @@ def test_incorrect_uuid(api: TestClient) -> None:
     for endpoint in uuid_endpoints:
         response = api.get(endpoint.format("1234"))
         assert response.status_code == 422, "Failed to get vm status"
-
-
-@pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
-@pytest.mark.impure
-def test_create(
-    api: TestClient,
-    monkeypatch: pytest.MonkeyPatch,
-    test_flake_with_core: Path,
-    age_keys: list["KeyPair"],
-) -> None:
-    monkeypatch.chdir(test_flake_with_core)
-    monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
-    cli = Cli()
-    cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
-    print(f"flake_url: {test_flake_with_core} ")
-    response = api.post(
-        "/api/vms/create",
-        json=dict(
-            flake_url=str(test_flake_with_core),
-            flake_attr="vm1",
-            cores=1,
-            memory_size=1024,
-            graphics=False,
-        ),
-    )
-    assert response.status_code == 200, "Failed to create vm"
-
-    uuid = response.json()["uuid"]
-    assert len(uuid) == 36
-    assert uuid.count("-") == 4
-
-    response = api.get(f"/api/vms/{uuid}/status")
-    assert response.status_code == 200, "Failed to get vm status"
-
-    response = api.get(f"/api/vms/{uuid}/logs")
-    print("=========VM LOGS==========")
-    assert isinstance(response.stream, SyncByteStream)
-    for line in response.stream:
-        print(line.decode("utf-8"))
-    print("=========END LOGS==========")
-    assert response.status_code == 200, "Failed to get vm logs"
-
-    response = api.get(f"/api/vms/{uuid}/status")
-    assert response.status_code == 200, "Failed to get vm status"
-    data = response.json()
-    assert (
-        data["status"] == "FINISHED"
-    ), f"Expected to be finished, but got {data['status']} ({data})"

pkgs/clan-cli/tests/test_vms_api_create.py

@@ -0,0 +1,106 @@
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, Iterator
+
+import pytest
+from api import TestClient
+from cli import Cli
+from fixtures_flakes import create_flake
+from httpx import SyncByteStream
+from root import CLAN_CORE
+
+if TYPE_CHECKING:
+    from age_keys import KeyPair
+
+
+@pytest.fixture
+def flake_with_vm_with_secrets(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
+    yield from create_flake(
+        monkeypatch,
+        "test_flake_with_core_dynamic_machines",
+        CLAN_CORE,
+        machines=["vm_with_secrets"],
+    )
+
+
+@pytest.fixture
+def remote_flake_with_vm_without_secrets(
+    monkeypatch: pytest.MonkeyPatch,
+) -> Iterator[Path]:
+    yield from create_flake(
+        monkeypatch,
+        "test_flake_with_core_dynamic_machines",
+        CLAN_CORE,
+        machines=["vm_without_secrets"],
+        remote=True,
+    )
+
+
+@pytest.fixture
+def create_user_with_age_key(
+    monkeypatch: pytest.MonkeyPatch,
+    age_keys: list["KeyPair"],
+) -> None:
+    monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
+    cli = Cli()
+    cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
+
+
+def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
+    print(f"flake_url: {flake} ")
+    response = api.post(
+        "/api/vms/create",
+        json=dict(
+            flake_url=str(flake),
+            flake_attr=vm,
+            cores=1,
+            memory_size=1024,
+            graphics=False,
+        ),
+    )
+    assert response.status_code == 200, "Failed to create vm"
+
+    uuid = response.json()["uuid"]
+    assert len(uuid) == 36
+    assert uuid.count("-") == 4
+
+    response = api.get(f"/api/vms/{uuid}/status")
+    assert response.status_code == 200, "Failed to get vm status"
+
+    response = api.get(f"/api/vms/{uuid}/logs")
+    print("=========VM LOGS==========")
+    assert isinstance(response.stream, SyncByteStream)
+    for line in response.stream:
+        print(line.decode("utf-8"))
+    print("=========END LOGS==========")
+    assert response.status_code == 200, "Failed to get vm logs"
+
+    response = api.get(f"/api/vms/{uuid}/status")
+    assert response.status_code == 200, "Failed to get vm status"
+    data = response.json()
+    assert (
+        data["status"] == "FINISHED"
+    ), f"Expected to be finished, but got {data['status']} ({data})"
+
+
+@pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
+@pytest.mark.impure
+def test_create_local(
+    api: TestClient,
+    monkeypatch: pytest.MonkeyPatch,
+    flake_with_vm_with_secrets: Path,
+    create_user_with_age_key: None,
+) -> None:
+    generic_create_vm_test(api, flake_with_vm_with_secrets, "vm_with_secrets")
+
+
+@pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
+@pytest.mark.impure
+def test_create_remote(
+    api: TestClient,
+    monkeypatch: pytest.MonkeyPatch,
+    remote_flake_with_vm_without_secrets: Path,
+) -> None:
+    generic_create_vm_test(
+        api, remote_flake_with_vm_without_secrets, "vm_without_secrets"
+    )

pkgs/theme/src/config.ts

@@ -24,10 +24,6 @@ export const config: PaletteConfig = {
             keyColor: "#92898a",
             tones: [2, 5, 8, 92, 95, 98],
         },
-        red: {
-            keyColor: "#e82439",
-            tones: [5, 95],
-        },
         green: {
             keyColor: "#7AC51B",
             tones: [2, 98],
@@ -40,6 +36,11 @@ export const config: PaletteConfig = {
             keyColor: "#661bc5",
             tones: [2, 98],
         },
+
+        red: {
+            keyColor: "#e82439",
+            tones: [95],
+        },
         blue: {
             keyColor: "#1B7AC5",
             tones: [5, 95],

pkgs/ui/nix/ui-assets.nix

@@ -1,5 +1,5 @@
 { fetchzip }:
 fetchzip {
-  url = "https://git.clan.lol/api/packages/clan/generic/ui/0l0hjjpvqyfiz5jk1yrqdhi50jc79v02fhdi7p7s39v1nfxzh3yn/assets.tar.gz";
-  sha256 = "0l0hjjpvqyfiz5jk1yrqdhi50jc79v02fhdi7p7s39v1nfxzh3yn";
+  url = "https://git.clan.lol/api/packages/clan/generic/ui/044pm5casi89nrbzp06l2akn797cdjcj49yyf495fspqfya3kxvz/assets.tar.gz";
+  sha256 = "044pm5casi89nrbzp06l2akn797cdjcj49yyf495fspqfya3kxvz";
 }