api/schema: get schema without having a machine

Also done:
- add @pytest.mark.with_core indicating that a test depends on clan-core and cannot be cached
- separate unit tests into two derivations, ones that can be cached and ones that cannot.
- fix frontend typescript code

checks/impure/flake-module.nix

@@ -13,7 +13,7 @@
         ]}"
         ROOT=$(git rev-parse --show-toplevel)
         cd "$ROOT/pkgs/clan-cli"
-        nix develop "$ROOT#clan-cli" -c bash -c 'TMPDIR=/tmp python -m pytest -m impure -s ./tests'
+        nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure -s ./tests $@"
       '';
 
       runMockApi = pkgs.writeShellScriptBin "run-mock-api" ''

clanModules/flake-module.nix

@@ -1,13 +1,11 @@
-{ self, lib, ... }: {
+{ inputs, ... }: {
   flake.clanModules = {
-    diskLayouts = lib.mapAttrs'
+    diskLayouts = {
-      (name: _: lib.nameValuePair (lib.removeSuffix ".nix" name) {
+      imports = [
-        imports = [
+        ./diskLayouts.nix
-          self.inputs.disko.nixosModules.disko
+        inputs.disko.nixosModules.default
-          ./diskLayouts/${name}
+      ];
-        ];
+    };
-      })
-      (builtins.readDir ./diskLayouts);
     deltachat = ./deltachat.nix;
     xfce = ./xfce.nix;
   };

pkgs/clan-cli/clan_cli/config/machine.py

@@ -1,7 +1,6 @@
 import json
 import os
 import subprocess
-import sys
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Optional
@@ -10,7 +9,6 @@ from fastapi import HTTPException
 
 from clan_cli.dirs import (
     machine_settings_file,
-    nixpkgs_source,
     specific_flake_dir,
     specific_machine_dir,
 )
@@ -91,51 +89,3 @@ def set_config_for_machine(
 
     if repo_dir is not None:
         commit_file(settings_path, repo_dir)
-
-
-def schema_for_machine(
-    flake_name: FlakeName, machine_name: str, config: Optional[dict] = None
-) -> dict:
-    flake = specific_flake_dir(flake_name)
-    # use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
-    with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file:
-        env = os.environ.copy()
-        inject_config_flags = []
-        if config is not None:
-            json.dump(config, clan_machine_settings_file, indent=2)
-            clan_machine_settings_file.seek(0)
-            env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
-            inject_config_flags = [
-                "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
-            ]
-        proc = subprocess.run(
-            nix_eval(
-                flags=inject_config_flags
-                + [
-                    "--impure",
-                    "--show-trace",
-                    "--expr",
-                    f"""
-                    let
-                        flake = builtins.getFlake (toString {flake});
-                        lib = import {nixpkgs_source()}/lib;
-                        options = flake.nixosConfigurations.{machine_name}.options;
-                        clanOptions = options.clan;
-                        jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
-                        jsonschema = jsonschemaLib.parseOptions clanOptions;
-                    in
-                        jsonschema
-                    """,
-                ],
-            ),
-            capture_output=True,
-            text=True,
-            cwd=flake,
-            env=env,
-        )
-    if proc.returncode != 0:
-        print(proc.stderr, file=sys.stderr)
-        raise Exception(
-            f"Failed to read schema for machine {machine_name}:\n{proc.stderr}"
-        )
-    return json.loads(proc.stdout)

pkgs/clan-cli/clan_cli/config/schema.py

@@ -0,0 +1,79 @@
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Optional
+
+from clan_cli.dirs import (
+    nixpkgs_source,
+    specific_flake_dir,
+)
+from clan_cli.nix import nix_eval
+
+from ..types import FlakeName
+
+
+def machine_schema(
+    flake_name: FlakeName,
+    config: dict,
+    clan_imports: Optional[list[str]] = None,
+) -> dict:
+    flake = specific_flake_dir(flake_name)
+    # use nix eval to lib.evalModules .#nixosConfigurations.<machine_name>.options.clan
+    with NamedTemporaryFile(mode="w", dir=flake) as clan_machine_settings_file:
+        env = os.environ.copy()
+        inject_config_flags = []
+        if clan_imports is not None:
+            config["clanImports"] = clan_imports
+        json.dump(config, clan_machine_settings_file, indent=2)
+        clan_machine_settings_file.seek(0)
+        env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
+        inject_config_flags = [
+            "--impure",  # needed to access CLAN_MACHINE_SETTINGS_FILE
+        ]
+        proc = subprocess.run(
+            nix_eval(
+                flags=inject_config_flags
+                + [
+                    "--impure",
+                    "--show-trace",
+                    "--expr",
+                    f"""
+                    let
+                        system = builtins.currentSystem;
+                        flake = builtins.getFlake (toString {flake});
+                        clan-core = flake.inputs.clan-core;
+                        nixpkgsSrc = flake.inputs.nixpkgs or {nixpkgs_source()};
+                        lib = import (nixpkgsSrc + /lib);
+                        pkgs = import nixpkgsSrc {{ inherit system; }};
+                        config = lib.importJSON (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE");
+                        fakeMachine = pkgs.nixos {{
+                            imports =
+                                [
+                                    clan-core.nixosModules.clanCore
+                                    # potentially the config might affect submodule options,
+                                    #   therefore we need to import it
+                                    config
+                                ]
+                                # add all clan modules specified via clanImports
+                                ++ (map (name: clan-core.clanModules.${{name}}) config.clanImports or []);
+                        }};
+                        clanOptions = fakeMachine.options.clan;
+                        jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }};
+                        jsonschema = jsonschemaLib.parseOptions clanOptions;
+                    in
+                        jsonschema
+                    """,
+                ],
+            ),
+            capture_output=True,
+            text=True,
+            cwd=flake,
+            env=env,
+        )
+    if proc.returncode != 0:
+        print(proc.stderr, file=sys.stderr)
+        raise Exception(f"Failed to read schema:\n{proc.stderr}")
+    return json.loads(proc.stdout)

pkgs/clan-cli/clan_cli/webui/routers/machines.py

@@ -8,10 +8,10 @@ from clan_cli.webui.api_inputs import MachineConfig
 
 from ...config.machine import (
     config_for_machine,
-    schema_for_machine,
     set_config_for_machine,
     verify_machine_config,
 )
+from ...config.schema import machine_schema
 from ...machines.create import create_machine as _create_machine
 from ...machines.list import list_machines as _list_machines
 from ...types import FlakeName
@@ -68,17 +68,11 @@ async def set_machine_config(
     set_config_for_machine(flake_name, name, conf)
 
 
-@router.get("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
+@router.put("/api/{flake_name}/schema", tags=[Tags.machine])
-async def get_machine_schema(flake_name: FlakeName, name: str) -> SchemaResponse:
+async def get_machine_schema(
-    schema = schema_for_machine(flake_name, name)
+    flake_name: FlakeName, config: Annotated[dict, Body()]
-    return SchemaResponse(schema=schema)
-
-
-@router.put("/api/{flake_name}/machines/{name}/schema", tags=[Tags.machine])
-async def set_machine_schema(
-    flake_name: FlakeName, name: str, config: Annotated[dict, Body()]
 ) -> SchemaResponse:
-    schema = schema_for_machine(flake_name, name, config)
+    schema = machine_schema(flake_name, config=config)
     return SchemaResponse(schema=schema)
 
 

pkgs/clan-cli/default.nix

@@ -36,6 +36,8 @@
 , mypy
 , deal
 , schemathesis
+, rope
+, clan-core-path
 }:
 let
 
@@ -132,14 +134,30 @@ python3.pkgs.buildPythonApplication {
   propagatedBuildInputs = dependencies;
 
   # also re-expose dependencies so we test them in CI
-  passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesAsSet) // {
+  passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "clan-dep-${n}") runtimeDependenciesAsSet) // rec {
-    clan-pytest = runCommand "clan-pytest" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } ''
+    clan-pytest-without-core = runCommand "clan-pytest-without-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } ''
       cp -r ${source} ./src
       chmod +w -R ./src
       cd ./src
 
       export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
-      ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
+      ${checkPython}/bin/python -m pytest -m "not impure and not with_core" -s ./tests
+      touch $out
+    '';
+    # separate the tests that can never be cached
+    clan-pytest-with-core = runCommand "clan-pytest-with-core" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } ''
+      cp -r ${source} ./src
+      chmod +w -R ./src
+      cd ./src
+
+      export CLAN_CORE=${clan-core-path}
+      export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
+      ${checkPython}/bin/python -m pytest -m "not impure and with_core" -s ./tests
+      touch $out
+    '';
+    clan-pytest = runCommand "clan-pytest" { } ''
+      echo ${clan-pytest-without-core}
+      echo ${clan-pytest-with-core}
       touch $out
     '';
     check-for-breakpoints = runCommand "breakpoints" { } ''
@@ -164,6 +182,7 @@ python3.pkgs.buildPythonApplication {
   passthru.checkPython = checkPython;
 
   passthru.devDependencies = [
+    rope
     setuptools
     wheel
   ] ++ pytestDependencies;

pkgs/clan-cli/flake-module.nix

@@ -1,8 +1,33 @@
-{ inputs, ... }:
+{ inputs, self, lib, ... }:
 {
   perSystem = { self', pkgs, system, ... }:
     let
       luisPythonPkgs = inputs.luispkgs.legacyPackages.${system}.python3Packages;
+      flakeLock = lib.importJSON (self + /flake.lock);
+      flakeInputs = (builtins.removeAttrs inputs [ "self" ]);
+      flakeLockVendoredDeps = flakeLock // {
+        nodes = flakeLock.nodes // (
+          lib.flip lib.mapAttrs flakeInputs (name: _: flakeLock.nodes.${name} // {
+            locked = {
+              inherit (flakeLock.nodes.${name}.locked) narHash;
+              lastModified =
+                # lol, nixpkgs has a different timestamp on the fs???
+                if name == "nixpkgs"
+                then 0
+                else 1;
+              path = "${inputs.${name}}";
+              type = "path";
+            };
+          })
+        );
+      };
+      flakeLockFile = builtins.toFile "clan-core-flake.lock"
+        (builtins.toJSON flakeLockVendoredDeps);
+      clanCoreWithVendoredDeps = lib.trace flakeLockFile pkgs.runCommand "clan-core-with-vendored-deps" { } ''
+        cp -r ${self} $out
+        chmod +w -R $out
+        cp ${flakeLockFile} $out/flake.lock
+      '';
     in
     {
       devShells.clan-cli = pkgs.callPackage ./shell.nix {
@@ -14,6 +39,7 @@
           inherit (inputs) nixpkgs;
           deal = luisPythonPkgs.deal;
           schemathesis = luisPythonPkgs.schemathesis;
+          clan-core-path = clanCoreWithVendoredDeps;
         };
         inherit (self'.packages.clan-cli) clan-openapi;
         default = self'.packages.clan-cli;

pkgs/clan-cli/tests/test_clan_modules.py

@@ -3,7 +3,7 @@ from api import TestClient
 from fixtures_flakes import FlakeForTest
 
 
-@pytest.mark.impure()
+@pytest.mark.with_core
 def test_configure_machine(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     # retrieve the list of available clanModules
     response = api.get(f"/api/{test_flake_with_core.name}/clan_modules")

pkgs/clan-cli/tests/test_machines_api.py

@@ -1,3 +1,4 @@
+import pytest
 from api import TestClient
 from fixtures_flakes import FlakeForTest
 
@@ -21,37 +22,46 @@ def test_machines(api: TestClient, test_flake: FlakeForTest) -> None:
     assert response.json() == {"machines": [{"name": "test", "status": "unknown"}]}
 
 
-def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
+@pytest.mark.with_core
+def test_configure_machine(api: TestClient, test_flake_with_core: FlakeForTest) -> None:
     # ensure error 404 if machine does not exist when accessing the config
-    response = api.get(f"/api/{test_flake.name}/machines/machine1/config")
+    response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config")
     assert response.status_code == 404
 
     # ensure error 404 if machine does not exist when writing to the config
-    response = api.put(f"/api/{test_flake.name}/machines/machine1/config", json={})
+    response = api.put(
+        f"/api/{test_flake_with_core.name}/machines/machine1/config", json={}
+    )
     assert response.status_code == 404
 
     # create the machine
-    response = api.post(f"/api/{test_flake.name}/machines", json={"name": "machine1"})
+    response = api.post(
+        f"/api/{test_flake_with_core.name}/machines", json={"name": "machine1"}
+    )
     assert response.status_code == 201
 
     # ensure an empty config is returned by default for a new machine
-    response = api.get(f"/api/{test_flake.name}/machines/machine1/config")
+    response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config")
     assert response.status_code == 200
     assert response.json() == {
         "clanImports": [],
         "clan": {},
     }
 
-    # get jsonschema for machine
+    # get jsonschema for without imports
-    response = api.get(f"/api/{test_flake.name}/machines/machine1/schema")
+    response = api.put(
+        f"/api/{test_flake_with_core.name}/schema",
+        json={"clanImports": []},
+    )
     assert response.status_code == 200
     json_response = response.json()
     assert "schema" in json_response and "properties" in json_response["schema"]
 
     # an invalid config missing the fileSystems
     invalid_config = dict(
-        clan=dict(
+        clan=dict(),
-            jitsi=dict(
+        services=dict(
+            nginx=dict(
                 enable=True,
             ),
         ),
@@ -59,7 +69,7 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # verify an invalid config (fileSystems missing) fails
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/verify",
+        f"/api/{test_flake_with_core.name}/machines/machine1/verify",
         json=invalid_config,
     )
     assert response.status_code == 200
@@ -70,13 +80,13 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # set come invalid config (fileSystems missing)
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
         json=invalid_config,
     )
     assert response.status_code == 200
 
     # ensure the config has actually been updated
-    response = api.get(f"/api/{test_flake.name}/machines/machine1/config")
+    response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config")
     assert response.status_code == 200
     assert response.json() == dict(clanImports=[], **invalid_config)
 
@@ -99,8 +109,9 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # set some valid config
     config2 = dict(
-        clan=dict(
+        clan=dict(),
-            jitsi=dict(
+        services=dict(
+            nginx=dict(
                 enable=True,
             ),
         ),
@@ -108,20 +119,20 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
     )
 
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
         json=config2,
     )
     assert response.status_code == 200
 
     # ensure the config has been applied
     response = api.get(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
     )
     assert response.status_code == 200
     assert response.json() == dict(clanImports=[], **config2)
 
     # get the config again
-    response = api.get(f"/api/{test_flake.name}/machines/machine1/config")
+    response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/config")
     assert response.status_code == 200
     assert response.json() == {"clanImports": [], **config2}
 
@@ -129,36 +140,36 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
     # For example, this should not result in the boot.loader.grub.devices being
     #   set twice (eg. merged)
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
         json=config2,
     )
     assert response.status_code == 200
 
     # ensure the config has been applied
     response = api.get(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
     )
     assert response.status_code == 200
     assert response.json() == dict(clanImports=[], **config2)
 
     # verify the machine config evaluates
-    response = api.get(f"/api/{test_flake.name}/machines/machine1/verify")
+    response = api.get(f"/api/{test_flake_with_core.name}/machines/machine1/verify")
     assert response.status_code == 200
 
     assert response.json() == {"success": True, "error": None}
 
     # get the schema with an extra module imported
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/schema",
+        f"/api/{test_flake_with_core.name}/schema",
-        json={"clanImports": ["fake-module"]},
+        json={"clanImports": ["diskLayouts"]},
     )
-    # expect the result schema to contain the fake-module.fake-flag option
+    # expect the result schema to contain the deltachat option
     assert response.status_code == 200
     assert (
-        response.json()["schema"]["properties"]["fake-module"]["properties"][
+        response.json()["schema"]["properties"]["diskLayouts"]["properties"][
-            "fake-flag"
+            "singleDiskExt4"
-        ]["type"]
+        ]["properties"]["device"]["type"]
-        == "boolean"
+        == "string"
     )
 
     # new config importing an extra clanModule (clanModules.fake-module)
@@ -174,14 +185,14 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
 
     # set the fake-module.fake-flag option to true
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
         json=config_with_imports,
     )
     assert response.status_code == 200
 
     # ensure the config has been applied
     response = api.get(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
     )
     assert response.status_code == 200
     assert response.json() == {
@@ -200,14 +211,14 @@ def test_configure_machine(api: TestClient, test_flake: FlakeForTest) -> None:
         **fs_config,
     )
     response = api.put(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
         json=config_with_empty_imports,
     )
     assert response.status_code == 200
 
     # ensure the config has been applied
     response = api.get(
-        f"/api/{test_flake.name}/machines/machine1/config",
+        f"/api/{test_flake_with_core.name}/machines/machine1/config",
     )
     assert response.status_code == 200
     assert response.json() == {

pkgs/clan-cli/tests/test_machines_config.py

@@ -1,8 +1,10 @@
+import pytest
 from fixtures_flakes import FlakeForTest
 
-from clan_cli.config import machine
+from clan_cli.config.schema import machine_schema
 
 
-def test_schema_for_machine(test_flake: FlakeForTest) -> None:
+@pytest.mark.with_core
-    schema = machine.schema_for_machine(test_flake.name, "machine1")
+def test_schema_for_machine(test_flake_with_core: FlakeForTest) -> None:
+    schema = machine_schema(test_flake_with_core.name, config={})
     assert "properties" in schema

pkgs/ui/src/components/createMachineForm/clanModules.tsx

@@ -1,4 +1,4 @@
-import { setMachineSchema } from "@/api/machine/machine";
+import { getMachineSchema } from "@/api/machine/machine";
 import { useListClanModules } from "@/api/modules/modules";
 import { Alert, AlertTitle, FormHelperText, Typography } from "@mui/material";
 import Box from "@mui/material/Box";
@@ -32,7 +32,7 @@ export default function ClanModules(props: ClanModulesProps) {
   const selectedModules = formHooks.watch("modules");
 
   useEffect(() => {
-    setMachineSchema(clanName, "example_machine", {
+    getMachineSchema(clanName, {
       imports: [],
     }).then((response) => {
       if (response.statusText == "OK") {
@@ -52,7 +52,7 @@ export default function ClanModules(props: ClanModulesProps) {
     } = event;
     const newValue = typeof value === "string" ? value.split(",") : value;
     formHooks.setValue("modules", newValue);
-    setMachineSchema(clanName, "example_machine", {
+    getMachineSchema(clanName, {
       imports: newValue,
     })
       .then((response) => {