From bf176ad2774b47de91e8255649ae09275d639b47 Mon Sep 17 00:00:00 2001 From: DavHau Date: Wed, 25 Oct 2023 16:36:01 +0100 Subject: [PATCH] api/machines: allow importing extra modules - add top-level option `clanImports` to clanCore - clanImports can be set and checked as any other option - buildClan resolves the clanImports from the settings.json before calling evalModules to prevent infinite recursions - new endpoint PUT machines/{name}/schema to allow getting the schema for a specific list of imports - to retrieve the currently imported modules, cimply do a GET or PU on machines/{name}/config which will return `clanImports` as part of the config Still missing: get list of available modules --- lib/build-clan/default.nix | 40 ++++++--- lib/default.nix | 4 +- lib/flake-module.nix | 2 +- nixosModules/clanCore/flake-module.nix | 6 ++ nixosModules/clanImports/default.nix | 16 ++++ pkgs/clan-cli/clan_cli/config/machine.py | 69 +++++++++------ .../clan_cli/webui/routers/machines.py | 8 ++ .../clan-cli/tests/test_flake/fake-module.nix | 11 +++ pkgs/clan-cli/tests/test_flake/flake.nix | 83 ++++++++++++------- pkgs/clan-cli/tests/test_machines_api.py | 74 +++++++++++++++-- pkgs/clan-cli/tests/test_machines_config.py | 2 +- 11 files changed, 234 insertions(+), 81 deletions(-) create mode 100644 nixosModules/clanImports/default.nix create mode 100644 pkgs/clan-cli/tests/test_flake/fake-module.nix diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 770f7856..1a141955 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -1,4 +1,4 @@ -{ nixpkgs, self, lib }: +{ clan-core, nixpkgs, lib }: { directory # The directory containing the machines subdirectory , specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available , machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... } @@ -7,6 +7,9 @@ let machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (builtins.readDir (directory + /machines)); machineSettings = machineName: + # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily + # This is useful for doing a dry-run before writing changes into the settings.json + # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) else @@ -14,18 +17,33 @@ let (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))); + # Read additional imports specified via a config option in settings.json + # This is not an infinite recursion, because the imports are discovered here + # before calling evalModules. + # It is still useful to have the imports as an option, as this allows for type + # checking and easy integration with the config frontend(s) + machineImports = machineSettings: + map + (module: clan-core.clanModules.${module}) + (machineSettings.clanImports or [ ]); + # TODO: remove default system once we have a hardware-config mechanism nixosConfiguration = { system ? "x86_64-linux", name }: nixpkgs.lib.nixosSystem { - modules = [ - self.nixosModules.clanCore - (machineSettings name) - (machines.${name} or { }) - { - clanCore.machineName = name; - clanCore.clanDir = directory; - nixpkgs.hostPlatform = lib.mkForce system; - } - ]; + modules = + let + settings = machineSettings name; + in + (machineImports settings) + ++ [ + settings + clan-core.nixosModules.clanCore + (machines.${name} or { }) + { + clanCore.machineName = name; + clanCore.clanDir = directory; + nixpkgs.hostPlatform = lib.mkForce system; + } + ]; inherit specialArgs; }; diff --git a/lib/default.nix b/lib/default.nix index 39b83883..856a4dff 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,6 +1,6 @@ -{ lib, self, nixpkgs, ... }: +{ lib, clan-core, nixpkgs, ... }: { jsonschema = import ./jsonschema { inherit lib; }; - buildClan = import ./build-clan { inherit lib self nixpkgs; }; + buildClan = import ./build-clan { inherit clan-core lib nixpkgs; }; } diff --git a/lib/flake-module.nix b/lib/flake-module.nix index 1062e92c..089da6c9 100644 --- a/lib/flake-module.nix +++ b/lib/flake-module.nix @@ -8,7 +8,7 @@ ]; flake.lib = import ./default.nix { inherit lib; - inherit self; inherit (inputs) nixpkgs; + clan-core = self; }; } diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/clanCore/flake-module.nix index d3b494cc..dbfb22fd 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/clanCore/flake-module.nix @@ -1,6 +1,7 @@ { self, inputs, lib, ... }: { flake.nixosModules.clanCore = { config, pkgs, options, ... }: { imports = [ + ../clanImports ./secrets ./zerotier ./networking.nix @@ -34,6 +35,11 @@ internal = true; }; }; + # TODO: factor these out into a separate interface.nix. + # Also think about moving these options out of `system.clan`. + # Maybe we should not re-use the already polluted confg.system namespace + # and instead have a separate top-level namespace like `clanOutputs`, with + # well defined options marked as `internal = true;`. options.system.clan = lib.mkOption { type = lib.types.submodule { options = { diff --git a/nixosModules/clanImports/default.nix b/nixosModules/clanImports/default.nix new file mode 100644 index 00000000..e89eaf0f --- /dev/null +++ b/nixosModules/clanImports/default.nix @@ -0,0 +1,16 @@ +{ lib +, ... +}: { + /* + Declaring imports inside the module system does not trigger an infinite + recursion in this case because buildClan generates the imports from the + settings.json file before calling out to evalModules. + */ + options.clanImports = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of imported module names imported from clan-core.clanModules. + The buildClan function will automatically import these modules for the current machine. + ''; + }; +} diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index edd4cf93..e15a5f29 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -45,9 +45,9 @@ def verify_machine_config( cwd=flake, env=env, ) - if proc.returncode != 0: - return proc.stderr - return None + if proc.returncode != 0: + return proc.stderr + return None def config_for_machine(machine_name: str) -> dict: @@ -85,32 +85,47 @@ def set_config_for_machine(machine_name: str, config: dict) -> Optional[str]: return None -def schema_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict: +def schema_for_machine( + machine_name: str, config: Optional[dict] = None, flake: Optional[Path] = None +) -> dict: if flake is None: flake = get_clan_flake_toplevel() - # use nix eval to lib.evalModules .#nixosModules.machine-{machine_name} - proc = subprocess.run( - nix_eval( - flags=[ - "--impure", - "--show-trace", - "--expr", - f""" - let - flake = builtins.getFlake (toString {flake}); - lib = import {nixpkgs_source()}/lib; - options = flake.nixosConfigurations.{machine_name}.options; - clanOptions = options.clan; - jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; - jsonschema = jsonschemaLib.parseOptions clanOptions; - in - jsonschema - """, - ], - ), - capture_output=True, - text=True, - ) + # use nix eval to lib.evalModules .#nixosConfigurations..options.clan + with NamedTemporaryFile(mode="w") as clan_machine_settings_file: + env = os.environ.copy() + inject_config_flags = [] + if config is not None: + json.dump(config, clan_machine_settings_file, indent=2) + clan_machine_settings_file.seek(0) + env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name + inject_config_flags = [ + "--impure", # needed to access CLAN_MACHINE_SETTINGS_FILE + ] + proc = subprocess.run( + nix_eval( + flags=inject_config_flags + + [ + "--impure", + "--show-trace", + "--expr", + f""" + let + flake = builtins.getFlake (toString {flake}); + lib = import {nixpkgs_source()}/lib; + options = flake.nixosConfigurations.{machine_name}.options; + clanOptions = options.clan; + jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; + jsonschema = jsonschemaLib.parseOptions clanOptions; + in + jsonschema + """, + ], + ), + capture_output=True, + text=True, + cwd=flake, + env=env, + ) if proc.returncode != 0: print(proc.stderr, file=sys.stderr) raise Exception( diff --git a/pkgs/clan-cli/clan_cli/webui/routers/machines.py b/pkgs/clan-cli/clan_cli/webui/routers/machines.py index 753a7113..2cfca86e 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/machines.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/machines.py @@ -69,6 +69,14 @@ async def get_machine_schema(name: str) -> SchemaResponse: return SchemaResponse(schema=schema) +@router.put("/api/machines/{name}/schema") +async def set_machine_schema( + name: str, config: Annotated[dict, Body()] +) -> SchemaResponse: + schema = schema_for_machine(name, config) + return SchemaResponse(schema=schema) + + @router.get("/api/machines/{name}/verify") async def put_verify_machine_config(name: str) -> VerifyMachineResponse: error = verify_machine_config(name) diff --git a/pkgs/clan-cli/tests/test_flake/fake-module.nix b/pkgs/clan-cli/tests/test_flake/fake-module.nix new file mode 100644 index 00000000..b3c6580d --- /dev/null +++ b/pkgs/clan-cli/tests/test_flake/fake-module.nix @@ -0,0 +1,11 @@ +{ lib +, ... +}: { + options.clan.fake-module.fake-flag = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + A useless fake flag fro testing purposes. + ''; + }; +} diff --git a/pkgs/clan-cli/tests/test_flake/flake.nix b/pkgs/clan-cli/tests/test_flake/flake.nix index c8290365..3f335a4e 100644 --- a/pkgs/clan-cli/tests/test_flake/flake.nix +++ b/pkgs/clan-cli/tests/test_flake/flake.nix @@ -2,36 +2,57 @@ # this placeholder is replaced by the path to nixpkgs inputs.nixpkgs.url = "__NIXPKGS__"; - outputs = inputs: { - nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem { - modules = [ - ./nixosModules/machine1.nix - ( - if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" - then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) - else if builtins.pathExists ./machines/machine1/settings.json - then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json) - else { } - ) - ({ lib, options, pkgs, ... }: { - config = { - nixpkgs.hostPlatform = "x86_64-linux"; - # speed up by not instantiating nixpkgs twice and disable documentation - nixpkgs.pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux; - documentation.enable = false; - }; - options.clanCore.optionsNix = lib.mkOption { - type = lib.types.raw; - internal = true; - readOnly = true; - default = (pkgs.nixosOptionsDoc { inherit options; }).optionsNix; - defaultText = "optionsNix"; - description = '' - This is to export nixos options used for `clan config` - ''; - }; - }) - ]; + outputs = inputs': + let + # fake clan-core input + fake-clan-core = { + clanModules.fake-module = ./fake-module.nix; + }; + inputs = inputs' // { clan-core = fake-clan-core; }; + machineSettings = ( + if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" + then builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")) + else if builtins.pathExists ./machines/machine1/settings.json + then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json) + else { } + ); + machineImports = + map + (module: fake-clan-core.clanModules.${module}) + (machineSettings.clanImports or [ ]); + in + { + nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem { + modules = + machineImports ++ [ + ./nixosModules/machine1.nix + machineSettings + ({ lib, options, pkgs, ... }: { + config = { + nixpkgs.hostPlatform = "x86_64-linux"; + # speed up by not instantiating nixpkgs twice and disable documentation + nixpkgs.pkgs = inputs.nixpkgs.legacyPackages.x86_64-linux; + documentation.enable = false; + }; + options.clanCore.optionsNix = lib.mkOption { + type = lib.types.raw; + internal = true; + readOnly = true; + default = (pkgs.nixosOptionsDoc { inherit options; }).optionsNix; + defaultText = "optionsNix"; + description = '' + This is to export nixos options used for `clan config` + ''; + }; + options.clanImports = lib.mkOption { + type = lib.types.listOf lib.types.str; + description = '' + A list of imported module names imported from clan-core.clanModules. + The buildClan function will automatically import these modules for the current machine. + ''; + }; + }) + ]; + }; }; - }; } diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index 02984dc0..a9fcb6cc 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -68,20 +68,13 @@ def test_configure_machine(api: TestClient, test_flake: Path) -> None: assert response.status_code == 200 assert response.json() == {"config": {}} - # set some valid config - config2 = dict( - clan=dict( - jitsi=dict( - enable=True, - ), - ), + fs_config = dict( fileSystems={ "/": dict( device="/dev/fake_disk", fsType="ext4", ), }, - # set boot.loader.grub.devices boot=dict( loader=dict( grub=dict( @@ -90,6 +83,16 @@ def test_configure_machine(api: TestClient, test_flake: Path) -> None: ), ), ) + + # set some valid config + config2 = dict( + clan=dict( + jitsi=dict( + enable=True, + ), + ), + **fs_config, + ) response = api.put( "/api/machines/machine1/config", json=config2, @@ -116,3 +119,58 @@ def test_configure_machine(api: TestClient, test_flake: Path) -> None: response = api.get("/api/machines/machine1/verify") assert response.status_code == 200 assert response.json() == {"success": True, "error": None} + + # get the schema with an extra module imported + response = api.put( + "/api/machines/machine1/schema", + json={"clanImports": ["fake-module"]}, + ) + # expect the result schema to contain the fake-module.fake-flag option + assert response.status_code == 200 + assert ( + response.json()["schema"]["properties"]["fake-module"]["properties"][ + "fake-flag" + ]["type"] + == "boolean" + ) + + # new config importing an extra clanModule (clanModules.fake-module) + config_with_imports: dict = { + "clanImports": ["fake-module"], + "clan": { + "fake-module": { + "fake-flag": True, + }, + }, + **fs_config, + } + + # set the fake-module.fake-flag option to true + response = api.put( + "/api/machines/machine1/config", + json=config_with_imports, + ) + assert response.status_code == 200 + assert response.json() == { + "config": { + "clanImports": ["fake-module"], + "clan": { + "fake-module": { + "fake-flag": True, + }, + }, + **fs_config, + } + } + + # remove the import from the config + config_with_empty_imports = dict( + clanImports=[], + **fs_config, + ) + response = api.put( + "/api/machines/machine1/config", + json=config_with_empty_imports, + ) + assert response.status_code == 200 + assert response.json() == {"config": config_with_empty_imports} diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index 4d3a0a7b..cb942775 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -4,5 +4,5 @@ from clan_cli.config import machine def test_schema_for_machine(test_flake: Path) -> None: - schema = machine.schema_for_machine("machine1", test_flake) + schema = machine.schema_for_machine("machine1", flake=test_flake) assert "properties" in schema