From 108a37b0a350eff7bc4d95e625ccaab3a899327b Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 1 Mar 2024 07:37:47 +0100 Subject: [PATCH 1/8] clan-cli machines: cache machines_func via store --- pkgs/clan-cli/clan_cli/machines/machines.py | 93 +++++++++++---------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index f0173f2a..bd2a768a 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -166,6 +166,7 @@ class Machine: config = nix_config() system = config["system"] + file_info = dict() with NamedTemporaryFile(mode="w") as config_json: if extra_config is not None: json.dump(extra_config, config_json, indent=2) @@ -173,66 +174,66 @@ class Machine: json.dump({}, config_json) config_json.flush() - nar_hash = json.loads( + file_info = json.loads( run( nix_eval( [ "--impure", "--expr", - f'(builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}).narHash', + f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}', ] ) ).stdout.strip() ) - args = [] + args = [] - # get git commit from flake - if extra_config is not None: - metadata = nix_metadata(self.flake_dir) - url = metadata["url"] - if "dirtyRevision" in metadata: - # if not impure: - # raise ClanError( - # "The machine has a dirty revision, and impure mode is not allowed" - # ) - # else: - # args += ["--impure"] - args += ["--impure"] + # get git commit from flake + if extra_config is not None: + metadata = nix_metadata(self.flake_dir) + url = metadata["url"] + if "dirtyRevision" in metadata: + # if not impure: + # raise ClanError( + # "The machine has a dirty revision, and impure mode is not allowed" + # ) + # else: + # args += ["--impure"] + args += ["--impure"] - args += [ - "--expr", - f""" - ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{ - extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{ - type = "file"; - url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{config_json.name}" else "file:{config_json.name}"; - narHash = "{nar_hash}"; - }})); - }}).{attr} - """, - ] - else: - if isinstance(self.flake, Path): - if (self.flake / ".git").exists(): - flake = f"git+file://{self.flake}" - else: - flake = f"path:{self.flake}" + args += [ + "--expr", + f""" + ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{ + extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{ + type = "file"; + url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}"; + narHash = "{file_info["narHash"]}"; + }})); + }}).{attr} + """, + ] + else: + if isinstance(self.flake, Path): + if (self.flake / ".git").exists(): + flake = f"git+file://{self.flake}" else: - flake = self.flake - args += [ - f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}', - *nix_options, - ] - - if method == "eval": - output = run(nix_eval(args)).stdout.strip() - return output - elif method == "build": - outpath = run(nix_build(args)).stdout.strip() - return Path(outpath) + flake = f"path:{self.flake}" else: - raise ValueError(f"Unknown method {method}") + flake = self.flake + args += [ + f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}', + *nix_options, + ] + + if method == "eval": + output = run(nix_eval(args)).stdout.strip() + return output + elif method == "build": + outpath = run(nix_build(args)).stdout.strip() + return Path(outpath) + else: + raise ValueError(f"Unknown method {method}") def eval_nix( self, From b1a4b4de9613d6500ee9792ce1c0af6703be974f Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 2 Mar 2024 07:27:29 +0100 Subject: [PATCH 2/8] clan-cli vms run: remove unused vm arg --- pkgs/clan-cli/clan_cli/vms/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index 46fedfe0..c9049e12 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -37,7 +37,7 @@ def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict: # TODO move this to the Machines class def build_vm( - machine: Machine, vm: VmConfig, tmpdir: Path, nix_options: list[str] = [] + machine: Machine, tmpdir: Path, nix_options: list[str] = [] ) -> dict[str, str]: secrets_dir = get_secrets(machine, tmpdir) @@ -113,7 +113,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None: tmpdir = Path(cachedir) # TODO: We should get this from the vm argument - nixos_config = build_vm(machine, vm, tmpdir, nix_options) + nixos_config = build_vm(machine, tmpdir, nix_options) state_dir = vm_state_dir(str(vm.flake_url), machine.name) state_dir.mkdir(parents=True, exist_ok=True) From 4cfd58044779d77301f622ad5c34ba296cb53ad1 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 2 Mar 2024 10:24:49 +0100 Subject: [PATCH 3/8] outputs: pass secretsData directly --- nixosModules/clanCore/outputs.nix | 14 +------------- pkgs/clan-cli/clan_cli/secrets/generate.py | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/nixosModules/clanCore/outputs.nix b/nixosModules/clanCore/outputs.nix index c7a77f1f..75d2c063 100644 --- a/nixosModules/clanCore/outputs.nix +++ b/nixosModules/clanCore/outputs.nix @@ -62,19 +62,7 @@ description = '' secret data as json for the generator ''; - default = pkgs.writers.writeJSON "secrets.json" (lib.mapAttrs - (_name: secret: { - secrets = lib.mapAttrsToList - (name: secret: { - inherit name; - } // lib.optionalAttrs (secret ? groups) { - inherit (secret) groups; - }) - secret.secrets; - facts = lib.mapAttrs (_: secret: secret.path) secret.facts; - generator = secret.generator.finalScript; - }) - config.clanCore.secrets); + default = pkgs.writers.writeJSON "secrets.json" config.clanCore.secrets; }; vm.create = lib.mkOption { type = lib.types.path; diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index b38b39a7..bb9733f3 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -58,7 +58,7 @@ def generate_service_secrets( "--unshare-user", "--uid", "1000", "--", - "bash", "-c", machine.secrets_data[service]["generator"] + "bash", "-c", machine.secrets_data[service]["generator"]["finalScript"] ], ) # fmt: on From f500aee78684900f3e388daf13ce3495a8897a2f Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 2 Mar 2024 10:25:10 +0100 Subject: [PATCH 4/8] clanCore secrets: rename toplevel secret to service --- nixosModules/clanCore/secrets/default.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 99c3a94f..4ab8f197 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -35,13 +35,13 @@ options.clanCore.secrets = lib.mkOption { default = { }; type = lib.types.attrsOf - (lib.types.submodule (secret: { + (lib.types.submodule (service: { options = { name = lib.mkOption { type = lib.types.str; - default = secret.config._module.args.name; + default = service.config._module.args.name; description = '' - Namespace of the secret + Namespace of the service ''; }; generator = lib.mkOption { From a1dcddf9b4fa2176fda23714885189152551d068 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 1 Mar 2024 10:25:39 +0100 Subject: [PATCH 5/8] clan-cli: add interactive secrets/fact generation --- nixosModules/clanCore/secrets/default.nix | 12 +++++++-- pkgs/clan-cli/clan_cli/secrets/generate.py | 31 +++++++++++++++++++--- pkgs/clan-cli/clan_cli/vms/run.py | 2 +- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 4ab8f197..5670459f 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -54,6 +54,14 @@ Extra paths to add to the PATH environment variable when running the generator. ''; }; + prompt = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + prompt text to ask for a value. + This value will be passed to the script as the environment variabel $prompt_value. + ''; + }; script = lib.mkOption { type = lib.types.str; description = '' @@ -92,14 +100,14 @@ config' = config; in lib.mkOption { - type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: { + type = lib.types.attrsOf (lib.types.submodule ({ config, name, ... }: { options = { name = lib.mkOption { type = lib.types.str; description = '' name of the secret ''; - default = config._module.args.name; + default = name; }; path = lib.mkOption { type = lib.types.str; diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index bb9733f3..e478d459 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -2,6 +2,7 @@ import argparse import importlib import logging import os +from collections.abc import Callable from pathlib import Path from tempfile import TemporaryDirectory @@ -24,6 +25,7 @@ def generate_service_secrets( secret_store: SecretStoreBase, fact_store: FactStoreBase, tmpdir: Path, + prompt: Callable[[str], str], ) -> None: service_dir = tmpdir / service # check if all secrets exist and generate them if at least one is missing @@ -41,6 +43,16 @@ def generate_service_secrets( secrets_dir = service_dir / "secrets" secrets_dir.mkdir(parents=True) env["secrets"] = str(secrets_dir) + # compatibility for old outputs.nix users + if isinstance(machine.secrets_data[service]["generator"], str): + generator = machine.secrets_data[service]["generator"] + else: + generator = machine.secrets_data[service]["generator"]["finalScript"] + if machine.secrets_data[service]["generator"]["prompt"]: + prompt_value = prompt( + machine.secrets_data[service]["generator"]["prompt"] + ) + env["prompt_value"] = prompt_value # fmt: off cmd = nix_shell( [ @@ -58,7 +70,7 @@ def generate_service_secrets( "--unshare-user", "--uid", "1000", "--", - "bash", "-c", machine.secrets_data[service]["generator"]["finalScript"] + "bash", "-c", generator ], ) # fmt: on @@ -105,17 +117,30 @@ def generate_service_secrets( ) -def generate_secrets(machine: Machine) -> None: +def generate_secrets( + machine: Machine, + prompt: None | Callable[[str], str] = None, +) -> None: secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) facts_module = importlib.import_module(machine.facts_module) fact_store = facts_module.FactStore(machine=machine) + if prompt is None: + prompt = lambda text: input(f"{text}: ") + with TemporaryDirectory() as tmp: tmpdir = Path(tmp) for service in machine.secrets_data: - generate_service_secrets(machine, service, secret_store, fact_store, tmpdir) + generate_service_secrets( + machine=machine, + service=service, + secret_store=secret_store, + fact_store=fact_store, + tmpdir=tmpdir, + prompt=prompt, + ) print("successfully generated secrets") diff --git a/pkgs/clan-cli/clan_cli/vms/run.py b/pkgs/clan-cli/clan_cli/vms/run.py index c9049e12..f58db06c 100644 --- a/pkgs/clan-cli/clan_cli/vms/run.py +++ b/pkgs/clan-cli/clan_cli/vms/run.py @@ -39,6 +39,7 @@ def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict: def build_vm( machine: Machine, tmpdir: Path, nix_options: list[str] = [] ) -> dict[str, str]: + # TODO pass prompt here for the GTK gui secrets_dir = get_secrets(machine, tmpdir) facts_module = importlib.import_module(machine.facts_module) @@ -68,7 +69,6 @@ def get_secrets( secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) - # TODO Only generate secrets for local clans generate_secrets(machine) secret_store.upload(secrets_dir) From a23c251b09d4473c68907a2c7a53ca8a282d11b4 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 3 Mar 2024 04:05:56 +0100 Subject: [PATCH 6/8] clan-cli secrets: actually check if only service needs regeneration --- pkgs/clan-cli/clan_cli/secrets/check.py | 19 +++++++++++++++---- pkgs/clan-cli/clan_cli/secrets/generate.py | 2 +- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/check.py b/pkgs/clan-cli/clan_cli/secrets/check.py index ea348548..5c51d28a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/check.py +++ b/pkgs/clan-cli/clan_cli/secrets/check.py @@ -7,7 +7,7 @@ from ..machines.machines import Machine log = logging.getLogger(__name__) -def check_secrets(machine: Machine) -> bool: +def check_secrets(machine: Machine, service: None | str = None) -> bool: secrets_module = importlib.import_module(machine.secrets_module) secret_store = secrets_module.SecretStore(machine=machine) facts_module = importlib.import_module(machine.facts_module) @@ -15,7 +15,11 @@ def check_secrets(machine: Machine) -> bool: missing_secrets = [] missing_facts = [] - for service in machine.secrets_data: + if service: + services = [service] + else: + services = list(machine.secrets_data.keys()) + for service in services: for secret in machine.secrets_data[service]["secrets"]: if isinstance(secret, str): secret_name = secret @@ -38,8 +42,11 @@ def check_secrets(machine: Machine) -> bool: def check_command(args: argparse.Namespace) -> None: - machine = Machine(name=args.machine, flake=args.flake) - check_secrets(machine) + machine = Machine( + name=args.machine, + flake=args.flake, + ) + check_secrets(machine, service=args.service) def register_check_parser(parser: argparse.ArgumentParser) -> None: @@ -47,4 +54,8 @@ def register_check_parser(parser: argparse.ArgumentParser) -> None: "machine", help="The machine to check secrets for", ) + parser.add_argument( + "--service", + help="the service to check", + ) parser.set_defaults(func=check_command) diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index e478d459..16527b89 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -29,7 +29,7 @@ def generate_service_secrets( ) -> None: service_dir = tmpdir / service # check if all secrets exist and generate them if at least one is missing - needs_regeneration = not check_secrets(machine) + needs_regeneration = not check_secrets(machine, service=service) log.debug(f"{service} needs_regeneration: {needs_regeneration}") if needs_regeneration: if not isinstance(machine.flake, Path): From b8da1494533074f5048fa7a250f32801e7c98ac8 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 3 Mar 2024 04:07:01 +0100 Subject: [PATCH 7/8] clan-cli sops: fix super class interface compliance --- pkgs/clan-cli/clan_cli/secrets/modules/sops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/modules/sops.py b/pkgs/clan-cli/clan_cli/secrets/modules/sops.py index 3f91d604..e2c0bb07 100644 --- a/pkgs/clan-cli/clan_cli/secrets/modules/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/modules/sops.py @@ -45,7 +45,7 @@ class SecretStore(SecretStoreBase): ) return path - def get(self, service: str, _name: str) -> bytes: + def get(self, service: str, name: str) -> bytes: raise NotImplementedError() def exists(self, service: str, name: str) -> bool: From ed653fa8b90a5cff2e53e3cf0d7d0a8b48d2bc2b Mon Sep 17 00:00:00 2001 From: lassulus Date: Sun, 3 Mar 2024 06:19:55 +0100 Subject: [PATCH 8/8] fix pyproject syntax, ignore E731 --- checks/lib/container-driver/pyproject.toml | 4 ++-- pkgs/clan-cli/pyproject.toml | 4 ++-- pkgs/clan-vm-manager/pyproject.toml | 4 ++-- pyproject.toml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/checks/lib/container-driver/pyproject.toml b/checks/lib/container-driver/pyproject.toml index 5c26a4b7..d7d5ebf8 100644 --- a/checks/lib/container-driver/pyproject.toml +++ b/checks/lib/container-driver/pyproject.toml @@ -19,8 +19,8 @@ test_driver = ["py.typed"] target-version = "py311" line-length = 88 -select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] -ignore = ["E501", "ANN101", "ANN401", "A003"] +lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] +lint.ignore = ["E501", "ANN101", "ANN401", "A003"] [tool.mypy] python_version = "3.11" diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 26eec53a..7800c659 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -55,5 +55,5 @@ ignore_missing_imports = true [tool.ruff] target-version = "py311" line-length = 88 -select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] -ignore = ["E501", "E402", "ANN101", "ANN401", "A003"] +lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] +lint.ignore = ["E501", "E402", "E731", "ANN101", "ANN401", "A003"] diff --git a/pkgs/clan-vm-manager/pyproject.toml b/pkgs/clan-vm-manager/pyproject.toml index 800b4661..6f8a2f6f 100644 --- a/pkgs/clan-vm-manager/pyproject.toml +++ b/pkgs/clan-vm-manager/pyproject.toml @@ -33,5 +33,5 @@ ignore_missing_imports = true [tool.ruff] target-version = "py311" line-length = 88 -select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] -ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"] +lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] +lint.ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"] diff --git a/pyproject.toml b/pyproject.toml index 7ab17add..18fd26e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,5 +10,5 @@ exclude = "clan_cli.nixpkgs" [tool.ruff] line-length = 88 target-version = "py311" -select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] -ignore = [ "E501", "ANN101", "ANN401", "A003"] +lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ] +lint.ignore = [ "E501", "ANN101", "ANN401", "A003"]