Merge pull request 'interactive_secrets' (#885) from interactive_secrets into main

checks/lib/container-driver/pyproject.toml

@@ -19,8 +19,8 @@ test_driver = ["py.typed"]
 target-version = "py311"
 line-length = 88
 
-select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
+lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
-ignore = ["E501", "ANN101", "ANN401", "A003"]
+lint.ignore = ["E501", "ANN101", "ANN401", "A003"]
 
 [tool.mypy]
 python_version = "3.11"

nixosModules/clanCore/outputs.nix

@@ -62,19 +62,7 @@
           description = ''
             secret data as json for the generator
           '';
-          default = pkgs.writers.writeJSON "secrets.json" (lib.mapAttrs
+          default = pkgs.writers.writeJSON "secrets.json" config.clanCore.secrets;
-            (_name: secret: {
-              secrets = lib.mapAttrsToList
-                (name: secret: {
-                  inherit name;
-                } // lib.optionalAttrs (secret ? groups) {
-                  inherit (secret) groups;
-                })
-                secret.secrets;
-              facts = lib.mapAttrs (_: secret: secret.path) secret.facts;
-              generator = secret.generator.finalScript;
-            })
-            config.clanCore.secrets);
         };
         vm.create = lib.mkOption {
           type = lib.types.path;

nixosModules/clanCore/secrets/default.nix

@@ -35,13 +35,13 @@
   options.clanCore.secrets = lib.mkOption {
     default = { };
     type = lib.types.attrsOf
-      (lib.types.submodule (secret: {
+      (lib.types.submodule (service: {
         options = {
           name = lib.mkOption {
             type = lib.types.str;
-            default = secret.config._module.args.name;
+            default = service.config._module.args.name;
             description = ''
-              Namespace of the secret
+              Namespace of the service
             '';
           };
           generator = lib.mkOption {
@@ -54,6 +54,14 @@
                     Extra paths to add to the PATH environment variable when running the generator.
                   '';
                 };
+                prompt = lib.mkOption {
+                  type = lib.types.nullOr lib.types.str;
+                  default = null;
+                  description = ''
+                    prompt text to ask for a value.
+                    This value will be passed to the script as the environment variabel $prompt_value.
+                  '';
+                };
                 script = lib.mkOption {
                   type = lib.types.str;
                   description = ''
@@ -92,14 +100,14 @@
               config' = config;
             in
             lib.mkOption {
-              type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
+              type = lib.types.attrsOf (lib.types.submodule ({ config, name, ... }: {
                 options = {
                   name = lib.mkOption {
                     type = lib.types.str;
                     description = ''
                       name of the secret
                     '';
-                    default = config._module.args.name;
+                    default = name;
                   };
                   path = lib.mkOption {
                     type = lib.types.str;

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -166,6 +166,7 @@ class Machine:
         config = nix_config()
         system = config["system"]
 
+        file_info = dict()
         with NamedTemporaryFile(mode="w") as config_json:
             if extra_config is not None:
                 json.dump(extra_config, config_json, indent=2)
@@ -173,66 +174,66 @@ class Machine:
                 json.dump({}, config_json)
             config_json.flush()
 
-            nar_hash = json.loads(
+            file_info = json.loads(
                 run(
                     nix_eval(
                         [
                             "--impure",
                             "--expr",
-                            f'(builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}).narHash',
+                            f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}',
                         ]
                     )
                 ).stdout.strip()
             )
 
-            args = []
+        args = []
 
-            # get git commit from flake
+        # get git commit from flake
-            if extra_config is not None:
+        if extra_config is not None:
-                metadata = nix_metadata(self.flake_dir)
+            metadata = nix_metadata(self.flake_dir)
-                url = metadata["url"]
+            url = metadata["url"]
-                if "dirtyRevision" in metadata:
+            if "dirtyRevision" in metadata:
-                    # if not impure:
+                # if not impure:
-                    #     raise ClanError(
+                #     raise ClanError(
-                    #         "The machine has a dirty revision, and impure mode is not allowed"
+                #         "The machine has a dirty revision, and impure mode is not allowed"
-                    #     )
+                #     )
-                    # else:
+                # else:
-                    #     args += ["--impure"]
+                #     args += ["--impure"]
-                    args += ["--impure"]
+                args += ["--impure"]
 
-                args += [
+            args += [
-                    "--expr",
+                "--expr",
-                    f"""
+                f"""
-                        ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{
+                    ((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{
-                          extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{
+                      extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{
-                            type = "file";
+                        type = "file";
-                            url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{config_json.name}" else "file:{config_json.name}";
+                        url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}";
-                            narHash = "{nar_hash}";
+                        narHash = "{file_info["narHash"]}";
-                          }}));
+                      }}));
-                        }}).{attr}
+                    }}).{attr}
-                    """,
+                """,
-                ]
+            ]
-            else:
+        else:
-                if isinstance(self.flake, Path):
+            if isinstance(self.flake, Path):
-                    if (self.flake / ".git").exists():
+                if (self.flake / ".git").exists():
-                        flake = f"git+file://{self.flake}"
+                    flake = f"git+file://{self.flake}"
-                    else:
-                        flake = f"path:{self.flake}"
                 else:
-                    flake = self.flake
+                    flake = f"path:{self.flake}"
-                args += [
-                    f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}',
-                    *nix_options,
-                ]
-
-            if method == "eval":
-                output = run(nix_eval(args)).stdout.strip()
-                return output
-            elif method == "build":
-                outpath = run(nix_build(args)).stdout.strip()
-                return Path(outpath)
             else:
-                raise ValueError(f"Unknown method {method}")
+                flake = self.flake
+            args += [
+                f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}',
+                *nix_options,
+            ]
+
+        if method == "eval":
+            output = run(nix_eval(args)).stdout.strip()
+            return output
+        elif method == "build":
+            outpath = run(nix_build(args)).stdout.strip()
+            return Path(outpath)
+        else:
+            raise ValueError(f"Unknown method {method}")
 
     def eval_nix(
         self,

pkgs/clan-cli/clan_cli/secrets/check.py

@@ -7,7 +7,7 @@ from ..machines.machines import Machine
 log = logging.getLogger(__name__)
 
 
-def check_secrets(machine: Machine) -> bool:
+def check_secrets(machine: Machine, service: None | str = None) -> bool:
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
     facts_module = importlib.import_module(machine.facts_module)
@@ -15,7 +15,11 @@ def check_secrets(machine: Machine) -> bool:
 
     missing_secrets = []
     missing_facts = []
-    for service in machine.secrets_data:
+    if service:
+        services = [service]
+    else:
+        services = list(machine.secrets_data.keys())
+    for service in services:
         for secret in machine.secrets_data[service]["secrets"]:
             if isinstance(secret, str):
                 secret_name = secret
@@ -38,8 +42,11 @@ def check_secrets(machine: Machine) -> bool:
 
 
 def check_command(args: argparse.Namespace) -> None:
-    machine = Machine(name=args.machine, flake=args.flake)
+    machine = Machine(
-    check_secrets(machine)
+        name=args.machine,
+        flake=args.flake,
+    )
+    check_secrets(machine, service=args.service)
 
 
 def register_check_parser(parser: argparse.ArgumentParser) -> None:
@@ -47,4 +54,8 @@ def register_check_parser(parser: argparse.ArgumentParser) -> None:
         "machine",
         help="The machine to check secrets for",
     )
+    parser.add_argument(
+        "--service",
+        help="the service to check",
+    )
     parser.set_defaults(func=check_command)

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -2,6 +2,7 @@ import argparse
 import importlib
 import logging
 import os
+from collections.abc import Callable
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -24,10 +25,11 @@ def generate_service_secrets(
     secret_store: SecretStoreBase,
     fact_store: FactStoreBase,
     tmpdir: Path,
+    prompt: Callable[[str], str],
 ) -> None:
     service_dir = tmpdir / service
     # check if all secrets exist and generate them if at least one is missing
-    needs_regeneration = not check_secrets(machine)
+    needs_regeneration = not check_secrets(machine, service=service)
     log.debug(f"{service} needs_regeneration: {needs_regeneration}")
     if needs_regeneration:
         if not isinstance(machine.flake, Path):
@@ -41,6 +43,16 @@ def generate_service_secrets(
         secrets_dir = service_dir / "secrets"
         secrets_dir.mkdir(parents=True)
         env["secrets"] = str(secrets_dir)
+        # compatibility for old outputs.nix users
+        if isinstance(machine.secrets_data[service]["generator"], str):
+            generator = machine.secrets_data[service]["generator"]
+        else:
+            generator = machine.secrets_data[service]["generator"]["finalScript"]
+            if machine.secrets_data[service]["generator"]["prompt"]:
+                prompt_value = prompt(
+                    machine.secrets_data[service]["generator"]["prompt"]
+                )
+                env["prompt_value"] = prompt_value
         # fmt: off
         cmd = nix_shell(
             [
@@ -58,7 +70,7 @@ def generate_service_secrets(
                 "--unshare-user",
                 "--uid", "1000",
                 "--",
-                "bash", "-c", machine.secrets_data[service]["generator"]
+                "bash", "-c", generator
             ],
         )
         # fmt: on
@@ -105,17 +117,30 @@ def generate_service_secrets(
         )
 
 
-def generate_secrets(machine: Machine) -> None:
+def generate_secrets(
+    machine: Machine,
+    prompt: None | Callable[[str], str] = None,
+) -> None:
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
 
     facts_module = importlib.import_module(machine.facts_module)
     fact_store = facts_module.FactStore(machine=machine)
 
+    if prompt is None:
+        prompt = lambda text: input(f"{text}: ")
+
     with TemporaryDirectory() as tmp:
         tmpdir = Path(tmp)
         for service in machine.secrets_data:
-            generate_service_secrets(machine, service, secret_store, fact_store, tmpdir)
+            generate_service_secrets(
+                machine=machine,
+                service=service,
+                secret_store=secret_store,
+                fact_store=fact_store,
+                tmpdir=tmpdir,
+                prompt=prompt,
+            )
 
     print("successfully generated secrets")
 

pkgs/clan-cli/clan_cli/secrets/modules/sops.py

@@ -45,7 +45,7 @@ class SecretStore(SecretStoreBase):
         )
         return path
 
-    def get(self, service: str, _name: str) -> bytes:
+    def get(self, service: str, name: str) -> bytes:
         raise NotImplementedError()
 
     def exists(self, service: str, name: str) -> bool:

pkgs/clan-cli/clan_cli/vms/run.py

@@ -37,8 +37,9 @@ def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict:
 
 # TODO move this to the Machines class
 def build_vm(
-    machine: Machine, vm: VmConfig, tmpdir: Path, nix_options: list[str] = []
+    machine: Machine, tmpdir: Path, nix_options: list[str] = []
 ) -> dict[str, str]:
+    # TODO pass prompt here for the GTK gui
     secrets_dir = get_secrets(machine, tmpdir)
 
     facts_module = importlib.import_module(machine.facts_module)
@@ -68,7 +69,6 @@ def get_secrets(
     secrets_module = importlib.import_module(machine.secrets_module)
     secret_store = secrets_module.SecretStore(machine=machine)
 
-    # TODO Only generate secrets for local clans
     generate_secrets(machine)
 
     secret_store.upload(secrets_dir)
@@ -113,7 +113,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
         tmpdir = Path(cachedir)
 
         # TODO: We should get this from the vm argument
-        nixos_config = build_vm(machine, vm, tmpdir, nix_options)
+        nixos_config = build_vm(machine, tmpdir, nix_options)
 
         state_dir = vm_state_dir(str(vm.flake_url), machine.name)
         state_dir.mkdir(parents=True, exist_ok=True)

pkgs/clan-cli/pyproject.toml

@@ -55,5 +55,5 @@ ignore_missing_imports = true
 [tool.ruff]
 target-version = "py311"
 line-length = 88
-select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
+lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
-ignore = ["E501", "E402", "ANN101", "ANN401", "A003"]
+lint.ignore = ["E501", "E402", "E731", "ANN101", "ANN401", "A003"]

pkgs/clan-vm-manager/pyproject.toml

@@ -33,5 +33,5 @@ ignore_missing_imports = true
 [tool.ruff]
 target-version = "py311"
 line-length = 88
-select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
+lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
-ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"]
+lint.ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"]

pyproject.toml

@@ -10,5 +10,5 @@ exclude = "clan_cli.nixpkgs"
 [tool.ruff]
 line-length = 88
 target-version = "py311"
-select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
+lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
-ignore = [ "E501", "ANN101", "ANN401", "A003"]
+lint.ignore = [ "E501", "ANN101", "ANN401", "A003"]