refactor clanCore.secrets -> clanCore.facts

checks/backups/flake-module.nix

@@ -74,7 +74,7 @@
             };
           };
         };
-        clanCore.secretStore = "vm";
+        clanCore.facts.secretStore = "vm";
         clanCore.clanDir = ../..;
 
         environment.systemPackages = [

checks/borgbackup/default.nix

@@ -36,7 +36,7 @@
                 };
               };
             };
-            clanCore.secretStore = "vm";
+            clanCore.facts.secretStore = "vm";
 
             clan.borgbackup.destinations.test.repo = "borg@localhost:.";
           }

nixosModules/clanCore/default.nix

@@ -1,6 +1,7 @@
 {
   imports = [
     ./backups.nix
+    ./facts
     ./manual.nix
     ./imports.nix
     ./metadata.nix
@@ -10,7 +11,6 @@
     ./outputs.nix
     ./packages.nix
     ./schema.nix
-    ./secrets
     ./vm.nix
     ./wayland-proxy-virtwl.nix
     ./zerotier

nixosModules/clanCore/facts/compat.nix

@@ -1,47 +1,44 @@
+{ config, lib, ... }:
 {
-  config,
+  imports = [
-  lib,
+    (lib.mkRemovedOptionModule [
-  pkgs,
+      "clanCore"
-  ...
+      "secretsPrefix"
-}:
+    ] "secretsPrefix was only used by the sops module and the code is now integrated in there")
-{
+    (lib.mkRenamedOptionModule
-  options.clanCore.secretStore = lib.mkOption {
+      [
-    type = lib.types.enum [
+        "clanCore"
-      "sops"
+        "secretStore"
-      "password-store"
+      ]
-      "vm"
+      [
-      "custom"
+        "clanCore"
-    ];
+        "facts"
-    default = "sops";
+        "secretStore"
-    description = ''
+      ]
-      method to store secrets
+    )
-      custom can be used to define a custom secret store.
+    (lib.mkRenamedOptionModule
-    '';
+      [
-  };
+        "clanCore"
-
+        "secretsDirectory"
-  options.clanCore.secretsDirectory = lib.mkOption {
+      ]
-    type = lib.types.path;
+      [
-    description = ''
+        "clanCore"
-      The directory where secrets are installed to. This is backend specific.
+        "facts"
-    '';
+        "secretDirectory"
-  };
+      ]
-
+    )
-  options.clanCore.secretsUploadDirectory = lib.mkOption {
+    (lib.mkRenamedOptionModule
-    type = lib.types.nullOr lib.types.path;
+      [
-    default = null;
+        "clanCore"
-    description = ''
+        "secretsUploadDirectory"
-      The directory where secrets are uploaded into, This is backend specific.
+      ]
-    '';
+      [
-  };
+        "clanCore"
-
+        "facts"
-  options.clanCore.secretsPrefix = lib.mkOption {
+        "secretUploadDirectory"
-    type = lib.types.str;
+      ]
-    default = "";
+    )
-    description = ''
+  ];
-      Prefix for secrets. This is backend specific.
-    '';
-  };
-
   options.clanCore.secrets = lib.mkOption {
     default = { };
     type = lib.types.attrsOf (
@@ -56,7 +53,7 @@
           };
           generator = lib.mkOption {
             type = lib.types.submodule (
-              { config, ... }:
+              { ... }:
               {
                 options = {
                   path = lib.mkOption {
@@ -84,28 +81,6 @@
                       The script is expected to generate all secrets and facts defined in the module.
                     '';
                   };
-                  finalScript = lib.mkOption {
-                    type = lib.types.str;
-                    readOnly = true;
-                    internal = true;
-                    default = ''
-                      set -eu -o pipefail
-
-                      export PATH="${lib.makeBinPath config.path}:${pkgs.coreutils}/bin"
-
-                      # prepare sandbox user
-                      mkdir -p /etc
-                      cp ${
-                        pkgs.runCommand "fake-etc" { } ''
-                          export PATH="${pkgs.coreutils}/bin"
-                          mkdir -p $out
-                          cp /etc/* $out/
-                        ''
-                      }/* /etc/
-
-                      ${config.script}
-                    '';
-                  };
                 };
               }
             );
@@ -134,11 +109,11 @@
                           description = ''
                             path to a secret which is generated by the generator
                           '';
-                          default = "${config'.clanCore.secretsDirectory}/${config'.clanCore.secretsPrefix}${config.name}";
+                          default = "${config'.clanCore.facts.secretDirectory}/${config.name}";
-                          defaultText = lib.literalExpression "\${config'.clanCore.secretsDirectory}/\${config'.clanCore.secretsPrefix}\${config.name}";
+                          defaultText = lib.literalExpression "\${config'.clanCore.facts.secretDirectory}/\${config.name}";
                         };
                       }
-                      // lib.optionalAttrs (config'.clanCore.secretStore == "sops") {
+                      // lib.optionalAttrs (config'.clanCore.facts.secretStore == "sops") {
                         groups = lib.mkOption {
                           type = lib.types.listOf lib.types.str;
                           default = config'.clanCore.sops.defaultGroups;
@@ -190,9 +165,16 @@
       })
     );
   };
-  imports = [
+  config = lib.mkIf (config.clanCore.secrets != { }) {
-    ./sops.nix
+    clanCore.facts.services = lib.mapAttrs' (
-    ./password-store.nix
+      name: service:
-    ./vm.nix
+      lib.warn "clanCore.secrets.${name} is deprecated, use clanCore.facts.services.${name} instead" (
-  ];
+        lib.nameValuePair name ({
+          secret = service.secrets;
+          public = service.facts;
+          generator = service.generator;
+        })
+      )
+    ) config.clanCore.secrets;
+  };
 }

nixosModules/clanCore/facts/default.nix

@@ -0,0 +1,217 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.clanCore.facts = {
+    secretStore = lib.mkOption {
+      type = lib.types.enum [
+        "sops"
+        "password-store"
+        "vm"
+        "custom"
+      ];
+      default = "sops";
+      description = ''
+        method to store secret facts
+        custom can be used to define a custom secret fact store.
+      '';
+    };
+    secretModule = lib.mkOption {
+      type = lib.types.str;
+      internal = true;
+      description = ''
+        the python import path to the secret module
+      '';
+    };
+    secretDirectory = lib.mkOption {
+      type = lib.types.path;
+      description = ''
+        The directory where secrets are installed to. This is backend specific.
+      '';
+    };
+    secretUploadDirectory = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        The directory where secrets are uploaded into, This is backend specific.
+      '';
+    };
+
+    publicStore = lib.mkOption {
+      type = lib.types.enum [
+        "in_repo"
+        "vm"
+        "custom"
+      ];
+      default = "in_repo";
+      description = ''
+        method to store public facts.
+        custom can be used to define a custom public fact store.
+      '';
+    };
+    publicModule = lib.mkOption {
+      type = lib.types.str;
+      internal = true;
+      description = ''
+        the python import path to the public module
+      '';
+    };
+    publicDirectory = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+
+    services = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (service: {
+          options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              default = service.config._module.args.name;
+              description = ''
+                Namespace of the service
+              '';
+            };
+            generator = lib.mkOption {
+              type = lib.types.submodule (
+                { config, ... }:
+                {
+                  options = {
+                    path = lib.mkOption {
+                      type = lib.types.listOf (lib.types.either lib.types.path lib.types.package);
+                      default = [ ];
+                      description = ''
+                        Extra paths to add to the PATH environment variable when running the generator.
+                      '';
+                    };
+                    prompt = lib.mkOption {
+                      type = lib.types.nullOr lib.types.str;
+                      default = null;
+                      description = ''
+                        prompt text to ask for a value.
+                        This value will be passed to the script as the environment variable $prompt_value.
+                      '';
+                    };
+                    script = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        Script to generate the secret.
+                        The script will be called with the following variables:
+                        - facts: path to a directory where facts can be stored
+                        - secrets: path to a directory where secrets can be stored
+                        The script is expected to generate all secrets and facts defined in the module.
+                      '';
+                    };
+                    finalScript = lib.mkOption {
+                      type = lib.types.str;
+                      readOnly = true;
+                      internal = true;
+                      default = ''
+                        set -eu -o pipefail
+
+                        export PATH="${lib.makeBinPath config.path}:${pkgs.coreutils}/bin"
+
+                        # prepare sandbox user
+                        mkdir -p /etc
+                        cp ${
+                          pkgs.runCommand "fake-etc" { } ''
+                            export PATH="${pkgs.coreutils}/bin"
+                            mkdir -p $out
+                            cp /etc/* $out/
+                          ''
+                        }/* /etc/
+
+                        ${config.script}
+                      '';
+                    };
+                  };
+                }
+              );
+            };
+            secret = lib.mkOption {
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (secret: {
+                  options =
+                    {
+                      name = lib.mkOption {
+                        type = lib.types.str;
+                        description = ''
+                          name of the secret
+                        '';
+                        default = secret.config._module.args.name;
+                      };
+                      path = lib.mkOption {
+                        type = lib.types.str;
+                        description = ''
+                          path to a secret which is generated by the generator
+                        '';
+                        default = "${config.clanCore.facts.secretDirectory}/${secret.config.name}";
+                      };
+                    }
+                    // lib.optionalAttrs (config.clanCore.facts.secretModule == "clan_cli.facts.secret_modules.sops") {
+                      groups = lib.mkOption {
+                        type = lib.types.listOf lib.types.str;
+                        default = config.clanCore.sops.defaultGroups;
+                        description = ''
+                          Groups to decrypt the secret for. By default we always use the user's key.
+                        '';
+                      };
+                    };
+                })
+              );
+              description = ''
+                path where the secret is located in the filesystem
+              '';
+            };
+            public = lib.mkOption {
+              default = { };
+              type = lib.types.attrsOf (
+                lib.types.submodule (fact: {
+                  options = {
+                    name = lib.mkOption {
+                      type = lib.types.str;
+                      description = ''
+                        name of the public fact
+                      '';
+                      default = fact.config._module.args.name;
+                    };
+                    path = lib.mkOption {
+                      type = lib.types.path;
+                      description = ''
+                        path to a fact which is generated by the generator
+                      '';
+                      defaultText = lib.literalExpression "\${config.clanCore.clanDir}/machines/\${config.clanCore.machineName}/facts/\${fact.config.name}";
+                      default =
+                        config.clanCore.clanDir + "/machines/${config.clanCore.machineName}/facts/${fact.config.name}";
+                    };
+                    value = lib.mkOption {
+                      defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
+                      type = lib.types.nullOr lib.types.str;
+                      default =
+                        if builtins.pathExists fact.config.path then lib.strings.fileContents fact.config.path else null;
+                    };
+                  };
+                })
+              );
+            };
+          };
+        })
+      );
+    };
+  };
+  imports = [
+    ./compat.nix
+
+    ./secret/sops.nix
+    ./secret/password-store.nix
+    ./secret/vm.nix
+
+    ./public/in_repo.nix
+    ./public/vm.nix
+  ];
+}

nixosModules/clanCore/facts/public/in_repo.nix

@@ -0,0 +1,6 @@
+{ config, lib, ... }:
+{
+  config = lib.mkIf (config.clanCore.facts.publicStore == "in_repo") {
+    clanCore.facts.publicModule = "clan_cli.facts.public_modules.in_repo";
+  };
+}

nixosModules/clanCore/facts/public/vm.nix

@@ -0,0 +1,6 @@
+{ config, lib, ... }:
+{
+  config = lib.mkIf (config.clanCore.facts.publicStore == "vm") {
+    clanCore.facts.publicModule = "clan_cli.facts.public_modules.vm";
+  };
+}

nixosModules/clanCore/facts/secret/password-store.nix

@@ -0,0 +1,15 @@
+{ config, lib, ... }:
+{
+  options.clan.password-store.targetDirectory = lib.mkOption {
+    type = lib.types.path;
+    default = "/etc/secrets";
+    description = ''
+      The directory where the password store is uploaded to.
+    '';
+  };
+  config = lib.mkIf (config.clanCore.facts.secretStore == "password-store") {
+    clanCore.facts.secretDirectory = config.clan.password-store.targetDirectory;
+    clanCore.facts.secretUploadDirectory = config.clan.password-store.targetDirectory;
+    clanCore.facts.secretModule = "clan_cli.facts.secret_modules.password_store";
+  };
+}

nixosModules/clanCore/facts/secret/sops.nix

@@ -41,10 +41,10 @@ in
       description = "The default groups to for encryption use when no groups are specified.";
     };
   };
-  config = lib.mkIf (config.clanCore.secretStore == "sops") {
+  config = lib.mkIf (config.clanCore.facts.secretStore == "sops") {
-    clanCore.secretsDirectory = "/run/secrets";
+    clanCore.facts.secretDirectory = "/run/secrets";
-    clanCore.secretsPrefix = config.clanCore.machineName + "-";
+    clanCore.facts.secretModule = "clan_cli.facts.secret_modules.sops";
-    system.clan.secretFactsModule = "clan_cli.facts.secret_modules.sops";
+    clanCore.facts.secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
     sops.secrets = builtins.mapAttrs (name: _: {
       sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
       format = "binary";
@@ -57,6 +57,5 @@ in
     sops.age.keyFile = lib.mkIf (builtins.pathExists (
       config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"
     )) (lib.mkDefault "/var/lib/sops-nix/key.txt");
-    clanCore.secretsUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
   };
 }

nixosModules/clanCore/facts/secret/vm.nix

@@ -0,0 +1,8 @@
+{ config, lib, ... }:
+{
+  config = lib.mkIf (config.clanCore.facts.secretStore == "vm") {
+    clanCore.facts.secretDirectory = "/etc/secrets";
+    clanCore.facts.secretUploadDirectory = "/etc/secrets";
+    clanCore.facts.secretModule = "clan_cli.facts.secret_modules.vm";
+  };
+}

nixosModules/clanCore/metadata.nix

@@ -23,6 +23,7 @@
     };
     clanDir = lib.mkOption {
       type = lib.types.either lib.types.path lib.types.str;
+      default = ".";
       description = ''
         the location of the flake repo, used to calculate the location of facts and secrets
       '';

nixosModules/clanCore/outputs.nix

@@ -44,33 +44,6 @@
           '';
           default = false;
         };
-        secretsUploadDirectory = lib.mkOption {
-          type = lib.types.path;
-          description = ''
-            the directory on the deployment server where secrets are uploaded
-          '';
-        };
-        publicFactsModule = lib.mkOption {
-          type = lib.types.str;
-          description = ''
-            the python import path to the facts module
-          '';
-          default = "clan_cli.facts.public_modules.in_repo";
-        };
-        secretFactsModule = lib.mkOption {
-          type = lib.types.str;
-          description = ''
-            the python import path to the secrets module
-          '';
-          default = "clan_cli.facts.secret_modules.sops";
-        };
-        secretsData = lib.mkOption {
-          type = lib.types.path;
-          description = ''
-            secret data as json for the generator
-          '';
-          default = pkgs.writers.writeJSON "secrets.json" config.clanCore.secrets;
-        };
         vm.create = lib.mkOption {
           type = lib.types.path;
           description = ''
@@ -92,10 +65,9 @@
   # optimization for faster secret generate/upload and machines update
   config = {
     system.clan.deployment.data = {
-      inherit (config.system.clan) publicFactsModule secretFactsModule secretsData;
+      inherit (config.clanCore) facts;
       inherit (config.clan.networking) targetHost buildHost;
       inherit (config.clan.deployment) requireExplicitUpdate;
-      inherit (config.clanCore) secretsUploadDirectory;
     };
     system.clan.deployment.file = pkgs.writeText "deployment.json" (
       builtins.toJSON config.system.clan.deployment.data

nixosModules/clanCore/secrets/password-store.nix

@@ -1,15 +0,0 @@
-{ config, lib, ... }:
-{
-  options.clan.password-store.targetDirectory = lib.mkOption {
-    type = lib.types.path;
-    default = "/etc/secrets";
-    description = ''
-      The directory where the password store is uploaded to.
-    '';
-  };
-  config = lib.mkIf (config.clanCore.secretStore == "password-store") {
-    clanCore.secretsDirectory = config.clan.password-store.targetDirectory;
-    clanCore.secretsUploadDirectory = config.clan.password-store.targetDirectory;
-    system.clan.secretFactsModule = "clan_cli.facts.secret_modules.password_store";
-  };
-}

nixosModules/clanCore/secrets/vm.nix

@@ -1,9 +0,0 @@
-{ config, lib, ... }:
-{
-  config = lib.mkIf (config.clanCore.secretStore == "vm") {
-    clanCore.secretsDirectory = "/etc/secrets";
-    clanCore.secretsUploadDirectory = "/etc/secrets";
-    system.clan.secretFactsModule = "clan_cli.facts.secret_modules.vm";
-    system.clan.publicFactsModule = "clan_cli.facts.public_modules.vm";
-  };
-}

pkgs/clan-cli/clan_cli/facts/check.py

@@ -18,9 +18,9 @@ def check_secrets(machine: Machine, service: None | str = None) -> bool:
     if service:
         services = [service]
     else:
-        services = list(machine.secrets_data.keys())
+        services = list(machine.facts_data.keys())
     for service in services:
-        for secret_fact in machine.secrets_data[service]["secrets"]:
+        for secret_fact in machine.facts_data[service]["secret"]:
             if isinstance(secret_fact, str):
                 secret_name = secret_fact
             else:
@@ -31,7 +31,7 @@ def check_secrets(machine: Machine, service: None | str = None) -> bool:
                 )
                 missing_secret_facts.append((service, secret_name))
 
-        for public_fact in machine.secrets_data[service]["facts"]:
+        for public_fact in machine.facts_data[service]["public"]:
             if not public_facts_store.exists(service, public_fact):
                 log.info(
                     f"Public fact '{public_fact}' for service {service} is missing."

pkgs/clan-cli/clan_cli/facts/generate.py

@@ -54,13 +54,13 @@ def generate_service_facts(
         secrets_dir.mkdir(parents=True)
         env["secrets"] = str(secrets_dir)
         # compatibility for old outputs.nix users
-        if isinstance(machine.secrets_data[service]["generator"], str):
+        if isinstance(machine.facts_data[service]["generator"], str):
-            generator = machine.secrets_data[service]["generator"]
+            generator = machine.facts_data[service]["generator"]
         else:
-            generator = machine.secrets_data[service]["generator"]["finalScript"]
+            generator = machine.facts_data[service]["generator"]["finalScript"]
-            if machine.secrets_data[service]["generator"]["prompt"]:
+            if machine.facts_data[service]["generator"]["prompt"]:
                 prompt_value = prompt(
-                    machine.secrets_data[service]["generator"]["prompt"]
+                    machine.facts_data[service]["generator"]["prompt"]
                 )
                 env["prompt_value"] = prompt_value
         # fmt: off
@@ -90,7 +90,7 @@ def generate_service_facts(
         )
         files_to_commit = []
         # store secrets
-        for secret in machine.secrets_data[service]["secrets"]:
+        for secret in machine.facts_data[service]["secret"]:
             if isinstance(secret, str):
                 # TODO: This is the old NixOS module, can be dropped everyone has updated.
                 secret_name = secret
@@ -111,11 +111,11 @@ def generate_service_facts(
                 files_to_commit.append(secret_path)
 
         # store facts
-        for name in machine.secrets_data[service]["facts"]:
+        for name in machine.facts_data[service]["public"]:
             fact_file = facts_dir / name
             if not fact_file.is_file():
                 msg = f"did not generate a file for '{name}' when running the following command:\n"
-                msg += machine.secrets_data[service]["generator"]
+                msg += machine.facts_data[service]["generator"]
                 raise ClanError(msg)
             fact_file = public_facts_store.set(service, name, fact_file.read_bytes())
             if fact_file:
@@ -147,7 +147,7 @@ def generate_facts(
 
     with TemporaryDirectory() as tmp:
         tmpdir = Path(tmp)
-        for service in machine.secrets_data:
+        for service in machine.facts_data:
             generate_service_facts(
                 machine=machine,
                 service=service,

pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py

@@ -106,8 +106,8 @@ class SecretStore(SecretStoreBase):
         return local_hash.decode() == remote_hash
 
     def upload(self, output_dir: Path) -> None:
-        for service in self.machine.secrets_data:
+        for service in self.machine.facts_data:
-            for secret in self.machine.secrets_data[service]["secrets"]:
+            for secret in self.machine.facts_data[service]["secret"]:
                 if isinstance(secret, dict):
                     secret_name = secret["name"]
                 else:

pkgs/clan-cli/clan_cli/facts/secret_modules/sops.py

@@ -14,9 +14,9 @@ class SecretStore(SecretStoreBase):
         self.machine = machine
 
         # no need to generate keys if we don't manage secrets
-        if not hasattr(self.machine, "secrets_data"):
+        if not hasattr(self.machine, "facts_data"):
             return
-        if not self.machine.secrets_data:
+        if not self.machine.facts_data:
             return
 
         if has_machine(self.machine.flake_dir, self.machine.name):

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -47,7 +47,7 @@ class Machine:
     eval_cache: dict[str, str]
     build_cache: dict[str, Path]
     _flake_path: Path | None
-    _deployment_info: None | dict[str, str]
+    _deployment_info: None | dict
     vm: QMPWrapper
 
     def __init__(
@@ -75,7 +75,7 @@ class Machine:
         self.eval_cache: dict[str, str] = {}
         self.build_cache: dict[str, Path] = {}
         self._flake_path: Path | None = None
-        self._deployment_info: None | dict[str, str] = deployment_info
+        self._deployment_info: None | dict = deployment_info
 
         state_dir = vm_state_dir(flake_url=str(self.flake), vm_name=self.data.name)
 
@@ -88,7 +88,7 @@ class Machine:
         return str(self)
 
     @property
-    def deployment_info(self) -> dict[str, str]:
+    def deployment_info(self) -> dict:
         if self._deployment_info is not None:
             return self._deployment_info
         self._deployment_info = json.loads(
@@ -113,26 +113,21 @@ class Machine:
 
     @property
     def secret_facts_module(self) -> str:
-        return self.deployment_info["secretFactsModule"]
+        return self.deployment_info["facts"]["secretModule"]
 
     @property
     def public_facts_module(self) -> str:
-        return self.deployment_info["publicFactsModule"]
+        return self.deployment_info["facts"]["publicModule"]
 
     @property
-    def secrets_data(self) -> dict[str, dict[str, Any]]:
+    def facts_data(self) -> dict[str, dict[str, Any]]:
-        if self.deployment_info["secretsData"]:
+        if self.deployment_info["facts"]["services"]:
-            try:
+            return self.deployment_info["facts"]["services"]
-                return json.loads(Path(self.deployment_info["secretsData"]).read_text())
-            except json.JSONDecodeError as e:
-                raise ClanError(
-                    f"Failed to parse secretsData for machine {self.data.name} as json"
-                ) from e
         return {}
 
     @property
     def secrets_upload_directory(self) -> str:
-        return self.deployment_info["secretsUploadDirectory"]
+        return self.deployment_info["facts"]["secretUploadDirectory"]
 
     @property
     def flake_dir(self) -> Path: