refactor secrets & facts -> secret_facts & public_facts

nixosModules/clanCore/outputs.nix

@@ -50,18 +50,19 @@
             the directory on the deployment server where secrets are uploaded
           '';
         };
-        factsModule = lib.mkOption {
+        publicFactsModule = lib.mkOption {
           type = lib.types.str;
           description = ''
             the python import path to the facts module
           '';
-          default = "clan_cli.facts.modules.in_repo";
+          default = "clan_cli.facts.public_modules.in_repo";
         };
-        secretsModule = lib.mkOption {
+        secretFactsModule = lib.mkOption {
           type = lib.types.str;
           description = ''
             the python import path to the secrets module
           '';
+          default = "clan_cli.facts.secret_modules.sops";
         };
         secretsData = lib.mkOption {
           type = lib.types.path;
@@ -91,7 +92,7 @@
   # optimization for faster secret generate/upload and machines update
   config = {
     system.clan.deployment.data = {
-      inherit (config.system.clan) factsModule secretsModule secretsData;
+      inherit (config.system.clan) publicFactsModule secretFactsModule secretsData;
       inherit (config.clan.networking) targetHost buildHost;
       inherit (config.clan.deployment) requireExplicitUpdate;
       inherit (config.clanCore) secretsUploadDirectory;

nixosModules/clanCore/secrets/password-store.nix

@@ -10,6 +10,6 @@
   config = lib.mkIf (config.clanCore.secretStore == "password-store") {
     clanCore.secretsDirectory = config.clan.password-store.targetDirectory;
     clanCore.secretsUploadDirectory = config.clan.password-store.targetDirectory;
-    system.clan.secretsModule = "clan_cli.secrets.modules.password_store";
+    system.clan.secretFactsModule = "clan_cli.facts.secret_modules.password_store";
   };
 }

nixosModules/clanCore/secrets/sops.nix

@@ -44,7 +44,7 @@ in
   config = lib.mkIf (config.clanCore.secretStore == "sops") {
     clanCore.secretsDirectory = "/run/secrets";
     clanCore.secretsPrefix = config.clanCore.machineName + "-";
-    system.clan.secretsModule = "clan_cli.secrets.modules.sops";
+    system.clan.secretFactsModule = "clan_cli.facts.secret_modules.sops";
     sops.secrets = builtins.mapAttrs (name: _: {
       sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
       format = "binary";

nixosModules/clanCore/secrets/vm.nix

@@ -3,7 +3,7 @@
   config = lib.mkIf (config.clanCore.secretStore == "vm") {
     clanCore.secretsDirectory = "/etc/secrets";
     clanCore.secretsUploadDirectory = "/etc/secrets";
-    system.clan.secretsModule = "clan_cli.secrets.modules.vm";
+    system.clan.secretFactsModule = "clan_cli.facts.secret_modules.vm";
-    system.clan.factsModule = "clan_cli.facts.modules.vm";
+    system.clan.publicFactsModule = "clan_cli.facts.public_modules.vm";
   };
 }

pkgs/clan-cli/clan_cli/facts/__init__.py

@@ -2,7 +2,9 @@
 import argparse
 
 from .check import register_check_parser
+from .generate import register_generate_parser
 from .list import register_list_parser
+from .upload import register_upload_parser
 
 
 # takes a (sub)parser and configures it
@@ -19,3 +21,11 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
 
     list_parser = subparser.add_parser("list", help="list all facts")
     register_list_parser(list_parser)
+
+    parser_generate = subparser.add_parser(
+        "generate", help="generate secrets for machines if they don't exist yet"
+    )
+    register_generate_parser(parser_generate)
+
+    parser_upload = subparser.add_parser("upload", help="upload secrets for machines")
+    register_upload_parser(parser_upload)

pkgs/clan-cli/clan_cli/facts/check.py

@@ -7,32 +7,55 @@ from ..machines.machines import Machine
 log = logging.getLogger(__name__)
 
 
-def check_facts(machine: Machine) -> bool:
+def check_secrets(machine: Machine, service: None | str = None) -> bool:
-    facts_module = importlib.import_module(machine.facts_module)
+    secret_facts_module = importlib.import_module(machine.secret_facts_module)
-    fact_store = facts_module.FactStore(machine=machine)
+    secret_facts_store = secret_facts_module.SecretStore(machine=machine)
+    public_facts_module = importlib.import_module(machine.public_facts_module)
+    public_facts_store = public_facts_module.FactStore(machine=machine)
 
-    existing_facts = fact_store.get_all()
+    missing_secret_facts = []
-    missing_facts = []
+    missing_public_facts = []
-    for service in machine.secrets_data:
+    if service:
-        for fact in machine.secrets_data[service]["facts"]:
+        services = [service]
-            if fact not in existing_facts.get(service, {}):
+    else:
-                log.info(f"Fact {fact} for service {service} is missing")
+        services = list(machine.secrets_data.keys())
-                missing_facts.append((service, fact))
+    for service in services:
+        for secret_fact in machine.secrets_data[service]["secrets"]:
+            if isinstance(secret_fact, str):
+                secret_name = secret_fact
+            else:
+                secret_name = secret_fact["name"]
+            if not secret_facts_store.exists(service, secret_name):
+                log.info(f"Secret fact {secret_fact} for service {service} is missing")
+                missing_secret_facts.append((service, secret_name))
 
-    if missing_facts:
+        for public_fact in machine.secrets_data[service]["facts"]:
+            if not public_facts_store.exists(service, public_fact):
+                log.info(f"public Fact {public_fact} for service {service} is missing")
+                missing_public_facts.append((service, public_fact))
+
+    log.debug(f"missing_secret_facts: {missing_secret_facts}")
+    log.debug(f"missing_public_facts: {missing_public_facts}")
+    if missing_secret_facts or missing_public_facts:
         return False
     return True
 
 
 def check_command(args: argparse.Namespace) -> None:
-    machine = Machine(name=args.machine, flake=args.flake)
+    machine = Machine(
-    if check_facts(machine):
+        name=args.machine,
-        print("All facts are present")
+        flake=args.flake,
+    )
+    check_secrets(machine, service=args.service)
 
 
 def register_check_parser(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "machine",
-        help="The machine to check facts for",
+        help="The machine to check secrets for",
+    )
+    parser.add_argument(
+        "--service",
+        help="the service to check",
     )
     parser.set_defaults(func=check_command)

pkgs/clan-cli/clan_cli/facts/generate.py

@@ -10,12 +10,12 @@ from tempfile import TemporaryDirectory
 from clan_cli.cmd import run
 
 from ..errors import ClanError
-from ..facts.modules import FactStoreBase
 from ..git import commit_files
 from ..machines.machines import Machine
 from ..nix import nix_shell
 from .check import check_secrets
-from .modules import SecretStoreBase
+from .public_modules import FactStoreBase
+from .secret_modules import SecretStoreBase
 
 log = logging.getLogger(__name__)
 
@@ -29,11 +29,11 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
     return proc.stdout
 
 
-def generate_service_secrets(
+def generate_service_facts(
     machine: Machine,
     service: str,
-    secret_store: SecretStoreBase,
+    secret_facts_store: SecretStoreBase,
-    fact_store: FactStoreBase,
+    public_facts_store: FactStoreBase,
     tmpdir: Path,
     prompt: Callable[[str], str],
 ) -> None:
@@ -104,7 +104,7 @@ def generate_service_secrets(
                 msg = f"did not generate a file for '{secret_name}' when running the following command:\n"
                 msg += generator
                 raise ClanError(msg)
-            secret_path = secret_store.set(
+            secret_path = secret_facts_store.set(
                 service, secret_name, secret_file.read_bytes(), groups
             )
             if secret_path:
@@ -117,7 +117,7 @@ def generate_service_secrets(
                 msg = f"did not generate a file for '{name}' when running the following command:\n"
                 msg += machine.secrets_data[service]["generator"]
                 raise ClanError(msg)
-            fact_file = fact_store.set(service, name, fact_file.read_bytes())
+            fact_file = public_facts_store.set(service, name, fact_file.read_bytes())
             if fact_file:
                 files_to_commit.append(fact_file)
         commit_files(
@@ -127,15 +127,15 @@ def generate_service_secrets(
         )
 
 
-def generate_secrets(
+def generate_facts(
     machine: Machine,
     prompt: None | Callable[[str], str] = None,
 ) -> None:
-    secrets_module = importlib.import_module(machine.secrets_module)
+    secret_facts_module = importlib.import_module(machine.secret_facts_module)
-    secret_store = secrets_module.SecretStore(machine=machine)
+    secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
-    facts_module = importlib.import_module(machine.facts_module)
+    public_facts_module = importlib.import_module(machine.public_facts_module)
-    fact_store = facts_module.FactStore(machine=machine)
+    public_facts_store = public_facts_module.FactStore(machine=machine)
 
     if prompt is None:
 
@@ -148,11 +148,11 @@ def generate_secrets(
     with TemporaryDirectory() as tmp:
         tmpdir = Path(tmp)
         for service in machine.secrets_data:
-            generate_service_secrets(
+            generate_service_facts(
                 machine=machine,
                 service=service,
-                secret_store=secret_store,
+                secret_facts_store=secret_facts_store,
-                fact_store=fact_store,
+                public_facts_store=public_facts_store,
                 tmpdir=tmpdir,
                 prompt=prompt,
             )
@@ -162,12 +162,12 @@ def generate_secrets(
 
 def generate_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake=args.flake)
-    generate_secrets(machine)
+    generate_facts(machine)
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "machine",
-        help="The machine to generate secrets for",
+        help="The machine to generate facts for",
     )
     parser.set_defaults(func=generate_command)

pkgs/clan-cli/clan_cli/facts/list.py

@@ -8,9 +8,10 @@ from ..machines.machines import Machine
 log = logging.getLogger(__name__)
 
 
+# TODO get also secret facts
 def get_all_facts(machine: Machine) -> dict:
-    facts_module = importlib.import_module(machine.facts_module)
+    public_facts_module = importlib.import_module(machine.public_facts_module)
-    fact_store = facts_module.FactStore(machine=machine)
+    public_facts_store = public_facts_module.FactStore(machine=machine)
 
     # for service in machine.secrets_data:
     #     facts[service] = {}
@@ -20,7 +21,7 @@ def get_all_facts(machine: Machine) -> dict:
     #             facts[service][fact] = fact_content.decode()
     #         else:
     #             log.error(f"Fact {fact} for service {service} is missing")
-    return fact_store.get_all()
+    return public_facts_store.get_all()
 
 
 def get_command(args: argparse.Namespace) -> None:

pkgs/clan-cli/clan_cli/facts/upload.py

@@ -12,14 +12,14 @@ log = logging.getLogger(__name__)
 
 
 def upload_secrets(machine: Machine) -> None:
-    secrets_module = importlib.import_module(machine.secrets_module)
+    secret_facts_module = importlib.import_module(machine.secret_facts_module)
-    secret_store = secrets_module.SecretStore(machine=machine)
+    secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
-    if secret_store.update_check():
+    if secret_facts_store.update_check():
         log.info("Secrets already up to date")
         return
     with TemporaryDirectory() as tempdir:
-        secret_store.upload(Path(tempdir))
+        secret_facts_store.upload(Path(tempdir))
         host = machine.target_host
 
         ssh_cmd = host.ssh_cmd()

pkgs/clan-cli/clan_cli/flash.py

@@ -12,9 +12,9 @@ from typing import Any
 
 from .cmd import Log, run
 from .errors import ClanError
+from .facts.secret_modules import SecretStoreBase
 from .machines.machines import Machine
 from .nix import nix_shell
-from .secrets.modules import SecretStoreBase
 
 log = logging.getLogger(__name__)
 
@@ -22,8 +22,10 @@ log = logging.getLogger(__name__)
 def flash_machine(
     machine: Machine, disks: dict[str, str], dry_run: bool, debug: bool
 ) -> None:
-    secrets_module = importlib.import_module(machine.secrets_module)
+    secret_facts_module = importlib.import_module(machine.secret_facts_module)
-    secret_store: SecretStoreBase = secrets_module.SecretStore(machine=machine)
+    secret_facts_store: SecretStoreBase = secret_facts_module.SecretStore(
+        machine=machine
+    )
     with TemporaryDirectory() as tmpdir_:
         tmpdir = Path(tmpdir_)
         upload_dir = machine.secrets_upload_directory
@@ -34,7 +36,7 @@ def flash_machine(
             local_dir = tmpdir / upload_dir
 
         local_dir.mkdir(parents=True)
-        secret_store.upload(local_dir)
+        secret_facts_store.upload(local_dir)
         disko_install = []
 
         if os.geteuid() != 0:

pkgs/clan-cli/clan_cli/machines/install.py

@@ -6,9 +6,9 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..cmd import Log, run
+from ..facts.generate import generate_facts
 from ..machines.machines import Machine
 from ..nix import nix_shell
-from ..secrets.generate import generate_secrets
 
 log = logging.getLogger(__name__)
 
@@ -16,15 +16,15 @@ log = logging.getLogger(__name__)
 def install_nixos(
     machine: Machine, kexec: str | None = None, debug: bool = False
 ) -> None:
-    secrets_module = importlib.import_module(machine.secrets_module)
+    secret_facts_module = importlib.import_module(machine.secret_facts_module)
     log.info(f"installing {machine.name}")
-    secret_store = secrets_module.SecretStore(machine=machine)
+    secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
     h = machine.target_host
     target_host = f"{h.user or 'root'}@{h.host}"
     log.info(f"target host: {target_host}")
 
-    generate_secrets(machine)
+    generate_facts(machine)
 
     with TemporaryDirectory() as tmpdir_:
         tmpdir = Path(tmpdir_)
@@ -34,7 +34,7 @@ def install_nixos(
             upload_dir_ = upload_dir_[1:]
         upload_dir = tmpdir / upload_dir_
         upload_dir.mkdir(parents=True)
-        secret_store.upload(upload_dir)
+        secret_facts_store.upload(upload_dir)
 
         cmd = [
             "nixos-anywhere",

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -112,12 +112,12 @@ class Machine:
         self.deployment_info["targetHost"] = value
 
     @property
-    def secrets_module(self) -> str:
+    def secret_facts_module(self) -> str:
-        return self.deployment_info["secretsModule"]
+        return self.deployment_info["secretFactsModule"]
 
     @property
-    def facts_module(self) -> str:
+    def public_facts_module(self) -> str:
-        return self.deployment_info["factsModule"]
+        return self.deployment_info["publicFactsModule"]
 
     @property
     def secrets_data(self) -> dict[str, dict[str, Any]]:

pkgs/clan-cli/clan_cli/machines/update.py

@@ -9,10 +9,10 @@ from pathlib import Path
 
 from ..cmd import run
 from ..errors import ClanError
+from ..facts.generate import generate_facts
+from ..facts.upload import upload_secrets
 from ..machines.machines import Machine
 from ..nix import nix_build, nix_command, nix_config, nix_metadata
-from ..secrets.generate import generate_secrets
-from ..secrets.upload import upload_secrets
 from ..ssh import Host, HostGroup, HostKeyCheck, parse_deployment_address
 
 log = logging.getLogger(__name__)
@@ -107,7 +107,7 @@ def deploy_nixos(hosts: HostGroup) -> None:
 
         machine: Machine = h.meta["machine"]
 
-        generate_secrets(machine)
+        generate_facts(machine)
         upload_secrets(machine)
 
         extra_args = h.meta.get("extra_args", [])

pkgs/clan-cli/clan_cli/secrets/__init__.py

@@ -1,14 +1,11 @@
 # !/usr/bin/env python3
 import argparse
 
-from .check import register_check_parser
-from .generate import register_generate_parser
 from .groups import register_groups_parser
 from .import_sops import register_import_sops_parser
 from .key import register_key_parser
 from .machines import register_machines_parser
 from .secrets import register_secrets_parser
-from .upload import register_upload_parser
 from .users import register_users_parser
 
 
@@ -33,17 +30,6 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
     import_sops_parser = subparser.add_parser("import-sops", help="import a sops file")
     register_import_sops_parser(import_sops_parser)
 
-    check_parser = subparser.add_parser("check", help="check if secrets are up to date")
-    register_check_parser(check_parser)
-
-    parser_generate = subparser.add_parser(
-        "generate", help="generate secrets for machines if they don't exist yet"
-    )
-    register_generate_parser(parser_generate)
-
-    parser_upload = subparser.add_parser("upload", help="upload secrets for machines")
-    register_upload_parser(parser_upload)
-
     parser_key = subparser.add_parser("key", help="create and show age keys")
     register_key_parser(parser_key)
 

pkgs/clan-cli/clan_cli/secrets/check.py

@@ -1,61 +0,0 @@
-import argparse
-import importlib
-import logging
-
-from ..machines.machines import Machine
-
-log = logging.getLogger(__name__)
-
-
-def check_secrets(machine: Machine, service: None | str = None) -> bool:
-    secrets_module = importlib.import_module(machine.secrets_module)
-    secret_store = secrets_module.SecretStore(machine=machine)
-    facts_module = importlib.import_module(machine.facts_module)
-    fact_store = facts_module.FactStore(machine=machine)
-
-    missing_secrets = []
-    missing_facts = []
-    if service:
-        services = [service]
-    else:
-        services = list(machine.secrets_data.keys())
-    for service in services:
-        for secret in machine.secrets_data[service]["secrets"]:
-            if isinstance(secret, str):
-                secret_name = secret
-            else:
-                secret_name = secret["name"]
-            if not secret_store.exists(service, secret_name):
-                log.info(f"Secret {secret} for service {service} is missing")
-                missing_secrets.append((service, secret_name))
-
-        for fact in machine.secrets_data[service]["facts"]:
-            if not fact_store.exists(service, fact):
-                log.info(f"Fact {fact} for service {service} is missing")
-                missing_facts.append((service, fact))
-
-    log.debug(f"missing_secrets: {missing_secrets}")
-    log.debug(f"missing_facts: {missing_facts}")
-    if missing_secrets or missing_facts:
-        return False
-    return True
-
-
-def check_command(args: argparse.Namespace) -> None:
-    machine = Machine(
-        name=args.machine,
-        flake=args.flake,
-    )
-    check_secrets(machine, service=args.service)
-
-
-def register_check_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
-        "machine",
-        help="The machine to check secrets for",
-    )
-    parser.add_argument(
-        "--service",
-        help="the service to check",
-    )
-    parser.set_defaults(func=check_command)

pkgs/clan-cli/clan_cli/vms/run.py

@@ -10,9 +10,9 @@ from tempfile import TemporaryDirectory
 from ..cmd import Log, run
 from ..dirs import module_root, user_cache_dir, vm_state_dir
 from ..errors import ClanError
+from ..facts.generate import generate_facts
 from ..machines.machines import Machine
 from ..nix import nix_shell
-from ..secrets.generate import generate_secrets
 from .inspect import VmConfig, inspect_vm
 from .qemu import qemu_command
 from .virtiofsd import start_virtiofsd
@@ -42,13 +42,13 @@ def build_vm(
     # TODO pass prompt here for the GTK gui
     secrets_dir = get_secrets(machine, tmpdir)
 
-    facts_module = importlib.import_module(machine.facts_module)
+    public_facts_module = importlib.import_module(machine.public_facts_module)
-    fact_store = facts_module.FactStore(machine=machine)
+    public_facts_store = public_facts_module.FactStore(machine=machine)
-    facts = fact_store.get_all()
+    public_facts = public_facts_store.get_all()
 
     nixos_config_file = machine.build_nix(
         "config.system.clan.vm.create",
-        extra_config=facts_to_nixos_config(facts),
+        extra_config=facts_to_nixos_config(public_facts),
         nix_options=nix_options,
     )
     try:
@@ -66,12 +66,12 @@ def get_secrets(
     secrets_dir = tmpdir / "secrets"
     secrets_dir.mkdir(parents=True, exist_ok=True)
 
-    secrets_module = importlib.import_module(machine.secrets_module)
+    secret_facts_module = importlib.import_module(machine.secret_facts_module)
-    secret_store = secrets_module.SecretStore(machine=machine)
+    secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
-    generate_secrets(machine)
+    generate_facts(machine)
 
-    secret_store.upload(secrets_dir)
+    secret_facts_store.upload(secrets_dir)
     return secrets_dir
 
 

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -44,7 +44,7 @@ def test_generate_secret(
             "user1",
         ]
     )
-    cmd = ["--flake", str(test_flake_with_core.path), "secrets", "generate", "vm1"]
+    cmd = ["--flake", str(test_flake_with_core.path), "facts", "generate", "vm1"]
     cli.run(cmd)
     has_secret(test_flake_with_core.path, "vm1-age.key")
     has_secret(test_flake_with_core.path, "vm1-zerotier-identity-secret")
@@ -60,7 +60,7 @@ def test_generate_secret(
     secret1_mtime = identity_secret.lstat().st_mtime_ns
 
     # test idempotency
-    cli.run(["secrets", "generate", "vm1"])
+    cli.run(["facts", "generate", "vm1"])
     assert age_key.lstat().st_mtime_ns == age_key_mtime
     assert identity_secret.lstat().st_mtime_ns == secret1_mtime
 
@@ -68,7 +68,7 @@ def test_generate_secret(
         secrets_folder / "vm1-zerotier-identity-secret" / "machines" / "vm1"
     ).exists()
 
-    cli.run(["secrets", "generate", "vm2"])
+    cli.run(["facts", "generate", "vm2"])
     assert has_secret(test_flake_with_core.path, "vm2-age.key")
     assert has_secret(test_flake_with_core.path, "vm2-zerotier-identity-secret")
     ip = machine_get_fact(test_flake_with_core.path, "vm1", "zerotier-ip")

pkgs/clan-cli/tests/test_secrets_password_store.py

@@ -43,7 +43,7 @@ def test_upload_secret(
     subprocess.run(
         nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True
     )
-    cli.run(["secrets", "generate", "vm1"])
+    cli.run(["facts", "generate", "vm1"])
     network_id = machine_get_fact(
         test_flake_with_core_and_pass.path, "vm1", "zerotier-network-id"
     )
@@ -54,7 +54,7 @@ def test_upload_secret(
     secret1_mtime = identity_secret.lstat().st_mtime_ns
 
     # test idempotency
-    cli.run(["secrets", "generate", "vm1"])
+    cli.run(["facts", "generate", "vm1"])
     assert identity_secret.lstat().st_mtime_ns == secret1_mtime
 
     flake = test_flake_with_core_and_pass.path.joinpath("flake.nix")
@@ -62,7 +62,7 @@ def test_upload_secret(
     addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
     new_text = flake.read_text().replace("__CLAN_TARGET_ADDRESS__", addr)
     flake.write_text(new_text)
-    cli.run(["secrets", "upload", "vm1"])
+    cli.run(["facts", "upload", "vm1"])
     zerotier_identity_secret = (
         test_flake_with_core_and_pass.path / "secrets" / "zerotier-identity-secret"
     )

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -55,7 +55,7 @@ def test_secrets_upload(
     new_text = flake.read_text().replace("__CLAN_TARGET_ADDRESS__", addr)
 
     flake.write_text(new_text)
-    cli.run(["--flake", str(test_flake_with_core.path), "secrets", "upload", "vm1"])
+    cli.run(["--flake", str(test_flake_with_core.path), "facts", "upload", "vm1"])
 
     # the flake defines this path as the location where the sops key should be installed
     sops_key = test_flake_with_core.path.joinpath("key.txt")