From 8114cebaa41342c44b83914175aca4f765e923ab Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 5 Feb 2024 08:40:02 +0100 Subject: [PATCH 1/3] clan-cli git: add commit_files function --- pkgs/clan-cli/clan_cli/git.py | 60 ++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/git.py b/pkgs/clan-cli/clan_cli/git.py index 7007413a..231ad54a 100644 --- a/pkgs/clan-cli/clan_cli/git.py +++ b/pkgs/clan-cli/clan_cli/git.py @@ -7,29 +7,49 @@ from clan_cli.nix import nix_shell from .cmd import Log, run -# generic vcs agnostic commit function def commit_file( file_path: Path, repo_dir: Path, commit_message: str | None = None, +) -> None: + """Commit a file to a git repository. + + :param file_path: The path to the file to commit. + :param repo_dir: The path to the git repository. + :param commit_message: The commit message. + :raises ClanError: If the file is not in the git repository. + """ + commit_files([file_path], repo_dir, commit_message) + + +# generic vcs agnostic commit function +def commit_files( + file_paths: list[Path], + repo_dir: Path, + commit_message: str | None = None, ) -> None: # check that the file is in the git repository and exists - if not Path(file_path).resolve().is_relative_to(repo_dir.resolve()): - raise ClanError(f"File {file_path} is not in the git repository {repo_dir}") - if not file_path.exists(): - raise ClanError(f"File {file_path} does not exist") + for file_path in file_paths: + if not Path(file_path).resolve().is_relative_to(repo_dir.resolve()): + raise ClanError(f"File {file_path} is not in the git repository {repo_dir}") + if not file_path.exists(): + raise ClanError(f"File {file_path} does not exist") # generate commit message if not provided if commit_message is None: - # ensure that mentioned file path is relative to repo - commit_message = f"Add {file_path.relative_to(repo_dir)}" + commit_message = "" + for file_path in file_paths: + # ensure that mentioned file path is relative to repo + commit_message += f"Add {file_path.relative_to(repo_dir)}" # check if the repo is a git repo and commit if (repo_dir / ".git").exists(): - _commit_file_to_git(repo_dir, file_path, commit_message) + _commit_file_to_git(repo_dir, file_paths, commit_message) else: return -def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> None: +def _commit_file_to_git( + repo_dir: Path, file_paths: list[Path], commit_message: str +) -> None: """Commit a file to a git repository. :param repo_dir: The path to the git repository. @@ -37,18 +57,20 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> :param commit_message: The commit message. :raises ClanError: If the file is not in the git repository. """ - cmd = nix_shell( - ["nixpkgs#git"], - ["git", "-C", str(repo_dir), "add", str(file_path)], - ) - # add the file to the git index + for file_path in file_paths: + cmd = nix_shell( + ["nixpkgs#git"], + ["git", "-C", str(repo_dir), "add", str(file_path)], + ) + # add the file to the git index - run(cmd, log=Log.BOTH, error_msg=f"Failed to add {file_path} file to git index") + run(cmd, log=Log.BOTH, error_msg=f"Failed to add {file_path} file to git index") # check if there is a diff cmd = nix_shell( ["nixpkgs#git"], - ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code", str(file_path)], + ["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code"] + + [str(file_path) for file_path in file_paths], ) result = run(cmd, check=False, cwd=repo_dir) # if there is no diff, return @@ -65,8 +87,8 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> "commit", "-m", commit_message, - str(file_path.relative_to(repo_dir)), - ], + ] + + [str(file_path) for file_path in file_paths], ) - run(cmd, error_msg=f"Failed to commit {file_path} to git repository {repo_dir}") + run(cmd, error_msg=f"Failed to commit {file_paths} to git repository {repo_dir}") From e265537f69670f201158c25de3802b5803e58d7a Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 5 Feb 2024 08:43:26 +0100 Subject: [PATCH 2/3] clan-cli secrets: remove debug output --- pkgs/clan-cli/clan_cli/secrets/generate.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index ead23e84..e90ac1d8 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -22,13 +22,9 @@ def generate_secrets(machine: Machine) -> None: with TemporaryDirectory() as d: for service in machine.secrets_data: - print(service) tmpdir = Path(d) / service # check if all secrets exist and generate them if at least one is missing needs_regeneration = not check_secrets(machine) - for fact in machine.secrets_data[service]["facts"].values(): - if not (machine.flake / fact).exists(): - print(f"fact {fact} is missing") if needs_regeneration: env = os.environ.copy() facts_dir = tmpdir / "facts" From 815527ec2b2f4298957170a56d3e2d533f915cba Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 5 Feb 2024 10:02:39 +0100 Subject: [PATCH 3/3] clan-cli secrets: commit facts/secrets after generating them --- pkgs/clan-cli/clan_cli/secrets/generate.py | 19 ++++++++++++++++++- .../secrets/modules/password_store.py | 7 ++++--- .../clan-cli/clan_cli/secrets/modules/sops.py | 8 ++++++-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index e90ac1d8..633ff264 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -9,6 +9,7 @@ from tempfile import TemporaryDirectory from clan_cli.cmd import run from ..errors import ClanError +from ..git import commit_files from ..machines.machines import Machine from ..nix import nix_shell from .check import check_secrets @@ -26,6 +27,10 @@ def generate_secrets(machine: Machine) -> None: # check if all secrets exist and generate them if at least one is missing needs_regeneration = not check_secrets(machine) if needs_regeneration: + if not isinstance(machine.flake, Path): + msg = f"flake is not a Path: {machine.flake}" + msg += "fact/secret generation is only supported for local flakes" + env = os.environ.copy() facts_dir = tmpdir / "facts" facts_dir.mkdir(parents=True) @@ -58,6 +63,7 @@ def generate_secrets(machine: Machine) -> None: cmd, env=env, ) + files_to_commit = [] # store secrets for secret in machine.secrets_data[service]["secrets"]: secret_file = secrets_dir / secret @@ -65,7 +71,12 @@ def generate_secrets(machine: Machine) -> None: msg = f"did not generate a file for '{secret}' when running the following command:\n" msg += machine.secrets_data[service]["generator"] raise ClanError(msg) - secret_store.set(service, secret, secret_file.read_bytes()) + secret_path = secret_store.set( + service, secret, secret_file.read_bytes() + ) + if secret_path: + files_to_commit.append(secret_path) + # store facts for name, fact_path in machine.secrets_data[service]["facts"].items(): fact_file = facts_dir / name @@ -76,6 +87,12 @@ def generate_secrets(machine: Machine) -> None: fact_path = machine.flake / fact_path fact_path.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(fact_file, fact_path) + files_to_commit.append(fact_path) + commit_files( + files_to_commit, + machine.flake_dir, + f"Update facts/secrets for service {service} in machine {machine.name}", + ) print("successfully generated secrets") diff --git a/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py b/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py index 15592e89..20d4abcd 100644 --- a/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py @@ -10,7 +10,7 @@ class SecretStore: def __init__(self, machine: Machine) -> None: self.machine = machine - def set(self, service: str, name: str, value: bytes) -> None: + def set(self, _service: str, name: str, value: bytes) -> Path | None: subprocess.run( nix_shell( ["nixpkgs#pass"], @@ -19,8 +19,9 @@ class SecretStore: input=value, check=True, ) + return None # we manage the files outside of the git repo - def get(self, service: str, name: str) -> bytes: + def get(self, _service: str, name: str) -> bytes: return subprocess.run( nix_shell( ["nixpkgs#pass"], @@ -30,7 +31,7 @@ class SecretStore: stdout=subprocess.PIPE, ).stdout - def exists(self, service: str, name: str) -> bool: + def exists(self, _service: str, name: str) -> bool: password_store = os.environ.get( "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" ) diff --git a/pkgs/clan-cli/clan_cli/secrets/modules/sops.py b/pkgs/clan-cli/clan_cli/secrets/modules/sops.py index 6c434e32..cb5ccda4 100644 --- a/pkgs/clan-cli/clan_cli/secrets/modules/sops.py +++ b/pkgs/clan-cli/clan_cli/secrets/modules/sops.py @@ -28,13 +28,17 @@ class SecretStore: ) add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) - def set(self, _service: str, name: str, value: bytes) -> None: + def set(self, _service: str, name: str, value: bytes) -> Path | None: + path = ( + sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}" + ) encrypt_secret( self.machine.flake_dir, - sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}", + path, value.decode(), add_machines=[self.machine.name], ) + return path def get(self, _service: str, _name: str) -> bytes: raise NotImplementedError()