clan-cli secrets upload: secrets are populated into tmpdir
This commit is contained in:
parent
7ca9c49163
commit
89b7ffce6c
|
|
@ -9,6 +9,7 @@
|
|||
export PATH="${lib.makeBinPath [
|
||||
pkgs.gitMinimal
|
||||
pkgs.nix
|
||||
pkgs.rsync # needed to have rsync installed on the dummy ssh server
|
||||
]}"
|
||||
ROOT=$(git rev-parse --show-toplevel)
|
||||
cd "$ROOT/pkgs/clan-cli"
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@
|
|||
system.clan.deployment.text = builtins.toJSON {
|
||||
inherit (config.system.clan) uploadSecrets generateSecrets;
|
||||
inherit (config.clan.networking) deploymentAddress;
|
||||
inherit (config.clanCore) secretsUploadDirectory;
|
||||
};
|
||||
system.clan.deployment.file = pkgs.writeText "deployment.json" config.system.clan.deployment.text;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
options.clanCore.secretStore = lib.mkOption {
|
||||
type = lib.types.enum [ "sops" "password-store" "custom" ];
|
||||
|
|
@ -17,6 +17,13 @@
|
|||
'';
|
||||
};
|
||||
|
||||
options.clanCore.secretsUploadDirectory = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
description = ''
|
||||
The directory where secrets are uploaded into, This is backend specific.
|
||||
'';
|
||||
};
|
||||
|
||||
options.clanCore.secretsPrefix = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "";
|
||||
|
|
@ -106,10 +113,6 @@
|
|||
};
|
||||
}));
|
||||
};
|
||||
config.system.build.generateUploadSecrets = pkgs.writeScript "generate_upload_secrets" ''
|
||||
${config.system.clan.generateSecrets}
|
||||
${config.system.clan.uploadSecrets}
|
||||
'';
|
||||
imports = [
|
||||
./sops.nix
|
||||
./password-store.nix
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ in
|
|||
};
|
||||
config = lib.mkIf (config.clanCore.secretStore == "password-store") {
|
||||
clanCore.secretsDirectory = config.clan.password-store.targetDirectory;
|
||||
clanCore.secretsUploadDirectory = config.clan.password-store.targetDirectory;
|
||||
system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
|
||||
#!/bin/sh
|
||||
set -efu
|
||||
|
|
@ -33,7 +34,7 @@ in
|
|||
trap "rm -rf $facts" EXIT
|
||||
secrets=$(mktemp -d)
|
||||
trap "rm -rf $secrets" EXIT
|
||||
${v.generator}
|
||||
( ${v.generator} )
|
||||
|
||||
${lib.concatMapStrings (fact: ''
|
||||
mkdir -p "$(dirname ${fact.path})"
|
||||
|
|
@ -50,8 +51,6 @@ in
|
|||
#!/bin/sh
|
||||
set -efu
|
||||
|
||||
target=$1
|
||||
|
||||
umask 0077
|
||||
|
||||
PATH=${lib.makeBinPath [
|
||||
|
|
@ -71,7 +70,7 @@ in
|
|||
sort |
|
||||
xargs -r -n 1 git -C ${passwordstoreDir} log -1 --format=%H
|
||||
)
|
||||
remote_pass_info=$(ssh "$target" -- ${lib.escapeShellArg ''
|
||||
remote_pass_info=$(ssh ${config.clan.networking.deploymentAddress} -- ${lib.escapeShellArg ''
|
||||
cat ${config.clan.password-store.targetDirectory}/.pass_info || :
|
||||
''})
|
||||
|
||||
|
|
@ -81,12 +80,6 @@ in
|
|||
fi
|
||||
fi
|
||||
|
||||
tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
|
||||
trap cleanup EXIT
|
||||
cleanup() {
|
||||
rm -fR "$tmp_dir"
|
||||
}
|
||||
|
||||
find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type f -follow ! -name .gpg-id |
|
||||
while read -r gpg_path; do
|
||||
|
||||
|
|
@ -99,7 +92,7 @@ in
|
|||
fi
|
||||
)
|
||||
pass_name=$rel_name
|
||||
tmp_path=$tmp_dir/$(basename $rel_name)
|
||||
tmp_path="$SECRETS_DIR"/$(basename $rel_name)
|
||||
|
||||
mkdir -p "$(dirname "$tmp_path")"
|
||||
pass show "$pass_name" > "$tmp_path"
|
||||
|
|
@ -109,10 +102,8 @@ in
|
|||
done
|
||||
|
||||
if test -n "''${local_pass_info-}"; then
|
||||
echo "$local_pass_info" > "$tmp_dir"/.pass_info
|
||||
echo "$local_pass_info" > "$SECRETS_DIR"/.pass_info
|
||||
fi
|
||||
|
||||
rsync --mkpath --delete -a "$tmp_dir"/ "$target":${config.clan.password-store.targetDirectory}/
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,12 +37,10 @@ in
|
|||
uploadSecrets = pkgs.writeScript "upload-secrets" ''
|
||||
#!${pkgs.python3}/bin/python
|
||||
import json
|
||||
import sys
|
||||
from clan_cli.secrets.sops_generate import upload_age_key_from_nix
|
||||
# the second toJSON is needed to escape the string for the python
|
||||
deployment_address = sys.argv[1]
|
||||
args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; age_key_file = config.sops.age.keyFile; })})
|
||||
upload_age_key_from_nix(**args, deployment_address=deployment_address)
|
||||
args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; })})
|
||||
upload_age_key_from_nix(**args)
|
||||
'';
|
||||
};
|
||||
sops.secrets = builtins.mapAttrs
|
||||
|
|
@ -56,5 +54,6 @@ in
|
|||
|
||||
sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"))
|
||||
(lib.mkDefault "/var/lib/sops-nix/key.txt");
|
||||
clanCore.secretsUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ def deploy_nixos(hosts: HostGroup, clan_dir: Path) -> None:
|
|||
h.meta["uploadSecrets"],
|
||||
clan_dir,
|
||||
target=target,
|
||||
target_directory=h.meta["targetDirectory"],
|
||||
target_directory=h.meta["secretsUploadDirectory"],
|
||||
)
|
||||
|
||||
target_host = h.meta.get("target_host")
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
|
|
@ -10,7 +11,6 @@ from clan_cli.nix import nix_shell
|
|||
|
||||
from ..dirs import get_clan_flake_toplevel
|
||||
from ..errors import ClanError
|
||||
from ..ssh import parse_deployment_address
|
||||
from .folders import sops_secrets_folder
|
||||
from .machines import add_machine, has_machine
|
||||
from .secrets import decrypt_secret, encrypt_secret, has_secret
|
||||
|
|
@ -102,27 +102,12 @@ def generate_secrets_from_nix(
|
|||
|
||||
# this is called by the sops.nix clan core module
|
||||
def upload_age_key_from_nix(
|
||||
machine_name: str, deployment_address: str, age_key_file: str
|
||||
machine_name: str,
|
||||
) -> None:
|
||||
secret_name = f"{machine_name}-age.key"
|
||||
if not has_secret(secret_name): # skip uploading the secret, not managed by us
|
||||
return
|
||||
secret = decrypt_secret(secret_name)
|
||||
|
||||
h = parse_deployment_address(machine_name, deployment_address)
|
||||
path = Path(age_key_file)
|
||||
|
||||
proc = h.run(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
'mkdir -p "$0" && echo -n "$1" > "$2"',
|
||||
str(path.parent),
|
||||
secret,
|
||||
age_key_file,
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(f"failed to upload age key to {deployment_address}")
|
||||
sys.exit(1)
|
||||
secrets_dir = Path(os.environ["SECRETS_DIR"])
|
||||
(secrets_dir / "key.txt").write_text(secret)
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@ import os
|
|||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from ..dirs import get_clan_flake_toplevel, module_root
|
||||
from ..errors import ClanError
|
||||
from ..nix import nix_build, nix_config, nix_eval
|
||||
from ..nix import nix_build, nix_config, nix_shell
|
||||
from ..ssh import parse_deployment_address
|
||||
|
||||
|
||||
def build_upload_script(machine: str, clan_dir: Path) -> str:
|
||||
|
|
@ -28,13 +30,13 @@ def build_upload_script(machine: str, clan_dir: Path) -> str:
|
|||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def get_deployment_address(machine: str, clan_dir: Path) -> str:
|
||||
def get_deployment_info(machine: str, clan_dir: Path) -> dict:
|
||||
config = nix_config()
|
||||
system = config["system"]
|
||||
|
||||
cmd = nix_eval(
|
||||
cmd = nix_build(
|
||||
[
|
||||
f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.clan.networking.deploymentAddress'
|
||||
f'{clan_dir}#clanInternals.machines."{system}"."{machine}".config.system.clan.deployment.file'
|
||||
]
|
||||
)
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
|
||||
|
|
@ -43,29 +45,60 @@ def get_deployment_address(machine: str, clan_dir: Path) -> str:
|
|||
f"failed to get deploymentAddress:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
|
||||
)
|
||||
|
||||
return json.loads(proc.stdout.strip())
|
||||
return json.load(open(proc.stdout.strip()))
|
||||
|
||||
|
||||
def run_upload_secrets(flake_attr: str, clan_dir: Path, target: str) -> None:
|
||||
def run_upload_secrets(
|
||||
flake_attr: str, clan_dir: Path, target: str, target_directory: str
|
||||
) -> None:
|
||||
env = os.environ.copy()
|
||||
env["CLAN_DIR"] = str(clan_dir)
|
||||
env["PYTHONPATH"] = str(module_root().parent) # TODO do this in the clanCore module
|
||||
print(f"uploading secrets... {flake_attr}")
|
||||
proc = subprocess.run(
|
||||
[flake_attr, target],
|
||||
env=env,
|
||||
)
|
||||
with TemporaryDirectory() as tempdir_:
|
||||
tempdir = Path(tempdir_)
|
||||
env["SECRETS_DIR"] = str(tempdir)
|
||||
proc = subprocess.run(
|
||||
[flake_attr],
|
||||
env=env,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise ClanError("failed to upload secrets")
|
||||
else:
|
||||
print("successfully uploaded secrets")
|
||||
if proc.returncode != 0:
|
||||
raise ClanError("failed to upload secrets")
|
||||
|
||||
h = parse_deployment_address(flake_attr, target)
|
||||
ssh_cmd = h.ssh_cmd()
|
||||
subprocess.run(
|
||||
nix_shell(
|
||||
["rsync"],
|
||||
[
|
||||
"rsync",
|
||||
"-e",
|
||||
" ".join(["ssh"] + ssh_cmd[2:]),
|
||||
"-az",
|
||||
"--delete",
|
||||
f"{str(tempdir)}/",
|
||||
f"{h.user}@{h.host}:{target_directory}/",
|
||||
],
|
||||
),
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def upload_secrets(machine: str) -> None:
|
||||
clan_dir = get_clan_flake_toplevel()
|
||||
target = get_deployment_address(machine, clan_dir)
|
||||
run_upload_secrets(build_upload_script(machine, clan_dir), clan_dir, target)
|
||||
deployment_info = get_deployment_info(machine, clan_dir)
|
||||
address = deployment_info.get("deploymentAddress", "")
|
||||
secrets_upload_directory = deployment_info.get("secretsUploadDirectory", "")
|
||||
run_upload_secrets(
|
||||
build_upload_script(machine, clan_dir),
|
||||
clan_dir,
|
||||
address,
|
||||
secrets_upload_directory,
|
||||
)
|
||||
|
||||
|
||||
def upload_command(args: argparse.Namespace) -> None:
|
||||
|
|
|
|||
|
|
@ -373,7 +373,7 @@ class Host:
|
|||
Command to run locally for the host
|
||||
|
||||
@cmd the commmand to run
|
||||
@stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE
|
||||
@stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocess.PIPE
|
||||
@stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE
|
||||
@extra_env environment variables to override whe running the command
|
||||
@cwd current working directory to run the process in
|
||||
|
|
@ -447,6 +447,33 @@ class Host:
|
|||
f"$ {displayed_cmd}", extra=dict(command_prefix=self.command_prefix)
|
||||
)
|
||||
|
||||
bash_cmd = export_cmd
|
||||
bash_args = []
|
||||
if isinstance(cmd, list):
|
||||
bash_cmd += 'exec "$@"'
|
||||
bash_args += cmd
|
||||
else:
|
||||
bash_cmd += cmd
|
||||
# FIXME we assume bash to be present here? Should be documented...
|
||||
ssh_cmd = self.ssh_cmd(verbose_ssh=verbose_ssh) + [
|
||||
"--",
|
||||
f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}",
|
||||
]
|
||||
return self._run(
|
||||
ssh_cmd,
|
||||
displayed_cmd,
|
||||
shell=False,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cwd=cwd,
|
||||
check=check,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
def ssh_cmd(
|
||||
self,
|
||||
verbose_ssh: bool = False,
|
||||
) -> List:
|
||||
if self.user is not None:
|
||||
ssh_target = f"{self.user}@{self.host}"
|
||||
else:
|
||||
|
|
@ -469,32 +496,7 @@ class Host:
|
|||
if verbose_ssh or self.verbose_ssh:
|
||||
ssh_opts.extend(["-v"])
|
||||
|
||||
bash_cmd = export_cmd
|
||||
bash_args = []
|
||||
if isinstance(cmd, list):
|
||||
bash_cmd += 'exec "$@"'
|
||||
bash_args += cmd
|
||||
else:
|
||||
bash_cmd += cmd
|
||||
# FIXME we assume bash to be present here? Should be documented...
|
||||
ssh_cmd = (
|
||||
["ssh", ssh_target]
|
||||
+ ssh_opts
|
||||
+ [
|
||||
"--",
|
||||
f"{sudo}bash -c {quote(bash_cmd)} -- {' '.join(map(quote, bash_args))}",
|
||||
]
|
||||
)
|
||||
return self._run(
|
||||
ssh_cmd,
|
||||
displayed_cmd,
|
||||
shell=False,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
cwd=cwd,
|
||||
check=check,
|
||||
timeout=timeout,
|
||||
)
|
||||
return ["ssh", ssh_target] + ssh_opts
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ def create_flake(
|
|||
if clan_core_flake:
|
||||
line = line.replace("__CLAN_CORE__", str(clan_core_flake))
|
||||
line = line.replace("__CLAN_SOPS_KEY_PATH__", sops_key)
|
||||
line = line.replace("__CLAN_SOPS_KEY_DIR__", str(flake))
|
||||
print(line, end="")
|
||||
monkeypatch.chdir(flake)
|
||||
monkeypatch.setenv("HOME", str(home))
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
|
||||
system.stateVersion = lib.version;
|
||||
sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
|
||||
clanCore.secretsUploadDirectory = "__CLAN_SOPS_KEY_DIR__";
|
||||
|
||||
clan.networking.zerotier.controller.enable = true;
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_upload_secret(
|
||||
def test_generate_secret(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
test_flake_with_core: Path,
|
||||
age_keys: list["KeyPair"],
|
||||
|
|
|
|||
|
|
@ -36,6 +36,6 @@ def test_secrets_upload(
|
|||
cli.run(["secrets", "upload", "vm1"])
|
||||
|
||||
# the flake defines this path as the location where the sops key should be installed
|
||||
sops_key = test_flake_with_core.joinpath("sops.key")
|
||||
sops_key = test_flake_with_core.joinpath("key.txt")
|
||||
assert sops_key.exists()
|
||||
assert sops_key.read_text() == age_keys[0].privkey
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user