Merge pull request 'init diskLayouts' (#302) from lassulus-HEAD into main
Reviewed-on: #302
This commit is contained in:
commit
4f3f7a4feb
4
.envrc
4
.envrc
@ -1 +1,5 @@
|
||||
if ! has nix_direnv_version || ! nix_direnv_version 2.3.0; then
|
||||
source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc" "sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8="
|
||||
fi
|
||||
|
||||
use flake
|
||||
|
@ -6,32 +6,20 @@
|
||||
#!${pkgs.bash}/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
|
||||
trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
|
||||
|
||||
export PATH="${lib.makeBinPath [
|
||||
pkgs.coreutils
|
||||
pkgs.gitMinimal
|
||||
pkgs.nix
|
||||
self'.packages.clan-cli.checkPython
|
||||
]}"
|
||||
|
||||
export CLAN_CORE=$TMPDIR/CLAN_CORE
|
||||
cp -r ${self} $CLAN_CORE
|
||||
chmod +w -R $CLAN_CORE
|
||||
|
||||
cp -r ${self'.packages.clan-cli.src} $TMPDIR/src
|
||||
chmod +w -R $TMPDIR/src
|
||||
cd $TMPDIR/src
|
||||
|
||||
python -m pytest -m "impure" -s ./tests --workers "" "$@"
|
||||
ROOT=$(git rev-parse --show-toplevel)
|
||||
cd "$ROOT/pkgs/clan-cli"
|
||||
nix develop "$ROOT#clan-cli" -c bash -c 'TMPDIR=/tmp python -m pytest -m impure -s ./tests'
|
||||
'';
|
||||
check-clan-template = pkgs.writeShellScriptBin "check-clan-template" ''
|
||||
#!${pkgs.bash}/bin/bash
|
||||
set -euo pipefail
|
||||
set -euox pipefail
|
||||
|
||||
export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
|
||||
trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
|
||||
export CLANTMP=$(${pkgs.coreutils}/bin/mktemp -d)
|
||||
trap "${pkgs.coreutils}/bin/chmod -R +w '$CLANTMP'; ${pkgs.coreutils}/bin/rm -rf '$CLANTMP'" EXIT
|
||||
|
||||
export PATH="${lib.makeBinPath [
|
||||
pkgs.coreutils
|
||||
@ -44,7 +32,7 @@
|
||||
self'.packages.clan-cli
|
||||
]}"
|
||||
|
||||
cd $TMPDIR
|
||||
cd $CLANTMP
|
||||
|
||||
echo initialize new clan
|
||||
nix flake init -t ${self}#new-clan
|
||||
|
44
clanModules/diskLayouts/singleDiskExt4.nix
Normal file
44
clanModules/diskLayouts/singleDiskExt4.nix
Normal file
@ -0,0 +1,44 @@
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
options.clan.diskLayouts.singleDiskExt4 = {
|
||||
device = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
example = "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345";
|
||||
};
|
||||
};
|
||||
config.disko.devices = {
|
||||
disk = {
|
||||
main = {
|
||||
type = "disk";
|
||||
device = config.clan.diskLayouts.singleDiskExt4.device;
|
||||
content = {
|
||||
type = "gpt";
|
||||
partitions = {
|
||||
boot = {
|
||||
size = "1M";
|
||||
type = "EF02"; # for grub MBR
|
||||
};
|
||||
ESP = {
|
||||
size = "512M";
|
||||
type = "EF00";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "vfat";
|
||||
mountpoint = "/boot";
|
||||
};
|
||||
};
|
||||
root = {
|
||||
size = "100%";
|
||||
content = {
|
||||
type = "filesystem";
|
||||
format = "ext4";
|
||||
mountpoint = "/";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
12
clanModules/flake-module.nix
Normal file
12
clanModules/flake-module.nix
Normal file
@ -0,0 +1,12 @@
|
||||
{ self, lib, ... }: {
|
||||
flake.clanModules = {
|
||||
diskLayouts = lib.mapAttrs'
|
||||
(name: _: lib.nameValuePair (lib.removeSuffix ".nix" name) {
|
||||
imports = [
|
||||
self.inputs.disko.nixosModules.disko
|
||||
./diskLayouts/${name}
|
||||
];
|
||||
})
|
||||
(builtins.readDir ./diskLayouts);
|
||||
};
|
||||
}
|
@ -24,12 +24,12 @@
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
];
|
||||
flake.clanModules = { };
|
||||
imports = [
|
||||
./checks/flake-module.nix
|
||||
./devShell.nix
|
||||
./formatter.nix
|
||||
./templates/flake-module.nix
|
||||
./clanModules/flake-module.nix
|
||||
|
||||
./pkgs/flake-module.nix
|
||||
|
||||
|
@ -21,7 +21,7 @@
|
||||
|
||||
treefmt.programs.mypy.enable = true;
|
||||
treefmt.programs.mypy.directories = {
|
||||
"pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
|
||||
"pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.pytestDependencies;
|
||||
};
|
||||
|
||||
treefmt.settings.formatter.nix = {
|
||||
|
@ -11,22 +11,44 @@ let
|
||||
(builtins.fromJSON
|
||||
(builtins.readFile (directory + /machines/${machineName}/settings.json)));
|
||||
|
||||
nixosConfiguration = { system ? "x86_64-linux", name }: nixpkgs.lib.nixosSystem {
|
||||
modules = [
|
||||
self.nixosModules.clanCore
|
||||
(machineSettings name)
|
||||
(machines.${name} or { })
|
||||
{
|
||||
clanCore.machineName = name;
|
||||
clanCore.clanDir = directory;
|
||||
# TODO: remove this once we have a hardware-config mechanism
|
||||
nixpkgs.hostPlatform = lib.mkDefault system;
|
||||
}
|
||||
];
|
||||
inherit specialArgs;
|
||||
};
|
||||
|
||||
nixosConfigurations = lib.mapAttrs
|
||||
(name: _:
|
||||
nixpkgs.lib.nixosSystem {
|
||||
modules = [
|
||||
self.nixosModules.clanCore
|
||||
(machineSettings name)
|
||||
(machines.${name} or { })
|
||||
{
|
||||
clanCore.machineName = name;
|
||||
clanCore.clanDir = directory;
|
||||
# TODO: remove this once we have a hardware-config mechanism
|
||||
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
|
||||
}
|
||||
];
|
||||
inherit specialArgs;
|
||||
})
|
||||
nixosConfiguration { inherit name; })
|
||||
(machinesDirs // machines);
|
||||
|
||||
systems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
"riscv64-linux"
|
||||
"x86_64-darwin"
|
||||
"aarch64-darwin"
|
||||
];
|
||||
|
||||
clanInternals = {
|
||||
machines = lib.mapAttrs
|
||||
(name: _:
|
||||
(builtins.listToAttrs (map
|
||||
(system:
|
||||
lib.nameValuePair system (nixosConfiguration { inherit name system; })
|
||||
)
|
||||
systems))
|
||||
)
|
||||
(machinesDirs // machines);
|
||||
};
|
||||
in
|
||||
nixosConfigurations
|
||||
{ inherit nixosConfigurations clanInternals; }
|
||||
|
@ -18,14 +18,17 @@
|
||||
type = lib.types.str;
|
||||
default = secret.config._module.args.name;
|
||||
description = ''
|
||||
namespace of the secret
|
||||
Namespace of the secret
|
||||
'';
|
||||
};
|
||||
generator = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
script to generate the secret.
|
||||
can be set to null. then the user has to provide the secret via the clan cli
|
||||
Script to generate the secret.
|
||||
The script will be called with the following variables:
|
||||
- facts: path to a directory where facts can be stored
|
||||
- secrets: path to a directory where secrets can be stored
|
||||
The script is expected to generate all secrets and facts defined in the module.
|
||||
'';
|
||||
};
|
||||
secrets = lib.mkOption {
|
||||
@ -63,7 +66,11 @@
|
||||
};
|
||||
value = lib.mkOption {
|
||||
defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
|
||||
default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}";
|
||||
default =
|
||||
if builtins.pathExists "${config.clanCore.clanDir}/${fact.config.path}" then
|
||||
builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}"
|
||||
else
|
||||
"";
|
||||
};
|
||||
};
|
||||
}));
|
||||
|
@ -22,52 +22,23 @@ let
|
||||
in
|
||||
{
|
||||
config = lib.mkIf (config.clanCore.secretStore == "sops") {
|
||||
system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
|
||||
#!/bin/sh
|
||||
set -efu
|
||||
|
||||
test -d "$CLAN_DIR"
|
||||
|
||||
PATH=$PATH:${lib.makeBinPath [
|
||||
config.clanCore.clanPkgs.clan-cli
|
||||
]}
|
||||
|
||||
# initialize secret store
|
||||
if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
|
||||
INITTMP=$(mktemp -d)
|
||||
trap 'rm -rf "$INITTMP"' EXIT
|
||||
${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
|
||||
PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
|
||||
clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
|
||||
tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
|
||||
) fi
|
||||
|
||||
${lib.foldlAttrs (acc: n: v: ''
|
||||
${acc}
|
||||
# ${n}
|
||||
# if any of the secrets are missing, we regenerate all connected facts/secrets
|
||||
(if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
|
||||
|
||||
facts=$(mktemp -d)
|
||||
trap "rm -rf $facts" EXIT
|
||||
secrets=$(mktemp -d)
|
||||
trap "rm -rf $secrets" EXIT
|
||||
${v.generator}
|
||||
|
||||
${lib.concatMapStrings (fact: ''
|
||||
mkdir -p "$(dirname ${fact.path})"
|
||||
cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
|
||||
'') (lib.attrValues v.facts)}
|
||||
|
||||
${lib.concatMapStrings (secret: ''
|
||||
cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
|
||||
'') (lib.attrValues v.secrets)}
|
||||
fi)
|
||||
'') "" config.clanCore.secrets}
|
||||
'';
|
||||
system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
|
||||
echo upload is not needed for sops secret store, since the secrets are part of the flake
|
||||
'';
|
||||
system.clan = {
|
||||
generateSecrets = pkgs.writeScript "generate-secrets" ''
|
||||
#!${pkgs.python3}/bin/python
|
||||
import json
|
||||
from clan_cli.secrets.sops_generate import generate_secrets_from_nix
|
||||
args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; secret_submodules = config.clanCore.secrets; })})
|
||||
generate_secrets_from_nix(**args)
|
||||
'';
|
||||
uploadSecrets = pkgs.writeScript "upload-secrets" ''
|
||||
#!${pkgs.python3}/bin/python
|
||||
import json
|
||||
from clan_cli.secrets.sops_generate import upload_age_key_from_nix
|
||||
# the second toJSON is needed to escape the string for the python
|
||||
args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })})
|
||||
upload_age_key_from_nix(**args)
|
||||
'';
|
||||
};
|
||||
sops.secrets = builtins.mapAttrs
|
||||
(name: _: {
|
||||
sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
|
||||
@ -76,5 +47,8 @@ in
|
||||
secrets;
|
||||
# To get proper error messages about missing secrets we need a dummy secret file that is always present
|
||||
sops.defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")));
|
||||
|
||||
sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"))
|
||||
(lib.mkDefault "/var/lib/sops-nix/key.txt");
|
||||
};
|
||||
}
|
||||
|
9
pkgs/clan-cli/clan_cli/machines/facts.py
Normal file
9
pkgs/clan-cli/clan_cli/machines/facts.py
Normal file
@ -0,0 +1,9 @@
|
||||
from .folders import machine_folder
|
||||
|
||||
|
||||
def machine_has_fact(machine: str, fact: str) -> bool:
|
||||
return (machine_folder(machine) / "facts" / fact).exists()
|
||||
|
||||
|
||||
def machine_get_fact(machine: str, fact: str) -> str:
|
||||
return (machine_folder(machine) / "facts" / fact).read_text()
|
@ -2,13 +2,12 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
from ..dirs import get_clan_flake_toplevel
|
||||
from ..nix import nix_command, nix_eval
|
||||
from ..secrets.generate import generate_secrets
|
||||
from ..secrets.upload import upload_secrets
|
||||
from ..ssh import Host, HostGroup, HostKeyCheck
|
||||
from ..ssh import Host, HostGroup, HostKeyCheck, parse_deployment_address
|
||||
|
||||
|
||||
def deploy_nixos(hosts: HostGroup) -> None:
|
||||
@ -78,11 +77,12 @@ def deploy_nixos(hosts: HostGroup) -> None:
|
||||
# FIXME: we want some kind of inventory here.
|
||||
def update(args: argparse.Namespace) -> None:
|
||||
clan_dir = get_clan_flake_toplevel().as_posix()
|
||||
host = json.loads(
|
||||
machine = args.machine
|
||||
address = json.loads(
|
||||
subprocess.run(
|
||||
nix_eval(
|
||||
[
|
||||
f'{clan_dir}#nixosConfigurations."{args.machine}".config.clan.networking.deploymentAddress'
|
||||
f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
|
||||
]
|
||||
),
|
||||
stdout=subprocess.PIPE,
|
||||
@ -90,31 +90,9 @@ def update(args: argparse.Namespace) -> None:
|
||||
text=True,
|
||||
).stdout
|
||||
)
|
||||
parts = host.split("@")
|
||||
user: Optional[str] = None
|
||||
if len(parts) > 1:
|
||||
user = parts[0]
|
||||
hostname = parts[1]
|
||||
else:
|
||||
hostname = parts[0]
|
||||
maybe_port = hostname.split(":")
|
||||
port = None
|
||||
if len(maybe_port) > 1:
|
||||
hostname = maybe_port[0]
|
||||
port = int(maybe_port[1])
|
||||
print(f"deploying {host}")
|
||||
deploy_nixos(
|
||||
HostGroup(
|
||||
[
|
||||
Host(
|
||||
host=hostname,
|
||||
port=port,
|
||||
user=user,
|
||||
meta=dict(flake_attr=args.machine),
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
host = parse_deployment_address(machine, address)
|
||||
print(f"deploying {machine}")
|
||||
deploy_nixos(HostGroup([host]))
|
||||
|
||||
|
||||
def register_update_parser(parser: argparse.ArgumentParser) -> None:
|
||||
|
@ -1,5 +1,8 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs
|
||||
|
||||
@ -25,6 +28,16 @@ def nix_build(
|
||||
)
|
||||
|
||||
|
||||
def nix_config() -> dict[str, Any]:
|
||||
cmd = nix_command(["show-config", "--json"])
|
||||
proc = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
|
||||
data = json.loads(proc.stdout)
|
||||
config = {}
|
||||
for key, value in data.items():
|
||||
config[key] = value["value"]
|
||||
return config
|
||||
|
||||
|
||||
def nix_eval(flags: list[str]) -> list[str]:
|
||||
default_flags = nix_command(
|
||||
[
|
||||
|
@ -1,31 +1,32 @@
|
||||
import argparse
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from clan_cli.errors import ClanError
|
||||
|
||||
from ..dirs import get_clan_flake_toplevel
|
||||
from ..nix import nix_build
|
||||
from ..dirs import get_clan_flake_toplevel, module_root
|
||||
from ..nix import nix_build, nix_config
|
||||
|
||||
|
||||
def generate_secrets(machine: str) -> None:
|
||||
clan_dir = get_clan_flake_toplevel().as_posix().strip()
|
||||
env = os.environ.copy()
|
||||
env["CLAN_DIR"] = clan_dir
|
||||
env["PYTHONPATH"] = str(module_root().parent) # TODO do this in the clanCore module
|
||||
config = nix_config()
|
||||
system = config["system"]
|
||||
|
||||
proc = subprocess.run(
|
||||
nix_build(
|
||||
[
|
||||
f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets'
|
||||
]
|
||||
),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cmd = nix_build(
|
||||
[
|
||||
f'path:{clan_dir}#clanInternals.machines."{machine}".{system}.config.system.clan.generateSecrets'
|
||||
]
|
||||
)
|
||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
|
||||
if proc.returncode != 0:
|
||||
print(proc.stderr, file=sys.stderr)
|
||||
raise ClanError(f"failed to generate secrets:\n{proc.stderr}")
|
||||
raise ClanError(
|
||||
f"failed to generate secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
|
||||
)
|
||||
|
||||
secret_generator_script = proc.stdout.strip()
|
||||
print(secret_generator_script)
|
||||
|
@ -9,8 +9,8 @@ def generate_key() -> str:
|
||||
path = default_sops_key_path()
|
||||
if path.exists():
|
||||
raise ClanError(f"Key already exists at {path}")
|
||||
generate_private_key(path)
|
||||
pub_key = get_public_key(path.read_text())
|
||||
priv_key, pub_key = generate_private_key()
|
||||
path.write_text(priv_key)
|
||||
return pub_key
|
||||
|
||||
|
||||
|
@ -19,11 +19,15 @@ def get_machine(name: str) -> str:
|
||||
return read_key(sops_machines_folder() / name)
|
||||
|
||||
|
||||
def has_machine(name: str) -> bool:
|
||||
return (sops_machines_folder() / name / "key.json").exists()
|
||||
|
||||
|
||||
def list_machines() -> list[str]:
|
||||
path = sops_machines_folder()
|
||||
|
||||
def validate(name: str) -> bool:
|
||||
return validate_hostname(name) and (path / name / "key.json").exists()
|
||||
return validate_hostname(name) and has_machine(name)
|
||||
|
||||
return list_objects(path, validate)
|
||||
|
||||
|
@ -171,14 +171,15 @@ def disallow_member(group_folder: Path, name: str) -> None:
|
||||
)
|
||||
|
||||
|
||||
def has_secret(secret: str) -> bool:
|
||||
return (sops_secrets_folder() / secret / "secret").exists()
|
||||
|
||||
|
||||
def list_secrets() -> list[str]:
|
||||
path = sops_secrets_folder()
|
||||
|
||||
def validate(name: str) -> bool:
|
||||
return (
|
||||
VALID_SECRET_NAME.match(name) is not None
|
||||
and (path / name / "secret").exists()
|
||||
)
|
||||
return VALID_SECRET_NAME.match(name) is not None and has_secret(name)
|
||||
|
||||
return list_objects(path, validate)
|
||||
|
||||
|
@ -30,10 +30,25 @@ def get_public_key(privkey: str) -> str:
|
||||
return res.stdout.strip()
|
||||
|
||||
|
||||
def generate_private_key(path: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cmd = nix_shell(["age"], ["age-keygen", "-o", str(path)])
|
||||
subprocess.run(cmd, check=True)
|
||||
def generate_private_key() -> tuple[str, str]:
|
||||
cmd = nix_shell(["age"], ["age-keygen"])
|
||||
try:
|
||||
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True)
|
||||
res = proc.stdout.strip()
|
||||
pubkey = None
|
||||
private_key = None
|
||||
for line in res.splitlines():
|
||||
if line.startswith("# public key:"):
|
||||
pubkey = line.split(":")[1].strip()
|
||||
if not line.startswith("#"):
|
||||
private_key = line
|
||||
if not pubkey:
|
||||
raise ClanError("Could not find public key in age-keygen output")
|
||||
if not private_key:
|
||||
raise ClanError("Could not find private key in age-keygen output")
|
||||
return private_key, pubkey
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise ClanError("Failed to generate private sops key") from e
|
||||
|
||||
|
||||
def get_user_name(user: str) -> str:
|
||||
|
124
pkgs/clan-cli/clan_cli/secrets/sops_generate.py
Normal file
124
pkgs/clan-cli/clan_cli/secrets/sops_generate.py
Normal file
@ -0,0 +1,124 @@
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
|
||||
from ..dirs import get_clan_flake_toplevel
|
||||
from ..errors import ClanError
|
||||
from ..ssh import parse_deployment_address
|
||||
from .folders import sops_secrets_folder
|
||||
from .machines import add_machine, has_machine
|
||||
from .secrets import decrypt_secret, encrypt_secret, has_secret
|
||||
from .sops import generate_private_key
|
||||
|
||||
|
||||
def generate_host_key(machine_name: str) -> None:
|
||||
if has_machine(machine_name):
|
||||
return
|
||||
priv_key, pub_key = generate_private_key()
|
||||
encrypt_secret(sops_secrets_folder() / f"{machine_name}-age.key", priv_key)
|
||||
add_machine(machine_name, pub_key, False)
|
||||
|
||||
|
||||
def generate_secrets_group(
|
||||
secret_group: str, machine_name: str, tempdir: Path, secret_options: dict[str, Any]
|
||||
) -> None:
|
||||
clan_dir = get_clan_flake_toplevel()
|
||||
secrets = secret_options["secrets"]
|
||||
needs_regeneration = any(
|
||||
not has_secret(f"{machine_name}-{secret['name']}")
|
||||
for secret in secrets.values()
|
||||
)
|
||||
generator = secret_options["generator"]
|
||||
subdir = tempdir / secret_group
|
||||
if needs_regeneration:
|
||||
facts_dir = subdir / "facts"
|
||||
facts_dir.mkdir(parents=True)
|
||||
secrets_dir = subdir / "secrets"
|
||||
secrets_dir.mkdir(parents=True)
|
||||
|
||||
text = f"""\
|
||||
set -euo pipefail
|
||||
facts={shlex.quote(str(facts_dir))}
|
||||
secrets={shlex.quote(str(secrets_dir))}
|
||||
{generator}
|
||||
"""
|
||||
try:
|
||||
subprocess.run(["bash", "-c", text], check=True)
|
||||
except subprocess.CalledProcessError:
|
||||
msg = "failed to the following command:\n"
|
||||
msg += text
|
||||
raise ClanError(msg)
|
||||
for secret in secrets.values():
|
||||
secret_file = secrets_dir / secret["name"]
|
||||
if not secret_file.is_file():
|
||||
msg = f"did not generate a file for '{secret['name']}' when running the following command:\n"
|
||||
msg += text
|
||||
raise ClanError(msg)
|
||||
encrypt_secret(
|
||||
sops_secrets_folder() / f"{machine_name}-{secret['name']}",
|
||||
secret_file.read_text(),
|
||||
)
|
||||
for fact in secret_options["facts"].values():
|
||||
fact_file = facts_dir / fact["name"]
|
||||
if not fact_file.is_file():
|
||||
msg = f"did not generate a file for '{fact['name']}' when running the following command:\n"
|
||||
msg += text
|
||||
raise ClanError(msg)
|
||||
fact_path = clan_dir.joinpath(fact["path"])
|
||||
fact_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(fact_file, fact_path)
|
||||
|
||||
|
||||
# this is called by the sops.nix clan core module
|
||||
def generate_secrets_from_nix(
|
||||
machine_name: str,
|
||||
secret_submodules: dict[str, Any],
|
||||
) -> None:
|
||||
generate_host_key(machine_name)
|
||||
errors = {}
|
||||
with TemporaryDirectory() as d:
|
||||
# if any of the secrets are missing, we regenerate all connected facts/secrets
|
||||
for secret_group, secret_options in secret_submodules.items():
|
||||
try:
|
||||
generate_secrets_group(
|
||||
secret_group, machine_name, Path(d), secret_options
|
||||
)
|
||||
except ClanError as e:
|
||||
errors[secret_group] = e
|
||||
for secret_group, error in errors.items():
|
||||
print(f"failed to generate secrets for {machine_name}/{secret_group}:")
|
||||
print(error, file=sys.stderr)
|
||||
if len(errors) > 0:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# this is called by the sops.nix clan core module
|
||||
def upload_age_key_from_nix(
|
||||
machine_name: str, deployment_address: str, age_key_file: str
|
||||
) -> None:
|
||||
secret_name = f"{machine_name}-age.key"
|
||||
if not has_secret(secret_name): # skip uploading the secret, not managed by us
|
||||
return
|
||||
secret = decrypt_secret(secret_name)
|
||||
|
||||
h = parse_deployment_address(machine_name, deployment_address)
|
||||
path = Path(age_key_file)
|
||||
|
||||
proc = h.run(
|
||||
[
|
||||
"bash",
|
||||
"-c",
|
||||
'mkdir -p "$0" && echo -n "$1" > "$2"',
|
||||
str(path.parent),
|
||||
secret,
|
||||
age_key_file,
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(f"failed to upload age key to {deployment_address}")
|
||||
sys.exit(1)
|
@ -1,31 +1,36 @@
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
from clan_cli.errors import ClanError
|
||||
|
||||
from ..dirs import get_clan_flake_toplevel
|
||||
from ..nix import nix_build, nix_eval
|
||||
from ..dirs import get_clan_flake_toplevel, module_root
|
||||
from ..errors import ClanError
|
||||
from ..nix import nix_build, nix_config, nix_eval
|
||||
|
||||
|
||||
def upload_secrets(machine: str) -> None:
|
||||
clan_dir = get_clan_flake_toplevel().as_posix()
|
||||
config = nix_config()
|
||||
system = config["system"]
|
||||
|
||||
proc = subprocess.run(
|
||||
nix_build(
|
||||
[
|
||||
f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets'
|
||||
f'{clan_dir}#clanInternals.machines."{machine}".{system}.config.system.clan.uploadSecrets'
|
||||
]
|
||||
),
|
||||
stdout=subprocess.PIPE,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["PYTHONPATH"] = str(module_root().parent) # TODO do this in the clanCore module
|
||||
host = json.loads(
|
||||
subprocess.run(
|
||||
nix_eval(
|
||||
[
|
||||
f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
|
||||
f'{clan_dir}#clanInternals.machines."{machine}".{system}.config.clan.networking.deploymentAddress'
|
||||
]
|
||||
),
|
||||
stdout=subprocess.PIPE,
|
||||
@ -40,6 +45,7 @@ def upload_secrets(machine: str) -> None:
|
||||
secret_upload_script,
|
||||
host,
|
||||
],
|
||||
env=env,
|
||||
)
|
||||
|
||||
if secret_upload.returncode != 0:
|
||||
|
@ -756,6 +756,36 @@ class HostGroup:
|
||||
return HostGroup(list(filter(pred, self.hosts)))
|
||||
|
||||
|
||||
def parse_deployment_address(machine_name: str, host: str) -> Host:
|
||||
parts = host.split("@")
|
||||
user: Optional[str] = None
|
||||
if len(parts) > 1:
|
||||
user = parts[0]
|
||||
hostname = parts[1]
|
||||
else:
|
||||
hostname = parts[0]
|
||||
maybe_options = hostname.split("?")
|
||||
options: Dict[str, str] = {}
|
||||
if len(maybe_options) > 1:
|
||||
hostname = maybe_options[0]
|
||||
for option in maybe_options[1].split("&"):
|
||||
k, v = option.split("=")
|
||||
options[k] = v
|
||||
maybe_port = hostname.split(":")
|
||||
port = None
|
||||
if len(maybe_port) > 1:
|
||||
hostname = maybe_port[0]
|
||||
port = int(maybe_port[1])
|
||||
return Host(
|
||||
hostname,
|
||||
user=user,
|
||||
port=port,
|
||||
command_prefix=machine_name,
|
||||
meta=dict(flake_attr=machine_name),
|
||||
ssh_options=options,
|
||||
)
|
||||
|
||||
|
||||
@overload
|
||||
def run(
|
||||
cmd: Union[List[str], str],
|
||||
|
@ -1,4 +1,5 @@
|
||||
{ age
|
||||
, lib
|
||||
, argcomplete
|
||||
, fastapi
|
||||
, uvicorn
|
||||
@ -20,7 +21,12 @@
|
||||
, rsync
|
||||
, pkgs
|
||||
, ui-assets
|
||||
, lib
|
||||
, bash
|
||||
, sshpass
|
||||
, zbar
|
||||
, tor
|
||||
, git
|
||||
, ipdb
|
||||
}:
|
||||
let
|
||||
|
||||
@ -30,16 +36,36 @@ let
|
||||
uvicorn # optional dependencies: if not enabled, webui subcommand will not work
|
||||
];
|
||||
|
||||
testDependencies = [
|
||||
pytestDependencies = runtimeDependencies ++ dependencies ++ [
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-subprocess
|
||||
pytest-parallel
|
||||
openssh
|
||||
git
|
||||
stdenv.cc
|
||||
ipdb # used for debugging
|
||||
];
|
||||
|
||||
checkPython = python3.withPackages (_ps: dependencies ++ testDependencies);
|
||||
# Optional dependencies for clan cli, we re-expose them here to make sure they all build.
|
||||
runtimeDependencies = [
|
||||
bash
|
||||
nix
|
||||
zerotierone
|
||||
bubblewrap
|
||||
openssh
|
||||
sshpass
|
||||
zbar
|
||||
tor
|
||||
age
|
||||
rsync
|
||||
sops
|
||||
git
|
||||
];
|
||||
|
||||
runtimeDependenciesAsSet = builtins.listToAttrs (builtins.map (p: lib.nameValuePair (lib.getName p.name) p) runtimeDependencies);
|
||||
|
||||
checkPython = python3.withPackages (_ps: pytestDependencies);
|
||||
|
||||
# - vendor the jsonschema nix lib (copy instead of symlink).
|
||||
source = runCommand "clan-cli-source" { } ''
|
||||
@ -73,6 +99,7 @@ let
|
||||
--experimental-features 'nix-command flakes' \
|
||||
--override-input nixpkgs ${pkgs.path}
|
||||
'';
|
||||
|
||||
in
|
||||
python3.pkgs.buildPythonPackage {
|
||||
name = "clan-cli";
|
||||
@ -85,25 +112,24 @@ python3.pkgs.buildPythonPackage {
|
||||
];
|
||||
propagatedBuildInputs = dependencies;
|
||||
|
||||
passthru.tests.clan-pytest = runCommand "clan-pytest"
|
||||
{
|
||||
nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ];
|
||||
} ''
|
||||
cp -r ${source} ./src
|
||||
chmod +w -R ./src
|
||||
cd ./src
|
||||
# also re-expose dependencies so we test them in CI
|
||||
passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "package-${n}") runtimeDependenciesAsSet) // {
|
||||
clan-pytest = runCommand "clan-pytest" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } ''
|
||||
cp -r ${source} ./src
|
||||
chmod +w -R ./src
|
||||
cd ./src
|
||||
|
||||
# git is needed for test_git.py
|
||||
export PATH="${lib.makeBinPath [pkgs.git]}:$PATH"
|
||||
|
||||
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
|
||||
${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
|
||||
touch $out
|
||||
'';
|
||||
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
|
||||
${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
passthru.clan-openapi = runCommand "clan-openapi" { } ''
|
||||
cp -r ${source} ./src
|
||||
chmod +w -R ./src
|
||||
cd ./src
|
||||
export PATH=${checkPython}/bin:$PATH
|
||||
|
||||
${checkPython}/bin/python ./bin/gen-openapi --out $out/openapi.json --app-dir . clan_cli.webui.app:app
|
||||
touch $out
|
||||
'';
|
||||
@ -113,9 +139,10 @@ python3.pkgs.buildPythonPackage {
|
||||
passthru.devDependencies = [
|
||||
setuptools
|
||||
wheel
|
||||
] ++ testDependencies;
|
||||
] ++ pytestDependencies;
|
||||
|
||||
passthru.testDependencies = dependencies ++ testDependencies;
|
||||
passthru.pytestDependencies = pytestDependencies;
|
||||
passthru.runtimeDependencies = runtimeDependencies;
|
||||
|
||||
postInstall = ''
|
||||
cp -r ${nixpkgs} $out/${python3.sitePackages}/clan_cli/nixpkgs
|
||||
|
@ -1,3 +1,4 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
perSystem = { self', pkgs, ... }: {
|
||||
devShells.clan-cli = pkgs.callPackage ./shell.nix {
|
||||
@ -10,26 +11,13 @@
|
||||
clan-openapi = self'.packages.clan-cli.clan-openapi;
|
||||
default = self'.packages.clan-cli;
|
||||
|
||||
## Optional dependencies for clan cli, we re-expose them here to make sure they all build.
|
||||
inherit (pkgs)
|
||||
age
|
||||
bash
|
||||
bubblewrap
|
||||
git
|
||||
openssh
|
||||
rsync
|
||||
sops
|
||||
sshpass
|
||||
tor
|
||||
zbar
|
||||
;
|
||||
# Override license so that we can build zerotierone without
|
||||
# having to re-import nixpkgs.
|
||||
zerotierone = pkgs.zerotierone.overrideAttrs (_old: { meta = { }; });
|
||||
## End optional dependencies
|
||||
};
|
||||
|
||||
checks = self'.packages.clan-cli.tests;
|
||||
checks = lib.mkDefault self'.packages.clan-cli.tests;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -1,26 +1,16 @@
|
||||
{ nix-unit, clan-cli, ui-assets, python3, system, ruff, mkShell, writeScriptBin }:
|
||||
{ nix-unit, clan-cli, ui-assets, system, mkShell, writeScriptBin, openssh }:
|
||||
let
|
||||
pythonWithDeps = python3.withPackages (
|
||||
ps:
|
||||
clan-cli.propagatedBuildInputs
|
||||
++ clan-cli.devDependencies
|
||||
++ [
|
||||
ps.pip
|
||||
ps.ipdb
|
||||
]
|
||||
);
|
||||
checkScript = writeScriptBin "check" ''
|
||||
nix build .#checks.${system}.{treefmt,clan-pytest} -L "$@"
|
||||
'';
|
||||
in
|
||||
mkShell {
|
||||
packages = [
|
||||
ruff
|
||||
nix-unit
|
||||
pythonWithDeps
|
||||
openssh
|
||||
clan-cli.checkPython
|
||||
];
|
||||
# sets up an editable install and add enty points to $PATH
|
||||
PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}";
|
||||
PYTHONBREAKPOINT = "ipdb.set_trace";
|
||||
|
||||
shellHook = ''
|
||||
|
@ -59,6 +59,7 @@ def sshd_config(project_root: Path, test_root: Path) -> Iterator[SshdConfig]:
|
||||
MaxStartups 64:30:256
|
||||
AuthorizedKeysFile {host_key}.pub
|
||||
AcceptEnv REALPATH
|
||||
PasswordAuthentication no
|
||||
"""
|
||||
)
|
||||
login_shell = dir / "shell"
|
||||
@ -109,7 +110,6 @@ def sshd(
|
||||
) -> Iterator[Sshd]:
|
||||
import subprocess
|
||||
|
||||
subprocess.run(["echo", "hello"], check=True)
|
||||
port = unused_tcp_port()
|
||||
sshd = shutil.which("sshd")
|
||||
assert sshd is not None, "no sshd binary found"
|
||||
@ -123,6 +123,7 @@ def sshd(
|
||||
)
|
||||
|
||||
while True:
|
||||
print(sshd_config.path)
|
||||
if (
|
||||
subprocess.run(
|
||||
[
|
||||
@ -137,7 +138,7 @@ def sshd(
|
||||
"-p",
|
||||
str(port),
|
||||
"true",
|
||||
]
|
||||
],
|
||||
).returncode
|
||||
== 0
|
||||
):
|
||||
|
@ -5,14 +5,29 @@
|
||||
# this placeholder is replaced by the path to nixpkgs
|
||||
inputs.clan-core.url = "__CLAN_CORE__";
|
||||
|
||||
outputs = { self, clan-core }: {
|
||||
nixosConfigurations = clan-core.lib.buildClan {
|
||||
directory = self;
|
||||
machines = {
|
||||
vm1 = { modulesPath, ... }: {
|
||||
imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
|
||||
outputs = { self, clan-core }:
|
||||
let
|
||||
clan = clan-core.lib.buildClan {
|
||||
directory = self;
|
||||
machines = {
|
||||
vm1 = { modulesPath, ... }: {
|
||||
imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
|
||||
clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
|
||||
sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
|
||||
|
||||
clanCore.secrets.testpassword = {
|
||||
generator = ''
|
||||
echo "secret1" > "$secrets/secret1"
|
||||
echo "fact1" > "$facts/fact1"
|
||||
'';
|
||||
secrets.secret1 = { };
|
||||
facts.fact1 = { };
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
in
|
||||
{
|
||||
inherit (clan) nixosConfigurations clanInternals;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
38
pkgs/clan-cli/tests/test_secrets_generate.py
Normal file
38
pkgs/clan-cli/tests/test_secrets_generate.py
Normal file
@ -0,0 +1,38 @@
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from cli import Cli
|
||||
|
||||
from clan_cli.machines.facts import machine_get_fact
|
||||
from clan_cli.secrets.folders import sops_secrets_folder
|
||||
from clan_cli.secrets.secrets import has_secret
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from age_keys import KeyPair
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_upload_secret(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
test_flake_with_core: Path,
|
||||
age_keys: list["KeyPair"],
|
||||
) -> None:
|
||||
monkeypatch.chdir(test_flake_with_core)
|
||||
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
|
||||
cli = Cli()
|
||||
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
|
||||
cli.run(["secrets", "generate", "vm1"])
|
||||
has_secret("vm1-age.key")
|
||||
has_secret("vm1-secret1")
|
||||
fact1 = machine_get_fact("vm1", "fact1")
|
||||
assert fact1 == "fact1\n"
|
||||
age_key = sops_secrets_folder().joinpath("vm1-age.key").joinpath("secret")
|
||||
secret1 = sops_secrets_folder().joinpath("vm1-secret1").joinpath("secret")
|
||||
age_key_mtime = age_key.lstat().st_mtime_ns
|
||||
secret1_mtime = secret1.lstat().st_mtime_ns
|
||||
|
||||
# test idempotency
|
||||
cli.run(["secrets", "generate", "vm1"])
|
||||
assert age_key.lstat().st_mtime_ns == age_key_mtime
|
||||
assert secret1.lstat().st_mtime_ns == secret1_mtime
|
40
pkgs/clan-cli/tests/test_secrets_upload.py
Normal file
40
pkgs/clan-cli/tests/test_secrets_upload.py
Normal file
@ -0,0 +1,40 @@
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from cli import Cli
|
||||
|
||||
from clan_cli.ssh import HostGroup
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from age_keys import KeyPair
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_secrets_upload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
test_flake_with_core: Path,
|
||||
host_group: HostGroup,
|
||||
age_keys: list["KeyPair"],
|
||||
) -> None:
|
||||
monkeypatch.chdir(test_flake_with_core)
|
||||
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
|
||||
|
||||
cli = Cli()
|
||||
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
|
||||
|
||||
cli.run(["secrets", "machines", "add", "vm1", age_keys[1].pubkey])
|
||||
monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
|
||||
cli.run(["secrets", "set", "vm1-age.key"])
|
||||
|
||||
flake = test_flake_with_core.joinpath("flake.nix")
|
||||
host = host_group.hosts[0]
|
||||
addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
|
||||
new_text = flake.read_text().replace("__CLAN_DEPLOYMENT_ADDRESS__", addr)
|
||||
sops_key = test_flake_with_core.joinpath("sops.key")
|
||||
new_text = new_text.replace("__CLAN_SOPS_KEY_PATH__", str(sops_key))
|
||||
|
||||
flake.write_text(new_text)
|
||||
cli.run(["secrets", "upload", "vm1"])
|
||||
assert sops_key.exists()
|
||||
assert sops_key.read_text() == age_keys[0].privkey
|
@ -7,12 +7,13 @@
|
||||
let
|
||||
system = "x86_64-linux";
|
||||
pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
|
||||
clan = clan-core.lib.buildClan {
|
||||
directory = self;
|
||||
};
|
||||
in
|
||||
{
|
||||
# all machines managed by cLAN
|
||||
nixosConfigurations = clan-core.lib.buildClan {
|
||||
directory = self;
|
||||
};
|
||||
inherit (clan) nixosConfigurations clanInternals;
|
||||
# add the cLAN cli tool to the dev shell
|
||||
devShells.${system}.default = pkgs.mkShell {
|
||||
packages = [
|
||||
|
Loading…
Reference in New Issue
Block a user