clan/clan-core
12
4
Fork 18
Code Issues 99 Pull Requests 3 Actions Packages Projects 1 Releases Wiki Activity

Merge pull request 'init diskLayouts' (#302) from lassulus-HEAD into main
All checks were successful
checks-impure / test (push) Successful in 13s
Details
checks / test (push) Successful in 31s
Details
assets1 / test (push) Successful in 6s
Details

Browse Source 
Reviewed-on: #302
This commit is contained in:
Mic92 2023-09-21 16:33:21 +00:00
commit 4f3f7a4feb
28 changed files with 536 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.envrc
View File

@ -1 +1,5 @@
if ! has nix_direnv_version || ! nix_direnv_version 2.3.0; then
source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc" "sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8="
fi
use flake

26
checks/impure/flake-module.nix
View File

@ -6,32 +6,20 @@
#!${pkgs.bash}/bin/bash
set -euo pipefail
export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
export PATH="${lib.makeBinPath [
pkgs.coreutils
pkgs.gitMinimal
pkgs.nix
self'.packages.clan-cli.checkPython
]}"
export CLAN_CORE=$TMPDIR/CLAN_CORE
cp -r ${self} $CLAN_CORE
chmod +w -R $CLAN_CORE
cp -r ${self'.packages.clan-cli.src} $TMPDIR/src
chmod +w -R $TMPDIR/src
cd $TMPDIR/src
python -m pytest -m "impure" -s ./tests --workers "" "$@"
ROOT=$(git rev-parse --show-toplevel)
cd "$ROOT/pkgs/clan-cli"
nix develop "$ROOT#clan-cli" -c bash -c 'TMPDIR=/tmp python -m pytest -m impure -s ./tests'
'';
check-clan-template = pkgs.writeShellScriptBin "check-clan-template" ''
#!${pkgs.bash}/bin/bash
set -euo pipefail
set -euox pipefail
export TMPDIR=$(${pkgs.coreutils}/bin/mktemp -d)
trap "${pkgs.coreutils}/bin/chmod -R +w '$TMPDIR'; ${pkgs.coreutils}/bin/rm -rf '$TMPDIR'" EXIT
export CLANTMP=$(${pkgs.coreutils}/bin/mktemp -d)
trap "${pkgs.coreutils}/bin/chmod -R +w '$CLANTMP'; ${pkgs.coreutils}/bin/rm -rf '$CLANTMP'" EXIT
export PATH="${lib.makeBinPath [
pkgs.coreutils
@ -44,7 +32,7 @@
self'.packages.clan-cli
]}"
cd $TMPDIR
cd $CLANTMP
echo initialize new clan
nix flake init -t ${self}#new-clan

44
clanModules/diskLayouts/singleDiskExt4.nix Normal file
View File

@ -0,0 +1,44 @@
{ config, lib, ... }:
{
options.clan.diskLayouts.singleDiskExt4 = {
device = lib.mkOption {
type = lib.types.str;
example = "/dev/disk/by-id/ata-Samsung_SSD_850_EVO_250GB_S21PNXAGB12345";
};
};
config.disko.devices = {
disk = {
main = {
type = "disk";
device = config.clan.diskLayouts.singleDiskExt4.device;
content = {
type = "gpt";
partitions = {
boot = {
size = "1M";
type = "EF02"; # for grub MBR
};
ESP = {
size = "512M";
type = "EF00";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
};
root = {
size = "100%";
content = {
type = "filesystem";
format = "ext4";
mountpoint = "/";
};
};
};
};
};
};
};
}

12
clanModules/flake-module.nix Normal file
View File

@ -0,0 +1,12 @@
{ self, lib, ... }: {
flake.clanModules = {
diskLayouts = lib.mapAttrs'
(name: _: lib.nameValuePair (lib.removeSuffix ".nix" name) {
imports = [
self.inputs.disko.nixosModules.disko
./diskLayouts/${name}
];
})
(builtins.readDir ./diskLayouts);
};
}

2
flake.nix
View File

@ -24,12 +24,12 @@
"x86_64-linux"
"aarch64-linux"
];
flake.clanModules = { };
imports = [
./checks/flake-module.nix
./devShell.nix
./formatter.nix
./templates/flake-module.nix
./clanModules/flake-module.nix
./pkgs/flake-module.nix

2
formatter.nix
View File

@ -21,7 +21,7 @@
treefmt.programs.mypy.enable = true;
treefmt.programs.mypy.directories = {
"pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
"pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.pytestDependencies;
};
treefmt.settings.formatter.nix = {

52
lib/build-clan/default.nix
View File

@ -11,22 +11,44 @@ let
(builtins.fromJSON
(builtins.readFile (directory + /machines/${machineName}/settings.json)));
nixosConfiguration = { system ? "x86_64-linux", name }: nixpkgs.lib.nixosSystem {
modules = [
self.nixosModules.clanCore
(machineSettings name)
(machines.${name} or { })
{
clanCore.machineName = name;
clanCore.clanDir = directory;
# TODO: remove this once we have a hardware-config mechanism
nixpkgs.hostPlatform = lib.mkDefault system;
}
];
inherit specialArgs;
};
nixosConfigurations = lib.mapAttrs
(name: _:
nixpkgs.lib.nixosSystem {
modules = [
self.nixosModules.clanCore
(machineSettings name)
(machines.${name} or { })
{
clanCore.machineName = name;
clanCore.clanDir = directory;
# TODO: remove this once we have a hardware-config mechanism
nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
}
];
inherit specialArgs;
})
nixosConfiguration { inherit name; })
(machinesDirs // machines);
systems = [
"x86_64-linux"
"aarch64-linux"
"riscv64-linux"
"x86_64-darwin"
"aarch64-darwin"
];
clanInternals = {
machines = lib.mapAttrs
(name: _:
(builtins.listToAttrs (map
(system:
lib.nameValuePair system (nixosConfiguration { inherit name system; })
)
systems))
)
(machinesDirs // machines);
};
in
nixosConfigurations
{ inherit nixosConfigurations clanInternals; }

17
nixosModules/clanCore/secrets/default.nix
View File

@ -18,14 +18,17 @@
type = lib.types.str;
default = secret.config._module.args.name;
description = ''
namespace of the secret
Namespace of the secret
'';
};
generator = lib.mkOption {
type = lib.types.nullOr lib.types.str;
type = lib.types.str;
description = ''
script to generate the secret.
can be set to null. then the user has to provide the secret via the clan cli
Script to generate the secret.
The script will be called with the following variables:
- facts: path to a directory where facts can be stored
- secrets: path to a directory where secrets can be stored
The script is expected to generate all secrets and facts defined in the module.
'';
};
secrets = lib.mkOption {
@ -63,7 +66,11 @@
};
value = lib.mkOption {
defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}";
default =
if builtins.pathExists "${config.clanCore.clanDir}/${fact.config.path}" then
builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}"
else
"";
};
};
}));

66
nixosModules/clanCore/secrets/sops.nix
View File

@ -22,52 +22,23 @@ let
in
{
config = lib.mkIf (config.clanCore.secretStore == "sops") {
system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
#!/bin/sh
set -efu
test -d "$CLAN_DIR"
PATH=$PATH:${lib.makeBinPath [
config.clanCore.clanPkgs.clan-cli
]}
# initialize secret store
if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
INITTMP=$(mktemp -d)
trap 'rm -rf "$INITTMP"' EXIT
${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
) fi
${lib.foldlAttrs (acc: n: v: ''
${acc}
# ${n}
# if any of the secrets are missing, we regenerate all connected facts/secrets
(if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
facts=$(mktemp -d)
trap "rm -rf $facts" EXIT
secrets=$(mktemp -d)
trap "rm -rf $secrets" EXIT
${v.generator}
${lib.concatMapStrings (fact: ''
mkdir -p "$(dirname ${fact.path})"
cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
'') (lib.attrValues v.facts)}
${lib.concatMapStrings (secret: ''
cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
'') (lib.attrValues v.secrets)}
fi)
'') "" config.clanCore.secrets}
'';
system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
echo upload is not needed for sops secret store, since the secrets are part of the flake
'';
system.clan = {
generateSecrets = pkgs.writeScript "generate-secrets" ''
#!${pkgs.python3}/bin/python
import json
from clan_cli.secrets.sops_generate import generate_secrets_from_nix
args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; secret_submodules = config.clanCore.secrets; })})
generate_secrets_from_nix(**args)
'';
uploadSecrets = pkgs.writeScript "upload-secrets" ''
#!${pkgs.python3}/bin/python
import json
from clan_cli.secrets.sops_generate import upload_age_key_from_nix
# the second toJSON is needed to escape the string for the python
args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })})
upload_age_key_from_nix(**args)
'';
};
sops.secrets = builtins.mapAttrs
(name: _: {
sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
@ -76,5 +47,8 @@ in
secrets;
# To get proper error messages about missing secrets we need a dummy secret file that is always present
sops.defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")));
sops.age.keyFile = lib.mkIf (builtins.pathExists (config.clanCore.clanDir + "/sops/secrets/${config.clanCore.machineName}-age.key/secret"))
(lib.mkDefault "/var/lib/sops-nix/key.txt");
};
}

9
pkgs/clan-cli/clan_cli/machines/facts.py Normal file
View File

@ -0,0 +1,9 @@
from .folders import machine_folder
def machine_has_fact(machine: str, fact: str) -> bool:
return (machine_folder(machine) / "facts" / fact).exists()
def machine_get_fact(machine: str, fact: str) -> str:
return (machine_folder(machine) / "facts" / fact).read_text()

36
pkgs/clan-cli/clan_cli/machines/update.py
View File

@ -2,13 +2,12 @@ import argparse
import json
import os
import subprocess
from typing import Optional
from ..dirs import get_clan_flake_toplevel
from ..nix import nix_command, nix_eval
from ..secrets.generate import generate_secrets
from ..secrets.upload import upload_secrets
from ..ssh import Host, HostGroup, HostKeyCheck
from ..ssh import Host, HostGroup, HostKeyCheck, parse_deployment_address
def deploy_nixos(hosts: HostGroup) -> None:
@ -78,11 +77,12 @@ def deploy_nixos(hosts: HostGroup) -> None:
# FIXME: we want some kind of inventory here.
def update(args: argparse.Namespace) -> None:
clan_dir = get_clan_flake_toplevel().as_posix()
host = json.loads(
machine = args.machine
address = json.loads(
subprocess.run(
nix_eval(
[
f'{clan_dir}#nixosConfigurations."{args.machine}".config.clan.networking.deploymentAddress'
f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
]
),
stdout=subprocess.PIPE,
@ -90,31 +90,9 @@ def update(args: argparse.Namespace) -> None:
text=True,
).stdout
)
parts = host.split("@")
user: Optional[str] = None
if len(parts) > 1:
user = parts[0]
hostname = parts[1]
else:
hostname = parts[0]
maybe_port = hostname.split(":")
port = None
if len(maybe_port) > 1:
hostname = maybe_port[0]
port = int(maybe_port[1])
print(f"deploying {host}")
deploy_nixos(
HostGroup(
[
Host(
host=hostname,
port=port,
user=user,
meta=dict(flake_attr=args.machine),
)
]
)
)
host = parse_deployment_address(machine, address)
print(f"deploying {machine}")
deploy_nixos(HostGroup([host]))
def register_update_parser(parser: argparse.ArgumentParser) -> None:

13
pkgs/clan-cli/clan_cli/nix.py
View File

@ -1,5 +1,8 @@
import json
import os
import subprocess
import tempfile
from typing import Any
from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs
@ -25,6 +28,16 @@ def nix_build(
)
def nix_config() -> dict[str, Any]:
cmd = nix_command(["show-config", "--json"])
proc = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
data = json.loads(proc.stdout)
config = {}
for key, value in data.items():
config[key] = value["value"]
return config
def nix_eval(flags: list[str]) -> list[str]:
default_flags = nix_command(
[

27
pkgs/clan-cli/clan_cli/secrets/generate.py
View File

@ -1,31 +1,32 @@
import argparse
import os
import shlex
import subprocess
import sys
from clan_cli.errors import ClanError
from ..dirs import get_clan_flake_toplevel
from ..nix import nix_build
from ..dirs import get_clan_flake_toplevel, module_root
from ..nix import nix_build, nix_config
def generate_secrets(machine: str) -> None:
clan_dir = get_clan_flake_toplevel().as_posix().strip()
env = os.environ.copy()
env["CLAN_DIR"] = clan_dir
env["PYTHONPATH"] = str(module_root().parent) # TODO do this in the clanCore module
config = nix_config()
system = config["system"]
proc = subprocess.run(
nix_build(
[
f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets'
]
),
capture_output=True,
text=True,
cmd = nix_build(
[
f'path:{clan_dir}#clanInternals.machines."{machine}".{system}.config.system.clan.generateSecrets'
]
)
proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
if proc.returncode != 0:
print(proc.stderr, file=sys.stderr)
raise ClanError(f"failed to generate secrets:\n{proc.stderr}")
raise ClanError(
f"failed to generate secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
)
secret_generator_script = proc.stdout.strip()
print(secret_generator_script)

4
pkgs/clan-cli/clan_cli/secrets/key.py
View File

@ -9,8 +9,8 @@ def generate_key() -> str:
path = default_sops_key_path()
if path.exists():
raise ClanError(f"Key already exists at {path}")
generate_private_key(path)
pub_key = get_public_key(path.read_text())
priv_key, pub_key = generate_private_key()
path.write_text(priv_key)
return pub_key

6
pkgs/clan-cli/clan_cli/secrets/machines.py
View File

@ -19,11 +19,15 @@ def get_machine(name: str) -> str:
return read_key(sops_machines_folder() / name)
def has_machine(name: str) -> bool:
return (sops_machines_folder() / name / "key.json").exists()
def list_machines() -> list[str]:
path = sops_machines_folder()
def validate(name: str) -> bool:
return validate_hostname(name) and (path / name / "key.json").exists()
return validate_hostname(name) and has_machine(name)
return list_objects(path, validate)

9
pkgs/clan-cli/clan_cli/secrets/secrets.py
View File

@ -171,14 +171,15 @@ def disallow_member(group_folder: Path, name: str) -> None:
)
def has_secret(secret: str) -> bool:
return (sops_secrets_folder() / secret / "secret").exists()
def list_secrets() -> list[str]:
path = sops_secrets_folder()
def validate(name: str) -> bool:
return (
VALID_SECRET_NAME.match(name) is not None
and (path / name / "secret").exists()
)
return VALID_SECRET_NAME.match(name) is not None and has_secret(name)
return list_objects(path, validate)

23
pkgs/clan-cli/clan_cli/secrets/sops.py
View File

@ -30,10 +30,25 @@ def get_public_key(privkey: str) -> str:
return res.stdout.strip()
def generate_private_key(path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
cmd = nix_shell(["age"], ["age-keygen", "-o", str(path)])
subprocess.run(cmd, check=True)
def generate_private_key() -> tuple[str, str]:
cmd = nix_shell(["age"], ["age-keygen"])
try:
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, text=True)
res = proc.stdout.strip()
pubkey = None
private_key = None
for line in res.splitlines():
if line.startswith("# public key:"):
pubkey = line.split(":")[1].strip()
if not line.startswith("#"):
private_key = line
if not pubkey:
raise ClanError("Could not find public key in age-keygen output")
if not private_key:
raise ClanError("Could not find private key in age-keygen output")
return private_key, pubkey
except subprocess.CalledProcessError as e:
raise ClanError("Failed to generate private sops key") from e
def get_user_name(user: str) -> str:

124
pkgs/clan-cli/clan_cli/secrets/sops_generate.py Normal file
View File

@ -0,0 +1,124 @@
import shlex
import shutil
import subprocess
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
from ..dirs import get_clan_flake_toplevel
from ..errors import ClanError
from ..ssh import parse_deployment_address
from .folders import sops_secrets_folder
from .machines import add_machine, has_machine
from .secrets import decrypt_secret, encrypt_secret, has_secret
from .sops import generate_private_key
def generate_host_key(machine_name: str) -> None:
if has_machine(machine_name):
return
priv_key, pub_key = generate_private_key()
encrypt_secret(sops_secrets_folder() / f"{machine_name}-age.key", priv_key)
add_machine(machine_name, pub_key, False)
def generate_secrets_group(
secret_group: str, machine_name: str, tempdir: Path, secret_options: dict[str, Any]
) -> None:
clan_dir = get_clan_flake_toplevel()
secrets = secret_options["secrets"]
needs_regeneration = any(
not has_secret(f"{machine_name}-{secret['name']}")
for secret in secrets.values()
)
generator = secret_options["generator"]
subdir = tempdir / secret_group
if needs_regeneration:
facts_dir = subdir / "facts"
facts_dir.mkdir(parents=True)
secrets_dir = subdir / "secrets"
secrets_dir.mkdir(parents=True)
text = f"""\
set -euo pipefail
facts={shlex.quote(str(facts_dir))}
secrets={shlex.quote(str(secrets_dir))}
{generator}
"""
try:
subprocess.run(["bash", "-c", text], check=True)
except subprocess.CalledProcessError:
msg = "failed to the following command:\n"
msg += text
raise ClanError(msg)
for secret in secrets.values():
secret_file = secrets_dir / secret["name"]
if not secret_file.is_file():
msg = f"did not generate a file for '{secret['name']}' when running the following command:\n"
msg += text
raise ClanError(msg)
encrypt_secret(
sops_secrets_folder() / f"{machine_name}-{secret['name']}",
secret_file.read_text(),
)
for fact in secret_options["facts"].values():
fact_file = facts_dir / fact["name"]
if not fact_file.is_file():
msg = f"did not generate a file for '{fact['name']}' when running the following command:\n"
msg += text
raise ClanError(msg)
fact_path = clan_dir.joinpath(fact["path"])
fact_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(fact_file, fact_path)
# this is called by the sops.nix clan core module
def generate_secrets_from_nix(
machine_name: str,
secret_submodules: dict[str, Any],
) -> None:
generate_host_key(machine_name)
errors = {}
with TemporaryDirectory() as d:
# if any of the secrets are missing, we regenerate all connected facts/secrets
for secret_group, secret_options in secret_submodules.items():
try:
generate_secrets_group(
secret_group, machine_name, Path(d), secret_options
)
except ClanError as e:
errors[secret_group] = e
for secret_group, error in errors.items():
print(f"failed to generate secrets for {machine_name}/{secret_group}:")
print(error, file=sys.stderr)
if len(errors) > 0:
sys.exit(1)
# this is called by the sops.nix clan core module
def upload_age_key_from_nix(
machine_name: str, deployment_address: str, age_key_file: str
) -> None:
secret_name = f"{machine_name}-age.key"
if not has_secret(secret_name): # skip uploading the secret, not managed by us
return
secret = decrypt_secret(secret_name)
h = parse_deployment_address(machine_name, deployment_address)
path = Path(age_key_file)
proc = h.run(
[
"bash",
"-c",
'mkdir -p "$0" && echo -n "$1" > "$2"',
str(path.parent),
secret,
age_key_file,
],
check=False,
)
if proc.returncode != 0:
print(f"failed to upload age key to {deployment_address}")
sys.exit(1)

18
pkgs/clan-cli/clan_cli/secrets/upload.py
View File

@ -1,31 +1,36 @@
import argparse
import json
import os
import subprocess
from clan_cli.errors import ClanError
from ..dirs import get_clan_flake_toplevel
from ..nix import nix_build, nix_eval
from ..dirs import get_clan_flake_toplevel, module_root
from ..errors import ClanError
from ..nix import nix_build, nix_config, nix_eval
def upload_secrets(machine: str) -> None:
clan_dir = get_clan_flake_toplevel().as_posix()
config = nix_config()
system = config["system"]
proc = subprocess.run(
nix_build(
[
f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets'
f'{clan_dir}#clanInternals.machines."{machine}".{system}.config.system.clan.uploadSecrets'
]
),
stdout=subprocess.PIPE,
text=True,
check=True,
)
env = os.environ.copy()
env["PYTHONPATH"] = str(module_root().parent) # TODO do this in the clanCore module
host = json.loads(
subprocess.run(
nix_eval(
[
f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
f'{clan_dir}#clanInternals.machines."{machine}".{system}.config.clan.networking.deploymentAddress'
]
),
stdout=subprocess.PIPE,
@ -40,6 +45,7 @@ def upload_secrets(machine: str) -> None:
secret_upload_script,
host,
],
env=env,
)
if secret_upload.returncode != 0:

30
pkgs/clan-cli/clan_cli/ssh/__init__.py
View File

@ -756,6 +756,36 @@ class HostGroup:
return HostGroup(list(filter(pred, self.hosts)))
def parse_deployment_address(machine_name: str, host: str) -> Host:
parts = host.split("@")
user: Optional[str] = None
if len(parts) > 1:
user = parts[0]
hostname = parts[1]
else:
hostname = parts[0]
maybe_options = hostname.split("?")
options: Dict[str, str] = {}
if len(maybe_options) > 1:
hostname = maybe_options[0]
for option in maybe_options[1].split("&"):
k, v = option.split("=")
options[k] = v
maybe_port = hostname.split(":")
port = None
if len(maybe_port) > 1:
hostname = maybe_port[0]
port = int(maybe_port[1])
return Host(
hostname,
user=user,
port=port,
command_prefix=machine_name,
meta=dict(flake_attr=machine_name),
ssh_options=options,
)
@overload
def run(
cmd: Union[List[str], str],

65
pkgs/clan-cli/default.nix
View File

@ -1,4 +1,5 @@
{ age
, lib
, argcomplete
, fastapi
, uvicorn
@ -20,7 +21,12 @@
, rsync
, pkgs
, ui-assets
, lib
, bash
, sshpass
, zbar
, tor
, git
, ipdb
}:
let
@ -30,16 +36,36 @@ let
uvicorn # optional dependencies: if not enabled, webui subcommand will not work
];
testDependencies = [
pytestDependencies = runtimeDependencies ++ dependencies ++ [
pytest
pytest-cov
pytest-subprocess
pytest-parallel
openssh
git
stdenv.cc
ipdb # used for debugging
];
checkPython = python3.withPackages (_ps: dependencies ++ testDependencies);
# Optional dependencies for clan cli, we re-expose them here to make sure they all build.
runtimeDependencies = [
bash
nix
zerotierone
bubblewrap
openssh
sshpass
zbar
tor
age
rsync
sops
git
];
runtimeDependenciesAsSet = builtins.listToAttrs (builtins.map (p: lib.nameValuePair (lib.getName p.name) p) runtimeDependencies);
checkPython = python3.withPackages (_ps: pytestDependencies);
# - vendor the jsonschema nix lib (copy instead of symlink).
source = runCommand "clan-cli-source" { } ''
@ -73,6 +99,7 @@ let
--experimental-features 'nix-command flakes' \
--override-input nixpkgs ${pkgs.path}
'';
in
python3.pkgs.buildPythonPackage {
name = "clan-cli";
@ -85,25 +112,24 @@ python3.pkgs.buildPythonPackage {
];
propagatedBuildInputs = dependencies;
passthru.tests.clan-pytest = runCommand "clan-pytest"
{
nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ];
} ''
cp -r ${source} ./src
chmod +w -R ./src
cd ./src
# also re-expose dependencies so we test them in CI
passthru.tests = (lib.mapAttrs' (n: lib.nameValuePair "package-${n}") runtimeDependenciesAsSet) // {
clan-pytest = runCommand "clan-pytest" { nativeBuildInputs = [ checkPython ] ++ pytestDependencies; } ''
cp -r ${source} ./src
chmod +w -R ./src
cd ./src
# git is needed for test_git.py
export PATH="${lib.makeBinPath [pkgs.git]}:$PATH"
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
touch $out
'';
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
touch $out
'';
};
passthru.clan-openapi = runCommand "clan-openapi" { } ''
cp -r ${source} ./src
chmod +w -R ./src
cd ./src
export PATH=${checkPython}/bin:$PATH
${checkPython}/bin/python ./bin/gen-openapi --out $out/openapi.json --app-dir . clan_cli.webui.app:app
touch $out
'';
@ -113,9 +139,10 @@ python3.pkgs.buildPythonPackage {
passthru.devDependencies = [
setuptools
wheel
] ++ testDependencies;
] ++ pytestDependencies;
passthru.testDependencies = dependencies ++ testDependencies;
passthru.pytestDependencies = pytestDependencies;
passthru.runtimeDependencies = runtimeDependencies;
postInstall = ''
cp -r ${nixpkgs} $out/${python3.sitePackages}/clan_cli/nixpkgs

16
pkgs/clan-cli/flake-module.nix
View File

@ -1,3 +1,4 @@
{ lib, ... }:
{
perSystem = { self', pkgs, ... }: {
devShells.clan-cli = pkgs.callPackage ./shell.nix {
@ -10,26 +11,13 @@
clan-openapi = self'.packages.clan-cli.clan-openapi;
default = self'.packages.clan-cli;
## Optional dependencies for clan cli, we re-expose them here to make sure they all build.
inherit (pkgs)
age
bash
bubblewrap
git
openssh
rsync
sops
sshpass
tor
zbar
;
# Override license so that we can build zerotierone without
# having to re-import nixpkgs.
zerotierone = pkgs.zerotierone.overrideAttrs (_old: { meta = { }; });
## End optional dependencies
};
checks = self'.packages.clan-cli.tests;
checks = lib.mkDefault self'.packages.clan-cli.tests;
};
}

16
pkgs/clan-cli/shell.nix
View File

@ -1,26 +1,16 @@
{ nix-unit, clan-cli, ui-assets, python3, system, ruff, mkShell, writeScriptBin }:
{ nix-unit, clan-cli, ui-assets, system, mkShell, writeScriptBin, openssh }:
let
pythonWithDeps = python3.withPackages (
ps:
clan-cli.propagatedBuildInputs
++ clan-cli.devDependencies
++ [
ps.pip
ps.ipdb
]
);
checkScript = writeScriptBin "check" ''
nix build .#checks.${system}.{treefmt,clan-pytest} -L "$@"
'';
in
mkShell {
packages = [
ruff
nix-unit
pythonWithDeps
openssh
clan-cli.checkPython
];
# sets up an editable install and add enty points to $PATH
PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}";
PYTHONBREAKPOINT = "ipdb.set_trace";
shellHook = ''

5
pkgs/clan-cli/tests/sshd.py
View File

@ -59,6 +59,7 @@ def sshd_config(project_root: Path, test_root: Path) -> Iterator[SshdConfig]:
MaxStartups 64:30:256
AuthorizedKeysFile {host_key}.pub
AcceptEnv REALPATH
PasswordAuthentication no
"""
)
login_shell = dir / "shell"
@ -109,7 +110,6 @@ def sshd(
) -> Iterator[Sshd]:
import subprocess
subprocess.run(["echo", "hello"], check=True)
port = unused_tcp_port()
sshd = shutil.which("sshd")
assert sshd is not None, "no sshd binary found"
@ -123,6 +123,7 @@ def sshd(
)
while True:
print(sshd_config.path)
if (
subprocess.run(
[
@ -137,7 +138,7 @@ def sshd(
"-p",
str(port),
"true",
]
],
).returncode
== 0
):

29
pkgs/clan-cli/tests/test_flake_with_core/flake.nix
View File

@ -5,14 +5,29 @@
# this placeholder is replaced by the path to nixpkgs
inputs.clan-core.url = "__CLAN_CORE__";
outputs = { self, clan-core }: {
nixosConfigurations = clan-core.lib.buildClan {
directory = self;
machines = {
vm1 = { modulesPath, ... }: {
imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
outputs = { self, clan-core }:
let
clan = clan-core.lib.buildClan {
directory = self;
machines = {
vm1 = { modulesPath, ... }: {
imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
clanCore.secrets.testpassword = {
generator = ''
echo "secret1" > "$secrets/secret1"
echo "fact1" > "$facts/fact1"
'';
secrets.secret1 = { };
facts.fact1 = { };
};
};
};
};
in
{
inherit (clan) nixosConfigurations clanInternals;
};
};
}

38
pkgs/clan-cli/tests/test_secrets_generate.py Normal file
View File

@ -0,0 +1,38 @@
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from cli import Cli
from clan_cli.machines.facts import machine_get_fact
from clan_cli.secrets.folders import sops_secrets_folder
from clan_cli.secrets.secrets import has_secret
if TYPE_CHECKING:
from age_keys import KeyPair
@pytest.mark.impure
def test_upload_secret(
monkeypatch: pytest.MonkeyPatch,
test_flake_with_core: Path,
age_keys: list["KeyPair"],
) -> None:
monkeypatch.chdir(test_flake_with_core)
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
cli = Cli()
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
cli.run(["secrets", "generate", "vm1"])
has_secret("vm1-age.key")
has_secret("vm1-secret1")
fact1 = machine_get_fact("vm1", "fact1")
assert fact1 == "fact1\n"
age_key = sops_secrets_folder().joinpath("vm1-age.key").joinpath("secret")
secret1 = sops_secrets_folder().joinpath("vm1-secret1").joinpath("secret")
age_key_mtime = age_key.lstat().st_mtime_ns
secret1_mtime = secret1.lstat().st_mtime_ns
# test idempotency
cli.run(["secrets", "generate", "vm1"])
assert age_key.lstat().st_mtime_ns == age_key_mtime
assert secret1.lstat().st_mtime_ns == secret1_mtime

40
pkgs/clan-cli/tests/test_secrets_upload.py Normal file
View File

@ -0,0 +1,40 @@
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from cli import Cli
from clan_cli.ssh import HostGroup
if TYPE_CHECKING:
from age_keys import KeyPair
@pytest.mark.impure
def test_secrets_upload(
monkeypatch: pytest.MonkeyPatch,
test_flake_with_core: Path,
host_group: HostGroup,
age_keys: list["KeyPair"],
) -> None:
monkeypatch.chdir(test_flake_with_core)
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
cli = Cli()
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
cli.run(["secrets", "machines", "add", "vm1", age_keys[1].pubkey])
monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
cli.run(["secrets", "set", "vm1-age.key"])
flake = test_flake_with_core.joinpath("flake.nix")
host = host_group.hosts[0]
addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
new_text = flake.read_text().replace("__CLAN_DEPLOYMENT_ADDRESS__", addr)
sops_key = test_flake_with_core.joinpath("sops.key")
new_text = new_text.replace("__CLAN_SOPS_KEY_PATH__", str(sops_key))
flake.write_text(new_text)
cli.run(["secrets", "upload", "vm1"])
assert sops_key.exists()
assert sops_key.read_text() == age_keys[0].privkey

7
templates/new-clan/flake.nix
View File

@ -7,12 +7,13 @@
let
system = "x86_64-linux";
pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
clan = clan-core.lib.buildClan {
directory = self;
};
in
{
# all machines managed by cLAN
nixosConfigurations = clan-core.lib.buildClan {
directory = self;
};
inherit (clan) nixosConfigurations clanInternals;
# add the cLAN cli tool to the dev shell
devShells.${system}.default = pkgs.mkShell {
packages = [