clan/clan-core
11
2
Fork 13
Code Issues 91 Pull Requests 5 Actions Packages Projects 1 Releases Wiki Activity

Merge pull request 'interactive_secrets' (#885) from interactive_secrets into main
All checks were successful
checks / check-links (push) Successful in 22s
Details
checks / checks (push) Successful in 31s
Details
checks / checks-impure (push) Successful in 1m53s
Details

Browse Source
This commit is contained in:
clan-bot 2024-03-03 06:15:44 +00:00
commit c228d72da2
11 changed files with 117 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
checks/lib/container-driver/pyproject.toml
View File

@ -19,8 +19,8 @@ test_driver = ["py.typed"]
target-version = "py311"
line-length = 88
select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
ignore = ["E501", "ANN101", "ANN401", "A003"]
lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
lint.ignore = ["E501", "ANN101", "ANN401", "A003"]
[tool.mypy]
python_version = "3.11"

14
nixosModules/clanCore/outputs.nix
View File

@ -62,19 +62,7 @@
description = ''
secret data as json for the generator
'';
default = pkgs.writers.writeJSON "secrets.json" (lib.mapAttrs
(_name: secret: {
secrets = lib.mapAttrsToList
(name: secret: {
inherit name;
} // lib.optionalAttrs (secret ? groups) {
inherit (secret) groups;
})
secret.secrets;
facts = lib.mapAttrs (_: secret: secret.path) secret.facts;
generator = secret.generator.finalScript;
})
config.clanCore.secrets);
default = pkgs.writers.writeJSON "secrets.json" config.clanCore.secrets;
};
vm.create = lib.mkOption {
type = lib.types.path;

18
nixosModules/clanCore/secrets/default.nix
View File

@ -35,13 +35,13 @@
options.clanCore.secrets = lib.mkOption {
default = { };
type = lib.types.attrsOf
(lib.types.submodule (secret: {
(lib.types.submodule (service: {
options = {
name = lib.mkOption {
type = lib.types.str;
default = secret.config._module.args.name;
default = service.config._module.args.name;
description = ''
Namespace of the secret
Namespace of the service
'';
};
generator = lib.mkOption {
@ -54,6 +54,14 @@
Extra paths to add to the PATH environment variable when running the generator.
'';
};
prompt = lib.mkOption {
type = lib.types.nullOr lib.types.str;
default = null;
description = ''
prompt text to ask for a value.
This value will be passed to the script as the environment variabel $prompt_value.
'';
};
script = lib.mkOption {
type = lib.types.str;
description = ''
@ -92,14 +100,14 @@
config' = config;
in
lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
type = lib.types.attrsOf (lib.types.submodule ({ config, name, ... }: {
options = {
name = lib.mkOption {
type = lib.types.str;
description = ''
name of the secret
'';
default = config._module.args.name;
default = name;
};
path = lib.mkOption {
type = lib.types.str;

93
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@ -166,6 +166,7 @@ class Machine:
config = nix_config()
system = config["system"]
file_info = dict()
with NamedTemporaryFile(mode="w") as config_json:
if extra_config is not None:
json.dump(extra_config, config_json, indent=2)
@ -173,66 +174,66 @@ class Machine:
json.dump({}, config_json)
config_json.flush()
nar_hash = json.loads(
file_info = json.loads(
run(
nix_eval(
[
"--impure",
"--expr",
f'(builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}).narHash',
f'let x = (builtins.fetchTree {{ type = "file"; url = "file://{config_json.name}"; }}); in {{ narHash = x.narHash; path = x.outPath; }}',
]
)
).stdout.strip()
)
args = []
args = []
# get git commit from flake
if extra_config is not None:
metadata = nix_metadata(self.flake_dir)
url = metadata["url"]
if "dirtyRevision" in metadata:
# if not impure:
# raise ClanError(
# "The machine has a dirty revision, and impure mode is not allowed"
# )
# else:
# args += ["--impure"]
args += ["--impure"]
# get git commit from flake
if extra_config is not None:
metadata = nix_metadata(self.flake_dir)
url = metadata["url"]
if "dirtyRevision" in metadata:
# if not impure:
# raise ClanError(
# "The machine has a dirty revision, and impure mode is not allowed"
# )
# else:
# args += ["--impure"]
args += ["--impure"]
args += [
"--expr",
f"""
((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{
extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{
type = "file";
url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{config_json.name}" else "file:{config_json.name}";
narHash = "{nar_hash}";
}}));
}}).{attr}
""",
]
else:
if isinstance(self.flake, Path):
if (self.flake / ".git").exists():
flake = f"git+file://{self.flake}"
else:
flake = f"path:{self.flake}"
args += [
"--expr",
f"""
((builtins.getFlake "{url}").clanInternals.machinesFunc."{system}"."{self.name}" {{
extraConfig = builtins.fromJSON (builtins.readFile (builtins.fetchTree {{
type = "file";
url = if (builtins.compareVersions builtins.nixVersion "2.19") == -1 then "{file_info["path"]}" else "file:{file_info["path"]}";
narHash = "{file_info["narHash"]}";
}}));
}}).{attr}
""",
]
else:
if isinstance(self.flake, Path):
if (self.flake / ".git").exists():
flake = f"git+file://{self.flake}"
else:
flake = self.flake
args += [
f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}',
*nix_options,
]
if method == "eval":
output = run(nix_eval(args)).stdout.strip()
return output
elif method == "build":
outpath = run(nix_build(args)).stdout.strip()
return Path(outpath)
flake = f"path:{self.flake}"
else:
raise ValueError(f"Unknown method {method}")
flake = self.flake
args += [
f'{flake}#clanInternals.machines."{system}".{self.name}.{attr}',
*nix_options,
]
if method == "eval":
output = run(nix_eval(args)).stdout.strip()
return output
elif method == "build":
outpath = run(nix_build(args)).stdout.strip()
return Path(outpath)
else:
raise ValueError(f"Unknown method {method}")
def eval_nix(
self,

19
pkgs/clan-cli/clan_cli/secrets/check.py
View File

@ -7,7 +7,7 @@ from ..machines.machines import Machine
log = logging.getLogger(__name__)
def check_secrets(machine: Machine) -> bool:
def check_secrets(machine: Machine, service: None | str = None) -> bool:
secrets_module = importlib.import_module(machine.secrets_module)
secret_store = secrets_module.SecretStore(machine=machine)
facts_module = importlib.import_module(machine.facts_module)
@ -15,7 +15,11 @@ def check_secrets(machine: Machine) -> bool:
missing_secrets = []
missing_facts = []
for service in machine.secrets_data:
if service:
services = [service]
else:
services = list(machine.secrets_data.keys())
for service in services:
for secret in machine.secrets_data[service]["secrets"]:
if isinstance(secret, str):
secret_name = secret
@ -38,8 +42,11 @@ def check_secrets(machine: Machine) -> bool:
def check_command(args: argparse.Namespace) -> None:
machine = Machine(name=args.machine, flake=args.flake)
check_secrets(machine)
machine = Machine(
name=args.machine,
flake=args.flake,
)
check_secrets(machine, service=args.service)
def register_check_parser(parser: argparse.ArgumentParser) -> None:
@ -47,4 +54,8 @@ def register_check_parser(parser: argparse.ArgumentParser) -> None:
"machine",
help="The machine to check secrets for",
)
parser.add_argument(
"--service",
help="the service to check",
)
parser.set_defaults(func=check_command)

33
pkgs/clan-cli/clan_cli/secrets/generate.py
View File

@ -2,6 +2,7 @@ import argparse
import importlib
import logging
import os
from collections.abc import Callable
from pathlib import Path
from tempfile import TemporaryDirectory
@ -24,10 +25,11 @@ def generate_service_secrets(
secret_store: SecretStoreBase,
fact_store: FactStoreBase,
tmpdir: Path,
prompt: Callable[[str], str],
) -> None:
service_dir = tmpdir / service
# check if all secrets exist and generate them if at least one is missing
needs_regeneration = not check_secrets(machine)
needs_regeneration = not check_secrets(machine, service=service)
log.debug(f"{service} needs_regeneration: {needs_regeneration}")
if needs_regeneration:
if not isinstance(machine.flake, Path):
@ -41,6 +43,16 @@ def generate_service_secrets(
secrets_dir = service_dir / "secrets"
secrets_dir.mkdir(parents=True)
env["secrets"] = str(secrets_dir)
# compatibility for old outputs.nix users
if isinstance(machine.secrets_data[service]["generator"], str):
generator = machine.secrets_data[service]["generator"]
else:
generator = machine.secrets_data[service]["generator"]["finalScript"]
if machine.secrets_data[service]["generator"]["prompt"]:
prompt_value = prompt(
machine.secrets_data[service]["generator"]["prompt"]
)
env["prompt_value"] = prompt_value
# fmt: off
cmd = nix_shell(
[
@ -58,7 +70,7 @@ def generate_service_secrets(
"--unshare-user",
"--uid", "1000",
"--",
"bash", "-c", machine.secrets_data[service]["generator"]
"bash", "-c", generator
],
)
# fmt: on
@ -105,17 +117,30 @@ def generate_service_secrets(
)
def generate_secrets(machine: Machine) -> None:
def generate_secrets(
machine: Machine,
prompt: None | Callable[[str], str] = None,
) -> None:
secrets_module = importlib.import_module(machine.secrets_module)
secret_store = secrets_module.SecretStore(machine=machine)
facts_module = importlib.import_module(machine.facts_module)
fact_store = facts_module.FactStore(machine=machine)
if prompt is None:
prompt = lambda text: input(f"{text}: ")
with TemporaryDirectory() as tmp:
tmpdir = Path(tmp)
for service in machine.secrets_data:
generate_service_secrets(machine, service, secret_store, fact_store, tmpdir)
generate_service_secrets(
machine=machine,
service=service,
secret_store=secret_store,
fact_store=fact_store,
tmpdir=tmpdir,
prompt=prompt,
)
print("successfully generated secrets")

2
pkgs/clan-cli/clan_cli/secrets/modules/sops.py
View File

@ -45,7 +45,7 @@ class SecretStore(SecretStoreBase):
)
return path
def get(self, service: str, _name: str) -> bytes:
def get(self, service: str, name: str) -> bytes:
raise NotImplementedError()
def exists(self, service: str, name: str) -> bool:

6
pkgs/clan-cli/clan_cli/vms/run.py
View File

@ -37,8 +37,9 @@ def facts_to_nixos_config(facts: dict[str, dict[str, bytes]]) -> dict:
# TODO move this to the Machines class
def build_vm(
machine: Machine, vm: VmConfig, tmpdir: Path, nix_options: list[str] = []
machine: Machine, tmpdir: Path, nix_options: list[str] = []
) -> dict[str, str]:
# TODO pass prompt here for the GTK gui
secrets_dir = get_secrets(machine, tmpdir)
facts_module = importlib.import_module(machine.facts_module)
@ -68,7 +69,6 @@ def get_secrets(
secrets_module = importlib.import_module(machine.secrets_module)
secret_store = secrets_module.SecretStore(machine=machine)
# TODO Only generate secrets for local clans
generate_secrets(machine)
secret_store.upload(secrets_dir)
@ -113,7 +113,7 @@ def run_vm(vm: VmConfig, nix_options: list[str] = []) -> None:
tmpdir = Path(cachedir)
# TODO: We should get this from the vm argument
nixos_config = build_vm(machine, vm, tmpdir, nix_options)
nixos_config = build_vm(machine, tmpdir, nix_options)
state_dir = vm_state_dir(str(vm.flake_url), machine.name)
state_dir.mkdir(parents=True, exist_ok=True)

4
pkgs/clan-cli/pyproject.toml
View File

@ -55,5 +55,5 @@ ignore_missing_imports = true
[tool.ruff]
target-version = "py311"
line-length = 88
select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
ignore = ["E501", "E402", "ANN101", "ANN401", "A003"]
lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
lint.ignore = ["E501", "E402", "E731", "ANN101", "ANN401", "A003"]

4
pkgs/clan-vm-manager/pyproject.toml
View File

@ -33,5 +33,5 @@ ignore_missing_imports = true
[tool.ruff]
target-version = "py311"
line-length = 88
select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"]
lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
lint.ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"]

4
pyproject.toml
View File

@ -10,5 +10,5 @@ exclude = "clan_cli.nixpkgs"
[tool.ruff]
line-length = 88
target-version = "py311"
select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
ignore = [ "E501", "ANN101", "ANN401", "A003"]
lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
lint.ignore = [ "E501", "ANN101", "ANN401", "A003"]