2023-08-28 09:09:05 +00:00
|
|
|
import argparse
|
2024-01-17 17:00:30 +00:00
|
|
|
import importlib
|
2023-10-03 10:50:47 +00:00
|
|
|
import logging
|
2023-09-14 13:22:13 +00:00
|
|
|
import os
|
2024-03-13 09:06:10 +00:00
|
|
|
import subprocess
|
2024-03-01 09:25:39 +00:00
|
|
|
from collections.abc import Callable
|
2024-01-15 18:34:04 +00:00
|
|
|
from pathlib import Path
|
|
|
|
from tempfile import TemporaryDirectory
|
2023-08-28 09:09:05 +00:00
|
|
|
|
2024-01-15 18:34:04 +00:00
|
|
|
from clan_cli.cmd import run
|
2024-01-11 21:28:35 +00:00
|
|
|
|
2024-05-31 13:21:07 +00:00
|
|
|
from ..completions import (
|
|
|
|
add_dynamic_completer,
|
|
|
|
complete_machines,
|
|
|
|
complete_services_for_machine,
|
|
|
|
)
|
2024-01-15 18:34:04 +00:00
|
|
|
from ..errors import ClanError
|
2024-02-05 09:02:39 +00:00
|
|
|
from ..git import commit_files
|
2024-04-15 19:06:03 +00:00
|
|
|
from ..machines.inventory import get_all_machines, get_selected_machines
|
2023-10-04 13:32:04 +00:00
|
|
|
from ..machines.machines import Machine
|
2024-01-15 18:34:04 +00:00
|
|
|
from ..nix import nix_shell
|
2024-02-02 16:28:33 +00:00
|
|
|
from .check import check_secrets
|
2024-03-23 04:05:31 +00:00
|
|
|
from .public_modules import FactStoreBase
|
|
|
|
from .secret_modules import SecretStoreBase
|
2023-08-28 09:09:05 +00:00
|
|
|
|
2023-10-03 10:50:47 +00:00
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
2023-09-09 13:38:28 +00:00
|
|
|
|
2024-03-13 09:06:10 +00:00
|
|
|
def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
|
|
|
|
"""
|
|
|
|
Read multi-line input from stdin.
|
|
|
|
"""
|
|
|
|
print(prompt, flush=True)
|
|
|
|
proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True)
|
2024-06-02 07:53:24 +00:00
|
|
|
log.info("Input received. Processing...")
|
2024-03-13 09:06:10 +00:00
|
|
|
return proc.stdout
|
|
|
|
|
|
|
|
|
2024-03-23 04:05:31 +00:00
|
|
|
def generate_service_facts(
|
2024-02-16 13:48:46 +00:00
|
|
|
machine: Machine,
|
|
|
|
service: str,
|
2024-05-26 20:55:48 +00:00
|
|
|
regenerate: bool,
|
2024-03-23 04:05:31 +00:00
|
|
|
secret_facts_store: SecretStoreBase,
|
|
|
|
public_facts_store: FactStoreBase,
|
2024-02-16 13:48:46 +00:00
|
|
|
tmpdir: Path,
|
2024-03-01 09:25:39 +00:00
|
|
|
prompt: Callable[[str], str],
|
2024-04-12 12:38:21 +00:00
|
|
|
) -> bool:
|
2024-02-16 13:48:46 +00:00
|
|
|
service_dir = tmpdir / service
|
|
|
|
# check if all secrets exist and generate them if at least one is missing
|
2024-03-03 03:05:56 +00:00
|
|
|
needs_regeneration = not check_secrets(machine, service=service)
|
2024-02-16 13:48:46 +00:00
|
|
|
log.debug(f"{service} needs_regeneration: {needs_regeneration}")
|
2024-05-26 20:55:48 +00:00
|
|
|
if not (needs_regeneration or regenerate):
|
2024-04-12 12:38:21 +00:00
|
|
|
return False
|
|
|
|
if not isinstance(machine.flake, Path):
|
|
|
|
msg = f"flake is not a Path: {machine.flake}"
|
|
|
|
msg += "fact/secret generation is only supported for local flakes"
|
|
|
|
|
|
|
|
env = os.environ.copy()
|
|
|
|
facts_dir = service_dir / "facts"
|
|
|
|
facts_dir.mkdir(parents=True)
|
|
|
|
env["facts"] = str(facts_dir)
|
|
|
|
secrets_dir = service_dir / "secrets"
|
|
|
|
secrets_dir.mkdir(parents=True)
|
|
|
|
env["secrets"] = str(secrets_dir)
|
|
|
|
# compatibility for old outputs.nix users
|
|
|
|
if isinstance(machine.facts_data[service]["generator"], str):
|
|
|
|
generator = machine.facts_data[service]["generator"]
|
|
|
|
else:
|
|
|
|
generator = machine.facts_data[service]["generator"]["finalScript"]
|
|
|
|
if machine.facts_data[service]["generator"]["prompt"]:
|
|
|
|
prompt_value = prompt(machine.facts_data[service]["generator"]["prompt"])
|
|
|
|
env["prompt_value"] = prompt_value
|
|
|
|
# fmt: off
|
|
|
|
cmd = nix_shell(
|
|
|
|
[
|
|
|
|
"nixpkgs#bash",
|
|
|
|
"nixpkgs#bubblewrap",
|
|
|
|
],
|
|
|
|
[
|
|
|
|
"bwrap",
|
|
|
|
"--ro-bind", "/nix/store", "/nix/store",
|
|
|
|
"--tmpfs", "/usr/lib/systemd",
|
|
|
|
"--dev", "/dev",
|
|
|
|
"--bind", str(facts_dir), str(facts_dir),
|
|
|
|
"--bind", str(secrets_dir), str(secrets_dir),
|
|
|
|
"--unshare-all",
|
|
|
|
"--unshare-user",
|
|
|
|
"--uid", "1000",
|
|
|
|
"--",
|
|
|
|
"bash", "-c", generator
|
|
|
|
],
|
|
|
|
)
|
|
|
|
# fmt: on
|
|
|
|
run(
|
|
|
|
cmd,
|
|
|
|
env=env,
|
|
|
|
)
|
|
|
|
files_to_commit = []
|
|
|
|
# store secrets
|
|
|
|
for secret in machine.facts_data[service]["secret"]:
|
|
|
|
if isinstance(secret, str):
|
|
|
|
# TODO: This is the old NixOS module, can be dropped everyone has updated.
|
|
|
|
secret_name = secret
|
|
|
|
groups = []
|
2024-03-01 09:25:39 +00:00
|
|
|
else:
|
2024-04-12 12:38:21 +00:00
|
|
|
secret_name = secret["name"]
|
|
|
|
groups = secret.get("groups", [])
|
|
|
|
|
|
|
|
secret_file = secrets_dir / secret_name
|
|
|
|
if not secret_file.is_file():
|
|
|
|
msg = f"did not generate a file for '{secret_name}' when running the following command:\n"
|
|
|
|
msg += generator
|
|
|
|
raise ClanError(msg)
|
|
|
|
secret_path = secret_facts_store.set(
|
|
|
|
service, secret_name, secret_file.read_bytes(), groups
|
2024-02-16 13:48:46 +00:00
|
|
|
)
|
2024-04-12 12:38:21 +00:00
|
|
|
if secret_path:
|
|
|
|
files_to_commit.append(secret_path)
|
|
|
|
|
|
|
|
# store facts
|
|
|
|
for name in machine.facts_data[service]["public"]:
|
|
|
|
fact_file = facts_dir / name
|
|
|
|
if not fact_file.is_file():
|
|
|
|
msg = f"did not generate a file for '{name}' when running the following command:\n"
|
|
|
|
msg += machine.facts_data[service]["generator"]
|
|
|
|
raise ClanError(msg)
|
|
|
|
fact_file = public_facts_store.set(service, name, fact_file.read_bytes())
|
|
|
|
if fact_file:
|
|
|
|
files_to_commit.append(fact_file)
|
|
|
|
commit_files(
|
|
|
|
files_to_commit,
|
|
|
|
machine.flake_dir,
|
|
|
|
f"Update facts/secrets for service {service} in machine {machine.name}",
|
|
|
|
)
|
|
|
|
return True
|
2024-02-16 13:48:46 +00:00
|
|
|
|
|
|
|
|
2024-04-15 19:06:03 +00:00
|
|
|
def prompt_func(text: str) -> str:
|
|
|
|
print(f"{text}: ")
|
|
|
|
return read_multiline_input()
|
|
|
|
|
|
|
|
|
|
|
|
def _generate_facts_for_machine(
|
2024-05-26 13:21:54 +00:00
|
|
|
machine: Machine,
|
|
|
|
service: str | None,
|
2024-05-26 20:55:48 +00:00
|
|
|
regenerate: bool,
|
2024-05-26 13:21:54 +00:00
|
|
|
tmpdir: Path,
|
|
|
|
prompt: Callable[[str], str] = prompt_func,
|
2024-04-12 12:38:21 +00:00
|
|
|
) -> bool:
|
2024-04-15 19:06:03 +00:00
|
|
|
local_temp = tmpdir / machine.name
|
|
|
|
local_temp.mkdir()
|
2024-03-23 04:05:31 +00:00
|
|
|
secret_facts_module = importlib.import_module(machine.secret_facts_module)
|
|
|
|
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
|
2024-01-15 18:34:04 +00:00
|
|
|
|
2024-03-23 04:05:31 +00:00
|
|
|
public_facts_module = importlib.import_module(machine.public_facts_module)
|
|
|
|
public_facts_store = public_facts_module.FactStore(machine=machine)
|
2024-02-12 12:31:12 +00:00
|
|
|
|
2024-04-15 19:06:03 +00:00
|
|
|
machine_updated = False
|
2024-05-26 13:21:54 +00:00
|
|
|
|
|
|
|
if service and service not in machine.facts_data:
|
|
|
|
services = list(machine.facts_data.keys())
|
|
|
|
raise ClanError(
|
|
|
|
f"Could not find service with name: {service}. The following services are available: {services}"
|
|
|
|
)
|
|
|
|
|
|
|
|
if service:
|
|
|
|
machine_service_facts = {service: machine.facts_data[service]}
|
|
|
|
else:
|
|
|
|
machine_service_facts = machine.facts_data
|
|
|
|
|
|
|
|
for service in machine_service_facts:
|
2024-04-15 19:06:03 +00:00
|
|
|
machine_updated |= generate_service_facts(
|
|
|
|
machine=machine,
|
|
|
|
service=service,
|
2024-05-26 20:55:48 +00:00
|
|
|
regenerate=regenerate,
|
2024-04-15 19:06:03 +00:00
|
|
|
secret_facts_store=secret_facts_store,
|
|
|
|
public_facts_store=public_facts_store,
|
|
|
|
tmpdir=local_temp,
|
|
|
|
prompt=prompt,
|
|
|
|
)
|
|
|
|
if machine_updated:
|
|
|
|
# flush caches to make sure the new secrets are available in evaluation
|
|
|
|
machine.flush_caches()
|
|
|
|
return machine_updated
|
2024-03-13 09:06:10 +00:00
|
|
|
|
2024-03-01 09:25:39 +00:00
|
|
|
|
2024-04-15 19:06:03 +00:00
|
|
|
def generate_facts(
|
2024-05-26 13:21:54 +00:00
|
|
|
machines: list[Machine],
|
|
|
|
service: str | None,
|
2024-05-26 20:55:48 +00:00
|
|
|
regenerate: bool,
|
2024-05-26 13:21:54 +00:00
|
|
|
prompt: Callable[[str], str] = prompt_func,
|
2024-04-15 19:06:03 +00:00
|
|
|
) -> bool:
|
2024-04-12 12:38:21 +00:00
|
|
|
was_regenerated = False
|
2024-02-16 13:48:46 +00:00
|
|
|
with TemporaryDirectory() as tmp:
|
|
|
|
tmpdir = Path(tmp)
|
2024-04-15 19:06:03 +00:00
|
|
|
|
|
|
|
for machine in machines:
|
|
|
|
errors = 0
|
|
|
|
try:
|
2024-05-26 13:21:54 +00:00
|
|
|
was_regenerated |= _generate_facts_for_machine(
|
2024-05-26 20:55:48 +00:00
|
|
|
machine, service, regenerate, tmpdir, prompt
|
2024-05-26 13:21:54 +00:00
|
|
|
)
|
2024-04-15 19:06:03 +00:00
|
|
|
except Exception as exc:
|
|
|
|
log.error(f"Failed to generate facts for {machine.name}: {exc}")
|
|
|
|
errors += 1
|
|
|
|
if errors > 0:
|
|
|
|
raise ClanError(
|
|
|
|
f"Failed to generate facts for {errors} hosts. Check the logs above"
|
|
|
|
)
|
|
|
|
|
|
|
|
if not was_regenerated:
|
2024-04-12 12:38:21 +00:00
|
|
|
print("All secrets and facts are already up to date")
|
|
|
|
return was_regenerated
|
2023-08-28 09:09:05 +00:00
|
|
|
|
|
|
|
|
|
|
|
def generate_command(args: argparse.Namespace) -> None:
|
2024-04-15 19:06:03 +00:00
|
|
|
if len(args.machines) == 0:
|
2024-05-29 07:08:23 +00:00
|
|
|
machines = get_all_machines(args.flake, args.option)
|
2024-04-15 19:06:03 +00:00
|
|
|
else:
|
2024-05-29 07:08:23 +00:00
|
|
|
machines = get_selected_machines(args.flake, args.option, args.machines)
|
2024-05-26 20:55:48 +00:00
|
|
|
generate_facts(machines, args.service, args.regenerate)
|
2023-08-28 09:09:05 +00:00
|
|
|
|
|
|
|
|
|
|
|
def register_generate_parser(parser: argparse.ArgumentParser) -> None:
|
2024-05-30 17:51:53 +00:00
|
|
|
machines_parser = parser.add_argument(
|
2024-04-15 19:06:03 +00:00
|
|
|
"machines",
|
|
|
|
type=str,
|
|
|
|
help="machine to generate facts for. if empty, generate facts for all machines",
|
|
|
|
nargs="*",
|
|
|
|
default=[],
|
2023-08-28 09:09:05 +00:00
|
|
|
)
|
2024-05-30 17:51:53 +00:00
|
|
|
add_dynamic_completer(machines_parser, complete_machines)
|
|
|
|
|
2024-05-31 13:21:07 +00:00
|
|
|
service_parser = parser.add_argument(
|
2024-05-26 13:21:54 +00:00
|
|
|
"--service",
|
|
|
|
type=str,
|
|
|
|
help="service to generate facts for, if empty, generate facts for every service",
|
|
|
|
default=None,
|
|
|
|
)
|
2024-05-31 13:21:07 +00:00
|
|
|
add_dynamic_completer(service_parser, complete_services_for_machine)
|
|
|
|
|
2024-05-26 20:55:48 +00:00
|
|
|
parser.add_argument(
|
|
|
|
"--regenerate",
|
|
|
|
type=bool,
|
|
|
|
action=argparse.BooleanOptionalAction,
|
|
|
|
help="whether to regenerate facts for the specified machine",
|
|
|
|
default=None,
|
|
|
|
)
|
2023-08-28 09:09:05 +00:00
|
|
|
parser.set_defaults(func=generate_command)
|