Merge pull request 'vars: implement dependencies' (#1771) from DavHau/clan-core:DavHau-vars into main
All checks were successful
deploy / deploy-docs (push) Successful in 20s
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-avahi Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bubblewrap Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-gnupg Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-disko Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-mypy Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-pass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-util-linux Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-virtiofsd Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nixos-anywhere Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-qemu Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-examples-cue Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-full Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.x86_64-linux.module-clan-vars-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema-pretty Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.template-minimal Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-eval Build done.
checks / checks-impure (push) Successful in 2m0s
All checks were successful
deploy / deploy-docs (push) Successful in 20s
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-avahi Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bubblewrap Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-gnupg Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-disko Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-mypy Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-pass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-util-linux Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-virtiofsd Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nixos-anywhere Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-qemu Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-examples-cue Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-full Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.x86_64-linux.module-clan-vars-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema-pretty Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.template-minimal Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-eval Build done.
checks / checks-impure (push) Successful in 2m0s
This commit is contained in:
commit
af04e513a0
@ -39,7 +39,7 @@ in
|
||||
vars = {
|
||||
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
|
||||
_name: generator: {
|
||||
inherit (generator) finalScript;
|
||||
inherit (generator) dependencies finalScript;
|
||||
files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
|
||||
}
|
||||
);
|
||||
|
@ -44,7 +44,7 @@ in
|
||||
description = ''
|
||||
A list of other generators that this generator depends on.
|
||||
The output values of these generators will be available to the generator script as files.
|
||||
For example, the file 'file1' of a dependency named 'dep1' will be available via $dependencies/dep1/file1.
|
||||
For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1.
|
||||
'';
|
||||
type = listOf str;
|
||||
default = [ ];
|
||||
@ -147,7 +147,7 @@ in
|
||||
description = ''
|
||||
The script to run to generate the files.
|
||||
The script will be run with the following environment variables:
|
||||
- $dependencies: The directory containing the output values of all declared dependencies
|
||||
- $in: The directory containing the output values of all declared dependencies
|
||||
- $out: The output directory to put the generated files
|
||||
- $prompts: The directory containing the prompted values as files
|
||||
The script should produce the files specified in the 'files' attribute under $out.
|
||||
|
@ -5,6 +5,7 @@ import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from graphlib import TopologicalSorter
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
@ -37,7 +38,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
|
||||
def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> list[str]:
|
||||
# fmt: off
|
||||
return run_cmd(
|
||||
[
|
||||
@ -50,6 +51,7 @@ def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
|
||||
"--tmpfs", "/usr/lib/systemd",
|
||||
"--dev", "/dev",
|
||||
"--bind", str(generator_dir), str(generator_dir),
|
||||
"--ro-bind", str(dep_tmpdir), str(dep_tmpdir),
|
||||
"--unshare-all",
|
||||
"--unshare-user",
|
||||
"--uid", "1000",
|
||||
@ -60,16 +62,58 @@ def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
|
||||
# fmt: on
|
||||
|
||||
|
||||
# TODO: implement caching to not decrypt the same secret multiple times
|
||||
def decrypt_dependencies(
|
||||
machine: Machine,
|
||||
generator_name: str,
|
||||
secret_vars_store: SecretStoreBase,
|
||||
public_vars_store: FactStoreBase,
|
||||
) -> dict[str, dict[str, bytes]]:
|
||||
generator = machine.vars_generators[generator_name]
|
||||
dependencies = set(generator["dependencies"])
|
||||
decrypted_dependencies = {}
|
||||
for dep_generator in dependencies:
|
||||
decrypted_dependencies[dep_generator] = {}
|
||||
dep_files = machine.vars_generators[dep_generator]["files"]
|
||||
for file_name, file in dep_files.items():
|
||||
if file["secret"]:
|
||||
decrypted_dependencies[dep_generator][file_name] = (
|
||||
secret_vars_store.get(dep_generator, file_name)
|
||||
)
|
||||
else:
|
||||
decrypted_dependencies[dep_generator][file_name] = (
|
||||
public_vars_store.get(dep_generator, file_name)
|
||||
)
|
||||
return decrypted_dependencies
|
||||
|
||||
|
||||
# decrypt dependencies and return temporary file tree
|
||||
def dependencies_as_dir(
|
||||
decrypted_dependencies: dict[str, dict[str, bytes]],
|
||||
tmpdir: Path,
|
||||
) -> Path:
|
||||
for dep_generator, files in decrypted_dependencies.items():
|
||||
dep_generator_dir = tmpdir / dep_generator
|
||||
dep_generator_dir.mkdir()
|
||||
dep_generator_dir.chmod(0o700)
|
||||
for file_name, file in files.items():
|
||||
file_path = dep_generator_dir / file_name
|
||||
file_path.touch()
|
||||
file_path.chmod(0o600)
|
||||
file_path.write_bytes(file)
|
||||
return tmpdir
|
||||
|
||||
|
||||
def execute_generator(
|
||||
machine: Machine,
|
||||
generator_name: str,
|
||||
regenerate: bool,
|
||||
secret_vars_store: SecretStoreBase,
|
||||
public_vars_store: FactStoreBase,
|
||||
tmpdir: Path,
|
||||
dep_tmpdir: Path,
|
||||
prompt: Callable[[str], str],
|
||||
) -> bool:
|
||||
generator_dir = tmpdir / generator_name
|
||||
generator_dir = dep_tmpdir / generator_name
|
||||
# check if all secrets exist and generate them if at least one is missing
|
||||
needs_regeneration = not check_secrets(machine, generator_name=generator_name)
|
||||
log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}")
|
||||
@ -79,22 +123,30 @@ def execute_generator(
|
||||
msg = f"flake is not a Path: {machine.flake}"
|
||||
msg += "fact/secret generation is only supported for local flakes"
|
||||
|
||||
env = os.environ.copy()
|
||||
generator_dir.mkdir(parents=True)
|
||||
env["out"] = str(generator_dir)
|
||||
# compatibility for old outputs.nix users
|
||||
generator = machine.vars_generators[generator_name]["finalScript"]
|
||||
# if machine.vars_data[generator_name]["generator"]["prompt"]:
|
||||
# prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"])
|
||||
# env["prompt_value"] = prompt_value
|
||||
if sys.platform == "linux":
|
||||
cmd = bubblewrap_cmd(generator, generator_dir)
|
||||
else:
|
||||
cmd = ["bash", "-c", generator]
|
||||
run(
|
||||
cmd,
|
||||
env=env,
|
||||
|
||||
# build temporary file tree of dependencies
|
||||
decrypted_dependencies = decrypt_dependencies(
|
||||
machine, generator_name, secret_vars_store, public_vars_store
|
||||
)
|
||||
env = os.environ.copy()
|
||||
generator_dir.mkdir(parents=True)
|
||||
env["out"] = str(generator_dir)
|
||||
with TemporaryDirectory() as tmp:
|
||||
dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmp))
|
||||
env["in"] = str(dep_tmpdir)
|
||||
if sys.platform == "linux":
|
||||
cmd = bubblewrap_cmd(generator, generator_dir, dep_tmpdir=dep_tmpdir)
|
||||
else:
|
||||
cmd = ["bash", "-c", generator]
|
||||
run(
|
||||
cmd,
|
||||
env=env,
|
||||
)
|
||||
files_to_commit = []
|
||||
# store secrets
|
||||
files = machine.vars_generators[generator_name]["files"]
|
||||
@ -129,6 +181,17 @@ def prompt_func(text: str) -> str:
|
||||
return read_multiline_input()
|
||||
|
||||
|
||||
def _get_subgraph(graph: dict[str, set], vertex: str) -> dict[str, set]:
|
||||
visited = set()
|
||||
queue = [vertex]
|
||||
while queue:
|
||||
vertex = queue.pop(0)
|
||||
if vertex not in visited:
|
||||
visited.add(vertex)
|
||||
queue.extend(graph[vertex] - visited)
|
||||
return {k: v for k, v in graph.items() if k in visited}
|
||||
|
||||
|
||||
def _generate_vars_for_machine(
|
||||
machine: Machine,
|
||||
generator_name: str | None,
|
||||
@ -152,21 +215,40 @@ def _generate_vars_for_machine(
|
||||
f"Could not find generator with name: {generator_name}. The following generators are available: {generators}"
|
||||
)
|
||||
|
||||
if generator_name:
|
||||
machine_generator_facts = {
|
||||
generator_name: machine.vars_generators[generator_name]
|
||||
}
|
||||
else:
|
||||
machine_generator_facts = machine.vars_generators
|
||||
# if generator_name:
|
||||
# machine_generator_facts = {
|
||||
# generator_name: machine.vars_generators[generator_name]
|
||||
# }
|
||||
# else:
|
||||
# machine_generator_facts = machine.vars_generators
|
||||
|
||||
for generator_name in machine_generator_facts:
|
||||
graph = {
|
||||
gen_name: set(generator["dependencies"])
|
||||
for gen_name, generator in machine.vars_generators.items()
|
||||
}
|
||||
|
||||
# extract sub-graph if specific generator selected
|
||||
if generator_name:
|
||||
graph = _get_subgraph(graph, generator_name)
|
||||
|
||||
# check if all dependencies actually exist
|
||||
for gen_name, dependencies in graph.items():
|
||||
for dep in dependencies:
|
||||
if dep not in graph:
|
||||
raise ClanError(
|
||||
f"Generator {gen_name} has a dependency on {dep}, which does not exist"
|
||||
)
|
||||
|
||||
# process generators in topological order
|
||||
sorter = TopologicalSorter(graph)
|
||||
for generator_name in sorter.static_order():
|
||||
machine_updated |= execute_generator(
|
||||
machine=machine,
|
||||
generator_name=generator_name,
|
||||
regenerate=regenerate,
|
||||
secret_vars_store=secret_vars_store,
|
||||
public_vars_store=public_vars_store,
|
||||
tmpdir=local_temp,
|
||||
dep_tmpdir=local_temp,
|
||||
prompt=prompt,
|
||||
)
|
||||
if machine_updated:
|
||||
|
@ -2,6 +2,7 @@ import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@ -23,6 +24,50 @@ def def_value() -> defaultdict:
|
||||
nested_dict: Callable[[], dict[str, Any]] = lambda: defaultdict(def_value)
|
||||
|
||||
|
||||
def test_get_subgraph() -> None:
|
||||
from clan_cli.vars.generate import _get_subgraph
|
||||
|
||||
graph = dict(
|
||||
a={"b", "c"},
|
||||
b={"c"},
|
||||
c=set(),
|
||||
d=set(),
|
||||
)
|
||||
assert _get_subgraph(graph, "a") == {
|
||||
"a": {"b", "c"},
|
||||
"b": {"c"},
|
||||
"c": set(),
|
||||
}
|
||||
assert _get_subgraph(graph, "b") == {"b": {"c"}, "c": set()}
|
||||
|
||||
|
||||
def test_dependencies_as_files() -> None:
|
||||
from clan_cli.vars.generate import dependencies_as_dir
|
||||
|
||||
decrypted_dependencies = dict(
|
||||
gen_1=dict(
|
||||
var_1a=b"var_1a",
|
||||
var_1b=b"var_1b",
|
||||
),
|
||||
gen_2=dict(
|
||||
var_2a=b"var_2a",
|
||||
var_2b=b"var_2b",
|
||||
),
|
||||
)
|
||||
with TemporaryDirectory() as tmpdir:
|
||||
dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmpdir))
|
||||
assert dep_tmpdir.is_dir()
|
||||
assert (dep_tmpdir / "gen_1" / "var_1a").read_bytes() == b"var_1a"
|
||||
assert (dep_tmpdir / "gen_1" / "var_1b").read_bytes() == b"var_1b"
|
||||
assert (dep_tmpdir / "gen_2" / "var_2a").read_bytes() == b"var_2a"
|
||||
assert (dep_tmpdir / "gen_2" / "var_2b").read_bytes() == b"var_2b"
|
||||
# ensure the files are not world readable
|
||||
assert (dep_tmpdir / "gen_1" / "var_1a").stat().st_mode & 0o777 == 0o600
|
||||
assert (dep_tmpdir / "gen_1" / "var_1b").stat().st_mode & 0o777 == 0o600
|
||||
assert (dep_tmpdir / "gen_2" / "var_2a").stat().st_mode & 0o777 == 0o600
|
||||
assert (dep_tmpdir / "gen_2" / "var_2b").stat().st_mode & 0o777 == 0o600
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_generate_public_var(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@ -155,3 +200,35 @@ def test_generate_secret_for_multiple_machines(
|
||||
assert sops_store2.exists("my_generator", "my_secret")
|
||||
assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n"
|
||||
assert sops_store2.get("my_generator", "my_secret").decode() == "machine2\n"
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_dependant_generators(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
temporary_home: Path,
|
||||
) -> None:
|
||||
config = nested_dict()
|
||||
parent_gen = config["clan"]["core"]["vars"]["generators"]["parent_generator"]
|
||||
parent_gen["files"]["my_value"]["secret"] = False
|
||||
parent_gen["script"] = "echo hello > $out/my_value"
|
||||
child_gen = config["clan"]["core"]["vars"]["generators"]["child_generator"]
|
||||
child_gen["files"]["my_value"]["secret"] = False
|
||||
child_gen["dependencies"] = ["parent_generator"]
|
||||
child_gen["script"] = "cat $in/parent_generator/my_value > $out/my_value"
|
||||
flake = generate_flake(
|
||||
temporary_home,
|
||||
flake_template=CLAN_CORE / "templates" / "minimal",
|
||||
machine_configs=dict(my_machine=config),
|
||||
)
|
||||
monkeypatch.chdir(flake.path)
|
||||
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
|
||||
parent_file_path = (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value"
|
||||
)
|
||||
assert parent_file_path.is_file()
|
||||
assert parent_file_path.read_text() == "hello\n"
|
||||
child_file_path = (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "child_generator" / "my_value"
|
||||
)
|
||||
assert child_file_path.is_file()
|
||||
assert child_file_path.read_text() == "hello\n"
|
||||
|
Loading…
Reference in New Issue
Block a user