From a2f0d077c87925d7806db3d8c7af1a7c29b7569e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 6 Feb 2024 15:36:11 +0100 Subject: [PATCH 1/6] remove unused ssh.run method --- pkgs/clan-cli/clan_cli/ssh/__init__.py | 73 -------------------------- pkgs/clan-cli/tests/test_ssh_local.py | 34 +----------- 2 files changed, 1 insertion(+), 106 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/ssh/__init__.py b/pkgs/clan-cli/clan_cli/ssh/__init__.py index a05077c7..920c85a0 100644 --- a/pkgs/clan-cli/clan_cli/ssh/__init__.py +++ b/pkgs/clan-cli/clan_cli/ssh/__init__.py @@ -20,9 +20,7 @@ from typing import ( IO, Any, Generic, - Literal, TypeVar, - overload, ) # https://no-color.org @@ -786,74 +784,3 @@ def parse_deployment_address( meta=meta, ssh_options=options, ) - - -@overload -def run( - cmd: list[str] | str, - text: Literal[True] = ..., - stdout: FILE = ..., - stderr: FILE = ..., - extra_env: dict[str, str] = ..., - cwd: None | str | Path = ..., - check: bool = ..., -) -> subprocess.CompletedProcess[str]: - ... - - -@overload -def run( - cmd: list[str] | str, - text: Literal[False], - stdout: FILE = ..., - stderr: FILE = ..., - extra_env: dict[str, str] = ..., - cwd: None | str | Path = ..., - check: bool = ..., -) -> subprocess.CompletedProcess[bytes]: - ... - - -def run( - cmd: list[str] | str, - text: bool = True, - stdout: FILE = None, - stderr: FILE = None, - extra_env: dict[str, str] = {}, - cwd: None | str | Path = None, - check: bool = True, -) -> subprocess.CompletedProcess[Any]: - """ - Run command locally - - @cmd if this parameter is a string the command is interpreted as a shell command, - otherwise if it is a list, than the first list element is the command - and the remaining list elements are passed as arguments to the - command. - @text when true, file objects for stdout and stderr are opened in text mode. - @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE - @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE - @extra_env environment variables to override whe running the command - @cwd current working directory to run the process in - @check If check is true, and the process exits with a non-zero exit code, a - CalledProcessError exception will be raised. Attributes of that exception - hold the arguments, the exit code, and stdout and stderr if they were - captured. - """ - if isinstance(cmd, list): - info("$ " + " ".join(cmd)) - else: - info(f"$ {cmd}") - env = os.environ.copy() - env.update(extra_env) - - return subprocess.run( - cmd, - stdout=stdout, - stderr=stderr, - env=env, - cwd=cwd, - check=check, - shell=not isinstance(cmd, list), - text=text, - ) diff --git a/pkgs/clan-cli/tests/test_ssh_local.py b/pkgs/clan-cli/tests/test_ssh_local.py index 3eee6ce9..99ec1a6f 100644 --- a/pkgs/clan-cli/tests/test_ssh_local.py +++ b/pkgs/clan-cli/tests/test_ssh_local.py @@ -1,32 +1,11 @@ import subprocess -from clan_cli.ssh import Host, HostGroup, run - - -def test_run() -> None: - p = run("echo hello") - assert p.stdout is None - - -def test_run_failure() -> None: - p = run("exit 1", check=False) - assert p.returncode == 1 - - try: - p = run("exit 1") - except Exception: - pass - else: - assert False, "Command should have raised an error" - +from clan_cli.ssh import Host, HostGroup hosts = HostGroup([Host("some_host")]) def test_run_environment() -> None: - p1 = run("echo $env_var", stdout=subprocess.PIPE, extra_env=dict(env_var="true")) - assert p1.stdout == "true\n" - p2 = hosts.run_local( "echo $env_var", extra_env=dict(env_var="true"), stdout=subprocess.PIPE ) @@ -38,17 +17,6 @@ def test_run_environment() -> None: assert "env_var=true" in p3[0].result.stdout -def test_run_non_shell() -> None: - p = run(["echo", "$hello"], stdout=subprocess.PIPE) - assert p.stdout == "$hello\n" - - -def test_run_stderr_stdout() -> None: - p = run("echo 1; echo 2 >&2", stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert p.stdout == "1\n" - assert p.stderr == "2\n" - - def test_run_local() -> None: hosts.run_local("echo hello") From 301a6b6a23e51299a46c87d7e67b6c718064cecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 6 Feb 2024 15:44:07 +0100 Subject: [PATCH 2/6] machines/update: get flake_attr from machine class --- pkgs/clan-cli/clan_cli/machines/install.py | 4 +--- pkgs/clan-cli/clan_cli/machines/update.py | 8 ++++---- pkgs/clan-cli/clan_cli/ssh/__init__.py | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 2d00a1d4..7968dbed 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -23,8 +23,6 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None: target_host = f"{h.user or 'root'}@{h.host}" log.info(f"target host: {target_host}") - flake_attr = h.meta.get("flake_attr", "") - generate_secrets(machine) with TemporaryDirectory() as tmpdir_: @@ -40,7 +38,7 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None: cmd = [ "nixos-anywhere", "-f", - f"{machine.flake}#{flake_attr}", + f"{machine.flake}#{machine.name}", "-t", "--no-reboot", "--extra-files", diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 13fc925e..5db27efa 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -105,10 +105,10 @@ def deploy_nixos(hosts: HostGroup) -> None: ssh_arg += " -i " + h.key if h.key else "" - flake_attr = h.meta.get("flake_attr", "") + machine: Machine = h.meta["machine"] - generate_secrets(h.meta["machine"]) - upload_secrets(h.meta["machine"]) + generate_secrets(machine) + upload_secrets(machine) target_host = h.meta.get("target_host") if target_host: @@ -130,7 +130,7 @@ def deploy_nixos(hosts: HostGroup) -> None: "--build-host", "", "--flake", - f"{path}#{flake_attr}", + f"{path}#{machine.name}", ] if target_host: cmd.extend(["--target-host", target_host]) diff --git a/pkgs/clan-cli/clan_cli/ssh/__init__.py b/pkgs/clan-cli/clan_cli/ssh/__init__.py index 920c85a0..0755e661 100644 --- a/pkgs/clan-cli/clan_cli/ssh/__init__.py +++ b/pkgs/clan-cli/clan_cli/ssh/__init__.py @@ -775,7 +775,6 @@ def parse_deployment_address( hostname = result.hostname port = result.port meta = meta.copy() - meta["flake_attr"] = machine_name return Host( hostname, user=user, From 6e57122da897c21f627309b5c5715f7e51fad706 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 6 Feb 2024 15:54:12 +0100 Subject: [PATCH 3/6] rename target_host to target_host_address --- pkgs/clan-cli/clan_cli/machines/install.py | 2 +- pkgs/clan-cli/clan_cli/machines/machines.py | 8 ++++---- pkgs/clan-cli/clan_cli/machines/update.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 7968dbed..36230b1f 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -73,7 +73,7 @@ def install_command(args: argparse.Namespace) -> None: kexec=args.kexec, ) machine = Machine(opts.machine, flake=opts.flake) - machine.target_host = opts.target_host + machine.target_host_address = opts.target_host install_nixos(machine, kexec=opts.kexec) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 13caca09..26d6e53e 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -47,7 +47,7 @@ class Machine: return self._deployment_info @property - def target_host(self) -> str: + def target_host_address(self) -> str: # deploymentAddress is deprecated. val = self.deployment_info.get("targetHost") or self.deployment_info.get( "deploymentAddress" @@ -57,8 +57,8 @@ class Machine: raise ClanError(msg) return val - @target_host.setter - def target_host(self, value: str) -> None: + @target_host_address.setter + def target_host_address(self, value: str) -> None: self.deployment_info["targetHost"] = value @property @@ -94,7 +94,7 @@ class Machine: @property def host(self) -> Host: return parse_deployment_address( - self.name, self.target_host, meta={"machine": self} + self.name, self.target_host_address, meta={"machine": self} ) def eval_nix(self, attr: str, refresh: bool = False) -> str: diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 5db27efa..5d63e2a0 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -157,13 +157,13 @@ def get_all_machines(clan_dir: Path) -> HostGroup: for name, machine_data in machines.items(): machine = Machine(name=name, flake=clan_dir, deployment_info=machine_data) try: - machine.target_host + machine.target_host_address except ClanError: ignored_machines.append(name) continue host = parse_deployment_address( name, - host=machine.target_host, + host=machine.target_host_address, meta={"machine": machine}, ) hosts.append(host) @@ -192,7 +192,7 @@ def update(args: argparse.Namespace) -> None: raise ClanError("Could not find clan flake toplevel directory") if len(args.machines) == 1 and args.target_host is not None: machine = Machine(name=args.machines[0], flake=args.flake) - machine.target_host = args.target_host + machine.target_host_address = args.target_host host = parse_deployment_address( args.machines[0], args.target_host, From 2315dba2a986193bbfec612daab60661ae865832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 6 Feb 2024 15:55:34 +0100 Subject: [PATCH 4/6] rename machine.host to machine.target_host --- pkgs/clan-cli/clan_cli/backups/create.py | 4 ++-- pkgs/clan-cli/clan_cli/backups/list.py | 2 +- pkgs/clan-cli/clan_cli/backups/restore.py | 6 +++--- pkgs/clan-cli/clan_cli/machines/install.py | 2 +- pkgs/clan-cli/clan_cli/machines/machines.py | 2 +- pkgs/clan-cli/clan_cli/machines/update.py | 2 +- pkgs/clan-cli/clan_cli/secrets/modules/password_store.py | 2 +- pkgs/clan-cli/clan_cli/secrets/upload.py | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/backups/create.py b/pkgs/clan-cli/clan_cli/backups/create.py index b119b58d..b8573794 100644 --- a/pkgs/clan-cli/clan_cli/backups/create.py +++ b/pkgs/clan-cli/clan_cli/backups/create.py @@ -13,7 +13,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None: backup_scripts = json.loads(machine.eval_nix("config.clanCore.backups")) if provider is None: for provider in backup_scripts["providers"]: - proc = machine.host.run( + proc = machine.target_host.run( ["bash", "-c", backup_scripts["providers"][provider]["create"]], ) if proc.returncode != 0: @@ -23,7 +23,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None: else: if provider not in backup_scripts["providers"]: raise ClanError(f"provider {provider} not found") - proc = machine.host.run( + proc = machine.target_host.run( ["bash", "-c", backup_scripts["providers"][provider]["create"]], ) if proc.returncode != 0: diff --git a/pkgs/clan-cli/clan_cli/backups/list.py b/pkgs/clan-cli/clan_cli/backups/list.py index 039bb792..b357cf36 100644 --- a/pkgs/clan-cli/clan_cli/backups/list.py +++ b/pkgs/clan-cli/clan_cli/backups/list.py @@ -19,7 +19,7 @@ class Backup: def list_provider(machine: Machine, provider: str) -> list[Backup]: results = [] backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups")) - proc = machine.host.run( + proc = machine.target_host.run( ["bash", "-c", backup_metadata["providers"][provider]["list"]], stdout=subprocess.PIPE, check=False, diff --git a/pkgs/clan-cli/clan_cli/backups/restore.py b/pkgs/clan-cli/clan_cli/backups/restore.py index 7f168f91..3a24a1f9 100644 --- a/pkgs/clan-cli/clan_cli/backups/restore.py +++ b/pkgs/clan-cli/clan_cli/backups/restore.py @@ -20,7 +20,7 @@ def restore_service( env["JOB"] = backup.job_name env["FOLDERS"] = ":".join(folders) - proc = machine.host.run( + proc = machine.target_host.run( [ "bash", "-c", @@ -34,7 +34,7 @@ def restore_service( f"failed to run preRestoreScript: {backup_folders[service]['preRestoreScript']}, error was: {proc.stdout}" ) - proc = machine.host.run( + proc = machine.target_host.run( [ "bash", "-c", @@ -48,7 +48,7 @@ def restore_service( f"failed to restore backup: {backup_metadata['providers'][provider]['restore']}" ) - proc = machine.host.run( + proc = machine.target_host.run( [ "bash", "-c", diff --git a/pkgs/clan-cli/clan_cli/machines/install.py b/pkgs/clan-cli/clan_cli/machines/install.py index 36230b1f..701303fe 100644 --- a/pkgs/clan-cli/clan_cli/machines/install.py +++ b/pkgs/clan-cli/clan_cli/machines/install.py @@ -19,7 +19,7 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None: log.info(f"using secret store: {secrets_module.SecretStore}") secret_store = secrets_module.SecretStore(machine=machine) - h = machine.host + h = machine.target_host target_host = f"{h.user or 'root'}@{h.host}" log.info(f"target host: {target_host}") diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 26d6e53e..d02319e0 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -92,7 +92,7 @@ class Machine: return Path(self.flake_path) @property - def host(self) -> Host: + def target_host(self) -> Host: return parse_deployment_address( self.name, self.target_host_address, meta={"machine": self} ) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 5d63e2a0..a83b1732 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -182,7 +182,7 @@ def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGrou hosts = [] for name in machine_names: machine = Machine(name=name, flake=flake_dir) - hosts.append(machine.host) + hosts.append(machine.target_host) return HostGroup(hosts) diff --git a/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py b/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py index 20d4abcd..cc06c34b 100644 --- a/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py +++ b/pkgs/clan-cli/clan_cli/secrets/modules/password_store.py @@ -86,7 +86,7 @@ class SecretStore: def update_check(self) -> bool: local_hash = self.generate_hash() - remote_hash = self.machine.host.run( + remote_hash = self.machine.target_host.run( # TODO get the path to the secrets from the machine ["cat", f"{self.machine.secrets_upload_directory}/.pass_info"], check=False, diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py index 796b73f0..420136e3 100644 --- a/pkgs/clan-cli/clan_cli/secrets/upload.py +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -22,7 +22,7 @@ def upload_secrets(machine: Machine) -> None: return with TemporaryDirectory() as tempdir: secret_store.upload(Path(tempdir)) - host = machine.host + host = machine.target_host ssh_cmd = host.ssh_cmd() run( From be3a75bbd77a015d4bd180489d5e6f42772650ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 6 Feb 2024 16:11:55 +0100 Subject: [PATCH 5/6] add support for build machines --- pkgs/clan-cli/clan_cli/machines/machines.py | 17 +++++++++++++++++ pkgs/clan-cli/clan_cli/machines/update.py | 18 ++++-------------- pkgs/clan-cli/clan_cli/ssh/__init__.py | 10 +++------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index d02319e0..07874dc1 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -97,6 +97,23 @@ class Machine: self.name, self.target_host_address, meta={"machine": self} ) + @property + def build_host(self) -> Host: + """ + The host where the machine is built and deployed from. + Can be the same as the target host. + """ + build_host = self.deployment_info.get("buildHost") + if build_host is None: + return self.target_host + # enable ssh agent forwarding to allow the build host to access the target host + return parse_deployment_address( + self.name, + build_host, + forward_agent=True, + meta={"machine": self, "target_host": self.target_host}, + ) + def eval_nix(self, attr: str, refresh: bool = False) -> str: """ eval a nix attribute of the machine diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index a83b1732..49397010 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -110,11 +110,6 @@ def deploy_nixos(hosts: HostGroup) -> None: generate_secrets(machine) upload_secrets(machine) - target_host = h.meta.get("target_host") - if target_host: - target_user = h.meta.get("target_user") - if target_user: - target_host = f"{target_user}@{target_host}" extra_args = h.meta.get("extra_args", []) cmd = [ "nixos-rebuild", @@ -132,7 +127,8 @@ def deploy_nixos(hosts: HostGroup) -> None: "--flake", f"{path}#{machine.name}", ] - if target_host: + if target_host := h.meta.get("target_host"): + target_host = f"{target_host.user or 'root'}@{target_host.host}" cmd.extend(["--target-host", target_host]) ret = h.run(cmd, check=False) # re-retry switch if the first time fails @@ -157,16 +153,10 @@ def get_all_machines(clan_dir: Path) -> HostGroup: for name, machine_data in machines.items(): machine = Machine(name=name, flake=clan_dir, deployment_info=machine_data) try: - machine.target_host_address + hosts.append(machine.build_host) except ClanError: ignored_machines.append(name) continue - host = parse_deployment_address( - name, - host=machine.target_host_address, - meta={"machine": machine}, - ) - hosts.append(host) if not hosts and ignored_machines != []: print( "WARNING: No machines to update. The following defined machines were ignored because they do not have `clan.networking.targetHost` nixos option set:", @@ -182,7 +172,7 @@ def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGrou hosts = [] for name in machine_names: machine = Machine(name=name, flake=flake_dir) - hosts.append(machine.target_host) + hosts.append(machine.build_host) return HostGroup(hosts) diff --git a/pkgs/clan-cli/clan_cli/ssh/__init__.py b/pkgs/clan-cli/clan_cli/ssh/__init__.py index 0755e661..b0c3b3f0 100644 --- a/pkgs/clan-cli/clan_cli/ssh/__init__.py +++ b/pkgs/clan-cli/clan_cli/ssh/__init__.py @@ -16,12 +16,7 @@ from enum import Enum from pathlib import Path from shlex import quote from threading import Thread -from typing import ( - IO, - Any, - Generic, - TypeVar, -) +from typing import IO, Any, Generic, TypeVar # https://no-color.org DISABLE_COLOR = not sys.stderr.isatty() or os.environ.get("NO_COLOR", "") != "" @@ -753,7 +748,7 @@ class HostGroup: def parse_deployment_address( - machine_name: str, host: str, meta: dict[str, Any] = {} + machine_name: str, host: str, forward_agent: bool = True, meta: dict[str, Any] = {} ) -> Host: parts = host.split("@") user: str | None = None @@ -780,6 +775,7 @@ def parse_deployment_address( user=user, port=port, command_prefix=machine_name, + forward_agent=forward_agent, meta=meta, ssh_options=options, ) From 614d1aecfd736594c9dc16abf837ee95bf20416f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 6 Feb 2024 16:47:57 +0100 Subject: [PATCH 6/6] set nixpkgs.pkgs for secrets generation This allows us to use the same nixpkgs instance for all machines. --- lib/build-clan/default.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 9625689d..37110c0d 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -30,7 +30,7 @@ let (machineSettings.clanImports or [ ]); # TODO: remove default system once we have a hardware-config mechanism - nixosConfiguration = { system ? "x86_64-linux", name, forceSystem ? false }: nixpkgs.lib.nixosSystem { + nixosConfiguration = { system ? "x86_64-linux", name, pkgs ? null }: nixpkgs.lib.nixosSystem { modules = let settings = machineSettings name; @@ -40,19 +40,21 @@ let settings clan-core.nixosModules.clanCore (machines.${name} or { }) - { + ({ clanCore.clanName = clanName; clanCore.clanIcon = clanIcon; clanCore.clanDir = directory; clanCore.machineName = name; - nixpkgs.hostPlatform = if forceSystem then lib.mkForce system else lib.mkDefault system; + nixpkgs.hostPlatform = lib.mkDefault system; # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs) nix.registry.nixpkgs.to = { type = "path"; path = lib.mkDefault nixpkgs; }; - } + } // lib.optionalAttrs (pkgs != null) { + nixpkgs.pkgs = lib.mkForce pkgs; + }) ]; inherit specialArgs; }; @@ -75,7 +77,7 @@ let configsPerSystem = builtins.listToAttrs (builtins.map (system: lib.nameValuePair system - (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; forceSystem = true; }) allMachines)) + (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; pkgs = nixpkgs.legacyPackages.${system}; }) allMachines)) supportedSystems); in {