Merge pull request 'remove unused ssh.run method' (#818) from Mic92-target_host into main

lib/build-clan/default.nix

@@ -30,7 +30,7 @@ let
       (machineSettings.clanImports or [ ]);
 
   # TODO: remove default system once we have a hardware-config mechanism
-  nixosConfiguration = { system ? "x86_64-linux", name, forceSystem ? false }: nixpkgs.lib.nixosSystem {
+  nixosConfiguration = { system ? "x86_64-linux", name, pkgs ? null }: nixpkgs.lib.nixosSystem {
     modules =
       let
         settings = machineSettings name;
@@ -40,19 +40,21 @@ let
         settings
         clan-core.nixosModules.clanCore
         (machines.${name} or { })
-        {
+        ({
           clanCore.clanName = clanName;
           clanCore.clanIcon = clanIcon;
           clanCore.clanDir = directory;
           clanCore.machineName = name;
-          nixpkgs.hostPlatform = if forceSystem then lib.mkForce system else lib.mkDefault system;
+          nixpkgs.hostPlatform = lib.mkDefault system;
 
           # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
           nix.registry.nixpkgs.to = {
             type = "path";
             path = lib.mkDefault nixpkgs;
           };
-        }
+        } // lib.optionalAttrs (pkgs != null) {
+          nixpkgs.pkgs = lib.mkForce pkgs;
+        })
       ];
     inherit specialArgs;
   };
@@ -75,7 +77,7 @@ let
   configsPerSystem = builtins.listToAttrs
     (builtins.map
       (system: lib.nameValuePair system
-        (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; forceSystem = true; }) allMachines))
+        (lib.mapAttrs (name: _: nixosConfiguration { inherit name system; pkgs = nixpkgs.legacyPackages.${system}; }) allMachines))
       supportedSystems);
 in
 {

pkgs/clan-cli/clan_cli/backups/create.py

@@ -13,7 +13,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
     backup_scripts = json.loads(machine.eval_nix("config.clanCore.backups"))
     if provider is None:
         for provider in backup_scripts["providers"]:
-            proc = machine.host.run(
+            proc = machine.target_host.run(
                 ["bash", "-c", backup_scripts["providers"][provider]["create"]],
             )
             if proc.returncode != 0:
@@ -23,7 +23,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
     else:
         if provider not in backup_scripts["providers"]:
             raise ClanError(f"provider {provider} not found")
-        proc = machine.host.run(
+        proc = machine.target_host.run(
             ["bash", "-c", backup_scripts["providers"][provider]["create"]],
         )
         if proc.returncode != 0:

pkgs/clan-cli/clan_cli/backups/list.py

@@ -19,7 +19,7 @@ class Backup:
 def list_provider(machine: Machine, provider: str) -> list[Backup]:
     results = []
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
-    proc = machine.host.run(
+    proc = machine.target_host.run(
         ["bash", "-c", backup_metadata["providers"][provider]["list"]],
         stdout=subprocess.PIPE,
         check=False,

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -20,7 +20,7 @@ def restore_service(
     env["JOB"] = backup.job_name
     env["FOLDERS"] = ":".join(folders)
 
-    proc = machine.host.run(
+    proc = machine.target_host.run(
         [
             "bash",
             "-c",
@@ -34,7 +34,7 @@ def restore_service(
             f"failed to run preRestoreScript: {backup_folders[service]['preRestoreScript']}, error was: {proc.stdout}"
         )
 
-    proc = machine.host.run(
+    proc = machine.target_host.run(
         [
             "bash",
             "-c",
@@ -48,7 +48,7 @@ def restore_service(
             f"failed to restore backup: {backup_metadata['providers'][provider]['restore']}"
         )
 
-    proc = machine.host.run(
+    proc = machine.target_host.run(
         [
             "bash",
             "-c",

pkgs/clan-cli/clan_cli/machines/install.py

@@ -19,12 +19,10 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None:
     log.info(f"using secret store: {secrets_module.SecretStore}")
     secret_store = secrets_module.SecretStore(machine=machine)
 
-    h = machine.host
+    h = machine.target_host
     target_host = f"{h.user or 'root'}@{h.host}"
     log.info(f"target host: {target_host}")
 
-    flake_attr = h.meta.get("flake_attr", "")
-
     generate_secrets(machine)
 
     with TemporaryDirectory() as tmpdir_:
@@ -40,7 +38,7 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None:
         cmd = [
             "nixos-anywhere",
             "-f",
-            f"{machine.flake}#{flake_attr}",
+            f"{machine.flake}#{machine.name}",
             "-t",
             "--no-reboot",
             "--extra-files",
@@ -75,7 +73,7 @@ def install_command(args: argparse.Namespace) -> None:
         kexec=args.kexec,
     )
     machine = Machine(opts.machine, flake=opts.flake)
-    machine.target_host = opts.target_host
+    machine.target_host_address = opts.target_host
 
     install_nixos(machine, kexec=opts.kexec)
 

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -47,7 +47,7 @@ class Machine:
         return self._deployment_info
 
     @property
-    def target_host(self) -> str:
+    def target_host_address(self) -> str:
         # deploymentAddress is deprecated.
         val = self.deployment_info.get("targetHost") or self.deployment_info.get(
             "deploymentAddress"
@@ -57,8 +57,8 @@ class Machine:
             raise ClanError(msg)
         return val
 
-    @target_host.setter
+    @target_host_address.setter
-    def target_host(self, value: str) -> None:
+    def target_host_address(self, value: str) -> None:
         self.deployment_info["targetHost"] = value
 
     @property
@@ -92,9 +92,26 @@ class Machine:
         return Path(self.flake_path)
 
     @property
-    def host(self) -> Host:
+    def target_host(self) -> Host:
         return parse_deployment_address(
-            self.name, self.target_host, meta={"machine": self}
+            self.name, self.target_host_address, meta={"machine": self}
+        )
+
+    @property
+    def build_host(self) -> Host:
+        """
+        The host where the machine is built and deployed from.
+        Can be the same as the target host.
+        """
+        build_host = self.deployment_info.get("buildHost")
+        if build_host is None:
+            return self.target_host
+        # enable ssh agent forwarding to allow the build host to access the target host
+        return parse_deployment_address(
+            self.name,
+            build_host,
+            forward_agent=True,
+            meta={"machine": self, "target_host": self.target_host},
         )
 
     def eval_nix(self, attr: str, refresh: bool = False) -> str:

pkgs/clan-cli/clan_cli/machines/update.py

@@ -105,16 +105,11 @@ def deploy_nixos(hosts: HostGroup) -> None:
 
         ssh_arg += " -i " + h.key if h.key else ""
 
-        flake_attr = h.meta.get("flake_attr", "")
+        machine: Machine = h.meta["machine"]
 
-        generate_secrets(h.meta["machine"])
+        generate_secrets(machine)
-        upload_secrets(h.meta["machine"])
+        upload_secrets(machine)
 
-        target_host = h.meta.get("target_host")
-        if target_host:
-            target_user = h.meta.get("target_user")
-            if target_user:
-                target_host = f"{target_user}@{target_host}"
         extra_args = h.meta.get("extra_args", [])
         cmd = [
             "nixos-rebuild",
@@ -130,9 +125,10 @@ def deploy_nixos(hosts: HostGroup) -> None:
             "--build-host",
             "",
             "--flake",
-            f"{path}#{flake_attr}",
+            f"{path}#{machine.name}",
         ]
-        if target_host:
+        if target_host := h.meta.get("target_host"):
+            target_host = f"{target_host.user or 'root'}@{target_host.host}"
             cmd.extend(["--target-host", target_host])
         ret = h.run(cmd, check=False)
         # re-retry switch if the first time fails
@@ -157,16 +153,10 @@ def get_all_machines(clan_dir: Path) -> HostGroup:
     for name, machine_data in machines.items():
         machine = Machine(name=name, flake=clan_dir, deployment_info=machine_data)
         try:
-            machine.target_host
+            hosts.append(machine.build_host)
         except ClanError:
             ignored_machines.append(name)
             continue
-        host = parse_deployment_address(
-            name,
-            host=machine.target_host,
-            meta={"machine": machine},
-        )
-        hosts.append(host)
     if not hosts and ignored_machines != []:
         print(
             "WARNING: No machines to update. The following defined machines were ignored because they do not have `clan.networking.targetHost` nixos option set:",
@@ -182,7 +172,7 @@ def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGrou
     hosts = []
     for name in machine_names:
         machine = Machine(name=name, flake=flake_dir)
-        hosts.append(machine.host)
+        hosts.append(machine.build_host)
     return HostGroup(hosts)
 
 
@@ -192,7 +182,7 @@ def update(args: argparse.Namespace) -> None:
         raise ClanError("Could not find clan flake toplevel directory")
     if len(args.machines) == 1 and args.target_host is not None:
         machine = Machine(name=args.machines[0], flake=args.flake)
-        machine.target_host = args.target_host
+        machine.target_host_address = args.target_host
         host = parse_deployment_address(
             args.machines[0],
             args.target_host,

pkgs/clan-cli/clan_cli/secrets/modules/password_store.py

@@ -86,7 +86,7 @@ class SecretStore:
 
     def update_check(self) -> bool:
         local_hash = self.generate_hash()
-        remote_hash = self.machine.host.run(
+        remote_hash = self.machine.target_host.run(
             # TODO get the path to the secrets from the machine
             ["cat", f"{self.machine.secrets_upload_directory}/.pass_info"],
             check=False,

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -22,7 +22,7 @@ def upload_secrets(machine: Machine) -> None:
             return
     with TemporaryDirectory() as tempdir:
         secret_store.upload(Path(tempdir))
-        host = machine.host
+        host = machine.target_host
 
         ssh_cmd = host.ssh_cmd()
         run(

pkgs/clan-cli/clan_cli/ssh/__init__.py

@@ -16,14 +16,7 @@ from enum import Enum
 from pathlib import Path
 from shlex import quote
 from threading import Thread
-from typing import (
+from typing import IO, Any, Generic, TypeVar
-    IO,
-    Any,
-    Generic,
-    Literal,
-    TypeVar,
-    overload,
-)
 
 # https://no-color.org
 DISABLE_COLOR = not sys.stderr.isatty() or os.environ.get("NO_COLOR", "") != ""
@@ -755,7 +748,7 @@ class HostGroup:
 
 
 def parse_deployment_address(
-    machine_name: str, host: str, meta: dict[str, Any] = {}
+    machine_name: str, host: str, forward_agent: bool = True, meta: dict[str, Any] = {}
 ) -> Host:
     parts = host.split("@")
     user: str | None = None
@@ -777,83 +770,12 @@ def parse_deployment_address(
     hostname = result.hostname
     port = result.port
     meta = meta.copy()
-    meta["flake_attr"] = machine_name
     return Host(
         hostname,
         user=user,
         port=port,
         command_prefix=machine_name,
+        forward_agent=forward_agent,
         meta=meta,
         ssh_options=options,
     )
-
-
-@overload
-def run(
-    cmd: list[str] | str,
-    text: Literal[True] = ...,
-    stdout: FILE = ...,
-    stderr: FILE = ...,
-    extra_env: dict[str, str] = ...,
-    cwd: None | str | Path = ...,
-    check: bool = ...,
-) -> subprocess.CompletedProcess[str]:
-    ...
-
-
-@overload
-def run(
-    cmd: list[str] | str,
-    text: Literal[False],
-    stdout: FILE = ...,
-    stderr: FILE = ...,
-    extra_env: dict[str, str] = ...,
-    cwd: None | str | Path = ...,
-    check: bool = ...,
-) -> subprocess.CompletedProcess[bytes]:
-    ...
-
-
-def run(
-    cmd: list[str] | str,
-    text: bool = True,
-    stdout: FILE = None,
-    stderr: FILE = None,
-    extra_env: dict[str, str] = {},
-    cwd: None | str | Path = None,
-    check: bool = True,
-) -> subprocess.CompletedProcess[Any]:
-    """
-    Run command locally
-
-    @cmd if this parameter is a string the command is interpreted as a shell command,
-         otherwise if it is a list, than the first list element is the command
-         and the remaining list elements are passed as arguments to the
-         command.
-    @text when true, file objects for stdout and stderr are opened in text mode.
-    @stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE
-    @stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE
-    @extra_env environment variables to override whe running the command
-    @cwd current working directory to run the process in
-    @check If check is true, and the process exits with a non-zero exit code, a
-           CalledProcessError exception will be raised. Attributes of that exception
-           hold the arguments, the exit code, and stdout and stderr if they were
-           captured.
-    """
-    if isinstance(cmd, list):
-        info("$ " + " ".join(cmd))
-    else:
-        info(f"$ {cmd}")
-    env = os.environ.copy()
-    env.update(extra_env)
-
-    return subprocess.run(
-        cmd,
-        stdout=stdout,
-        stderr=stderr,
-        env=env,
-        cwd=cwd,
-        check=check,
-        shell=not isinstance(cmd, list),
-        text=text,
-    )

pkgs/clan-cli/tests/test_ssh_local.py

@@ -1,32 +1,11 @@
 import subprocess
 
-from clan_cli.ssh import Host, HostGroup, run
+from clan_cli.ssh import Host, HostGroup
-
-
-def test_run() -> None:
-    p = run("echo hello")
-    assert p.stdout is None
-
-
-def test_run_failure() -> None:
-    p = run("exit 1", check=False)
-    assert p.returncode == 1
-
-    try:
-        p = run("exit 1")
-    except Exception:
-        pass
-    else:
-        assert False, "Command should have raised an error"
-
 
 hosts = HostGroup([Host("some_host")])
 
 
 def test_run_environment() -> None:
-    p1 = run("echo $env_var", stdout=subprocess.PIPE, extra_env=dict(env_var="true"))
-    assert p1.stdout == "true\n"
-
     p2 = hosts.run_local(
         "echo $env_var", extra_env=dict(env_var="true"), stdout=subprocess.PIPE
     )
@@ -38,17 +17,6 @@ def test_run_environment() -> None:
     assert "env_var=true" in p3[0].result.stdout
 
 
-def test_run_non_shell() -> None:
-    p = run(["echo", "$hello"], stdout=subprocess.PIPE)
-    assert p.stdout == "$hello\n"
-
-
-def test_run_stderr_stdout() -> None:
-    p = run("echo 1; echo 2 >&2", stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    assert p.stdout == "1\n"
-    assert p.stderr == "2\n"
-
-
 def test_run_local() -> None:
     hosts.run_local("echo hello")