DavHau/clan-core
1
0
Fork 0
forked from clan/clan-core
Code Pull Requests Activity

Merge pull request 'remove unused ssh.run method' (#818) from Mic92-target_host into main

Browse Source
This commit is contained in:
clan-bot 2024-02-06 16:24:47 +00:00
commit e912b125c3
11 changed files with 53 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
lib/build-clan/default.nix
View File

@ -30,7 +30,7 @@ let
(machineSettings.clanImports or [ ]);
# TODO: remove default system once we have a hardware-config mechanism
nixosConfiguration = { system ? "x86_64-linux", name, forceSystem ? false }: nixpkgs.lib.nixosSystem {
nixosConfiguration = { system ? "x86_64-linux", name, pkgs ? null }: nixpkgs.lib.nixosSystem {
modules =
let
settings = machineSettings name;
@ -40,19 +40,21 @@ let
settings
clan-core.nixosModules.clanCore
(machines.${name} or { })
{
({
clanCore.clanName = clanName;
clanCore.clanIcon = clanIcon;
clanCore.clanDir = directory;
clanCore.machineName = name;
nixpkgs.hostPlatform = if forceSystem then lib.mkForce system else lib.mkDefault system;
nixpkgs.hostPlatform = lib.mkDefault system;
# speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
nix.registry.nixpkgs.to = {
type = "path";
path = lib.mkDefault nixpkgs;
};
}
} // lib.optionalAttrs (pkgs != null) {
nixpkgs.pkgs = lib.mkForce pkgs;
})
];
inherit specialArgs;
};
@ -75,7 +77,7 @@ let
configsPerSystem = builtins.listToAttrs
(builtins.map
(system: lib.nameValuePair system
(lib.mapAttrs (name: _: nixosConfiguration { inherit name system; forceSystem = true; }) allMachines))
(lib.mapAttrs (name: _: nixosConfiguration { inherit name system; pkgs = nixpkgs.legacyPackages.${system}; }) allMachines))
supportedSystems);
in
{

4
pkgs/clan-cli/clan_cli/backups/create.py
View File

@ -13,7 +13,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
backup_scripts = json.loads(machine.eval_nix("config.clanCore.backups"))
if provider is None:
for provider in backup_scripts["providers"]:
proc = machine.host.run(
proc = machine.target_host.run(
["bash", "-c", backup_scripts["providers"][provider]["create"]],
)
if proc.returncode != 0:
@ -23,7 +23,7 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
else:
if provider not in backup_scripts["providers"]:
raise ClanError(f"provider {provider} not found")
proc = machine.host.run(
proc = machine.target_host.run(
["bash", "-c", backup_scripts["providers"][provider]["create"]],
)
if proc.returncode != 0:

2
pkgs/clan-cli/clan_cli/backups/list.py
View File

@ -19,7 +19,7 @@ class Backup:
def list_provider(machine: Machine, provider: str) -> list[Backup]:
results = []
backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
proc = machine.host.run(
proc = machine.target_host.run(
["bash", "-c", backup_metadata["providers"][provider]["list"]],
stdout=subprocess.PIPE,
check=False,

6
pkgs/clan-cli/clan_cli/backups/restore.py
View File

@ -20,7 +20,7 @@ def restore_service(
env["JOB"] = backup.job_name
env["FOLDERS"] = ":".join(folders)
proc = machine.host.run(
proc = machine.target_host.run(
[
"bash",
"-c",
@ -34,7 +34,7 @@ def restore_service(
f"failed to run preRestoreScript: {backup_folders[service]['preRestoreScript']}, error was: {proc.stdout}"
)
proc = machine.host.run(
proc = machine.target_host.run(
[
"bash",
"-c",
@ -48,7 +48,7 @@ def restore_service(
f"failed to restore backup: {backup_metadata['providers'][provider]['restore']}"
)
proc = machine.host.run(
proc = machine.target_host.run(
[
"bash",
"-c",

8
pkgs/clan-cli/clan_cli/machines/install.py
View File

@ -19,12 +19,10 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None:
log.info(f"using secret store: {secrets_module.SecretStore}")
secret_store = secrets_module.SecretStore(machine=machine)
h = machine.host
h = machine.target_host
target_host = f"{h.user or 'root'}@{h.host}"
log.info(f"target host: {target_host}")
flake_attr = h.meta.get("flake_attr", "")
generate_secrets(machine)
with TemporaryDirectory() as tmpdir_:
@ -40,7 +38,7 @@ def install_nixos(machine: Machine, kexec: str | None = None) -> None:
cmd = [
"nixos-anywhere",
"-f",
f"{machine.flake}#{flake_attr}",
f"{machine.flake}#{machine.name}",
"-t",
"--no-reboot",
"--extra-files",
@ -75,7 +73,7 @@ def install_command(args: argparse.Namespace) -> None:
kexec=args.kexec,
)
machine = Machine(opts.machine, flake=opts.flake)
machine.target_host = opts.target_host
machine.target_host_address = opts.target_host
install_nixos(machine, kexec=opts.kexec)

27
pkgs/clan-cli/clan_cli/machines/machines.py
View File

@ -47,7 +47,7 @@ class Machine:
return self._deployment_info
@property
def target_host(self) -> str:
def target_host_address(self) -> str:
# deploymentAddress is deprecated.
val = self.deployment_info.get("targetHost") or self.deployment_info.get(
"deploymentAddress"
@ -57,8 +57,8 @@ class Machine:
raise ClanError(msg)
return val
@target_host.setter
def target_host(self, value: str) -> None:
@target_host_address.setter
def target_host_address(self, value: str) -> None:
self.deployment_info["targetHost"] = value
@property
@ -92,9 +92,26 @@ class Machine:
return Path(self.flake_path)
@property
def host(self) -> Host:
def target_host(self) -> Host:
return parse_deployment_address(
self.name, self.target_host, meta={"machine": self}
self.name, self.target_host_address, meta={"machine": self}
)
@property
def build_host(self) -> Host:
"""
The host where the machine is built and deployed from.
Can be the same as the target host.
"""
build_host = self.deployment_info.get("buildHost")
if build_host is None:
return self.target_host
# enable ssh agent forwarding to allow the build host to access the target host
return parse_deployment_address(
self.name,
build_host,
forward_agent=True,
meta={"machine": self, "target_host": self.target_host},
)
def eval_nix(self, attr: str, refresh: bool = False) -> str:

28
pkgs/clan-cli/clan_cli/machines/update.py
View File

@ -105,16 +105,11 @@ def deploy_nixos(hosts: HostGroup) -> None:
ssh_arg += " -i " + h.key if h.key else ""
flake_attr = h.meta.get("flake_attr", "")
machine: Machine = h.meta["machine"]
generate_secrets(h.meta["machine"])
upload_secrets(h.meta["machine"])
generate_secrets(machine)
upload_secrets(machine)
target_host = h.meta.get("target_host")
if target_host:
target_user = h.meta.get("target_user")
if target_user:
target_host = f"{target_user}@{target_host}"
extra_args = h.meta.get("extra_args", [])
cmd = [
"nixos-rebuild",
@ -130,9 +125,10 @@ def deploy_nixos(hosts: HostGroup) -> None:
"--build-host",
"",
"--flake",
f"{path}#{flake_attr}",
f"{path}#{machine.name}",
]
if target_host:
if target_host := h.meta.get("target_host"):
target_host = f"{target_host.user or 'root'}@{target_host.host}"
cmd.extend(["--target-host", target_host])
ret = h.run(cmd, check=False)
# re-retry switch if the first time fails
@ -157,16 +153,10 @@ def get_all_machines(clan_dir: Path) -> HostGroup:
for name, machine_data in machines.items():
machine = Machine(name=name, flake=clan_dir, deployment_info=machine_data)
try:
machine.target_host
hosts.append(machine.build_host)
except ClanError:
ignored_machines.append(name)
continue
host = parse_deployment_address(
name,
host=machine.target_host,
meta={"machine": machine},
)
hosts.append(host)
if not hosts and ignored_machines != []:
print(
"WARNING: No machines to update. The following defined machines were ignored because they do not have `clan.networking.targetHost` nixos option set:",
@ -182,7 +172,7 @@ def get_selected_machines(machine_names: list[str], flake_dir: Path) -> HostGrou
hosts = []
for name in machine_names:
machine = Machine(name=name, flake=flake_dir)
hosts.append(machine.host)
hosts.append(machine.build_host)
return HostGroup(hosts)
@ -192,7 +182,7 @@ def update(args: argparse.Namespace) -> None:
raise ClanError("Could not find clan flake toplevel directory")
if len(args.machines) == 1 and args.target_host is not None:
machine = Machine(name=args.machines[0], flake=args.flake)
machine.target_host = args.target_host
machine.target_host_address = args.target_host
host = parse_deployment_address(
args.machines[0],
args.target_host,

2
pkgs/clan-cli/clan_cli/secrets/modules/password_store.py
View File

@ -86,7 +86,7 @@ class SecretStore:
def update_check(self) -> bool:
local_hash = self.generate_hash()
remote_hash = self.machine.host.run(
remote_hash = self.machine.target_host.run(
# TODO get the path to the secrets from the machine
["cat", f"{self.machine.secrets_upload_directory}/.pass_info"],
check=False,

2
pkgs/clan-cli/clan_cli/secrets/upload.py
View File

@ -22,7 +22,7 @@ def upload_secrets(machine: Machine) -> None:
return
with TemporaryDirectory() as tempdir:
secret_store.upload(Path(tempdir))
host = machine.host
host = machine.target_host
ssh_cmd = host.ssh_cmd()
run(

84
pkgs/clan-cli/clan_cli/ssh/__init__.py
View File

@ -16,14 +16,7 @@ from enum import Enum
from pathlib import Path
from shlex import quote
from threading import Thread
from typing import (
IO,
Any,
Generic,
Literal,
TypeVar,
overload,
)
from typing import IO, Any, Generic, TypeVar
# https://no-color.org
DISABLE_COLOR = not sys.stderr.isatty() or os.environ.get("NO_COLOR", "") != ""
@ -755,7 +748,7 @@ class HostGroup:
def parse_deployment_address(
machine_name: str, host: str, meta: dict[str, Any] = {}
machine_name: str, host: str, forward_agent: bool = True, meta: dict[str, Any] = {}
) -> Host:
parts = host.split("@")
user: str | None = None
@ -777,83 +770,12 @@ def parse_deployment_address(
hostname = result.hostname
port = result.port
meta = meta.copy()
meta["flake_attr"] = machine_name
return Host(
hostname,
user=user,
port=port,
command_prefix=machine_name,
forward_agent=forward_agent,
meta=meta,
ssh_options=options,
)
@overload
def run(
cmd: list[str] | str,
text: Literal[True] = ...,
stdout: FILE = ...,
stderr: FILE = ...,
extra_env: dict[str, str] = ...,
cwd: None | str | Path = ...,
check: bool = ...,
) -> subprocess.CompletedProcess[str]:
...
@overload
def run(
cmd: list[str] | str,
text: Literal[False],
stdout: FILE = ...,
stderr: FILE = ...,
extra_env: dict[str, str] = ...,
cwd: None | str | Path = ...,
check: bool = ...,
) -> subprocess.CompletedProcess[bytes]:
...
def run(
cmd: list[str] | str,
text: bool = True,
stdout: FILE = None,
stderr: FILE = None,
extra_env: dict[str, str] = {},
cwd: None | str | Path = None,
check: bool = True,
) -> subprocess.CompletedProcess[Any]:
"""
Run command locally
@cmd if this parameter is a string the command is interpreted as a shell command,
otherwise if it is a list, than the first list element is the command
and the remaining list elements are passed as arguments to the
command.
@text when true, file objects for stdout and stderr are opened in text mode.
@stdout if not None stdout of the command will be redirected to this file i.e. stdout=subprocss.PIPE
@stderr if not None stderr of the command will be redirected to this file i.e. stderr=subprocess.PIPE
@extra_env environment variables to override whe running the command
@cwd current working directory to run the process in
@check If check is true, and the process exits with a non-zero exit code, a
CalledProcessError exception will be raised. Attributes of that exception
hold the arguments, the exit code, and stdout and stderr if they were
captured.
"""
if isinstance(cmd, list):
info("$ " + " ".join(cmd))
else:
info(f"$ {cmd}")
env = os.environ.copy()
env.update(extra_env)
return subprocess.run(
cmd,
stdout=stdout,
stderr=stderr,
env=env,
cwd=cwd,
check=check,
shell=not isinstance(cmd, list),
text=text,
)

34
pkgs/clan-cli/tests/test_ssh_local.py
View File

@ -1,32 +1,11 @@
import subprocess
from clan_cli.ssh import Host, HostGroup, run
def test_run() -> None:
p = run("echo hello")
assert p.stdout is None
def test_run_failure() -> None:
p = run("exit 1", check=False)
assert p.returncode == 1
try:
p = run("exit 1")
except Exception:
pass
else:
assert False, "Command should have raised an error"
from clan_cli.ssh import Host, HostGroup
hosts = HostGroup([Host("some_host")])
def test_run_environment() -> None:
p1 = run("echo $env_var", stdout=subprocess.PIPE, extra_env=dict(env_var="true"))
assert p1.stdout == "true\n"
p2 = hosts.run_local(
"echo $env_var", extra_env=dict(env_var="true"), stdout=subprocess.PIPE
)
@ -38,17 +17,6 @@ def test_run_environment() -> None:
assert "env_var=true" in p3[0].result.stdout
def test_run_non_shell() -> None:
p = run(["echo", "$hello"], stdout=subprocess.PIPE)
assert p.stdout == "$hello\n"
def test_run_stderr_stdout() -> None:
p = run("echo 1; echo 2 >&2", stdout=subprocess.PIPE, stderr=subprocess.PIPE)
assert p.stdout == "1\n"
assert p.stderr == "2\n"
def test_run_local() -> None:
hosts.run_local("echo hello")