forked from clan/clan-core
Merge pull request 'cmd.py refactor part 4' (#707) from Qubasa-main into main
This commit is contained in:
commit
04b579f2d3
@ -12,7 +12,7 @@
|
|||||||
]}"
|
]}"
|
||||||
ROOT=$(git rev-parse --show-toplevel)
|
ROOT=$(git rev-parse --show-toplevel)
|
||||||
cd "$ROOT/pkgs/clan-cli"
|
cd "$ROOT/pkgs/clan-cli"
|
||||||
nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -m impure ./tests $@"
|
nix develop "$ROOT#clan-cli" -c bash -c "TMPDIR=/tmp python -m pytest -s -m impure ./tests $@"
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -124,6 +124,8 @@ def main() -> None:
|
|||||||
if args.debug:
|
if args.debug:
|
||||||
setup_logging(logging.DEBUG)
|
setup_logging(logging.DEBUG)
|
||||||
log.debug("Debug log activated")
|
log.debug("Debug log activated")
|
||||||
|
else:
|
||||||
|
setup_logging(logging.INFO)
|
||||||
|
|
||||||
if not hasattr(args, "func"):
|
if not hasattr(args, "func"):
|
||||||
return
|
return
|
||||||
|
@ -8,9 +8,10 @@ from enum import Enum
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import IO, Any
|
from typing import IO, Any
|
||||||
|
|
||||||
|
from .custom_logger import get_caller
|
||||||
from .errors import ClanCmdError, CmdOut
|
from .errors import ClanCmdError, CmdOut
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
glog = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class Log(Enum):
|
class Log(Enum):
|
||||||
@ -38,12 +39,14 @@ def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]:
|
|||||||
ret = handle_fd(process.stdout)
|
ret = handle_fd(process.stdout)
|
||||||
if log in [Log.STDOUT, Log.BOTH]:
|
if log in [Log.STDOUT, Log.BOTH]:
|
||||||
sys.stdout.buffer.write(ret)
|
sys.stdout.buffer.write(ret)
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
stdout_buf += ret
|
stdout_buf += ret
|
||||||
ret = handle_fd(process.stderr)
|
ret = handle_fd(process.stderr)
|
||||||
|
|
||||||
if log in [Log.STDERR, Log.BOTH]:
|
if log in [Log.STDERR, Log.BOTH]:
|
||||||
sys.stderr.buffer.write(ret)
|
sys.stderr.buffer.write(ret)
|
||||||
|
sys.stderr.flush()
|
||||||
stderr_buf += ret
|
stderr_buf += ret
|
||||||
return stdout_buf.decode("utf-8"), stderr_buf.decode("utf-8")
|
return stdout_buf.decode("utf-8"), stderr_buf.decode("utf-8")
|
||||||
|
|
||||||
@ -55,7 +58,9 @@ def run(
|
|||||||
cwd: Path = Path.cwd(),
|
cwd: Path = Path.cwd(),
|
||||||
log: Log = Log.STDERR,
|
log: Log = Log.STDERR,
|
||||||
check: bool = True,
|
check: bool = True,
|
||||||
|
error_msg: str | None = None,
|
||||||
) -> CmdOut:
|
) -> CmdOut:
|
||||||
|
glog.debug(f"running command: {shlex.join(cmd)}. Caller: {get_caller()}")
|
||||||
# Start the subprocess
|
# Start the subprocess
|
||||||
process = subprocess.Popen(
|
process = subprocess.Popen(
|
||||||
cmd,
|
cmd,
|
||||||
@ -67,7 +72,7 @@ def run(
|
|||||||
)
|
)
|
||||||
|
|
||||||
stdout_buf, stderr_buf = handle_output(process, log)
|
stdout_buf, stderr_buf = handle_output(process, log)
|
||||||
|
# stdout_buf, stderr_buf = process.communicate()
|
||||||
# Wait for the subprocess to finish
|
# Wait for the subprocess to finish
|
||||||
rc = process.wait()
|
rc = process.wait()
|
||||||
cmd_out = CmdOut(
|
cmd_out = CmdOut(
|
||||||
@ -76,6 +81,7 @@ def run(
|
|||||||
cwd=cwd,
|
cwd=cwd,
|
||||||
command=shlex.join(cmd),
|
command=shlex.join(cmd),
|
||||||
returncode=process.returncode,
|
returncode=process.returncode,
|
||||||
|
msg=error_msg,
|
||||||
)
|
)
|
||||||
|
|
||||||
if check and rc != 0:
|
if check and rc != 0:
|
||||||
|
@ -4,12 +4,11 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import shlex
|
|
||||||
import subprocess
|
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, get_origin
|
from typing import Any, get_origin
|
||||||
|
|
||||||
|
from clan_cli.cmd import run
|
||||||
from clan_cli.dirs import machine_settings_file
|
from clan_cli.dirs import machine_settings_file
|
||||||
from clan_cli.errors import ClanError
|
from clan_cli.errors import ClanError
|
||||||
from clan_cli.git import commit_file
|
from clan_cli.git import commit_file
|
||||||
@ -117,15 +116,11 @@ def options_for_machine(
|
|||||||
f"{clan_dir}#nixosConfigurations.{machine_name}.config.clanCore.optionsNix"
|
f"{clan_dir}#nixosConfigurations.{machine_name}.config.clanCore.optionsNix"
|
||||||
)
|
)
|
||||||
cmd = nix_eval(flags=flags)
|
cmd = nix_eval(flags=flags)
|
||||||
proc = subprocess.run(
|
proc = run(
|
||||||
cmd,
|
cmd,
|
||||||
stdout=subprocess.PIPE,
|
error_msg=f"Failed to read options for machine {machine_name}",
|
||||||
text=True,
|
|
||||||
)
|
)
|
||||||
if proc.returncode != 0:
|
|
||||||
raise ClanError(
|
|
||||||
f"Failed to read options for machine {machine_name}:\n{shlex.join(cmd)}\nexit with {proc.returncode}"
|
|
||||||
)
|
|
||||||
return json.loads(proc.stdout)
|
return json.loads(proc.stdout)
|
||||||
|
|
||||||
|
|
||||||
@ -141,11 +136,8 @@ def read_machine_option_value(
|
|||||||
f"{clan_dir}#nixosConfigurations.{machine_name}.config.{option}",
|
f"{clan_dir}#nixosConfigurations.{machine_name}.config.{option}",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
|
proc = run(cmd, error_msg=f"Failed to read option {option}")
|
||||||
if proc.returncode != 0:
|
|
||||||
raise ClanError(
|
|
||||||
f"Failed to read option {option}:\n{shlex.join(cmd)}\nexit with {proc.returncode}"
|
|
||||||
)
|
|
||||||
value = json.loads(proc.stdout)
|
value = json.loads(proc.stdout)
|
||||||
# print the value so that the output can be copied and fed as an input.
|
# print the value so that the output can be copied and fed as an input.
|
||||||
# for example a list should be displayed as space separated values surrounded by quotes.
|
# for example a list should be displayed as space separated values surrounded by quotes.
|
||||||
|
@ -4,7 +4,7 @@ import re
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
|
|
||||||
from clan_cli.cmd import run
|
from clan_cli.cmd import Log, run
|
||||||
from clan_cli.dirs import machine_settings_file, nixpkgs_source, specific_machine_dir
|
from clan_cli.dirs import machine_settings_file, nixpkgs_source, specific_machine_dir
|
||||||
from clan_cli.errors import ClanError, ClanHttpError
|
from clan_cli.errors import ClanError, ClanHttpError
|
||||||
from clan_cli.git import commit_file
|
from clan_cli.git import commit_file
|
||||||
@ -65,6 +65,7 @@ def verify_machine_config(
|
|||||||
cmd,
|
cmd,
|
||||||
cwd=flake,
|
cwd=flake,
|
||||||
env=env,
|
env=env,
|
||||||
|
log=Log.BOTH,
|
||||||
)
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
return proc.stderr
|
return proc.stderr
|
||||||
|
@ -1,10 +1,9 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
|
|
||||||
|
from clan_cli.cmd import run
|
||||||
from clan_cli.dirs import nixpkgs_source
|
from clan_cli.dirs import nixpkgs_source
|
||||||
from clan_cli.errors import ClanError, ClanHttpError
|
from clan_cli.errors import ClanError, ClanHttpError
|
||||||
from clan_cli.nix import nix_eval
|
from clan_cli.nix import nix_eval
|
||||||
@ -25,7 +24,7 @@ def machine_schema(
|
|||||||
clan_machine_settings_file.seek(0)
|
clan_machine_settings_file.seek(0)
|
||||||
env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
|
env["CLAN_MACHINE_SETTINGS_FILE"] = clan_machine_settings_file.name
|
||||||
# ensure that the requested clanImports exist
|
# ensure that the requested clanImports exist
|
||||||
proc = subprocess.run(
|
proc = run(
|
||||||
nix_eval(
|
nix_eval(
|
||||||
flags=[
|
flags=[
|
||||||
"--impure",
|
"--impure",
|
||||||
@ -47,13 +46,11 @@ def machine_schema(
|
|||||||
""",
|
""",
|
||||||
]
|
]
|
||||||
),
|
),
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
cwd=flake_dir,
|
cwd=flake_dir,
|
||||||
env=env,
|
env=env,
|
||||||
|
check=False,
|
||||||
)
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
print(proc.stderr, file=sys.stderr)
|
|
||||||
raise ClanHttpError(
|
raise ClanHttpError(
|
||||||
status_code=400,
|
status_code=400,
|
||||||
msg=f"Failed to check clanImports for existence:\n{proc.stderr}",
|
msg=f"Failed to check clanImports for existence:\n{proc.stderr}",
|
||||||
@ -65,7 +62,7 @@ def machine_schema(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# get the schema
|
# get the schema
|
||||||
proc = subprocess.run(
|
proc = run(
|
||||||
nix_eval(
|
nix_eval(
|
||||||
flags=[
|
flags=[
|
||||||
"--impure",
|
"--impure",
|
||||||
@ -100,12 +97,10 @@ def machine_schema(
|
|||||||
""",
|
""",
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
capture_output=True,
|
check=False,
|
||||||
text=True,
|
|
||||||
cwd=flake_dir,
|
cwd=flake_dir,
|
||||||
env=env,
|
env=env,
|
||||||
)
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
print(proc.stderr, file=sys.stderr)
|
|
||||||
raise ClanError(f"Failed to read schema:\n{proc.stderr}")
|
raise ClanError(f"Failed to read schema:\n{proc.stderr}")
|
||||||
return json.loads(proc.stdout)
|
return json.loads(proc.stdout)
|
||||||
|
@ -63,11 +63,18 @@ def get_caller() -> str:
|
|||||||
|
|
||||||
|
|
||||||
def setup_logging(level: Any) -> None:
|
def setup_logging(level: Any) -> None:
|
||||||
handler = logging.StreamHandler()
|
# Get the root logger and set its level
|
||||||
handler.setLevel(level)
|
root_logger = logging.getLogger()
|
||||||
handler.setFormatter(CustomFormatter())
|
root_logger.setLevel(level)
|
||||||
logger = logging.getLogger("registerHandler")
|
|
||||||
|
# Create and add the default handler
|
||||||
|
default_handler = logging.StreamHandler()
|
||||||
|
|
||||||
|
# Create and add your custom handler
|
||||||
|
default_handler.setLevel(level)
|
||||||
|
default_handler.setFormatter(CustomFormatter())
|
||||||
|
root_logger.addHandler(default_handler)
|
||||||
|
|
||||||
|
# Set logging level for other modules used by this module
|
||||||
logging.getLogger("asyncio").setLevel(logging.INFO)
|
logging.getLogger("asyncio").setLevel(logging.INFO)
|
||||||
logging.getLogger("httpx").setLevel(level=logging.WARNING)
|
logging.getLogger("httpx").setLevel(level=logging.WARNING)
|
||||||
logger.addHandler(handler)
|
|
||||||
# logging.basicConfig(level=level, handlers=[handler])
|
|
||||||
|
@ -8,9 +8,11 @@ class CmdOut(NamedTuple):
|
|||||||
cwd: Path
|
cwd: Path
|
||||||
command: str
|
command: str
|
||||||
returncode: int
|
returncode: int
|
||||||
|
msg: str | None = None
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return f"""
|
return f"""
|
||||||
|
Message: {self.msg}
|
||||||
Working Directory: '{self.cwd}'
|
Working Directory: '{self.cwd}'
|
||||||
Return Code: {self.returncode}
|
Return Code: {self.returncode}
|
||||||
=================== Command ===================
|
=================== Command ===================
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# from clan_cli.dirs import find_git_repo_root
|
# from clan_cli.dirs import find_git_repo_root
|
||||||
from clan_cli.errors import ClanCmdError, ClanError
|
from clan_cli.errors import ClanError
|
||||||
from clan_cli.nix import nix_shell
|
from clan_cli.nix import nix_shell
|
||||||
|
|
||||||
from .cmd import run
|
from .cmd import Log, run
|
||||||
|
|
||||||
|
|
||||||
# generic vcs agnostic commit function
|
# generic vcs agnostic commit function
|
||||||
@ -42,12 +42,8 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) ->
|
|||||||
["git", "-C", str(repo_dir), "add", str(file_path)],
|
["git", "-C", str(repo_dir), "add", str(file_path)],
|
||||||
)
|
)
|
||||||
# add the file to the git index
|
# add the file to the git index
|
||||||
try:
|
|
||||||
run(cmd)
|
run(cmd, log=Log.BOTH, error_msg=f"Failed to add {file_path} file to git index")
|
||||||
except ClanCmdError as e:
|
|
||||||
raise ClanError(
|
|
||||||
f"Failed to add {file_path} to git repository {repo_dir}:\n{e.cmd.command}\n exited with {e.cmd.returncode}"
|
|
||||||
) from e
|
|
||||||
|
|
||||||
# check if there is a diff
|
# check if there is a diff
|
||||||
cmd = nix_shell(
|
cmd = nix_shell(
|
||||||
@ -72,11 +68,5 @@ def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) ->
|
|||||||
str(file_path.relative_to(repo_dir)),
|
str(file_path.relative_to(repo_dir)),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
try:
|
|
||||||
run(
|
run(cmd, error_msg=f"Failed to commit {file_path} to git repository {repo_dir}")
|
||||||
cmd,
|
|
||||||
)
|
|
||||||
except ClanCmdError as e:
|
|
||||||
raise ClanError(
|
|
||||||
f"Failed to commit {file_path} to git repository {repo_dir}:\n{e.cmd.command}\n exited with {e.cmd.returncode}"
|
|
||||||
) from e
|
|
||||||
|
@ -1,11 +1,9 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import shlex
|
|
||||||
import subprocess
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..errors import ClanError
|
from ..cmd import run
|
||||||
from ..nix import nix_config, nix_eval
|
from ..nix import nix_config, nix_eval
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@ -22,17 +20,8 @@ def list_machines(flake_url: Path | str) -> list[str]:
|
|||||||
"--json",
|
"--json",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
proc = subprocess.run(cmd, text=True, stdout=subprocess.PIPE)
|
proc = run(cmd)
|
||||||
assert proc.stdout is not None
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise ClanError(
|
|
||||||
f"""
|
|
||||||
command: {shlex.join(cmd)}
|
|
||||||
exit code: {proc.returncode}
|
|
||||||
stdout:
|
|
||||||
{proc.stdout}
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
res = proc.stdout.strip()
|
res = proc.stdout.strip()
|
||||||
return json.loads(res)
|
return json.loads(res)
|
||||||
|
|
||||||
|
@ -1,10 +1,9 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..cmd import run
|
from ..cmd import Log, run
|
||||||
from ..nix import nix_build, nix_config, nix_eval
|
from ..nix import nix_build, nix_config, nix_eval
|
||||||
from ..ssh import Host, parse_deployment_address
|
from ..ssh import Host, parse_deployment_address
|
||||||
|
|
||||||
@ -19,6 +18,8 @@ def build_machine_data(machine_name: str, clan_dir: Path) -> dict:
|
|||||||
f'{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file'
|
f'{clan_dir}#clanInternals.machines."{system}"."{machine_name}".config.system.clan.deployment.file'
|
||||||
]
|
]
|
||||||
),
|
),
|
||||||
|
log=Log.BOTH,
|
||||||
|
error_msg="failed to build machine data",
|
||||||
)
|
)
|
||||||
|
|
||||||
return json.loads(Path(proc.stdout.strip()).read_text())
|
return json.loads(Path(proc.stdout.strip()).read_text())
|
||||||
@ -70,10 +71,10 @@ class Machine:
|
|||||||
) # TODO do this in the clanCore module
|
) # TODO do this in the clanCore module
|
||||||
env["SECRETS_DIR"] = str(secrets_dir)
|
env["SECRETS_DIR"] = str(secrets_dir)
|
||||||
print(f"uploading secrets... {self.upload_secrets}")
|
print(f"uploading secrets... {self.upload_secrets}")
|
||||||
proc = subprocess.run(
|
proc = run(
|
||||||
[self.upload_secrets],
|
[self.upload_secrets],
|
||||||
env=env,
|
env=env,
|
||||||
text=True,
|
check=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
if proc.returncode == 23:
|
if proc.returncode == 23:
|
||||||
|
@ -3,7 +3,7 @@ import logging
|
|||||||
import shlex
|
import shlex
|
||||||
|
|
||||||
from clan_cli import create_parser
|
from clan_cli import create_parser
|
||||||
from clan_cli.custom_logger import get_caller
|
from clan_cli.custom_logger import get_caller, setup_logging
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@ -11,10 +11,11 @@ log = logging.getLogger(__name__)
|
|||||||
class Cli:
|
class Cli:
|
||||||
def run(self, args: list[str]) -> argparse.Namespace:
|
def run(self, args: list[str]) -> argparse.Namespace:
|
||||||
parser = create_parser(prog="clan")
|
parser = create_parser(prog="clan")
|
||||||
cmd = shlex.join(["clan", *args])
|
parsed = parser.parse_args(args)
|
||||||
|
setup_logging(logging.DEBUG)
|
||||||
|
cmd = shlex.join(["clan", "--debug", *args])
|
||||||
log.debug(f"$ {cmd}")
|
log.debug(f"$ {cmd}")
|
||||||
log.debug(f"Caller {get_caller()}")
|
log.debug(f"Caller {get_caller()}")
|
||||||
parsed = parser.parse_args(args)
|
|
||||||
if hasattr(parsed, "func"):
|
if hasattr(parsed, "func"):
|
||||||
parsed.func(parsed)
|
parsed.func(parsed)
|
||||||
return parsed
|
return parsed
|
||||||
|
@ -33,7 +33,8 @@ def test_generate_secret(
|
|||||||
age_keys[0].pubkey,
|
age_keys[0].pubkey,
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
cli.run(["--flake", str(test_flake_with_core.path), "secrets", "generate", "vm1"])
|
cmd = ["--flake", str(test_flake_with_core.path), "secrets", "generate", "vm1"]
|
||||||
|
cli.run(cmd)
|
||||||
has_secret(test_flake_with_core.path, "vm1-age.key")
|
has_secret(test_flake_with_core.path, "vm1-age.key")
|
||||||
has_secret(test_flake_with_core.path, "vm1-zerotier-identity-secret")
|
has_secret(test_flake_with_core.path, "vm1-zerotier-identity-secret")
|
||||||
has_secret(test_flake_with_core.path, "vm1-zerotier-subnet")
|
has_secret(test_flake_with_core.path, "vm1-zerotier-subnet")
|
||||||
|
Loading…
Reference in New Issue
Block a user