Merge remote-tracking branch 'origin/main' into rework-installation
Some checks failed
buildbot/nix-build .#checks.x86_64-linux.devShell-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-eval Build done.
checks / checks-impure (pull_request) Failing after 27s
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-inventory-machine Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-avahi Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bubblewrap Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-full Build done.
buildbot/nix-build .#checks.x86_64-linux.module-clan-vars-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-mypy Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-pass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-qemu Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.x86_64-linux.inventory-classes-up-to-date Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-vm-manager Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-util-linux Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-virtiofsd Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-examples-cue Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-vm-manager-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.package-classgen Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-api-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema-pretty Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.template-minimal Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-vm-manager Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-vm-manager-pytest Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
30
flake.lock
@ -7,11 +7,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1720661479,
|
||||
"narHash": "sha256-nsGgA14vVn0GGiqEfomtVgviRJCuSR3UEopfP8ixW1I=",
|
||||
"lastModified": 1721417620,
|
||||
"narHash": "sha256-6q9b1h8fI3hXg2DG6/vrKWCeG8c5Wj2Kvv22RCgedzg=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "786965e1b1ed3fd2018d78399984f461e2a44689",
|
||||
"rev": "bec6e3cde912b8acb915fecdc509eda7c973fb42",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -48,11 +48,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1720659757,
|
||||
"narHash": "sha256-ltzUuCsEfPA9CYM9BAnwObBGqDyQIs2OLkbVMeOOk00=",
|
||||
"lastModified": 1721571445,
|
||||
"narHash": "sha256-2MnlPVcNJZ9Nbu90kFyo7+lng366gswErP4FExfrUbc=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-images",
|
||||
"rev": "5eddae0afbcfd4283af5d6676d08ad059ca04b70",
|
||||
"rev": "accee005735844d57b411d9969c5d0aabc6a55f6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -63,11 +63,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1720977633,
|
||||
"narHash": "sha256-if0qaFmAe8X01NsVRK5e9Asg9mEWVkHrA9WuqM5jB70=",
|
||||
"lastModified": 1721571961,
|
||||
"narHash": "sha256-jfF4gpRUpTBY2OxDB0FRySsgNGOiuDckEtu7YDQom3Y=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0af9d835c27984b3265145f8e3cbc6c153479196",
|
||||
"rev": "4cc8b29327bed3d52b40041f810f49734298af46",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -95,11 +95,11 @@
|
||||
"nixpkgs-stable": []
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1720926522,
|
||||
"narHash": "sha256-eTpnrT6yu1vp8C0B5fxHXhgKxHoYMoYTEikQx///jxY=",
|
||||
"lastModified": 1721531171,
|
||||
"narHash": "sha256-AsvPw7T0tBLb53xZGcUC3YPqlIpdxoSx56u8vPCr6gU=",
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"rev": "0703ba03fd9c1665f8ab68cc3487302475164617",
|
||||
"rev": "909e8cfb60d83321d85c8d17209d733658a21c95",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -115,11 +115,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1720930114,
|
||||
"narHash": "sha256-VZK73b5hG5bSeAn97TTcnPjXUXtV7j/AtS4KN8ggCS0=",
|
||||
"lastModified": 1721458737,
|
||||
"narHash": "sha256-wNXLQ/ATs1S4Opg1PmuNoJ+Wamqj93rgZYV3Di7kxkg=",
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"rev": "b92afa1501ac73f1d745526adc4f89b527595f14",
|
||||
"rev": "888bfb10a9b091d9ed2f5f8064de8d488f7b7c97",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -58,11 +58,11 @@ let
|
||||
{
|
||||
# { ${name} :: meta // { name, tags } }
|
||||
machines = lib.mapAttrs (
|
||||
name: config:
|
||||
name: machineConfig:
|
||||
(lib.attrByPath [
|
||||
"clan"
|
||||
"meta"
|
||||
] { } config)
|
||||
] { } machineConfig)
|
||||
// {
|
||||
# meta.name default is the attribute name of the machine
|
||||
name = lib.mkDefault (
|
||||
@ -70,11 +70,11 @@ let
|
||||
"clan"
|
||||
"meta"
|
||||
"name"
|
||||
] name config
|
||||
] name machineConfig
|
||||
);
|
||||
}
|
||||
# tags
|
||||
// (clanToInventory config {
|
||||
// (clanToInventory machineConfig {
|
||||
clanPath = [
|
||||
"clan"
|
||||
"tags"
|
||||
@ -82,15 +82,15 @@ let
|
||||
inventoryPath = [ "tags" ];
|
||||
})
|
||||
# system
|
||||
// (clanToInventory config {
|
||||
// (clanToInventory machineConfig {
|
||||
clanPath = [
|
||||
"nixpkgs"
|
||||
"hostSystem"
|
||||
"hostPlatform"
|
||||
];
|
||||
inventoryPath = [ "system" ];
|
||||
})
|
||||
# deploy.targetHost
|
||||
// (clanToInventory config {
|
||||
// (clanToInventory machineConfig {
|
||||
clanPath = [
|
||||
"clan"
|
||||
"core"
|
||||
|
@ -17,7 +17,7 @@ in
|
||||
imports = [
|
||||
./public/in_repo.nix
|
||||
# ./public/vm.nix
|
||||
# ./secret/password-store.nix
|
||||
./secret/password-store.nix
|
||||
./secret/sops.nix
|
||||
# ./secret/vm.nix
|
||||
];
|
||||
@ -39,7 +39,7 @@ in
|
||||
vars = {
|
||||
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
|
||||
_name: generator: {
|
||||
inherit (generator) dependencies finalScript;
|
||||
inherit (generator) dependencies finalScript prompts;
|
||||
files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
|
||||
}
|
||||
);
|
||||
|
@ -108,8 +108,9 @@ in
|
||||
Prompts are available to the generator script as files.
|
||||
For example, a prompt named 'prompt1' will be available via $prompts/prompt1
|
||||
'';
|
||||
default = { };
|
||||
type = attrsOf (submodule {
|
||||
options = {
|
||||
options = options {
|
||||
description = {
|
||||
description = ''
|
||||
The description of the prompted value
|
||||
|
12
nixosModules/clanCore/vars/secret/password-store.nix
Normal file
@ -0,0 +1,12 @@
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
config.clan.core.vars.settings =
|
||||
lib.mkIf (config.clan.core.vars.settings.secretStore == "password-store")
|
||||
{
|
||||
fileModule = file: {
|
||||
path = lib.mkIf file.secret "${config.clan.core.password-store.targetDirectory}/${config.clan.core.machineName}-${file.config.generatorName}-${file.config.name}";
|
||||
};
|
||||
secretUploadDirectory = lib.mkDefault "/etc/secrets";
|
||||
secretModule = "clan_cli.vars.secret_modules.password_store";
|
||||
};
|
||||
}
|
@ -17,8 +17,13 @@ let
|
||||
imports = [
|
||||
(modulesPath + "/virtualisation/qemu-vm.nix")
|
||||
./serial.nix
|
||||
./waypipe.nix
|
||||
];
|
||||
|
||||
clan.services.waypipe = {
|
||||
inherit (config.clan.core.vm.inspect.waypipe) enable command;
|
||||
};
|
||||
|
||||
# required for issuing shell commands via qga
|
||||
services.qemuGuest.enable = true;
|
||||
|
||||
@ -149,12 +154,19 @@ in
|
||||
'';
|
||||
};
|
||||
|
||||
waypipe = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Whether to use waypipe for native wayland passthrough, or not.
|
||||
'';
|
||||
waypipe = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
default = false;
|
||||
description = ''
|
||||
Whether to use waypipe for native wayland passthrough, or not.
|
||||
'';
|
||||
};
|
||||
command = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
default = [ ];
|
||||
description = "Commands that waypipe should run";
|
||||
};
|
||||
};
|
||||
};
|
||||
# All important VM config variables needed by the vm runner
|
||||
@ -193,13 +205,22 @@ in
|
||||
whether to enable graphics for the vm
|
||||
'';
|
||||
};
|
||||
waypipe = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
whether to enable native wayland window passthrough with waypipe for the vm
|
||||
'';
|
||||
|
||||
waypipe = {
|
||||
enable = lib.mkOption {
|
||||
type = lib.types.bool;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = ''
|
||||
Whether to use waypipe for native wayland passthrough, or not.
|
||||
'';
|
||||
};
|
||||
command = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
internal = true;
|
||||
readOnly = true;
|
||||
description = "Commands that waypipe should run";
|
||||
};
|
||||
};
|
||||
machine_icon = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
@ -245,7 +266,12 @@ in
|
||||
initrd = "${vmConfig.config.system.build.initialRamdisk}/${vmConfig.config.system.boot.loader.initrdFile}";
|
||||
toplevel = vmConfig.config.system.build.toplevel;
|
||||
regInfo = (pkgs.closureInfo { rootPaths = vmConfig.config.virtualisation.additionalPaths; });
|
||||
inherit (config.clan.virtualisation) memorySize cores graphics;
|
||||
inherit (config.clan.virtualisation)
|
||||
memorySize
|
||||
cores
|
||||
graphics
|
||||
waypipe
|
||||
;
|
||||
}
|
||||
);
|
||||
};
|
||||
|
@ -39,8 +39,6 @@
|
||||
# General default settings
|
||||
fonts.enableDefaultPackages = lib.mkDefault true;
|
||||
hardware.opengl.enable = lib.mkDefault true;
|
||||
# Assume it is run inside a clan context
|
||||
clan.virtualisation.waypipe = lib.mkDefault true;
|
||||
|
||||
# User account
|
||||
services.getty.autologinUser = lib.mkDefault config.clan.services.waypipe.user;
|
@ -34,7 +34,7 @@ class MainApplication(Adw.Application):
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
super().__init__(
|
||||
application_id="org.clan.clan-app",
|
||||
application_id="org.clan.app",
|
||||
flags=Gio.ApplicationFlags.HANDLES_COMMAND_LINE,
|
||||
)
|
||||
|
||||
|
@ -4,11 +4,10 @@ import logging
|
||||
from typing import Any
|
||||
|
||||
import gi
|
||||
from clan_cli.api import MethodRegistry
|
||||
from clan_cli.api import MethodRegistry, dataclass_to_dict, from_dict
|
||||
|
||||
from clan_app.api import GObjApi, GResult, ImplFunc
|
||||
from clan_app.api.file import open_file
|
||||
from clan_app.components.serializer import dataclass_to_dict, from_dict
|
||||
|
||||
gi.require_version("WebKit", "6.0")
|
||||
from gi.repository import GLib, GObject, WebKit
|
||||
|
@ -4,7 +4,7 @@
|
||||
setuptools,
|
||||
copyDesktopItems,
|
||||
pygobject3,
|
||||
wrapGAppsHook,
|
||||
wrapGAppsHook4,
|
||||
gtk4,
|
||||
adwaita-icon-theme,
|
||||
pygobject-stubs,
|
||||
@ -84,7 +84,7 @@ python3.pkgs.buildPythonApplication rec {
|
||||
nativeBuildInputs = [
|
||||
setuptools
|
||||
copyDesktopItems
|
||||
wrapGAppsHook
|
||||
wrapGAppsHook4
|
||||
|
||||
gobject-introspection
|
||||
];
|
||||
|
@ -1,5 +1,7 @@
|
||||
{
|
||||
lib,
|
||||
glib,
|
||||
gsettings-desktop-schemas,
|
||||
stdenv,
|
||||
clan-app,
|
||||
mkShell,
|
||||
@ -33,7 +35,9 @@ mkShell {
|
||||
|
||||
buildInputs =
|
||||
[
|
||||
glib
|
||||
ruff
|
||||
gtk4
|
||||
gtk4.dev # has the demo called 'gtk4-widget-factory'
|
||||
libadwaita.devdoc # has the demo called 'adwaita-1-demo'
|
||||
]
|
||||
@ -60,6 +64,9 @@ mkShell {
|
||||
# Add clan-cli to the python path so that we can import it without building it in nix first
|
||||
export PYTHONPATH="$GIT_ROOT/pkgs/clan-cli":"$PYTHONPATH"
|
||||
|
||||
export XDG_DATA_DIRS=${gtk4}/share/gsettings-schemas/gtk4-4.14.4:$XDG_DATA_DIRS
|
||||
export XDG_DATA_DIRS=${gsettings-desktop-schemas}/share/gsettings-schemas/gsettings-desktop-schemas-46.0:$XDG_DATA_DIRS
|
||||
|
||||
# Add the webview-ui to the .webui directory
|
||||
ln -nsf ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/ ./clan_app/.webui
|
||||
|
||||
|
@ -1,11 +1,141 @@
|
||||
import dataclasses
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, fields, is_dataclass
|
||||
from functools import wraps
|
||||
from inspect import Parameter, Signature, signature
|
||||
from typing import Annotated, Any, Generic, Literal, TypeVar, get_type_hints
|
||||
from pathlib import Path
|
||||
from types import UnionType
|
||||
from typing import (
|
||||
Annotated,
|
||||
Any,
|
||||
Generic,
|
||||
Literal,
|
||||
TypeVar,
|
||||
get_args,
|
||||
get_origin,
|
||||
get_type_hints,
|
||||
)
|
||||
|
||||
from clan_cli.errors import ClanError
|
||||
|
||||
|
||||
def sanitize_string(s: str) -> str:
|
||||
# Using the native string sanitizer to handle all edge cases
|
||||
# Remove the outer quotes '"string"'
|
||||
return json.dumps(s)[1:-1]
|
||||
|
||||
|
||||
def dataclass_to_dict(obj: Any) -> Any:
|
||||
"""
|
||||
Utility function to convert dataclasses to dictionaries
|
||||
It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
|
||||
|
||||
It does NOT convert member functions.
|
||||
"""
|
||||
if is_dataclass(obj):
|
||||
return {
|
||||
# Use either the original name or name
|
||||
sanitize_string(
|
||||
field.metadata.get("original_name", field.name)
|
||||
): dataclass_to_dict(getattr(obj, field.name))
|
||||
for field in fields(obj) # type: ignore
|
||||
}
|
||||
elif isinstance(obj, list | tuple):
|
||||
return [dataclass_to_dict(item) for item in obj]
|
||||
elif isinstance(obj, dict):
|
||||
return {sanitize_string(k): dataclass_to_dict(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, Path):
|
||||
return sanitize_string(str(obj))
|
||||
elif isinstance(obj, str):
|
||||
return sanitize_string(obj)
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
def is_union_type(type_hint: type) -> bool:
|
||||
return type(type_hint) is UnionType
|
||||
|
||||
|
||||
def get_inner_type(type_hint: type) -> type:
|
||||
if is_union_type(type_hint):
|
||||
# Return the first non-None type
|
||||
return next(t for t in get_args(type_hint) if t is not type(None))
|
||||
return type_hint
|
||||
|
||||
|
||||
def get_second_type(type_hint: type[dict]) -> type:
|
||||
"""
|
||||
Get the value type of a dictionary type hint
|
||||
"""
|
||||
args = get_args(type_hint)
|
||||
if len(args) == 2:
|
||||
# Return the second argument, which should be the value type (Machine)
|
||||
return args[1]
|
||||
|
||||
raise ValueError(f"Invalid type hint for dict: {type_hint}")
|
||||
|
||||
|
||||
def from_dict(t: type, data: dict[str, Any] | None) -> Any:
|
||||
"""
|
||||
Dynamically instantiate a data class from a dictionary, handling nested data classes.
|
||||
"""
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Attempt to create an instance of the data_class
|
||||
field_values = {}
|
||||
for field in fields(t):
|
||||
original_name = field.metadata.get("original_name", field.name)
|
||||
|
||||
field_value = data.get(original_name)
|
||||
|
||||
field_type = get_inner_type(field.type) # type: ignore
|
||||
|
||||
if original_name in data:
|
||||
# If the field is another dataclass, recursively instantiate it
|
||||
if is_dataclass(field_type):
|
||||
field_value = from_dict(field_type, field_value)
|
||||
elif isinstance(field_type, Path | str) and isinstance(
|
||||
field_value, str
|
||||
):
|
||||
field_value = (
|
||||
Path(field_value) if field_type == Path else field_value
|
||||
)
|
||||
elif get_origin(field_type) is dict and isinstance(field_value, dict):
|
||||
# The field is a dictionary with a specific type
|
||||
inner_type = get_second_type(field_type)
|
||||
field_value = {
|
||||
k: from_dict(inner_type, v) for k, v in field_value.items()
|
||||
}
|
||||
elif get_origin is list and isinstance(field_value, list):
|
||||
# The field is a list with a specific type
|
||||
inner_type = get_args(field_type)[0]
|
||||
field_value = [from_dict(inner_type, v) for v in field_value]
|
||||
|
||||
# Set the value
|
||||
if (
|
||||
field.default is not dataclasses.MISSING
|
||||
or field.default_factory is not dataclasses.MISSING
|
||||
):
|
||||
# Fields with default value
|
||||
# a: Int = 1
|
||||
# b: list = Field(default_factory=list)
|
||||
if original_name in data or field_value is not None:
|
||||
field_values[field.name] = field_value
|
||||
else:
|
||||
# Fields without default value
|
||||
# a: Int
|
||||
field_values[field.name] = field_value
|
||||
|
||||
return t(**field_values)
|
||||
|
||||
except (TypeError, ValueError) as e:
|
||||
print(f"Failed to instantiate {t.__name__}: {e} {data}")
|
||||
return None
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
ResponseDataType = TypeVar("ResponseDataType")
|
||||
|
@ -6,7 +6,7 @@ from pathlib import Path
|
||||
|
||||
from clan_cli.cmd import run_no_stdout
|
||||
from clan_cli.errors import ClanCmdError, ClanError
|
||||
from clan_cli.inventory import Inventory, load_inventory
|
||||
from clan_cli.inventory import Inventory, load_inventory_json
|
||||
from clan_cli.nix import nix_eval
|
||||
|
||||
from . import API
|
||||
@ -152,4 +152,4 @@ def get_module_info(
|
||||
|
||||
@API.register
|
||||
def get_inventory(base_path: str) -> Inventory:
|
||||
return load_inventory(base_path)
|
||||
return load_inventory_json(base_path)
|
||||
|
@ -1,12 +1,11 @@
|
||||
# !/usr/bin/env python3
|
||||
import argparse
|
||||
import os
|
||||
from dataclasses import dataclass, fields
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.api import API
|
||||
from clan_cli.arg_actions import AppendOptionAction
|
||||
from clan_cli.inventory import Meta, load_inventory, save_inventory
|
||||
from clan_cli.inventory import Inventory, init_inventory
|
||||
|
||||
from ..cmd import CmdOut, run
|
||||
from ..dirs import clan_templates
|
||||
@ -27,13 +26,11 @@ class CreateClanResponse:
|
||||
@dataclass
|
||||
class CreateOptions:
|
||||
directory: Path | str
|
||||
# Metadata for the clan
|
||||
# Metadata can be shown with `clan show`
|
||||
meta: Meta | None = None
|
||||
# URL to the template to use. Defaults to the "minimal" template
|
||||
template: str = "minimal"
|
||||
setup_json_inventory: bool = True
|
||||
setup_git: bool = True
|
||||
initial: Inventory | None = None
|
||||
|
||||
|
||||
def git_command(directory: Path, *args: str) -> list[str]:
|
||||
@ -67,10 +64,34 @@ def create_clan(options: CreateOptions) -> CreateClanResponse:
|
||||
]
|
||||
)
|
||||
flake_init = run(command, cwd=directory)
|
||||
|
||||
git_init = None
|
||||
if not directory.joinpath(".git").exists():
|
||||
git_init = run(git_command(directory, "init"))
|
||||
git_add = run(git_command(directory, "add", "."))
|
||||
|
||||
# check if username is set
|
||||
has_username = run(git_command(directory, "config", "user.name"), check=False)
|
||||
git_config_username = None
|
||||
if has_username.returncode != 0:
|
||||
git_config_username = run(
|
||||
git_command(directory, "config", "user.name", "clan-tool")
|
||||
)
|
||||
|
||||
has_username = run(git_command(directory, "config", "user.email"), check=False)
|
||||
git_config_email = None
|
||||
if has_username.returncode != 0:
|
||||
git_config_email = run(
|
||||
git_command(directory, "config", "user.email", "clan@example.com")
|
||||
)
|
||||
|
||||
flake_update = run(
|
||||
nix_shell(["nixpkgs#nix"], ["nix", "flake", "update"]), cwd=directory
|
||||
)
|
||||
|
||||
if options.initial:
|
||||
init_inventory(options.directory, init=options.initial)
|
||||
|
||||
response = CreateClanResponse(
|
||||
flake_init=flake_init,
|
||||
flake_update=flake_update,
|
||||
@ -116,6 +137,7 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
<<<<<<< HEAD
|
||||
"--meta",
|
||||
help=f"""Metadata to set for the clan. Available options are: {", ".join([f.name for f in fields(Meta)]) }""",
|
||||
nargs=2,
|
||||
@ -131,6 +153,18 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
||||||| a1a36606
|
||||
"--meta",
|
||||
help=f"""Metadata to set for the clan. Available options are: {", ".join([f.name for f in fields(Meta)]) }""",
|
||||
nargs=2,
|
||||
metavar=("name", "value"),
|
||||
action=AppendOptionAction,
|
||||
default=[],
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
=======
|
||||
>>>>>>> origin/main
|
||||
"path", type=Path, help="Path to the clan directory", default=Path(".")
|
||||
)
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
from clan_cli.api import API
|
||||
from clan_cli.inventory import Meta, load_inventory, save_inventory
|
||||
from clan_cli.inventory import Meta, load_inventory_json, save_inventory
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -12,7 +12,7 @@ class UpdateOptions:
|
||||
|
||||
@API.register
|
||||
def update_clan_meta(options: UpdateOptions) -> Meta:
|
||||
inventory = load_inventory(options.directory)
|
||||
inventory = load_inventory_json(options.directory)
|
||||
inventory.meta = options.meta
|
||||
|
||||
save_inventory(inventory, options.directory, "Update clan metadata")
|
||||
|
@ -9,32 +9,49 @@ from .errors import ClanError
|
||||
|
||||
@dataclass
|
||||
class FlakeId:
|
||||
# FIXME: this is such a footgun if you accidnetally pass a string
|
||||
_value: str | Path
|
||||
loc: str | Path
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
assert isinstance(
|
||||
self._value, str | Path
|
||||
), f"Flake {self._value} has an invalid type: {type(self._value)}"
|
||||
self.loc, str | Path
|
||||
), f"Flake {self.loc} has an invalid format: {type(self.loc)}"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return str(self._value)
|
||||
return str(self.loc)
|
||||
|
||||
@property
|
||||
def path(self) -> Path:
|
||||
assert isinstance(self._value, Path), f"Flake {self._value} is not a local path"
|
||||
return self._value
|
||||
assert self.is_local(), f"Flake {self.loc} is not a local path"
|
||||
return Path(self.loc)
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
assert isinstance(self._value, str), f"Flake {self._value} is not a remote url"
|
||||
return self._value
|
||||
assert self.is_remote(), f"Flake {self.loc} is not a remote url"
|
||||
return str(self.loc)
|
||||
|
||||
def is_local(self) -> bool:
|
||||
return isinstance(self._value, Path)
|
||||
"""
|
||||
https://nix.dev/manual/nix/2.22/language/builtins.html?highlight=urlS#source-types
|
||||
|
||||
Examples:
|
||||
|
||||
- file:///home/eelco/nix/README.md file LOCAL
|
||||
- git+file://git:github.com:NixOS/nixpkgs git+file LOCAL
|
||||
- https://example.com/index.html https REMOTE
|
||||
- github:nixos/nixpkgs github REMOTE
|
||||
- ftp://serv.file ftp REMOTE
|
||||
- ./. '' LOCAL
|
||||
|
||||
"""
|
||||
x = urllib.parse.urlparse(str(self.loc))
|
||||
if x.scheme == "" or "file" in x.scheme:
|
||||
# See above *file* or empty are the only local schemas
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def is_remote(self) -> bool:
|
||||
return isinstance(self._value, str)
|
||||
return not self.is_local()
|
||||
|
||||
|
||||
# Define the ClanURI class
|
||||
|
@ -48,19 +48,22 @@ class SecretStore(SecretStoreBase):
|
||||
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
return decrypt_secret(
|
||||
self.machine.flake_dir, f"{self.machine.name}-{name}"
|
||||
self.machine.flake_dir,
|
||||
sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}",
|
||||
).encode("utf-8")
|
||||
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
return has_secret(
|
||||
self.machine.flake_dir,
|
||||
f"{self.machine.name}-{name}",
|
||||
sops_secrets_folder(self.machine.flake_dir) / f"{self.machine.name}-{name}",
|
||||
)
|
||||
|
||||
def upload(self, output_dir: Path) -> None:
|
||||
key_name = f"{self.machine.name}-age.key"
|
||||
if not has_secret(self.machine.flake_dir, key_name):
|
||||
if not has_secret(sops_secrets_folder(self.machine.flake_dir) / key_name):
|
||||
# skip uploading the secret, not managed by us
|
||||
return
|
||||
key = decrypt_secret(self.machine.flake_dir, key_name)
|
||||
key = decrypt_secret(
|
||||
self.machine.flake_dir,
|
||||
sops_secrets_folder(self.machine.flake_dir) / key_name,
|
||||
)
|
||||
(output_dir / "key.txt").write_text(key)
|
||||
|
@ -62,7 +62,7 @@ def _commit_file_to_git(
|
||||
for file_path in file_paths:
|
||||
cmd = run_cmd(
|
||||
["git"],
|
||||
["git", "-C", str(repo_dir), "add", str(file_path)],
|
||||
["git", "-C", str(repo_dir), "add", "--", str(file_path)],
|
||||
)
|
||||
# add the file to the git index
|
||||
|
||||
@ -75,7 +75,7 @@ def _commit_file_to_git(
|
||||
# check if there is a diff
|
||||
cmd = run_cmd(
|
||||
["git"],
|
||||
["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code"]
|
||||
["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code", "--"]
|
||||
+ [str(file_path) for file_path in file_paths],
|
||||
)
|
||||
result = run(cmd, check=False, cwd=repo_dir)
|
||||
@ -93,6 +93,7 @@ def _commit_file_to_git(
|
||||
"commit",
|
||||
"-m",
|
||||
commit_message,
|
||||
"--no-verify", # dont run pre-commit hooks
|
||||
]
|
||||
+ [str(file_path) for file_path in file_paths],
|
||||
)
|
||||
|
@ -1,13 +1,26 @@
|
||||
import dataclasses
|
||||
import json
|
||||
from dataclasses import fields, is_dataclass
|
||||
from pathlib import Path
|
||||
from types import UnionType
|
||||
from typing import Any, get_args, get_origin
|
||||
"""
|
||||
All read/write operations MUST use the inventory.
|
||||
|
||||
from clan_cli.errors import ClanError
|
||||
Machine data, clan data or service data can be accessed in a performant way.
|
||||
|
||||
This file exports stable classnames for static & dynamic type safety.
|
||||
|
||||
Utilize:
|
||||
|
||||
- load_inventory_eval: To load the actual inventory with nix declarations merged.
|
||||
Operate on the returned inventory to make changes
|
||||
- save_inventory: To persist changes.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.api import API, dataclass_to_dict, from_dict
|
||||
from clan_cli.errors import ClanCmdError, ClanError
|
||||
from clan_cli.git import commit_file
|
||||
|
||||
from ..cmd import run_no_stdout
|
||||
from ..nix import nix_eval
|
||||
from .classes import (
|
||||
Inventory,
|
||||
Machine,
|
||||
@ -24,6 +37,8 @@ from .classes import (
|
||||
# Re export classes here
|
||||
# This allows to rename classes in the generated code
|
||||
__all__ = [
|
||||
"from_dict",
|
||||
"dataclass_to_dict",
|
||||
"Service",
|
||||
"Machine",
|
||||
"Meta",
|
||||
@ -37,121 +52,6 @@ __all__ = [
|
||||
]
|
||||
|
||||
|
||||
def sanitize_string(s: str) -> str:
|
||||
return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
|
||||
|
||||
|
||||
def dataclass_to_dict(obj: Any) -> Any:
|
||||
"""
|
||||
Utility function to convert dataclasses to dictionaries
|
||||
It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
|
||||
|
||||
It does NOT convert member functions.
|
||||
"""
|
||||
if is_dataclass(obj):
|
||||
return {
|
||||
# Use either the original name or name
|
||||
sanitize_string(
|
||||
field.metadata.get("original_name", field.name)
|
||||
): dataclass_to_dict(getattr(obj, field.name))
|
||||
for field in fields(obj) # type: ignore
|
||||
}
|
||||
elif isinstance(obj, list | tuple):
|
||||
return [dataclass_to_dict(item) for item in obj]
|
||||
elif isinstance(obj, dict):
|
||||
return {sanitize_string(k): dataclass_to_dict(v) for k, v in obj.items()}
|
||||
elif isinstance(obj, Path):
|
||||
return str(obj)
|
||||
elif isinstance(obj, str):
|
||||
return sanitize_string(obj)
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
def is_union_type(type_hint: type) -> bool:
|
||||
return type(type_hint) is UnionType
|
||||
|
||||
|
||||
def get_inner_type(type_hint: type) -> type:
|
||||
if is_union_type(type_hint):
|
||||
# Return the first non-None type
|
||||
return next(t for t in get_args(type_hint) if t is not type(None))
|
||||
return type_hint
|
||||
|
||||
|
||||
def get_second_type(type_hint: type[dict]) -> type:
|
||||
"""
|
||||
Get the value type of a dictionary type hint
|
||||
"""
|
||||
args = get_args(type_hint)
|
||||
if len(args) == 2:
|
||||
# Return the second argument, which should be the value type (Machine)
|
||||
return args[1]
|
||||
|
||||
raise ValueError(f"Invalid type hint for dict: {type_hint}")
|
||||
|
||||
|
||||
def from_dict(t: type, data: dict[str, Any] | None) -> Any:
|
||||
"""
|
||||
Dynamically instantiate a data class from a dictionary, handling nested data classes.
|
||||
"""
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Attempt to create an instance of the data_class
|
||||
field_values = {}
|
||||
for field in fields(t):
|
||||
original_name = field.metadata.get("original_name", field.name)
|
||||
|
||||
field_value = data.get(original_name)
|
||||
|
||||
field_type = get_inner_type(field.type) # type: ignore
|
||||
|
||||
if original_name in data:
|
||||
# If the field is another dataclass, recursively instantiate it
|
||||
if is_dataclass(field_type):
|
||||
field_value = from_dict(field_type, field_value)
|
||||
elif isinstance(field_type, Path | str) and isinstance(
|
||||
field_value, str
|
||||
):
|
||||
field_value = (
|
||||
Path(field_value) if field_type == Path else field_value
|
||||
)
|
||||
elif get_origin(field_type) is dict and isinstance(field_value, dict):
|
||||
# The field is a dictionary with a specific type
|
||||
inner_type = get_second_type(field_type)
|
||||
field_value = {
|
||||
k: from_dict(inner_type, v) for k, v in field_value.items()
|
||||
}
|
||||
elif get_origin is list and isinstance(field_value, list):
|
||||
# The field is a list with a specific type
|
||||
inner_type = get_args(field_type)[0]
|
||||
field_value = [from_dict(inner_type, v) for v in field_value]
|
||||
|
||||
# Set the value
|
||||
if (
|
||||
field.default is not dataclasses.MISSING
|
||||
or field.default_factory is not dataclasses.MISSING
|
||||
):
|
||||
# Fields with default value
|
||||
# a: Int = 1
|
||||
# b: list = Field(default_factory=list)
|
||||
if original_name in data or field_value is not None:
|
||||
field_values[field.name] = field_value
|
||||
else:
|
||||
# Fields without default value
|
||||
# a: Int
|
||||
field_values[field.name] = field_value
|
||||
|
||||
return t(**field_values)
|
||||
|
||||
except (TypeError, ValueError) as e:
|
||||
print(f"Failed to instantiate {t.__name__}: {e} {data}")
|
||||
return None
|
||||
# raise ClanError(f"Failed to instantiate {t.__name__}: {e}")
|
||||
|
||||
|
||||
def get_path(flake_dir: str | Path) -> Path:
|
||||
"""
|
||||
Get the path to the inventory file in the flake directory
|
||||
@ -165,14 +65,42 @@ default_inventory = Inventory(
|
||||
)
|
||||
|
||||
|
||||
def load_inventory(
|
||||
def load_inventory_eval(flake_dir: str | Path) -> Inventory:
|
||||
"""
|
||||
Loads the actual inventory.
|
||||
After all merge operations with eventual nix code in buildClan.
|
||||
|
||||
Evaluates clanInternals.inventory with nix. Which is performant.
|
||||
|
||||
- Contains all clan metadata
|
||||
- Contains all machines
|
||||
- and more
|
||||
"""
|
||||
cmd = nix_eval(
|
||||
[
|
||||
f"{flake_dir}#clanInternals.inventory",
|
||||
"--json",
|
||||
]
|
||||
)
|
||||
proc = run_no_stdout(cmd)
|
||||
|
||||
try:
|
||||
res = proc.stdout.strip()
|
||||
data = json.loads(res)
|
||||
inventory = from_dict(Inventory, data)
|
||||
return inventory
|
||||
except json.JSONDecodeError as e:
|
||||
raise ClanError(f"Error decoding inventory from flake: {e}")
|
||||
|
||||
|
||||
def load_inventory_json(
|
||||
flake_dir: str | Path, default: Inventory = default_inventory
|
||||
) -> Inventory:
|
||||
"""
|
||||
Load the inventory file from the flake directory
|
||||
If not file is found, returns the default inventory
|
||||
"""
|
||||
inventory = default_inventory
|
||||
inventory = default
|
||||
|
||||
inventory_file = get_path(flake_dir)
|
||||
if inventory_file.exists():
|
||||
@ -184,6 +112,10 @@ def load_inventory(
|
||||
# Error decoding the inventory file
|
||||
raise ClanError(f"Error decoding inventory file: {e}")
|
||||
|
||||
if not inventory_file.exists():
|
||||
# Copy over the meta from the flake if the inventory is not initialized
|
||||
inventory.meta = load_inventory_eval(flake_dir).meta
|
||||
|
||||
return inventory
|
||||
|
||||
|
||||
@ -198,3 +130,22 @@ def save_inventory(inventory: Inventory, flake_dir: str | Path, message: str) ->
|
||||
json.dump(dataclass_to_dict(inventory), f, indent=2)
|
||||
|
||||
commit_file(inventory_file, Path(flake_dir), commit_message=message)
|
||||
|
||||
|
||||
@API.register
|
||||
def init_inventory(directory: str, init: Inventory | None = None) -> None:
|
||||
inventory = None
|
||||
# Try reading the current flake
|
||||
if init is None:
|
||||
try:
|
||||
inventory = load_inventory_eval(directory)
|
||||
except ClanCmdError:
|
||||
pass
|
||||
|
||||
if init is not None:
|
||||
inventory = init
|
||||
|
||||
# Write inventory.json file
|
||||
if inventory is not None:
|
||||
# Persist creates a commit message for each change
|
||||
save_inventory(inventory, directory, "Init inventory")
|
||||
|
@ -1,13 +1,17 @@
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from ..api import API
|
||||
from ..clan_uri import FlakeId
|
||||
from ..errors import ClanError
|
||||
from ..git import commit_file
|
||||
from ..inventory import Machine, MachineDeploy, get_path, load_inventory, save_inventory
|
||||
from ..inventory import (
|
||||
Machine,
|
||||
MachineDeploy,
|
||||
load_inventory_eval,
|
||||
load_inventory_json,
|
||||
save_inventory,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@ -20,12 +24,16 @@ def create_machine(flake: FlakeId, machine: Machine) -> None:
|
||||
"Machine name must be a valid hostname", location="Create Machine"
|
||||
)
|
||||
|
||||
inventory = load_inventory(flake.path)
|
||||
inventory = load_inventory_json(flake.path)
|
||||
|
||||
full_inventory = load_inventory_eval(flake.path)
|
||||
|
||||
if machine.name in full_inventory.machines.keys():
|
||||
raise ClanError(f"Machine with the name {machine.name} already exists")
|
||||
|
||||
inventory.machines.update({machine.name: machine})
|
||||
save_inventory(inventory, flake.path, f"Create machine {machine.name}")
|
||||
|
||||
commit_file(get_path(flake.path), Path(flake.path))
|
||||
|
||||
|
||||
def create_command(args: argparse.Namespace) -> None:
|
||||
create_machine(
|
||||
|
@ -6,12 +6,12 @@ from ..clan_uri import FlakeId
|
||||
from ..completions import add_dynamic_completer, complete_machines
|
||||
from ..dirs import specific_machine_dir
|
||||
from ..errors import ClanError
|
||||
from ..inventory import load_inventory, save_inventory
|
||||
from ..inventory import load_inventory_json, save_inventory
|
||||
|
||||
|
||||
@API.register
|
||||
def delete_machine(flake: FlakeId, name: str) -> None:
|
||||
inventory = load_inventory(flake.path)
|
||||
inventory = load_inventory_json(flake.path)
|
||||
|
||||
machine = inventory.machines.pop(name, None)
|
||||
if machine is None:
|
||||
|
@ -1,31 +1,17 @@
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.api import API
|
||||
from clan_cli.inventory import Machine, from_dict
|
||||
|
||||
from ..cmd import run_no_stdout
|
||||
from ..nix import nix_eval
|
||||
from clan_cli.inventory import Machine, load_inventory_eval
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@API.register
|
||||
def list_machines(flake_url: str | Path, debug: bool = False) -> dict[str, Machine]:
|
||||
cmd = nix_eval(
|
||||
[
|
||||
f"{flake_url}#clanInternals.inventory.machines",
|
||||
"--json",
|
||||
]
|
||||
)
|
||||
|
||||
proc = run_no_stdout(cmd)
|
||||
|
||||
res = proc.stdout.strip()
|
||||
data = {name: from_dict(Machine, v) for name, v in json.loads(res).items()}
|
||||
return data
|
||||
inventory = load_inventory_eval(flake_url)
|
||||
return inventory.machines
|
||||
|
||||
|
||||
def list_command(args: argparse.Namespace) -> None:
|
||||
|
@ -49,27 +49,27 @@ class Group:
|
||||
|
||||
def list_groups(flake_dir: Path) -> list[Group]:
|
||||
groups: list[Group] = []
|
||||
folder = sops_groups_folder(flake_dir)
|
||||
if not folder.exists():
|
||||
groups_dir = sops_groups_folder(flake_dir)
|
||||
if not groups_dir.exists():
|
||||
return groups
|
||||
|
||||
for name in os.listdir(folder):
|
||||
group_folder = folder / name
|
||||
for group in os.listdir(groups_dir):
|
||||
group_folder = groups_dir / group
|
||||
if not group_folder.is_dir():
|
||||
continue
|
||||
machines_path = machines_folder(flake_dir, name)
|
||||
machines_path = machines_folder(flake_dir, group)
|
||||
machines = []
|
||||
if machines_path.is_dir():
|
||||
for f in machines_path.iterdir():
|
||||
if validate_hostname(f.name):
|
||||
machines.append(f.name)
|
||||
users_path = users_folder(flake_dir, name)
|
||||
users_path = users_folder(flake_dir, group)
|
||||
users = []
|
||||
if users_path.is_dir():
|
||||
for f in users_path.iterdir():
|
||||
if VALID_USER_NAME.match(f.name):
|
||||
users.append(f.name)
|
||||
groups.append(Group(flake_dir, name, machines, users))
|
||||
groups.append(Group(flake_dir, group, machines, users))
|
||||
return groups
|
||||
|
||||
|
||||
@ -204,7 +204,9 @@ def add_group_argument(parser: argparse.ArgumentParser) -> None:
|
||||
|
||||
def add_secret(flake_dir: Path, group: str, name: str) -> None:
|
||||
secrets.allow_member(
|
||||
secrets.groups_folder(flake_dir, name), sops_groups_folder(flake_dir), group
|
||||
secrets.groups_folder(sops_secrets_folder(flake_dir) / name),
|
||||
sops_groups_folder(flake_dir),
|
||||
group,
|
||||
)
|
||||
|
||||
|
||||
@ -214,7 +216,7 @@ def add_secret_command(args: argparse.Namespace) -> None:
|
||||
|
||||
def remove_secret(flake_dir: Path, group: str, name: str) -> None:
|
||||
updated_paths = secrets.disallow_member(
|
||||
secrets.groups_folder(flake_dir, name), group
|
||||
secrets.groups_folder(sops_secrets_folder(flake_dir) / name), group
|
||||
)
|
||||
commit_files(
|
||||
updated_paths,
|
||||
|
@ -6,7 +6,12 @@ from ..errors import ClanError
|
||||
from ..git import commit_files
|
||||
from ..machines.types import machine_name_type, validate_hostname
|
||||
from . import secrets
|
||||
from .folders import list_objects, remove_object, sops_machines_folder
|
||||
from .folders import (
|
||||
list_objects,
|
||||
remove_object,
|
||||
sops_machines_folder,
|
||||
sops_secrets_folder,
|
||||
)
|
||||
from .secrets import update_secrets
|
||||
from .sops import read_key, write_key
|
||||
from .types import public_or_private_age_key_type, secret_name_type
|
||||
@ -56,7 +61,7 @@ def list_machines(flake_dir: Path) -> list[str]:
|
||||
|
||||
def add_secret(flake_dir: Path, machine: str, secret: str) -> None:
|
||||
paths = secrets.allow_member(
|
||||
secrets.machines_folder(flake_dir, secret),
|
||||
secrets.machines_folder(sops_secrets_folder(flake_dir) / secret),
|
||||
sops_machines_folder(flake_dir),
|
||||
machine,
|
||||
)
|
||||
@ -69,7 +74,7 @@ def add_secret(flake_dir: Path, machine: str, secret: str) -> None:
|
||||
|
||||
def remove_secret(flake_dir: Path, machine: str, secret: str) -> None:
|
||||
updated_paths = secrets.disallow_member(
|
||||
secrets.machines_folder(flake_dir, secret), machine
|
||||
secrets.machines_folder(sops_secrets_folder(flake_dir) / secret), machine
|
||||
)
|
||||
commit_files(
|
||||
updated_paths,
|
||||
|
@ -82,7 +82,7 @@ def collect_keys_for_path(path: Path) -> set[str]:
|
||||
|
||||
def encrypt_secret(
|
||||
flake_dir: Path,
|
||||
secret: Path,
|
||||
secret_path: Path,
|
||||
value: IO[str] | str | bytes | None,
|
||||
add_users: list[str] = [],
|
||||
add_machines: list[str] = [],
|
||||
@ -95,7 +95,7 @@ def encrypt_secret(
|
||||
for user in add_users:
|
||||
files_to_commit.extend(
|
||||
allow_member(
|
||||
users_folder(flake_dir, secret.name),
|
||||
users_folder(secret_path),
|
||||
sops_users_folder(flake_dir),
|
||||
user,
|
||||
False,
|
||||
@ -105,7 +105,7 @@ def encrypt_secret(
|
||||
for machine in add_machines:
|
||||
files_to_commit.extend(
|
||||
allow_member(
|
||||
machines_folder(flake_dir, secret.name),
|
||||
machines_folder(secret_path),
|
||||
sops_machines_folder(flake_dir),
|
||||
machine,
|
||||
False,
|
||||
@ -115,33 +115,33 @@ def encrypt_secret(
|
||||
for group in add_groups:
|
||||
files_to_commit.extend(
|
||||
allow_member(
|
||||
groups_folder(flake_dir, secret.name),
|
||||
groups_folder(secret_path),
|
||||
sops_groups_folder(flake_dir),
|
||||
group,
|
||||
False,
|
||||
)
|
||||
)
|
||||
|
||||
keys = collect_keys_for_path(secret)
|
||||
keys = collect_keys_for_path(secret_path)
|
||||
|
||||
if key.pubkey not in keys:
|
||||
keys.add(key.pubkey)
|
||||
files_to_commit.extend(
|
||||
allow_member(
|
||||
users_folder(flake_dir, secret.name),
|
||||
users_folder(secret_path),
|
||||
sops_users_folder(flake_dir),
|
||||
key.username,
|
||||
False,
|
||||
)
|
||||
)
|
||||
|
||||
secret_path = secret / "secret"
|
||||
secret_path = secret_path / "secret"
|
||||
encrypt_file(secret_path, value, list(sorted(keys)))
|
||||
files_to_commit.append(secret_path)
|
||||
commit_files(
|
||||
files_to_commit,
|
||||
flake_dir,
|
||||
f"Update secret {secret.name}",
|
||||
f"Update secret {secret_path.name}",
|
||||
)
|
||||
|
||||
|
||||
@ -169,16 +169,16 @@ def add_secret_argument(parser: argparse.ArgumentParser, autocomplete: bool) ->
|
||||
add_dynamic_completer(secrets_parser, complete_secrets)
|
||||
|
||||
|
||||
def machines_folder(flake_dir: Path, group: str) -> Path:
|
||||
return sops_secrets_folder(flake_dir) / group / "machines"
|
||||
def machines_folder(secret_path: Path) -> Path:
|
||||
return secret_path / "machines"
|
||||
|
||||
|
||||
def users_folder(flake_dir: Path, group: str) -> Path:
|
||||
return sops_secrets_folder(flake_dir) / group / "users"
|
||||
def users_folder(secret_path: Path) -> Path:
|
||||
return secret_path / "users"
|
||||
|
||||
|
||||
def groups_folder(flake_dir: Path, group: str) -> Path:
|
||||
return sops_secrets_folder(flake_dir) / group / "groups"
|
||||
def groups_folder(secret_path: Path) -> Path:
|
||||
return secret_path / "groups"
|
||||
|
||||
|
||||
def list_directory(directory: Path) -> str:
|
||||
@ -245,8 +245,8 @@ def disallow_member(group_folder: Path, name: str) -> list[Path]:
|
||||
)
|
||||
|
||||
|
||||
def has_secret(flake_dir: Path, secret: str) -> bool:
|
||||
return (sops_secrets_folder(flake_dir) / secret / "secret").exists()
|
||||
def has_secret(secret_path: Path) -> bool:
|
||||
return (secret_path / "secret").exists()
|
||||
|
||||
|
||||
def list_secrets(flake_dir: Path, pattern: str | None = None) -> list[str]:
|
||||
@ -255,7 +255,7 @@ def list_secrets(flake_dir: Path, pattern: str | None = None) -> list[str]:
|
||||
def validate(name: str) -> bool:
|
||||
return (
|
||||
VALID_SECRET_NAME.match(name) is not None
|
||||
and has_secret(flake_dir, name)
|
||||
and has_secret(sops_secrets_folder(flake_dir) / name)
|
||||
and (pattern is None or pattern in name)
|
||||
)
|
||||
|
||||
@ -278,16 +278,21 @@ def list_command(args: argparse.Namespace) -> None:
|
||||
print("\n".join(lst))
|
||||
|
||||
|
||||
def decrypt_secret(flake_dir: Path, secret: str) -> str:
|
||||
def decrypt_secret(flake_dir: Path, secret_path: Path) -> str:
|
||||
ensure_sops_key(flake_dir)
|
||||
secret_path = sops_secrets_folder(flake_dir) / secret / "secret"
|
||||
if not secret_path.exists():
|
||||
raise ClanError(f"Secret '{secret}' does not exist")
|
||||
return decrypt_file(secret_path)
|
||||
path = secret_path / "secret"
|
||||
if not path.exists():
|
||||
raise ClanError(f"Secret '{secret_path!s}' does not exist")
|
||||
return decrypt_file(path)
|
||||
|
||||
|
||||
def get_command(args: argparse.Namespace) -> None:
|
||||
print(decrypt_secret(args.flake.path, args.secret), end="")
|
||||
print(
|
||||
decrypt_secret(
|
||||
args.flake.path, sops_secrets_folder(args.flake.path) / args.secret
|
||||
),
|
||||
end="",
|
||||
)
|
||||
|
||||
|
||||
def set_command(args: argparse.Namespace) -> None:
|
||||
|
@ -9,7 +9,7 @@ from ..completions import (
|
||||
from ..errors import ClanError
|
||||
from ..git import commit_files
|
||||
from . import secrets
|
||||
from .folders import list_objects, remove_object, sops_users_folder
|
||||
from .folders import list_objects, remove_object, sops_secrets_folder, sops_users_folder
|
||||
from .secrets import update_secrets
|
||||
from .sops import read_key, write_key
|
||||
from .types import (
|
||||
@ -63,7 +63,9 @@ def list_users(flake_dir: Path) -> list[str]:
|
||||
|
||||
def add_secret(flake_dir: Path, user: str, secret: str) -> None:
|
||||
updated_paths = secrets.allow_member(
|
||||
secrets.users_folder(flake_dir, secret), sops_users_folder(flake_dir), user
|
||||
secrets.users_folder(sops_secrets_folder(flake_dir) / secret),
|
||||
sops_users_folder(flake_dir),
|
||||
user,
|
||||
)
|
||||
commit_files(
|
||||
updated_paths,
|
||||
@ -74,7 +76,7 @@ def add_secret(flake_dir: Path, user: str, secret: str) -> None:
|
||||
|
||||
def remove_secret(flake_dir: Path, user: str, secret: str) -> None:
|
||||
updated_paths = secrets.disallow_member(
|
||||
secrets.users_folder(flake_dir, secret), user
|
||||
secrets.users_folder(sops_secrets_folder(flake_dir) / secret), user
|
||||
)
|
||||
commit_files(
|
||||
updated_paths,
|
||||
|
@ -2,9 +2,8 @@ import argparse
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from getpass import getpass
|
||||
from graphlib import TopologicalSorter
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
@ -29,17 +28,7 @@ from .secret_modules import SecretStoreBase
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
|
||||
"""
|
||||
Read multi-line input from stdin.
|
||||
"""
|
||||
print(prompt, flush=True)
|
||||
proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True)
|
||||
log.info("Input received. Processing...")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> list[str]:
|
||||
def bubblewrap_cmd(generator: str, tmpdir: Path) -> list[str]:
|
||||
# fmt: off
|
||||
return nix_shell(
|
||||
[
|
||||
@ -51,8 +40,7 @@ def bubblewrap_cmd(generator: str, generator_dir: Path, dep_tmpdir: Path) -> lis
|
||||
"--ro-bind", "/nix/store", "/nix/store",
|
||||
"--tmpfs", "/usr/lib/systemd",
|
||||
"--dev", "/dev",
|
||||
"--bind", str(generator_dir), str(generator_dir),
|
||||
"--ro-bind", str(dep_tmpdir), str(dep_tmpdir),
|
||||
"--bind", str(tmpdir), str(tmpdir),
|
||||
"--unshare-all",
|
||||
"--unshare-user",
|
||||
"--uid", "1000",
|
||||
@ -92,7 +80,7 @@ def decrypt_dependencies(
|
||||
def dependencies_as_dir(
|
||||
decrypted_dependencies: dict[str, dict[str, bytes]],
|
||||
tmpdir: Path,
|
||||
) -> Path:
|
||||
) -> None:
|
||||
for dep_generator, files in decrypted_dependencies.items():
|
||||
dep_generator_dir = tmpdir / dep_generator
|
||||
dep_generator_dir.mkdir()
|
||||
@ -102,7 +90,6 @@ def dependencies_as_dir(
|
||||
file_path.touch()
|
||||
file_path.chmod(0o600)
|
||||
file_path.write_bytes(file)
|
||||
return tmpdir
|
||||
|
||||
|
||||
def execute_generator(
|
||||
@ -111,10 +98,7 @@ def execute_generator(
|
||||
regenerate: bool,
|
||||
secret_vars_store: SecretStoreBase,
|
||||
public_vars_store: FactStoreBase,
|
||||
dep_tmpdir: Path,
|
||||
prompt: Callable[[str], str],
|
||||
) -> bool:
|
||||
generator_dir = dep_tmpdir / generator_name
|
||||
# check if all secrets exist and generate them if at least one is missing
|
||||
needs_regeneration = not check_secrets(machine, generator_name=generator_name)
|
||||
log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}")
|
||||
@ -124,51 +108,65 @@ def execute_generator(
|
||||
msg = f"flake is not a Path: {machine.flake}"
|
||||
msg += "fact/secret generation is only supported for local flakes"
|
||||
|
||||
# compatibility for old outputs.nix users
|
||||
generator = machine.vars_generators[generator_name]["finalScript"]
|
||||
# if machine.vars_data[generator_name]["generator"]["prompt"]:
|
||||
# prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"])
|
||||
# env["prompt_value"] = prompt_value
|
||||
|
||||
# build temporary file tree of dependencies
|
||||
decrypted_dependencies = decrypt_dependencies(
|
||||
machine, generator_name, secret_vars_store, public_vars_store
|
||||
)
|
||||
env = os.environ.copy()
|
||||
generator_dir.mkdir(parents=True)
|
||||
env["out"] = str(generator_dir)
|
||||
with TemporaryDirectory() as tmp:
|
||||
dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmp))
|
||||
env["in"] = str(dep_tmpdir)
|
||||
tmpdir = Path(tmp)
|
||||
tmpdir_in = tmpdir / "in"
|
||||
tmpdir_prompts = tmpdir / "prompts"
|
||||
tmpdir_out = tmpdir / "out"
|
||||
tmpdir_in.mkdir()
|
||||
tmpdir_out.mkdir()
|
||||
env["in"] = str(tmpdir_in)
|
||||
env["out"] = str(tmpdir_out)
|
||||
# populate dependency inputs
|
||||
dependencies_as_dir(decrypted_dependencies, tmpdir_in)
|
||||
# populate prompted values
|
||||
# TODO: make prompts rest API friendly
|
||||
if machine.vars_generators[generator_name]["prompts"]:
|
||||
tmpdir_prompts.mkdir()
|
||||
env["prompts"] = str(tmpdir_prompts)
|
||||
for prompt_name, prompt in machine.vars_generators[generator_name][
|
||||
"prompts"
|
||||
].items():
|
||||
prompt_file = tmpdir_prompts / prompt_name
|
||||
value = prompt_func(prompt["description"], prompt["type"])
|
||||
prompt_file.write_text(value)
|
||||
|
||||
if sys.platform == "linux":
|
||||
cmd = bubblewrap_cmd(generator, generator_dir, dep_tmpdir=dep_tmpdir)
|
||||
cmd = bubblewrap_cmd(generator, tmpdir)
|
||||
else:
|
||||
cmd = ["bash", "-c", generator]
|
||||
run(
|
||||
cmd,
|
||||
env=env,
|
||||
)
|
||||
files_to_commit = []
|
||||
# store secrets
|
||||
files = machine.vars_generators[generator_name]["files"]
|
||||
for file_name, file in files.items():
|
||||
groups = machine.deployment["sops"]["defaultGroups"]
|
||||
files_to_commit = []
|
||||
# store secrets
|
||||
files = machine.vars_generators[generator_name]["files"]
|
||||
for file_name, file in files.items():
|
||||
groups = machine.deployment["sops"]["defaultGroups"]
|
||||
|
||||
secret_file = generator_dir / file_name
|
||||
if not secret_file.is_file():
|
||||
msg = f"did not generate a file for '{file_name}' when running the following command:\n"
|
||||
msg += generator
|
||||
raise ClanError(msg)
|
||||
if file["secret"]:
|
||||
file_path = secret_vars_store.set(
|
||||
generator_name, file_name, secret_file.read_bytes(), groups
|
||||
)
|
||||
else:
|
||||
file_path = public_vars_store.set(
|
||||
generator_name, file_name, secret_file.read_bytes()
|
||||
)
|
||||
if file_path:
|
||||
files_to_commit.append(file_path)
|
||||
secret_file = tmpdir_out / file_name
|
||||
if not secret_file.is_file():
|
||||
msg = f"did not generate a file for '{file_name}' when running the following command:\n"
|
||||
msg += generator
|
||||
raise ClanError(msg)
|
||||
if file["secret"]:
|
||||
file_path = secret_vars_store.set(
|
||||
generator_name, file_name, secret_file.read_bytes(), groups
|
||||
)
|
||||
else:
|
||||
file_path = public_vars_store.set(
|
||||
generator_name, file_name, secret_file.read_bytes()
|
||||
)
|
||||
if file_path:
|
||||
files_to_commit.append(file_path)
|
||||
commit_files(
|
||||
files_to_commit,
|
||||
machine.flake_dir,
|
||||
@ -177,9 +175,18 @@ def execute_generator(
|
||||
return True
|
||||
|
||||
|
||||
def prompt_func(text: str) -> str:
|
||||
print(f"{text}: ")
|
||||
return read_multiline_input()
|
||||
def prompt_func(description: str, input_type: str) -> str:
|
||||
if input_type == "line":
|
||||
result = input(f"Enter the value for {description}: ")
|
||||
elif input_type == "multiline":
|
||||
print(f"Enter the value for {description} (Finish with Ctrl-D): ")
|
||||
result = sys.stdin.read()
|
||||
elif input_type == "hidden":
|
||||
result = getpass(f"Enter the value for {description} (hidden): ")
|
||||
else:
|
||||
raise ClanError(f"Unknown input type: {input_type} for prompt {description}")
|
||||
log.info("Input received. Processing...")
|
||||
return result
|
||||
|
||||
|
||||
def _get_subgraph(graph: dict[str, set], vertex: str) -> dict[str, set]:
|
||||
@ -197,11 +204,7 @@ def _generate_vars_for_machine(
|
||||
machine: Machine,
|
||||
generator_name: str | None,
|
||||
regenerate: bool,
|
||||
tmpdir: Path,
|
||||
prompt: Callable[[str], str] = prompt_func,
|
||||
) -> bool:
|
||||
local_temp = tmpdir / machine.name
|
||||
local_temp.mkdir()
|
||||
secret_vars_module = importlib.import_module(machine.secret_vars_module)
|
||||
secret_vars_store = secret_vars_module.SecretStore(machine=machine)
|
||||
|
||||
@ -216,13 +219,6 @@ def _generate_vars_for_machine(
|
||||
f"Could not find generator with name: {generator_name}. The following generators are available: {generators}"
|
||||
)
|
||||
|
||||
# if generator_name:
|
||||
# machine_generator_facts = {
|
||||
# generator_name: machine.vars_generators[generator_name]
|
||||
# }
|
||||
# else:
|
||||
# machine_generator_facts = machine.vars_generators
|
||||
|
||||
graph = {
|
||||
gen_name: set(generator["dependencies"])
|
||||
for gen_name, generator in machine.vars_generators.items()
|
||||
@ -250,8 +246,6 @@ def _generate_vars_for_machine(
|
||||
regenerate=regenerate,
|
||||
secret_vars_store=secret_vars_store,
|
||||
public_vars_store=public_vars_store,
|
||||
dep_tmpdir=local_temp,
|
||||
prompt=prompt,
|
||||
)
|
||||
if machine_updated:
|
||||
# flush caches to make sure the new secrets are available in evaluation
|
||||
@ -263,25 +257,21 @@ def generate_vars(
|
||||
machines: list[Machine],
|
||||
generator_name: str | None,
|
||||
regenerate: bool,
|
||||
prompt: Callable[[str], str] = prompt_func,
|
||||
) -> bool:
|
||||
was_regenerated = False
|
||||
with TemporaryDirectory() as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
|
||||
for machine in machines:
|
||||
errors = 0
|
||||
try:
|
||||
was_regenerated |= _generate_vars_for_machine(
|
||||
machine, generator_name, regenerate, tmpdir, prompt
|
||||
)
|
||||
except Exception as exc:
|
||||
log.error(f"Failed to generate facts for {machine.name}: {exc}")
|
||||
errors += 1
|
||||
if errors > 0:
|
||||
raise ClanError(
|
||||
f"Failed to generate facts for {errors} hosts. Check the logs above"
|
||||
)
|
||||
for machine in machines:
|
||||
errors = 0
|
||||
try:
|
||||
was_regenerated |= _generate_vars_for_machine(
|
||||
machine, generator_name, regenerate
|
||||
)
|
||||
except Exception as exc:
|
||||
log.error(f"Failed to generate facts for {machine.name}: {exc}")
|
||||
errors += 1
|
||||
if errors > 0:
|
||||
raise ClanError(
|
||||
f"Failed to generate facts for {errors} hosts. Check the logs above"
|
||||
)
|
||||
|
||||
if not was_regenerated:
|
||||
print("All secrets and facts are already up to date")
|
||||
|
@ -13,33 +13,45 @@ class SecretStore(SecretStoreBase):
|
||||
self.machine = machine
|
||||
|
||||
def set(
|
||||
self, service: str, name: str, value: bytes, groups: list[str]
|
||||
self, generator_name: str, name: str, value: bytes, groups: list[str]
|
||||
) -> Path | None:
|
||||
subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#pass"],
|
||||
["pass", "insert", "-m", f"machines/{self.machine.name}/{name}"],
|
||||
[
|
||||
"pass",
|
||||
"insert",
|
||||
"-m",
|
||||
f"machines/{self.machine.name}/{generator_name}/{name}",
|
||||
],
|
||||
),
|
||||
input=value,
|
||||
check=True,
|
||||
)
|
||||
return None # we manage the files outside of the git repo
|
||||
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
def get(self, generator_name: str, name: str) -> bytes:
|
||||
return subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#pass"],
|
||||
["pass", "show", f"machines/{self.machine.name}/{name}"],
|
||||
[
|
||||
"pass",
|
||||
"show",
|
||||
f"machines/{self.machine.name}/{generator_name}/{name}",
|
||||
],
|
||||
),
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout
|
||||
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
def exists(self, generator_name: str, name: str) -> bool:
|
||||
password_store = os.environ.get(
|
||||
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
|
||||
)
|
||||
secret_path = Path(password_store) / f"machines/{self.machine.name}/{name}.gpg"
|
||||
secret_path = (
|
||||
Path(password_store)
|
||||
/ f"machines/{self.machine.name}/{generator_name}/{name}.gpg"
|
||||
)
|
||||
return secret_path.exists()
|
||||
|
||||
def generate_hash(self) -> bytes:
|
||||
|
@ -36,13 +36,20 @@ class SecretStore(SecretStoreBase):
|
||||
)
|
||||
add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
|
||||
|
||||
def secret_path(self, generator_name: str, secret_name: str) -> Path:
|
||||
return (
|
||||
self.machine.flake_dir
|
||||
/ "sops"
|
||||
/ "vars"
|
||||
/ self.machine.name
|
||||
/ generator_name
|
||||
/ secret_name
|
||||
)
|
||||
|
||||
def set(
|
||||
self, generator_name: str, name: str, value: bytes, groups: list[str]
|
||||
) -> Path | None:
|
||||
path = (
|
||||
sops_secrets_folder(self.machine.flake_dir)
|
||||
/ f"vars-{self.machine.name}-{generator_name}-{name}"
|
||||
)
|
||||
path = self.secret_path(generator_name, name)
|
||||
encrypt_secret(
|
||||
self.machine.flake_dir,
|
||||
path,
|
||||
@ -54,19 +61,21 @@ class SecretStore(SecretStoreBase):
|
||||
|
||||
def get(self, generator_name: str, name: str) -> bytes:
|
||||
return decrypt_secret(
|
||||
self.machine.flake_dir, f"vars-{self.machine.name}-{generator_name}-{name}"
|
||||
self.machine.flake_dir, self.secret_path(generator_name, name)
|
||||
).encode("utf-8")
|
||||
|
||||
def exists(self, generator_name: str, name: str) -> bool:
|
||||
return has_secret(
|
||||
self.machine.flake_dir,
|
||||
f"vars-{self.machine.name}-{generator_name}-{name}",
|
||||
self.secret_path(generator_name, name),
|
||||
)
|
||||
|
||||
def upload(self, output_dir: Path) -> None:
|
||||
key_name = f"{self.machine.name}-age.key"
|
||||
if not has_secret(self.machine.flake_dir, key_name):
|
||||
if not has_secret(sops_secrets_folder(self.machine.flake_dir) / key_name):
|
||||
# skip uploading the secret, not managed by us
|
||||
return
|
||||
key = decrypt_secret(self.machine.flake_dir, key_name)
|
||||
key = decrypt_secret(
|
||||
self.machine.flake_dir,
|
||||
sops_secrets_folder(self.machine.flake_dir) / key_name,
|
||||
)
|
||||
(output_dir / "key.txt").write_text(key)
|
||||
|
@ -8,6 +8,12 @@ from ..completions import add_dynamic_completer, complete_machines
|
||||
from ..machines.machines import Machine
|
||||
|
||||
|
||||
@dataclass
|
||||
class WaypipeConfig:
|
||||
enable: bool
|
||||
command: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class VmConfig:
|
||||
machine_name: str
|
||||
@ -24,6 +30,8 @@ class VmConfig:
|
||||
def __post_init__(self) -> None:
|
||||
if isinstance(self.flake_url, str):
|
||||
self.flake_url = FlakeId(self.flake_url)
|
||||
if isinstance(self.waypipe, dict):
|
||||
self.waypipe = WaypipeConfig(**self.waypipe)
|
||||
|
||||
|
||||
def inspect_vm(machine: Machine) -> VmConfig:
|
||||
|
@ -56,6 +56,7 @@ def generate_flake(
|
||||
},
|
||||
# define the machines directly including their config
|
||||
machine_configs: dict[str, dict] = {},
|
||||
inventory: dict[str, dict] = {},
|
||||
) -> FlakeForTest:
|
||||
"""
|
||||
Creates a clan flake with the given name.
|
||||
@ -80,6 +81,12 @@ def generate_flake(
|
||||
shutil.copytree(flake_template, flake)
|
||||
sp.run(["chmod", "+w", "-R", str(flake)], check=True)
|
||||
|
||||
# initialize inventory
|
||||
if inventory:
|
||||
# check if inventory valid
|
||||
inventory_path = flake / "inventory.json"
|
||||
inventory_path.write_text(json.dumps(inventory, indent=2))
|
||||
|
||||
# substitute `substitutions` in all files of the template
|
||||
for file in flake.rglob("*"):
|
||||
if file.is_file():
|
||||
|
@ -20,12 +20,12 @@ def test_create_flake(
|
||||
cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
|
||||
|
||||
assert (flake_dir / ".clan-flake").exists()
|
||||
|
||||
# Replace the inputs.clan.url in the template flake.nix
|
||||
substitute(
|
||||
flake_dir / "flake.nix",
|
||||
clan_core,
|
||||
)
|
||||
# Dont evaluate the inventory before the substitute call
|
||||
|
||||
monkeypatch.chdir(flake_dir)
|
||||
cli.run(["machines", "create", "machine1"])
|
||||
|
@ -14,7 +14,7 @@ from clan_cli.inventory import (
|
||||
ServiceBorgbackupRoleClient,
|
||||
ServiceBorgbackupRoleServer,
|
||||
ServiceMeta,
|
||||
load_inventory,
|
||||
load_inventory_json,
|
||||
save_inventory,
|
||||
)
|
||||
from clan_cli.machines.create import create_machine
|
||||
@ -67,7 +67,7 @@ def test_add_module_to_inventory(
|
||||
),
|
||||
)
|
||||
|
||||
inventory = load_inventory(base_path)
|
||||
inventory = load_inventory_json(base_path)
|
||||
|
||||
inventory.services.borgbackup = {
|
||||
"borg1": ServiceBorgbackup(
|
||||
|
@ -1,6 +1,8 @@
|
||||
import os
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
@ -13,7 +15,8 @@ from root import CLAN_CORE
|
||||
|
||||
from clan_cli.clan_uri import FlakeId
|
||||
from clan_cli.machines.machines import Machine
|
||||
from clan_cli.vars.secret_modules.sops import SecretStore
|
||||
from clan_cli.nix import nix_shell
|
||||
from clan_cli.vars.secret_modules import password_store, sops
|
||||
|
||||
|
||||
def def_value() -> defaultdict:
|
||||
@ -55,7 +58,8 @@ def test_dependencies_as_files() -> None:
|
||||
),
|
||||
)
|
||||
with TemporaryDirectory() as tmpdir:
|
||||
dep_tmpdir = dependencies_as_dir(decrypted_dependencies, Path(tmpdir))
|
||||
dep_tmpdir = Path(tmpdir)
|
||||
dependencies_as_dir(decrypted_dependencies, dep_tmpdir)
|
||||
assert dep_tmpdir.is_dir()
|
||||
assert (dep_tmpdir / "gen_1" / "var_1a").read_bytes() == b"var_1a"
|
||||
assert (dep_tmpdir / "gen_1" / "var_1b").read_bytes() == b"var_1b"
|
||||
@ -93,7 +97,45 @@ def test_generate_public_var(
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_generate_secret_var_with_default_group(
|
||||
def test_generate_secret_var_sops(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
temporary_home: Path,
|
||||
sops_setup: SopsSetup,
|
||||
) -> None:
|
||||
user = os.environ.get("USER", "user")
|
||||
config = nested_dict()
|
||||
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
|
||||
my_generator["files"]["my_secret"]["secret"] = True
|
||||
my_generator["script"] = "echo hello > $out/my_secret"
|
||||
flake = generate_flake(
|
||||
temporary_home,
|
||||
flake_template=CLAN_CORE / "templates" / "minimal",
|
||||
machine_configs=dict(my_machine=config),
|
||||
)
|
||||
monkeypatch.chdir(flake.path)
|
||||
cli.run(
|
||||
[
|
||||
"secrets",
|
||||
"users",
|
||||
"add",
|
||||
"--flake",
|
||||
str(flake.path),
|
||||
user,
|
||||
sops_setup.keys[0].pubkey,
|
||||
]
|
||||
)
|
||||
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
|
||||
var_file_path = (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
|
||||
)
|
||||
assert not var_file_path.is_file()
|
||||
sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
|
||||
assert sops_store.exists("my_generator", "my_secret")
|
||||
assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_generate_secret_var_sops_with_default_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
temporary_home: Path,
|
||||
sops_setup: SopsSetup,
|
||||
@ -126,19 +168,75 @@ def test_generate_secret_var_with_default_group(
|
||||
assert not (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
|
||||
).is_file()
|
||||
sops_store = SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
|
||||
sops_store = sops.SecretStore(Machine(name="my_machine", flake=FlakeId(flake.path)))
|
||||
assert sops_store.exists("my_generator", "my_secret")
|
||||
assert (
|
||||
flake.path
|
||||
/ "sops"
|
||||
/ "secrets"
|
||||
/ "vars-my_machine-my_generator-my_secret"
|
||||
/ "vars"
|
||||
/ "my_machine"
|
||||
/ "my_generator"
|
||||
/ "my_secret"
|
||||
/ "groups"
|
||||
/ "my_group"
|
||||
).exists()
|
||||
assert sops_store.get("my_generator", "my_secret").decode() == "hello\n"
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_generate_secret_var_password_store(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
temporary_home: Path,
|
||||
) -> None:
|
||||
config = nested_dict()
|
||||
my_generator = config["clan"]["core"]["vars"]["settings"]["secretStore"] = (
|
||||
"password-store"
|
||||
)
|
||||
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
|
||||
my_generator["files"]["my_secret"]["secret"] = True
|
||||
my_generator["script"] = "echo hello > $out/my_secret"
|
||||
flake = generate_flake(
|
||||
temporary_home,
|
||||
flake_template=CLAN_CORE / "templates" / "minimal",
|
||||
machine_configs=dict(my_machine=config),
|
||||
)
|
||||
monkeypatch.chdir(flake.path)
|
||||
gnupghome = temporary_home / "gpg"
|
||||
gnupghome.mkdir(mode=0o700)
|
||||
monkeypatch.setenv("GNUPGHOME", str(gnupghome))
|
||||
monkeypatch.setenv("PASSWORD_STORE_DIR", str(temporary_home / "pass"))
|
||||
gpg_key_spec = temporary_home / "gpg_key_spec"
|
||||
gpg_key_spec.write_text(
|
||||
"""
|
||||
Key-Type: 1
|
||||
Key-Length: 1024
|
||||
Name-Real: Root Superuser
|
||||
Name-Email: test@local
|
||||
Expire-Date: 0
|
||||
%no-protection
|
||||
"""
|
||||
)
|
||||
subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#gnupg"], ["gpg", "--batch", "--gen-key", str(gpg_key_spec)]
|
||||
),
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
nix_shell(["nixpkgs#pass"], ["pass", "init", "test@local"]), check=True
|
||||
)
|
||||
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
|
||||
var_file_path = (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
|
||||
)
|
||||
assert not var_file_path.is_file()
|
||||
store = password_store.SecretStore(
|
||||
Machine(name="my_machine", flake=FlakeId(flake.path))
|
||||
)
|
||||
assert store.exists("my_generator", "my_secret")
|
||||
assert store.get("my_generator", "my_secret").decode() == "hello\n"
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_generate_secret_for_multiple_machines(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@ -194,8 +292,8 @@ def test_generate_secret_for_multiple_machines(
|
||||
assert machine2_var_file_path.is_file()
|
||||
assert machine2_var_file_path.read_text() == "machine2\n"
|
||||
# check if secret vars have been created correctly
|
||||
sops_store1 = SecretStore(Machine(name="machine1", flake=FlakeId(flake.path)))
|
||||
sops_store2 = SecretStore(Machine(name="machine2", flake=FlakeId(flake.path)))
|
||||
sops_store1 = sops.SecretStore(Machine(name="machine1", flake=FlakeId(flake.path)))
|
||||
sops_store2 = sops.SecretStore(Machine(name="machine2", flake=FlakeId(flake.path)))
|
||||
assert sops_store1.exists("my_generator", "my_secret")
|
||||
assert sops_store2.exists("my_generator", "my_secret")
|
||||
assert sops_store1.get("my_generator", "my_secret").decode() == "machine1\n"
|
||||
@ -232,3 +330,40 @@ def test_dependant_generators(
|
||||
)
|
||||
assert child_file_path.is_file()
|
||||
assert child_file_path.read_text() == "hello\n"
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_type", "input_value"),
|
||||
[
|
||||
("line", "my input"),
|
||||
("multiline", "my\nmultiline\ninput\n"),
|
||||
# The hidden type cannot easily be tested, as getpass() reads from /dev/tty directly
|
||||
# ("hidden", "my hidden input"),
|
||||
],
|
||||
)
|
||||
def test_prompt(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
temporary_home: Path,
|
||||
prompt_type: str,
|
||||
input_value: str,
|
||||
) -> None:
|
||||
config = nested_dict()
|
||||
my_generator = config["clan"]["core"]["vars"]["generators"]["my_generator"]
|
||||
my_generator["files"]["my_value"]["secret"] = False
|
||||
my_generator["prompts"]["prompt1"]["description"] = "dream2nix"
|
||||
my_generator["prompts"]["prompt1"]["type"] = prompt_type
|
||||
my_generator["script"] = "cat $prompts/prompt1 > $out/my_value"
|
||||
flake = generate_flake(
|
||||
temporary_home,
|
||||
flake_template=CLAN_CORE / "templates" / "minimal",
|
||||
machine_configs=dict(my_machine=config),
|
||||
)
|
||||
monkeypatch.chdir(flake.path)
|
||||
monkeypatch.setattr("sys.stdin", StringIO(input_value))
|
||||
cli.run(["vars", "generate", "--flake", str(flake.path), "my_machine"])
|
||||
var_file_path = (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_value"
|
||||
)
|
||||
assert var_file_path.is_file()
|
||||
assert var_file_path.read_text() == input_value
|
||||
|
@ -14,6 +14,9 @@
|
||||
},
|
||||
{
|
||||
"path": "../../lib/build-clan"
|
||||
},
|
||||
{
|
||||
"path": "../../../democlan"
|
||||
}
|
||||
],
|
||||
"settings": {
|
||||
|
Before Width: | Height: | Size: 108 KiB |
Before Width: | Height: | Size: 106 KiB |
Before Width: | Height: | Size: 104 KiB |
Before Width: | Height: | Size: 98 KiB |
Before Width: | Height: | Size: 155 KiB |
Before Width: | Height: | Size: 86 KiB |
Before Width: | Height: | Size: 163 KiB |
Before Width: | Height: | Size: 183 KiB |
Before Width: | Height: | Size: 3.1 KiB After Width: | Height: | Size: 3.1 KiB |
After Width: | Height: | Size: 375 B |
After Width: | Height: | Size: 717 B |
After Width: | Height: | Size: 717 B |
After Width: | Height: | Size: 1.5 KiB |
Before Width: | Height: | Size: 152 KiB |
@ -158,9 +158,15 @@ class VMObject(GObject.Object):
|
||||
flake=uri.flake,
|
||||
)
|
||||
assert self.machine is not None
|
||||
state_dir = vm_state_dir(
|
||||
flake_url=str(self.machine.flake.url), vm_name=self.machine.name
|
||||
)
|
||||
|
||||
if self.machine.flake.is_local():
|
||||
state_dir = vm_state_dir(
|
||||
flake_url=str(self.machine.flake.path), vm_name=self.machine.name
|
||||
)
|
||||
else:
|
||||
state_dir = vm_state_dir(
|
||||
flake_url=self.machine.flake.url, vm_name=self.machine.name
|
||||
)
|
||||
self.qmp_wrap = QMPWrapper(state_dir)
|
||||
assert self.machine is not None
|
||||
yield self.machine
|
||||
|
@ -1,156 +0,0 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any
|
||||
|
||||
import gi
|
||||
from clan_cli.api import API
|
||||
|
||||
gi.require_version("WebKit", "6.0")
|
||||
|
||||
from gi.repository import GLib, WebKit
|
||||
|
||||
site_index: Path = (
|
||||
Path(sys.argv[0]).absolute()
|
||||
/ Path("../..")
|
||||
/ Path("clan_vm_manager/.webui/index.html")
|
||||
).resolve()
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dataclass_to_dict(obj: Any) -> Any:
|
||||
"""
|
||||
Utility function to convert dataclasses to dictionaries
|
||||
It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
|
||||
|
||||
It does NOT convert member functions.
|
||||
"""
|
||||
if dataclasses.is_dataclass(obj):
|
||||
return {k: dataclass_to_dict(v) for k, v in dataclasses.asdict(obj).items()}
|
||||
elif isinstance(obj, list | tuple):
|
||||
return [dataclass_to_dict(item) for item in obj]
|
||||
elif isinstance(obj, dict):
|
||||
return {k: dataclass_to_dict(v) for k, v in obj.items()}
|
||||
else:
|
||||
return obj
|
||||
|
||||
|
||||
class WebView:
|
||||
def __init__(self, methods: dict[str, Callable]) -> None:
|
||||
self.method_registry: dict[str, Callable] = methods
|
||||
|
||||
self.webview = WebKit.WebView()
|
||||
|
||||
settings = self.webview.get_settings()
|
||||
# settings.
|
||||
settings.set_property("enable-developer-extras", True)
|
||||
self.webview.set_settings(settings)
|
||||
|
||||
self.manager = self.webview.get_user_content_manager()
|
||||
# Can be called with: window.webkit.messageHandlers.gtk.postMessage("...")
|
||||
# Important: it seems postMessage must be given some payload, otherwise it won't trigger the event
|
||||
self.manager.register_script_message_handler("gtk")
|
||||
self.manager.connect("script-message-received", self.on_message_received)
|
||||
|
||||
self.webview.load_uri(f"file://{site_index}")
|
||||
|
||||
# global mutex lock to ensure functions run sequentially
|
||||
self.mutex_lock = Lock()
|
||||
self.queue_size = 0
|
||||
|
||||
def on_message_received(
|
||||
self, user_content_manager: WebKit.UserContentManager, message: Any
|
||||
) -> None:
|
||||
payload = json.loads(message.to_json(0))
|
||||
method_name = payload["method"]
|
||||
handler_fn = self.method_registry[method_name]
|
||||
|
||||
log.debug(f"Received message: {payload}")
|
||||
log.debug(f"Queue size: {self.queue_size} (Wait)")
|
||||
|
||||
def threaded_wrapper() -> bool:
|
||||
"""
|
||||
Ensures only one function is executed at a time
|
||||
|
||||
Wait until there is no other function acquiring the global lock.
|
||||
|
||||
Starts a thread with the potentially long running API function within.
|
||||
"""
|
||||
if not self.mutex_lock.locked():
|
||||
thread = threading.Thread(
|
||||
target=self.threaded_handler,
|
||||
args=(
|
||||
handler_fn,
|
||||
payload.get("data"),
|
||||
method_name,
|
||||
),
|
||||
)
|
||||
thread.start()
|
||||
return GLib.SOURCE_REMOVE
|
||||
|
||||
return GLib.SOURCE_CONTINUE
|
||||
|
||||
GLib.idle_add(
|
||||
threaded_wrapper,
|
||||
)
|
||||
self.queue_size += 1
|
||||
|
||||
def threaded_handler(
|
||||
self,
|
||||
handler_fn: Callable[
|
||||
...,
|
||||
Any,
|
||||
],
|
||||
data: dict[str, Any] | None,
|
||||
method_name: str,
|
||||
) -> None:
|
||||
with self.mutex_lock:
|
||||
log.debug("Executing... ", method_name)
|
||||
log.debug(f"{data}")
|
||||
if data is None:
|
||||
result = handler_fn()
|
||||
else:
|
||||
reconciled_arguments = {}
|
||||
for k, v in data.items():
|
||||
# Some functions expect to be called with dataclass instances
|
||||
# But the js api returns dictionaries.
|
||||
# Introspect the function and create the expected dataclass from dict dynamically
|
||||
# Depending on the introspected argument_type
|
||||
arg_type = API.get_method_argtype(method_name, k)
|
||||
if dataclasses.is_dataclass(arg_type):
|
||||
reconciled_arguments[k] = arg_type(**v)
|
||||
else:
|
||||
reconciled_arguments[k] = v
|
||||
|
||||
result = handler_fn(**reconciled_arguments)
|
||||
|
||||
serialized = json.dumps(dataclass_to_dict(result))
|
||||
|
||||
# Use idle_add to queue the response call to js on the main GTK thread
|
||||
GLib.idle_add(self.return_data_to_js, method_name, serialized)
|
||||
self.queue_size -= 1
|
||||
log.debug(f"Done: Remaining queue size: {self.queue_size}")
|
||||
|
||||
def return_data_to_js(self, method_name: str, serialized: str) -> bool:
|
||||
# This function must be run on the main GTK thread to interact with the webview
|
||||
# result = method_fn(data) # takes very long
|
||||
# serialized = result
|
||||
self.webview.evaluate_javascript(
|
||||
f"""
|
||||
window.clan.{method_name}(`{serialized}`);
|
||||
""",
|
||||
-1,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
return GLib.SOURCE_REMOVE
|
||||
|
||||
def get_webview(self) -> WebKit.WebView:
|
||||
return self.webview
|
@ -2,7 +2,6 @@ import logging
|
||||
import threading
|
||||
|
||||
import gi
|
||||
from clan_cli.api import API
|
||||
from clan_cli.history.list import list_history
|
||||
|
||||
from clan_vm_manager.components.interfaces import ClanConfig
|
||||
@ -12,7 +11,6 @@ from clan_vm_manager.singletons.use_vms import ClanStore
|
||||
from clan_vm_manager.views.details import Details
|
||||
from clan_vm_manager.views.list import ClanList
|
||||
from clan_vm_manager.views.logs import Logs
|
||||
from clan_vm_manager.views.webview import WebView
|
||||
|
||||
gi.require_version("Adw", "1")
|
||||
|
||||
@ -61,9 +59,6 @@ class MainWindow(Adw.ApplicationWindow):
|
||||
stack_view.add_named(Details(), "details")
|
||||
stack_view.add_named(Logs(), "logs")
|
||||
|
||||
webview = WebView(methods=API._registry)
|
||||
stack_view.add_named(webview.get_webview(), "webview")
|
||||
|
||||
stack_view.set_visible_child_name(config.initial_view)
|
||||
|
||||
view.set_content(scroll)
|
||||
|
@ -18,7 +18,6 @@
|
||||
runCommand,
|
||||
setuptools,
|
||||
webkitgtk_6_0,
|
||||
webview-ui,
|
||||
wrapGAppsHook,
|
||||
}:
|
||||
let
|
||||
@ -26,7 +25,7 @@ let
|
||||
desktop-file = makeDesktopItem {
|
||||
name = "org.clan.vm-manager";
|
||||
exec = "clan-vm-manager %u";
|
||||
icon = ./clan_vm_manager/assets/clan_white.png;
|
||||
icon = "clan-white";
|
||||
desktopName = "Clan Manager";
|
||||
startupWMClass = "clan";
|
||||
mimeTypes = [ "x-scheme-handler/clan" ];
|
||||
@ -142,10 +141,9 @@ python3.pkgs.buildPythonApplication rec {
|
||||
passthru.runtimeDependencies = runtimeDependencies;
|
||||
passthru.testDependencies = testDependencies;
|
||||
|
||||
# TODO: place webui in lib/python3.11/site-packages/clan_vm_manager
|
||||
postInstall = ''
|
||||
mkdir -p $out/clan_vm_manager/.webui
|
||||
cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/clan_vm_manager/.webui
|
||||
mkdir -p $out/share/icons/hicolor
|
||||
cp -r ./clan_vm_manager/assets/white-favicons/* $out/share/icons/hicolor
|
||||
'';
|
||||
|
||||
# Don't leak python packages into a devshell.
|
||||
|
@ -13,10 +13,10 @@
|
||||
else
|
||||
{
|
||||
devShells.clan-vm-manager = pkgs.callPackage ./shell.nix {
|
||||
inherit (config.packages) clan-vm-manager webview-ui;
|
||||
inherit (config.packages) clan-vm-manager;
|
||||
};
|
||||
packages.clan-vm-manager = pkgs.python3.pkgs.callPackage ./default.nix {
|
||||
inherit (config.packages) clan-cli webview-ui;
|
||||
inherit (config.packages) clan-cli;
|
||||
};
|
||||
|
||||
checks = config.packages.clan-vm-manager.tests;
|
||||
|
@ -1,23 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
CLAN=$(nix build .#clan-vm-manager --print-out-paths)
|
||||
|
||||
if ! command -v xdg-mime &> /dev/null; then
|
||||
echo "Warning: 'xdg-mime' is not available. The desktop file cannot be installed."
|
||||
fi
|
||||
|
||||
ALREADY_INSTALLED=$(nix profile list --json | jq 'has("elements") and (.elements | has("clan-vm-manager"))')
|
||||
|
||||
if [ "$ALREADY_INSTALLED" = "true" ]; then
|
||||
echo "Upgrading installed clan-vm-manager"
|
||||
nix profile upgrade clan-vm-manager
|
||||
else
|
||||
nix profile install .#clan-vm-manager --priority 4
|
||||
fi
|
||||
|
||||
|
||||
# install desktop file
|
||||
set -eou pipefail
|
||||
DESKTOP_FILE_NAME=org.clan.vm-manager.desktop
|
||||
DESKTOP_DST=~/.local/share/applications/"$DESKTOP_FILE_NAME"
|
||||
DESKTOP_SRC="$CLAN/share/applications/$DESKTOP_FILE_NAME"
|
||||
UI_BIN="$CLAN/bin/clan-vm-manager"
|
||||
|
||||
cp -f "$DESKTOP_SRC" "$DESKTOP_DST"
|
||||
sleep 2
|
||||
sed -i "s|Exec=.*clan-vm-manager|Exec=$UI_BIN|" "$DESKTOP_DST"
|
||||
xdg-mime default "$DESKTOP_FILE_NAME" x-scheme-handler/clan
|
||||
echo "==== Validating desktop file installation ===="
|
||||
set -x
|
||||
desktop-file-validate "$DESKTOP_DST"
|
||||
set +xeou pipefail
|
||||
|
@ -10,7 +10,7 @@
|
||||
python3,
|
||||
gtk4,
|
||||
libadwaita,
|
||||
webview-ui,
|
||||
|
||||
}:
|
||||
|
||||
let
|
||||
@ -52,11 +52,5 @@ mkShell {
|
||||
|
||||
# Add clan-cli to the python path so that we can import it without building it in nix first
|
||||
export PYTHONPATH="$GIT_ROOT/pkgs/clan-cli":"$PYTHONPATH"
|
||||
|
||||
# Add the webview-ui to the .webui directory
|
||||
rm -rf ./clan_vm_manager/.webui/*
|
||||
mkdir -p ./clan_vm_manager/.webui
|
||||
cp -a ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* ./clan_vm_manager/.webui
|
||||
chmod -R +w ./clan_vm_manager/.webui
|
||||
'';
|
||||
}
|
||||
|
@ -32,7 +32,7 @@ let
|
||||
}
|
||||
// flashDiskoConfig;
|
||||
|
||||
# Important: The partition names need to be different to the clan install
|
||||
# Important: The partition names need to be different to the clan install
|
||||
flashDiskoConfig = {
|
||||
boot.loader.grub.efiSupport = lib.mkDefault true;
|
||||
boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
|
||||
|
4
pkgs/webview-ui/.gitignore
vendored
@ -1 +1,3 @@
|
||||
api
|
||||
api
|
||||
|
||||
.vite
|
@ -8,6 +8,7 @@ import { Flash } from "./routes/flash/view";
|
||||
import { Settings } from "./routes/settings";
|
||||
import { Welcome } from "./routes/welcome";
|
||||
import { Deploy } from "./routes/deploy";
|
||||
import { CreateMachine } from "./routes/machines/create";
|
||||
|
||||
export type Route = keyof typeof routes;
|
||||
|
||||
@ -22,6 +23,11 @@ export const routes = {
|
||||
label: "Machines",
|
||||
icon: "devices_other",
|
||||
},
|
||||
"machines/add": {
|
||||
child: CreateMachine,
|
||||
label: "create Machine",
|
||||
icon: "add",
|
||||
},
|
||||
hosts: {
|
||||
child: HostList,
|
||||
label: "hosts",
|
||||
|
@ -67,6 +67,17 @@ function createFunctions<K extends OperationNames>(
|
||||
dispatch: (args: OperationArgs<K>) => void;
|
||||
receive: (fn: (response: OperationResponse<K>) => void, id: string) => void;
|
||||
} {
|
||||
window.clan[operationName] = (s: string) => {
|
||||
const f = (response: OperationResponse<K>) => {
|
||||
// Get the correct receiver function for the op_key
|
||||
const receiver = registry[operationName][response.op_key];
|
||||
if (receiver) {
|
||||
receiver(response);
|
||||
}
|
||||
};
|
||||
deserialize(f)(s);
|
||||
};
|
||||
|
||||
return {
|
||||
dispatch: (args: OperationArgs<K>) => {
|
||||
// Send the data to the gtk app
|
||||
@ -78,15 +89,6 @@ function createFunctions<K extends OperationNames>(
|
||||
receive: (fn: (response: OperationResponse<K>) => void, id: string) => {
|
||||
// @ts-expect-error: This should work although typescript doesn't let us write
|
||||
registry[operationName][id] = fn;
|
||||
|
||||
window.clan[operationName] = (s: string) => {
|
||||
const f = (response: OperationResponse<K>) => {
|
||||
if (response.op_key === id) {
|
||||
registry[operationName][id](response);
|
||||
}
|
||||
};
|
||||
deserialize(f)(s);
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -141,8 +143,9 @@ const deserialize =
|
||||
fn(r);
|
||||
} catch (e) {
|
||||
console.log("Error parsing JSON: ", e);
|
||||
console.log({ download: () => download("error.json", str) });
|
||||
window.localStorage.setItem("error", str);
|
||||
console.error(str);
|
||||
console.error("See localStorage 'error'");
|
||||
alert(`Error parsing JSON: ${e}`);
|
||||
}
|
||||
};
|
||||
|
@ -4,7 +4,6 @@
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
|
||||
|
||||
html {
|
||||
overflow-x: hidden;
|
||||
overflow-y: scroll;
|
||||
|
@ -42,7 +42,15 @@ export const ClanForm = () => {
|
||||
await toast.promise(
|
||||
(async () => {
|
||||
await callApi("create_clan", {
|
||||
options: { directory: target_dir, meta, template_url },
|
||||
options: {
|
||||
directory: target_dir,
|
||||
template_url,
|
||||
initial: {
|
||||
meta,
|
||||
services: {},
|
||||
machines: {},
|
||||
},
|
||||
},
|
||||
});
|
||||
setActiveURI(target_dir);
|
||||
setRoute("machines");
|
||||
|
124
pkgs/webview-ui/app/src/routes/machines/create.tsx
Normal file
@ -0,0 +1,124 @@
|
||||
import { callApi, OperationArgs, pyApi } from "@/src/api";
|
||||
import { activeURI } from "@/src/App";
|
||||
import { createForm, required } from "@modular-forms/solid";
|
||||
import toast from "solid-toast";
|
||||
|
||||
type CreateMachineForm = OperationArgs<"create_machine">;
|
||||
|
||||
export function CreateMachine() {
|
||||
const [formStore, { Form, Field }] = createForm<CreateMachineForm>({});
|
||||
|
||||
const handleSubmit = async (values: CreateMachineForm) => {
|
||||
const active_dir = activeURI();
|
||||
if (!active_dir) {
|
||||
toast.error("Open a clan to create the machine in");
|
||||
return;
|
||||
}
|
||||
|
||||
callApi("create_machine", {
|
||||
flake: {
|
||||
loc: active_dir,
|
||||
},
|
||||
machine: {
|
||||
name: "jon",
|
||||
deploy: {
|
||||
targetHost: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
console.log("submit", values);
|
||||
};
|
||||
return (
|
||||
<div class="px-1">
|
||||
Create new Machine
|
||||
<Form onSubmit={handleSubmit}>
|
||||
<Field
|
||||
name="machine.name"
|
||||
validate={[required("This field is required")]}
|
||||
>
|
||||
{(field, props) => (
|
||||
<>
|
||||
<label class="input input-bordered flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
class="grow"
|
||||
placeholder="name"
|
||||
required
|
||||
{...props}
|
||||
/>
|
||||
</label>
|
||||
<div class="label">
|
||||
{field.error && (
|
||||
<span class="label-text-alt font-bold text-error">
|
||||
{field.error}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</Field>
|
||||
<Field name="machine.description">
|
||||
{(field, props) => (
|
||||
<>
|
||||
<label class="input input-bordered flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
class="grow"
|
||||
placeholder="description"
|
||||
required
|
||||
{...props}
|
||||
/>
|
||||
</label>
|
||||
<div class="label">
|
||||
{field.error && (
|
||||
<span class="label-text-alt font-bold text-error">
|
||||
{field.error}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</Field>
|
||||
<Field name="machine.deploy.targetHost">
|
||||
{(field, props) => (
|
||||
<>
|
||||
<label class="input input-bordered flex items-center gap-2">
|
||||
<input
|
||||
type="text"
|
||||
class="grow"
|
||||
placeholder="root@flash-installer.local"
|
||||
required
|
||||
{...props}
|
||||
/>
|
||||
</label>
|
||||
<div class="label">
|
||||
<span class="label-text-alt text-neutral">
|
||||
Must be set before deployment for the following tasks:
|
||||
<ul>
|
||||
<li>
|
||||
<span>Detect hardware config</span>
|
||||
</li>
|
||||
<li>
|
||||
<span>Detect disk layout</span>
|
||||
</li>
|
||||
<li>
|
||||
<span>Remote installation</span>
|
||||
</li>
|
||||
</ul>
|
||||
</span>
|
||||
{field.error && (
|
||||
<span class="label-text-alt font-bold text-error">
|
||||
{field.error}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</Field>
|
||||
<button class="btn btn-error float-right" type="submit">
|
||||
<span class="material-icons">add</span>Create
|
||||
</button>
|
||||
</Form>
|
||||
</div>
|
||||
);
|
||||
}
|
@ -7,7 +7,7 @@ import {
|
||||
createSignal,
|
||||
type Component,
|
||||
} from "solid-js";
|
||||
import { activeURI, route, setActiveURI } from "@/src/App";
|
||||
import { activeURI, route, setActiveURI, setRoute } from "@/src/App";
|
||||
import { OperationResponse, callApi, pyApi } from "@/src/api";
|
||||
import toast from "solid-toast";
|
||||
import { MachineListItem } from "@/src/components/MachineListItem";
|
||||
@ -86,6 +86,11 @@ export const MachineListView: Component = () => {
|
||||
<span class="material-icons ">refresh</span>
|
||||
</button>
|
||||
</div>
|
||||
<div class="tooltip tooltip-bottom" data-tip="Create machine">
|
||||
<button class="btn btn-ghost" onClick={() => setRoute("machines/add")}>
|
||||
<span class="material-icons ">add</span>
|
||||
</button>
|
||||
</div>
|
||||
{/* <Show when={services()}>
|
||||
{(services) => (
|
||||
<For each={Object.values(services())}>
|
||||
|
@ -12,7 +12,8 @@ import {
|
||||
setRoute,
|
||||
clanList,
|
||||
} from "@/src/App";
|
||||
import { For } from "solid-js";
|
||||
import { For, Show } from "solid-js";
|
||||
import { createQuery } from "@tanstack/solid-query";
|
||||
|
||||
export const registerClan = async () => {
|
||||
try {
|
||||
@ -26,6 +27,7 @@ export const registerClan = async () => {
|
||||
const res = new Set([...s, loc.data]);
|
||||
return Array.from(res);
|
||||
});
|
||||
setActiveURI(loc.data);
|
||||
setRoute((r) => {
|
||||
if (r === "welcome") return "machines";
|
||||
return r;
|
||||
@ -37,6 +39,87 @@ export const registerClan = async () => {
|
||||
}
|
||||
};
|
||||
|
||||
interface ClanDetailsProps {
|
||||
clan_dir: string;
|
||||
}
|
||||
const ClanDetails = (props: ClanDetailsProps) => {
|
||||
const { clan_dir } = props;
|
||||
|
||||
const details = createQuery(() => ({
|
||||
queryKey: [clan_dir, "meta"],
|
||||
queryFn: async () => {
|
||||
const result = await callApi("show_clan_meta", { uri: clan_dir });
|
||||
if (result.status === "error") throw new Error("Failed to fetch data");
|
||||
return result.data;
|
||||
},
|
||||
}));
|
||||
|
||||
return (
|
||||
<div class="stat">
|
||||
<div class="stat-figure text-primary">
|
||||
<div class="join">
|
||||
<button
|
||||
class=" join-item btn-sm"
|
||||
classList={{
|
||||
"btn btn-ghost btn-outline": activeURI() !== clan_dir,
|
||||
"badge badge-primary": activeURI() === clan_dir,
|
||||
}}
|
||||
disabled={activeURI() === clan_dir}
|
||||
onClick={() => {
|
||||
setActiveURI(clan_dir);
|
||||
}}
|
||||
>
|
||||
{activeURI() === clan_dir ? "active" : "select"}
|
||||
</button>
|
||||
<button
|
||||
class="btn btn-ghost btn-outline join-item btn-sm"
|
||||
onClick={() => {
|
||||
setClanList((s) =>
|
||||
s.filter((v, idx) => {
|
||||
if (v == clan_dir) {
|
||||
setActiveURI(
|
||||
clanList()[idx - 1] || clanList()[idx + 1] || null
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
);
|
||||
}}
|
||||
>
|
||||
Remove
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="stat-title">Clan URI</div>
|
||||
|
||||
<Show when={details.isSuccess}>
|
||||
<div
|
||||
class="stat-value"
|
||||
// classList={{
|
||||
// "text-primary": activeURI() === clan_dir,
|
||||
// }}
|
||||
>
|
||||
{details.data?.name}
|
||||
</div>
|
||||
</Show>
|
||||
<Show
|
||||
when={details.isSuccess && details.data?.description}
|
||||
fallback={<div class="stat-desc text-lg">{clan_dir}</div>}
|
||||
>
|
||||
<div
|
||||
class="stat-desc text-lg"
|
||||
// classList={{
|
||||
// "text-primary": activeURI() === clan_dir,
|
||||
// }}
|
||||
>
|
||||
{details.data?.description}
|
||||
</div>
|
||||
</Show>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export const Settings = () => {
|
||||
return (
|
||||
<div class="card card-normal">
|
||||
@ -54,60 +137,7 @@ export const Settings = () => {
|
||||
</div>
|
||||
<div class="stats stats-vertical shadow">
|
||||
<For each={clanList()}>
|
||||
{(value) => (
|
||||
<div class="stat">
|
||||
<div class="stat-figure text-primary">
|
||||
<div class="join">
|
||||
<button
|
||||
class=" join-item btn-sm"
|
||||
classList={{
|
||||
"btn btn-ghost btn-outline": activeURI() !== value,
|
||||
"badge badge-primary": activeURI() === value,
|
||||
}}
|
||||
disabled={activeURI() === value}
|
||||
onClick={() => {
|
||||
setActiveURI(value);
|
||||
}}
|
||||
>
|
||||
{activeURI() === value ? "active" : "select"}
|
||||
</button>
|
||||
<button
|
||||
class="btn btn-ghost btn-outline join-item btn-sm"
|
||||
onClick={() => {
|
||||
setClanList((s) =>
|
||||
s.filter((v, idx) => {
|
||||
if (v == value) {
|
||||
setActiveURI(
|
||||
clanList()[idx - 1] ||
|
||||
clanList()[idx + 1] ||
|
||||
null
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
);
|
||||
// if (activeURI() === value) {
|
||||
// setActiveURI();
|
||||
// }
|
||||
}}
|
||||
>
|
||||
Remove URI
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="stat-title">Clan URI</div>
|
||||
|
||||
<div
|
||||
class="stat-desc text-lg"
|
||||
classList={{
|
||||
"text-primary": activeURI() === value,
|
||||
}}
|
||||
>
|
||||
{value}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{(value) => <ClanDetails clan_dir={value} />}
|
||||
</For>
|
||||
</div>
|
||||
</div>
|
||||
|
@ -17,7 +17,6 @@
|
||||
systems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
|
||||
"x86_64-darwin"
|
||||
"aarch64-darwin"
|
||||
];
|
||||
|
5
templates/minimal/inventory.json
Normal file
@ -0,0 +1,5 @@
|
||||
{
|
||||
"meta": { "name": "__CHANGE_ME__" },
|
||||
"machines": {},
|
||||
"services": {}
|
||||
}
|