Merge branch 'main' into Mic92-main
This commit is contained in:
commit
adf404ee0f
3
.envrc
3
.envrc
@ -1,3 +1,4 @@
|
||||
# shellcheck shell=bash
|
||||
if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then
|
||||
source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4="
|
||||
fi
|
||||
@ -5,7 +6,7 @@ fi
|
||||
watch_file .direnv/selected-shell
|
||||
|
||||
if [ -e .direnv/selected-shell ]; then
|
||||
use flake .#$(cat .direnv/selected-shell)
|
||||
use flake ".#$(cat .direnv/selected-shell)"
|
||||
else
|
||||
use flake
|
||||
fi
|
||||
|
@ -1,100 +0,0 @@
|
||||
From bc199a27f23b0fcf175b116f7cf606c0d22b422a Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
|
||||
Date: Tue, 11 Jun 2024 11:40:47 +0200
|
||||
Subject: [PATCH 1/2] register_new_matrix_user: add password-file flag
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
getpass in python expects stdin to be a tty, hence we cannot just pipe
|
||||
into register_new_matrix_user. --password-file instead works better and
|
||||
it would also allow the use of stdin if /dev/stdin is passed.
|
||||
|
||||
Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
|
||||
Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
|
||||
---
|
||||
changelog.d/17294.feature | 2 ++
|
||||
debian/register_new_matrix_user.ronn | 9 +++++++--
|
||||
synapse/_scripts/register_new_matrix_user.py | 20 +++++++++++++++-----
|
||||
3 files changed, 24 insertions(+), 7 deletions(-)
|
||||
create mode 100644 changelog.d/17294.feature
|
||||
|
||||
diff --git a/changelog.d/17294.feature b/changelog.d/17294.feature
|
||||
new file mode 100644
|
||||
index 000000000..33aac7b0b
|
||||
--- /dev/null
|
||||
+++ b/changelog.d/17294.feature
|
||||
@@ -0,0 +1,2 @@
|
||||
+`register_new_matrix_user` now supports a --password-file flag, which
|
||||
+is useful for scripting.
|
||||
diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn
|
||||
index 0410b1f4c..d99e9215a 100644
|
||||
--- a/debian/register_new_matrix_user.ronn
|
||||
+++ b/debian/register_new_matrix_user.ronn
|
||||
@@ -31,8 +31,13 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
|
||||
Local part of the new user. Will prompt if omitted.
|
||||
|
||||
* `-p`, `--password`:
|
||||
- New password for user. Will prompt if omitted. Supplying the password
|
||||
- on the command line is not recommended. Use the STDIN instead.
|
||||
+ New password for user. Will prompt if this option and `--password-file` are omitted.
|
||||
+ Supplying the password on the command line is not recommended.
|
||||
+ Use `--password-file` if possible.
|
||||
+
|
||||
+ * `--password-file`:
|
||||
+ File containing the new password for user. If set, overrides `--password`.
|
||||
+ This is a more secure alternative to specifying the password on the command line.
|
||||
|
||||
* `-a`, `--admin`:
|
||||
Register new user as an admin. Will prompt if omitted.
|
||||
diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
|
||||
index 77a7129ee..972b35e2d 100644
|
||||
--- a/synapse/_scripts/register_new_matrix_user.py
|
||||
+++ b/synapse/_scripts/register_new_matrix_user.py
|
||||
@@ -173,11 +173,18 @@ def main() -> None:
|
||||
default=None,
|
||||
help="Local part of the new user. Will prompt if omitted.",
|
||||
)
|
||||
- parser.add_argument(
|
||||
+ password_group = parser.add_mutually_exclusive_group()
|
||||
+ password_group.add_argument(
|
||||
"-p",
|
||||
"--password",
|
||||
default=None,
|
||||
- help="New password for user. Will prompt if omitted.",
|
||||
+ help="New password for user. Will prompt for a password if "
|
||||
+ "this flag and `--password-file` are both omitted.",
|
||||
+ )
|
||||
+ password_group.add_argument(
|
||||
+ "--password-file",
|
||||
+ default=None,
|
||||
+ help="File containing the new password for user. If set, will override `--password`.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t",
|
||||
@@ -247,6 +254,11 @@ def main() -> None:
|
||||
print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
+ if args.password_file:
|
||||
+ password = _read_file(args.password_file, "password-file").strip()
|
||||
+ else:
|
||||
+ password = args.password
|
||||
+
|
||||
if args.server_url:
|
||||
server_url = args.server_url
|
||||
elif config is not None:
|
||||
@@ -269,9 +281,7 @@ def main() -> None:
|
||||
if args.admin or args.no_admin:
|
||||
admin = args.admin
|
||||
|
||||
- register_new_user(
|
||||
- args.user, args.password, server_url, secret, admin, args.user_type
|
||||
- )
|
||||
+ register_new_user(args.user, password, server_url, secret, admin, args.user_type)
|
||||
|
||||
|
||||
def _read_file(file_path: Any, config_path: str) -> str:
|
||||
--
|
||||
2.44.1
|
||||
|
@ -1,94 +0,0 @@
|
||||
From 1789416df425d22693b0055a6688d8686e0ee4a1 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
|
||||
Date: Thu, 13 Jun 2024 14:38:19 +0200
|
||||
Subject: [PATCH 2/2] register-new-matrix-user: add a flag to ignore already
|
||||
existing users
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
This allows to register users in a more declarative and stateless way.
|
||||
|
||||
Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
|
||||
---
|
||||
synapse/_scripts/register_new_matrix_user.py | 22 ++++++++++++++++++--
|
||||
1 file changed, 20 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
|
||||
index 972b35e2d..233e7267d 100644
|
||||
--- a/synapse/_scripts/register_new_matrix_user.py
|
||||
+++ b/synapse/_scripts/register_new_matrix_user.py
|
||||
@@ -52,6 +52,7 @@ def request_registration(
|
||||
user_type: Optional[str] = None,
|
||||
_print: Callable[[str], None] = print,
|
||||
exit: Callable[[int], None] = sys.exit,
|
||||
+ exists_ok: bool = False,
|
||||
) -> None:
|
||||
url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),)
|
||||
|
||||
@@ -97,6 +98,10 @@ def request_registration(
|
||||
r = requests.post(url, json=data)
|
||||
|
||||
if r.status_code != 200:
|
||||
+ response = r.json()
|
||||
+ if exists_ok and response["errcode"] == "M_USER_IN_USE":
|
||||
+ _print("User already exists. Skipping.")
|
||||
+ return
|
||||
_print("ERROR! Received %d %s" % (r.status_code, r.reason))
|
||||
if 400 <= r.status_code < 500:
|
||||
try:
|
||||
@@ -115,6 +120,7 @@ def register_new_user(
|
||||
shared_secret: str,
|
||||
admin: Optional[bool],
|
||||
user_type: Optional[str],
|
||||
+ exists_ok: bool = False,
|
||||
) -> None:
|
||||
if not user:
|
||||
try:
|
||||
@@ -154,7 +160,13 @@ def register_new_user(
|
||||
admin = False
|
||||
|
||||
request_registration(
|
||||
- user, password, server_location, shared_secret, bool(admin), user_type
|
||||
+ user,
|
||||
+ password,
|
||||
+ server_location,
|
||||
+ shared_secret,
|
||||
+ bool(admin),
|
||||
+ user_type,
|
||||
+ exists_ok=exists_ok,
|
||||
)
|
||||
|
||||
|
||||
@@ -173,6 +185,11 @@ def main() -> None:
|
||||
default=None,
|
||||
help="Local part of the new user. Will prompt if omitted.",
|
||||
)
|
||||
+ parser.add_argument(
|
||||
+ "--exists-ok",
|
||||
+ action="store_true",
|
||||
+ help="Do not fail if user already exists.",
|
||||
+ )
|
||||
password_group = parser.add_mutually_exclusive_group()
|
||||
password_group.add_argument(
|
||||
"-p",
|
||||
@@ -192,6 +209,7 @@ def main() -> None:
|
||||
default=None,
|
||||
help="User type as specified in synapse.api.constants.UserTypes",
|
||||
)
|
||||
+
|
||||
admin_group = parser.add_mutually_exclusive_group()
|
||||
admin_group.add_argument(
|
||||
"-a",
|
||||
@@ -281,7 +299,7 @@ def main() -> None:
|
||||
if args.admin or args.no_admin:
|
||||
admin = args.admin
|
||||
|
||||
- register_new_user(args.user, password, server_url, secret, admin, args.user_type)
|
||||
+ register_new_user(args.user, password, server_url, secret, admin, args.user_type, exists_ok=args.exists_ok)
|
||||
|
||||
|
||||
def _read_file(file_path: Any, config_path: str) -> str:
|
||||
--
|
||||
2.44.1
|
||||
|
@ -17,19 +17,8 @@ let
|
||||
ln -s $out/config.json $out/config.${nginx-vhost}.json
|
||||
'';
|
||||
|
||||
# FIXME: This was taken from upstream. Drop this when our patch is upstream
|
||||
synapseCfg = config.services.matrix-synapse;
|
||||
wantedExtras =
|
||||
synapseCfg.extras
|
||||
++ lib.optional (synapseCfg.settings ? oidc_providers) "oidc"
|
||||
++ lib.optional (synapseCfg.settings ? jwt_config) "jwt"
|
||||
++ lib.optional (synapseCfg.settings ? saml2_config) "saml2"
|
||||
++ lib.optional (synapseCfg.settings ? redis) "redis"
|
||||
++ lib.optional (synapseCfg.settings ? sentry) "sentry"
|
||||
++ lib.optional (synapseCfg.settings ? user_directory) "user-search"
|
||||
++ lib.optional (synapseCfg.settings.url_preview_enabled) "url-preview"
|
||||
++ lib.optional (synapseCfg.settings.database.name == "psycopg2") "postgres";
|
||||
in
|
||||
# FIXME: This was taken from upstream. Drop this when our patch is upstream
|
||||
{
|
||||
options.services.matrix-synapse.package = lib.mkOption { readOnly = false; };
|
||||
options.clan.matrix-synapse = {
|
||||
@ -78,21 +67,6 @@ in
|
||||
];
|
||||
config = {
|
||||
services.matrix-synapse = {
|
||||
package = lib.mkForce (
|
||||
pkgs.matrix-synapse.override {
|
||||
matrix-synapse-unwrapped = pkgs.matrix-synapse.unwrapped.overrideAttrs (_old: {
|
||||
doInstallCheck = false; # too slow, nixpkgs maintainer already run this.
|
||||
patches = [
|
||||
# see: https://github.com/element-hq/synapse/pull/17304
|
||||
./0001-register_new_matrix_user-add-password-file-flag.patch
|
||||
./0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch
|
||||
];
|
||||
});
|
||||
extras = wantedExtras;
|
||||
plugins = synapseCfg.plugins;
|
||||
}
|
||||
);
|
||||
|
||||
enable = true;
|
||||
settings = {
|
||||
server_name = cfg.domain;
|
||||
|
@ -1,6 +1,8 @@
|
||||
# shellcheck shell=bash
|
||||
source_up
|
||||
|
||||
watch_file $(find ./nix -name "*.nix" -printf '%p ')
|
||||
mapfile -d '' -t nix_files < <(find ./nix -name "*.nix" -print0)
|
||||
watch_file "${nix_files[@]}"
|
||||
|
||||
# Because we depend on nixpkgs sources, uploading to builders takes a long time
|
||||
use flake .#docs --builders ''
|
||||
|
@ -26,6 +26,7 @@ pkgs.stdenv.mkDerivation {
|
||||
mkdocs-material
|
||||
mkdocs-rss-plugin
|
||||
mkdocs-macros
|
||||
filelock # FIXME: this should be already provided by mkdocs-rss-plugin
|
||||
]);
|
||||
configurePhase = ''
|
||||
mkdir -p ./site/reference/cli
|
||||
|
@ -38,7 +38,7 @@
|
||||
patchShebangs --build $out
|
||||
|
||||
ruff format --check --diff $out
|
||||
ruff --line-length 88 $out
|
||||
ruff check --line-length 88 $out
|
||||
mypy --strict $out
|
||||
'';
|
||||
|
||||
|
60
flake.lock
60
flake.lock
@ -7,11 +7,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1718846788,
|
||||
"narHash": "sha256-9dtXYtEkmXoUJV+PGLqscqF7qTn4AIhAKpFWRFU2NYs=",
|
||||
"lastModified": 1720056646,
|
||||
"narHash": "sha256-BymcV4HWtx2VFuabDCM4/nEJcfivCx0S02wUCz11mAY=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "e1174d991944a01eaaa04bc59c6281edca4c0e6e",
|
||||
"rev": "64679cd7f318c9b6595902b47d4585b1d51d5f9e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -27,11 +27,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717285511,
|
||||
"narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
|
||||
"lastModified": 1719994518,
|
||||
"narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
|
||||
"rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -40,29 +40,6 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-generators": {
|
||||
"inputs": {
|
||||
"nixlib": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1718025593,
|
||||
"narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-generators",
|
||||
"rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-generators",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-images": {
|
||||
"inputs": {
|
||||
"nixos-stable": [],
|
||||
@ -71,11 +48,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1718845599,
|
||||
"narHash": "sha256-HbQ0iKohKJC5grC95HNjLxGPdgsc/BJgoENDYNbzkLo=",
|
||||
"lastModified": 1720055024,
|
||||
"narHash": "sha256-c5rsiI1R7tnCDpcgfsa7ouSdn6wpctbme9TUp53CFyU=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-images",
|
||||
"rev": "c1e6a5f7b08f1c9993de1cfc5f15f838bf783b88",
|
||||
"rev": "f8650460d37d9d1820a93ebb7f0db5b6c3621946",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -86,11 +63,11 @@
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1719451888,
|
||||
"narHash": "sha256-Ky0sgEEJMcBmNEJztY6KcVn+6bq74EKM7pd1CR1wnPQ=",
|
||||
"lastModified": 1720340162,
|
||||
"narHash": "sha256-iVLH0Ygtw/Iw9Q1cFFX7OhNnoPbc7/ZWW6J3c0zbiZw=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "3664857c48feacb35770c00abfdc671e55849be5",
|
||||
"rev": "60a94e515488e335bd5bce096431d490486915e3",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -104,7 +81,6 @@
|
||||
"inputs": {
|
||||
"disko": "disko",
|
||||
"flake-parts": "flake-parts",
|
||||
"nixos-generators": "nixos-generators",
|
||||
"nixos-images": "nixos-images",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"sops-nix": "sops-nix",
|
||||
@ -119,11 +95,11 @@
|
||||
"nixpkgs-stable": []
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1719111739,
|
||||
"narHash": "sha256-kr2QzRrplzlCP87ddayCZQS+dhGW98kw2zy7+jUXtF4=",
|
||||
"lastModified": 1720321395,
|
||||
"narHash": "sha256-kcI8q9Nh8/CSj0ygfWq1DLckHl8IHhFarL8ie6g7OEk=",
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"rev": "5e2e9421e9ed2b918be0a441c4535cfa45e04811",
|
||||
"rev": "c184aca4db5d71c3db0c8cbfcaaec337a5d065ea",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -139,11 +115,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1718522839,
|
||||
"narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=",
|
||||
"lastModified": 1720436211,
|
||||
"narHash": "sha256-/cKXod0oGLl+vH4bKBZnTV3qxrw4jgOLnyQ8KXey5J8=",
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81",
|
||||
"rev": "6fc8bded78715cdd43a3278a14ded226eb3a239e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
@ -13,9 +13,6 @@
|
||||
sops-nix.url = "github:Mic92/sops-nix";
|
||||
sops-nix.inputs.nixpkgs.follows = "nixpkgs";
|
||||
sops-nix.inputs.nixpkgs-stable.follows = "";
|
||||
nixos-generators.url = "github:nix-community/nixos-generators";
|
||||
nixos-generators.inputs.nixpkgs.follows = "nixpkgs";
|
||||
nixos-generators.inputs.nixlib.follows = "nixpkgs";
|
||||
nixos-images.url = "github:nix-community/nixos-images";
|
||||
nixos-images.inputs.nixos-unstable.follows = "nixpkgs";
|
||||
# unused input
|
||||
|
@ -1,4 +1,4 @@
|
||||
{ lib, inputs, ... }:
|
||||
{ inputs, ... }:
|
||||
{
|
||||
imports = [ inputs.treefmt-nix.flakeModule ];
|
||||
perSystem =
|
||||
@ -8,43 +8,19 @@
|
||||
treefmt.programs.shellcheck.enable = true;
|
||||
|
||||
treefmt.programs.mypy.enable = true;
|
||||
treefmt.programs.nixfmt.enable = true;
|
||||
treefmt.programs.nixfmt.package = pkgs.nixfmt-rfc-style;
|
||||
treefmt.programs.deadnix.enable = true;
|
||||
|
||||
treefmt.programs.mypy.directories = {
|
||||
"pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
|
||||
"pkgs/clan-app".extraPythonPackages =
|
||||
# clan-app currently only exists on linux
|
||||
(self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies;
|
||||
};
|
||||
treefmt.programs.ruff.check = true;
|
||||
treefmt.programs.ruff.format = true;
|
||||
|
||||
treefmt.settings.formatter.nix = {
|
||||
command = "sh";
|
||||
options = [
|
||||
"-eucx"
|
||||
''
|
||||
# First deadnix
|
||||
${lib.getExe pkgs.deadnix} --edit "$@"
|
||||
# Then nixpkgs-fmt
|
||||
${lib.getExe pkgs.nixfmt-rfc-style} "$@"
|
||||
''
|
||||
"--" # this argument is ignored by bash
|
||||
];
|
||||
includes = [ "*.nix" ];
|
||||
excludes = [
|
||||
# Was copied from nixpkgs. Keep diff minimal to simplify upstreaming.
|
||||
"pkgs/builders/script-writers.nix"
|
||||
];
|
||||
};
|
||||
treefmt.settings.formatter.python = {
|
||||
command = "sh";
|
||||
options = [
|
||||
"-eucx"
|
||||
''
|
||||
${lib.getExe pkgs.ruff} check --fix "$@"
|
||||
${lib.getExe pkgs.ruff} format "$@"
|
||||
''
|
||||
"--" # this argument is ignored by bash
|
||||
];
|
||||
includes = [ "*.py" ];
|
||||
};
|
||||
# FIXME: currently broken in CI
|
||||
#treefmt.settings.formatter.vale =
|
||||
# let
|
||||
|
@ -1,3 +1,4 @@
|
||||
# shellcheck shell=bash
|
||||
source_up
|
||||
|
||||
watch_file flake-module.nix
|
||||
|
@ -1,5 +1,26 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (lib.types) submoduleWith;
|
||||
submodule =
|
||||
module:
|
||||
submoduleWith {
|
||||
specialArgs.pkgs = pkgs;
|
||||
modules = [ module ];
|
||||
};
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
./public/in_repo.nix
|
||||
# ./public/vm.nix
|
||||
# ./secret/password-store.nix
|
||||
./secret/sops.nix
|
||||
# ./secret/vm.nix
|
||||
];
|
||||
options.clan.core.vars = lib.mkOption {
|
||||
visible = false;
|
||||
description = ''
|
||||
@ -11,6 +32,20 @@
|
||||
- generate secrets like private keys automatically when they are needed
|
||||
- output multiple values like private and public keys simultaneously
|
||||
'';
|
||||
type = lib.types.submoduleWith { modules = [ ./interface.nix ]; };
|
||||
type = submodule { imports = [ ./interface.nix ]; };
|
||||
};
|
||||
|
||||
config.system.clan.deployment.data = {
|
||||
vars = {
|
||||
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
|
||||
_name: generator: {
|
||||
inherit (generator) finalScript;
|
||||
files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
|
||||
}
|
||||
);
|
||||
inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;
|
||||
};
|
||||
inherit (config.clan.networking) targetHost buildHost;
|
||||
inherit (config.clan.deployment) requireExplicitUpdate;
|
||||
};
|
||||
}
|
||||
|
@ -54,21 +54,6 @@ in
|
||||
};
|
||||
};
|
||||
|
||||
# Ensure that generators.imports works
|
||||
# This allows importing generators from third party projects without providing
|
||||
# them access to other settings.
|
||||
test_generator_modules =
|
||||
let
|
||||
generator_module = {
|
||||
my-generator.files.password = { };
|
||||
};
|
||||
config = eval { generators.imports = [ generator_module ]; };
|
||||
in
|
||||
{
|
||||
expr = config.generators ? my-generator;
|
||||
expected = true;
|
||||
};
|
||||
|
||||
# script can be text
|
||||
test_script_text =
|
||||
let
|
||||
|
@ -1,8 +1,12 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
lib,
|
||||
config,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
inherit (lib) mkOption;
|
||||
inherit (lib.types)
|
||||
anything
|
||||
attrsOf
|
||||
bool
|
||||
either
|
||||
@ -14,30 +18,27 @@ let
|
||||
submoduleWith
|
||||
;
|
||||
# the original types.submodule has strange behavior
|
||||
submodule = module: submoduleWith { modules = [ module ]; };
|
||||
submodule =
|
||||
module:
|
||||
submoduleWith {
|
||||
specialArgs.pkgs = pkgs;
|
||||
modules = [ module ];
|
||||
};
|
||||
options = lib.mapAttrs (_: mkOption);
|
||||
subOptions = opts: submodule { options = options opts; };
|
||||
in
|
||||
{
|
||||
options = options {
|
||||
settings = {
|
||||
options = {
|
||||
settings = import ./settings-opts.nix { inherit lib; };
|
||||
generators = lib.mkOption {
|
||||
description = ''
|
||||
Settings for the generated variables.
|
||||
A set of generators that can be used to generate files.
|
||||
Generators are scripts that produce files based on the values of other generators and user input.
|
||||
Each generator is expected to produce a set of files under a directory.
|
||||
'';
|
||||
type = submodule {
|
||||
freeformType = anything;
|
||||
imports = [ ./settings.nix ];
|
||||
};
|
||||
};
|
||||
generators = {
|
||||
default = {
|
||||
imports = [
|
||||
# implementation of the generator
|
||||
./generator.nix
|
||||
];
|
||||
};
|
||||
type = submodule {
|
||||
freeformType = attrsOf (subOptions {
|
||||
default = { };
|
||||
type = attrsOf (submodule {
|
||||
imports = [ ./generator.nix ];
|
||||
options = options {
|
||||
dependencies = {
|
||||
description = ''
|
||||
A list of other generators that this generator depends on.
|
||||
@ -52,32 +53,45 @@ in
|
||||
A set of files to generate.
|
||||
The generator 'script' is expected to produce exactly these files under $out.
|
||||
'';
|
||||
type = attrsOf (subOptions {
|
||||
secret = {
|
||||
description = ''
|
||||
Whether the file should be treated as a secret.
|
||||
'';
|
||||
type = bool;
|
||||
default = true;
|
||||
};
|
||||
path = {
|
||||
description = ''
|
||||
The path to the file containing the content of the generated value.
|
||||
This will be set automatically
|
||||
'';
|
||||
type = str;
|
||||
readOnly = true;
|
||||
};
|
||||
value = {
|
||||
description = ''
|
||||
The content of the generated value.
|
||||
Only available if the file is not secret.
|
||||
'';
|
||||
type = str;
|
||||
default = throw "Cannot access value of secret file";
|
||||
defaultText = "Throws error because the value of a secret file is not accessible";
|
||||
};
|
||||
});
|
||||
type = attrsOf (
|
||||
submodule (file: {
|
||||
imports = [ config.settings.fileModule ];
|
||||
options = options {
|
||||
name = {
|
||||
type = lib.types.str;
|
||||
description = ''
|
||||
name of the public fact
|
||||
'';
|
||||
readOnly = true;
|
||||
default = file.config._module.args.name;
|
||||
};
|
||||
secret = {
|
||||
description = ''
|
||||
Whether the file should be treated as a secret.
|
||||
'';
|
||||
type = bool;
|
||||
default = true;
|
||||
};
|
||||
path = {
|
||||
description = ''
|
||||
The path to the file containing the content of the generated value.
|
||||
This will be set automatically
|
||||
'';
|
||||
type = str;
|
||||
readOnly = true;
|
||||
};
|
||||
value = {
|
||||
description = ''
|
||||
The content of the generated value.
|
||||
Only available if the file is not secret.
|
||||
'';
|
||||
type = str;
|
||||
default = throw "Cannot access value of secret file";
|
||||
defaultText = "Throws error because the value of a secret file is not accessible";
|
||||
};
|
||||
};
|
||||
})
|
||||
);
|
||||
};
|
||||
prompts = {
|
||||
description = ''
|
||||
@ -85,28 +99,30 @@ in
|
||||
Prompts are available to the generator script as files.
|
||||
For example, a prompt named 'prompt1' will be available via $prompts/prompt1
|
||||
'';
|
||||
type = attrsOf (subOptions {
|
||||
description = {
|
||||
description = ''
|
||||
The description of the prompted value
|
||||
'';
|
||||
type = str;
|
||||
example = "SSH private key";
|
||||
};
|
||||
type = {
|
||||
description = ''
|
||||
The input type of the prompt.
|
||||
The following types are available:
|
||||
- hidden: A hidden text (e.g. password)
|
||||
- line: A single line of text
|
||||
- multiline: A multiline text
|
||||
'';
|
||||
type = enum [
|
||||
"hidden"
|
||||
"line"
|
||||
"multiline"
|
||||
];
|
||||
default = "line";
|
||||
type = attrsOf (submodule {
|
||||
options = {
|
||||
description = {
|
||||
description = ''
|
||||
The description of the prompted value
|
||||
'';
|
||||
type = str;
|
||||
example = "SSH private key";
|
||||
};
|
||||
type = {
|
||||
description = ''
|
||||
The input type of the prompt.
|
||||
The following types are available:
|
||||
- hidden: A hidden text (e.g. password)
|
||||
- line: A single line of text
|
||||
- multiline: A multiline text
|
||||
'';
|
||||
type = enum [
|
||||
"hidden"
|
||||
"line"
|
||||
"multiline"
|
||||
];
|
||||
default = "line";
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
@ -140,8 +156,8 @@ in
|
||||
internal = true;
|
||||
visible = false;
|
||||
};
|
||||
});
|
||||
};
|
||||
};
|
||||
});
|
||||
};
|
||||
};
|
||||
}
|
||||
|
12
nixosModules/clanCore/vars/public/in_repo.nix
Normal file
12
nixosModules/clanCore/vars/public/in_repo.nix
Normal file
@ -0,0 +1,12 @@
|
||||
{ config, lib, ... }:
|
||||
{
|
||||
config.clan.core.vars.settings =
|
||||
lib.mkIf (config.clan.core.vars.settings.publicStore == "in_repo")
|
||||
{
|
||||
publicModule = "clan_cli.vars.public_modules.in_repo";
|
||||
fileModule = file: {
|
||||
path =
|
||||
config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}";
|
||||
};
|
||||
};
|
||||
}
|
61
nixosModules/clanCore/vars/secret/sops.nix
Normal file
61
nixosModules/clanCore/vars/secret/sops.nix
Normal file
@ -0,0 +1,61 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
secretsDir = config.clan.core.clanDir + "/sops/secrets";
|
||||
groupsDir = config.clan.core.clanDir + "/sops/groups";
|
||||
|
||||
# My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation?
|
||||
containsSymlink =
|
||||
path:
|
||||
builtins.pathExists path
|
||||
&& (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink");
|
||||
|
||||
containsMachine =
|
||||
parent: name: type:
|
||||
type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clan.core.machineName}";
|
||||
|
||||
containsMachineOrGroups =
|
||||
name: type:
|
||||
(containsMachine secretsDir name type)
|
||||
|| lib.any (
|
||||
group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}"
|
||||
) groups;
|
||||
|
||||
filterDir =
|
||||
filter: dir:
|
||||
lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir));
|
||||
|
||||
groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir);
|
||||
secrets = filterDir containsMachineOrGroups secretsDir;
|
||||
in
|
||||
{
|
||||
config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
|
||||
# Before we generate a secret we cannot know the path yet, so we need to set it to an empty string
|
||||
fileModule = file: {
|
||||
path =
|
||||
lib.mkIf file.secret
|
||||
config.sops.secrets.${"${config.clan.core.machineName}-${file.config.name}"}.path
|
||||
or "/no-such-path";
|
||||
};
|
||||
secretModule = "clan_cli.vars.secret_modules.sops";
|
||||
secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
|
||||
};
|
||||
|
||||
config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
|
||||
secrets = builtins.mapAttrs (name: _: {
|
||||
sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret";
|
||||
format = "binary";
|
||||
}) secrets;
|
||||
# To get proper error messages about missing secrets we need a dummy secret file that is always present
|
||||
defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (
|
||||
lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" ""))
|
||||
);
|
||||
age.keyFile = lib.mkIf (builtins.pathExists (
|
||||
config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret"
|
||||
)) (lib.mkDefault "/var/lib/sops-nix/key.txt");
|
||||
};
|
||||
}
|
71
nixosModules/clanCore/vars/settings-opts.nix
Normal file
71
nixosModules/clanCore/vars/settings-opts.nix
Normal file
@ -0,0 +1,71 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
secretStore = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"sops"
|
||||
"password-store"
|
||||
"vm"
|
||||
"custom"
|
||||
];
|
||||
default = "sops";
|
||||
description = ''
|
||||
method to store secret facts
|
||||
custom can be used to define a custom secret fact store.
|
||||
'';
|
||||
};
|
||||
|
||||
secretModule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
description = ''
|
||||
the python import path to the secret module
|
||||
'';
|
||||
};
|
||||
|
||||
secretUploadDirectory = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
description = ''
|
||||
The directory where secrets are uploaded into, This is backend specific.
|
||||
'';
|
||||
};
|
||||
|
||||
fileModule = lib.mkOption {
|
||||
type = lib.types.deferredModule;
|
||||
internal = true;
|
||||
description = ''
|
||||
A module to be imported in every vars.files.<name> submodule.
|
||||
Used by backends to define the `path` attribute.
|
||||
'';
|
||||
default = { };
|
||||
};
|
||||
|
||||
publicStore = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"in_repo"
|
||||
"vm"
|
||||
"custom"
|
||||
];
|
||||
default = "in_repo";
|
||||
description = ''
|
||||
method to store public facts.
|
||||
custom can be used to define a custom public fact store.
|
||||
'';
|
||||
};
|
||||
|
||||
publicModule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
description = ''
|
||||
the python import path to the public module
|
||||
'';
|
||||
};
|
||||
|
||||
publicDirectory = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
description = ''
|
||||
The directory where public facts are stored.
|
||||
'';
|
||||
};
|
||||
}
|
@ -1,72 +0,0 @@
|
||||
{ lib, ... }:
|
||||
{
|
||||
options = {
|
||||
secretStore = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"sops"
|
||||
"password-store"
|
||||
"vm"
|
||||
"custom"
|
||||
];
|
||||
default = "sops";
|
||||
description = ''
|
||||
method to store secret facts
|
||||
custom can be used to define a custom secret fact store.
|
||||
'';
|
||||
};
|
||||
|
||||
secretModule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
description = ''
|
||||
the python import path to the secret module
|
||||
'';
|
||||
};
|
||||
|
||||
secretUploadDirectory = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
description = ''
|
||||
The directory where secrets are uploaded into, This is backend specific.
|
||||
'';
|
||||
};
|
||||
|
||||
secretPathFunction = lib.mkOption {
|
||||
type = lib.types.raw;
|
||||
description = ''
|
||||
The function to use to generate the path for a secret.
|
||||
The default function will use the path attribute of the secret.
|
||||
The function will be called with the secret submodule as an argument.
|
||||
'';
|
||||
};
|
||||
|
||||
publicStore = lib.mkOption {
|
||||
type = lib.types.enum [
|
||||
"in_repo"
|
||||
"vm"
|
||||
"custom"
|
||||
];
|
||||
default = "in_repo";
|
||||
description = ''
|
||||
method to store public facts.
|
||||
custom can be used to define a custom public fact store.
|
||||
'';
|
||||
};
|
||||
|
||||
publicModule = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
internal = true;
|
||||
description = ''
|
||||
the python import path to the public module
|
||||
'';
|
||||
};
|
||||
|
||||
publicDirectory = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.path;
|
||||
default = null;
|
||||
description = ''
|
||||
The directory where public facts are stored.
|
||||
'';
|
||||
};
|
||||
};
|
||||
}
|
@ -27,107 +27,127 @@ rec {
|
||||
# Examples:
|
||||
# writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; }
|
||||
# makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world"
|
||||
makeScriptWriter = { interpreter, check ? "", makeWrapperArgs ? [], }: nameOrPath: content:
|
||||
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
|
||||
makeScriptWriter =
|
||||
{
|
||||
interpreter,
|
||||
check ? "",
|
||||
makeWrapperArgs ? [ ],
|
||||
}:
|
||||
nameOrPath: content:
|
||||
assert lib.or (types.path.check nameOrPath) (
|
||||
builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null
|
||||
);
|
||||
assert lib.or (types.path.check content) (types.str.check content);
|
||||
let
|
||||
name = last (builtins.split "/" nameOrPath);
|
||||
in
|
||||
|
||||
pkgs.runCommandLocal name (
|
||||
{
|
||||
inherit makeWrapperArgs;
|
||||
nativeBuildInputs = [
|
||||
makeWrapper
|
||||
];
|
||||
}
|
||||
// lib.optionalAttrs (nameOrPath == "/bin/${name}") {
|
||||
meta.mainProgram = name;
|
||||
}
|
||||
// (
|
||||
if (types.str.check content) then {
|
||||
inherit content interpreter;
|
||||
passAsFile = [ "content" ];
|
||||
} else {
|
||||
inherit interpreter;
|
||||
contentPath = content;
|
||||
pkgs.runCommandLocal name
|
||||
(
|
||||
{
|
||||
inherit makeWrapperArgs;
|
||||
nativeBuildInputs = [ makeWrapper ];
|
||||
}
|
||||
// lib.optionalAttrs (nameOrPath == "/bin/${name}") { meta.mainProgram = name; }
|
||||
// (
|
||||
if (types.str.check content) then
|
||||
{
|
||||
inherit content interpreter;
|
||||
passAsFile = [ "content" ];
|
||||
}
|
||||
else
|
||||
{
|
||||
inherit interpreter;
|
||||
contentPath = content;
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
''
|
||||
# On darwin a script cannot be used as an interpreter in a shebang but
|
||||
# there doesn't seem to be a limit to the size of shebang and multiple
|
||||
# arguments to the interpreter are allowed.
|
||||
if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
|
||||
then
|
||||
wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
|
||||
# Get first word from the line (note: xargs echo remove leading spaces)
|
||||
wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
|
||||
|
||||
if isScript $wrapperInterpreter
|
||||
''
|
||||
# On darwin a script cannot be used as an interpreter in a shebang but
|
||||
# there doesn't seem to be a limit to the size of shebang and multiple
|
||||
# arguments to the interpreter are allowed.
|
||||
if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
|
||||
then
|
||||
echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
|
||||
exit 1
|
||||
wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
|
||||
# Get first word from the line (note: xargs echo remove leading spaces)
|
||||
wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
|
||||
|
||||
if isScript $wrapperInterpreter
|
||||
then
|
||||
echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# This should work as long as wrapperInterpreter is a shell, which is
|
||||
# the case for programs wrapped with makeWrapper, like
|
||||
# python3.withPackages etc.
|
||||
interpreterLine="$wrapperInterpreterLine $interpreter"
|
||||
else
|
||||
interpreterLine=$interpreter
|
||||
fi
|
||||
|
||||
# This should work as long as wrapperInterpreter is a shell, which is
|
||||
# the case for programs wrapped with makeWrapper, like
|
||||
# python3.withPackages etc.
|
||||
interpreterLine="$wrapperInterpreterLine $interpreter"
|
||||
else
|
||||
interpreterLine=$interpreter
|
||||
fi
|
||||
echo "#! $interpreterLine" > $out
|
||||
cat "$contentPath" >> $out
|
||||
${optionalString (check != "") ''
|
||||
${check} $out
|
||||
''}
|
||||
chmod +x $out
|
||||
|
||||
echo "#! $interpreterLine" > $out
|
||||
cat "$contentPath" >> $out
|
||||
${optionalString (check != "") ''
|
||||
${check} $out
|
||||
''}
|
||||
chmod +x $out
|
||||
|
||||
# Relocate executable if path was specified instead of name.
|
||||
# Only in this case wrapProgram is applied, as it wouldn't work with a
|
||||
# single executable file under $out.
|
||||
${optionalString (types.path.check nameOrPath) ''
|
||||
mv $out tmp
|
||||
mkdir -p $out/$(dirname "${nameOrPath}")
|
||||
mv tmp $out/${nameOrPath}
|
||||
wrapProgram $out/${nameOrPath} ''${makeWrapperArgs[@]}
|
||||
''}
|
||||
'';
|
||||
# Relocate executable if path was specified instead of name.
|
||||
# Only in this case wrapProgram is applied, as it wouldn't work with a
|
||||
# single executable file under $out.
|
||||
${optionalString (types.path.check nameOrPath) ''
|
||||
mv $out tmp
|
||||
mkdir -p $out/$(dirname "${nameOrPath}")
|
||||
mv tmp $out/${nameOrPath}
|
||||
wrapProgram $out/${nameOrPath} ''${makeWrapperArgs[@]}
|
||||
''}
|
||||
'';
|
||||
|
||||
# Base implementation for compiled executables.
|
||||
# Takes a compile script, which in turn takes the name as an argument.
|
||||
#
|
||||
# Examples:
|
||||
# writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; }
|
||||
makeBinWriter = { compileScript, strip ? true }: nameOrPath: content:
|
||||
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
|
||||
makeBinWriter =
|
||||
{
|
||||
compileScript,
|
||||
strip ? true,
|
||||
}:
|
||||
nameOrPath: content:
|
||||
assert lib.or (types.path.check nameOrPath) (
|
||||
builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null
|
||||
);
|
||||
assert lib.or (types.path.check content) (types.str.check content);
|
||||
let
|
||||
name = last (builtins.split "/" nameOrPath);
|
||||
in
|
||||
pkgs.runCommand name ((if (types.str.check content) then {
|
||||
inherit content;
|
||||
passAsFile = [ "content" ];
|
||||
} else {
|
||||
contentPath = content;
|
||||
}) // lib.optionalAttrs (nameOrPath == "/bin/${name}") {
|
||||
meta.mainProgram = name;
|
||||
}) ''
|
||||
${compileScript}
|
||||
${lib.optionalString strip
|
||||
"${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"}
|
||||
# Sometimes binaries produced for darwin (e. g. by GHC) won't be valid
|
||||
# mach-o executables from the get-go, but need to be corrected somehow
|
||||
# which is done by fixupPhase.
|
||||
${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"}
|
||||
${optionalString (types.path.check nameOrPath) ''
|
||||
mv $out tmp
|
||||
mkdir -p $out/$(dirname "${nameOrPath}")
|
||||
mv tmp $out/${nameOrPath}
|
||||
''}
|
||||
'';
|
||||
pkgs.runCommand name
|
||||
(
|
||||
(
|
||||
if (types.str.check content) then
|
||||
{
|
||||
inherit content;
|
||||
passAsFile = [ "content" ];
|
||||
}
|
||||
else
|
||||
{ contentPath = content; }
|
||||
)
|
||||
// lib.optionalAttrs (nameOrPath == "/bin/${name}") { meta.mainProgram = name; }
|
||||
)
|
||||
''
|
||||
${compileScript}
|
||||
${lib.optionalString strip "${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"}
|
||||
# Sometimes binaries produced for darwin (e. g. by GHC) won't be valid
|
||||
# mach-o executables from the get-go, but need to be corrected somehow
|
||||
# which is done by fixupPhase.
|
||||
${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"}
|
||||
${optionalString (types.path.check nameOrPath) ''
|
||||
mv $out tmp
|
||||
mkdir -p $out/$(dirname "${nameOrPath}")
|
||||
mv tmp $out/${nameOrPath}
|
||||
''}
|
||||
'';
|
||||
|
||||
# Like writeScript but the first line is a shebang to bash
|
||||
#
|
||||
@ -135,13 +155,10 @@ rec {
|
||||
# writeBash "example" ''
|
||||
# echo hello world
|
||||
# ''
|
||||
writeBash = makeScriptWriter {
|
||||
interpreter = "${lib.getExe pkgs.bash}";
|
||||
};
|
||||
writeBash = makeScriptWriter { interpreter = "${lib.getExe pkgs.bash}"; };
|
||||
|
||||
# Like writeScriptBin but the first line is a shebang to bash
|
||||
writeBashBin = name:
|
||||
writeBash "/bin/${name}";
|
||||
writeBashBin = name: writeBash "/bin/${name}";
|
||||
|
||||
# Like writeScript but the first line is a shebang to dash
|
||||
#
|
||||
@ -149,13 +166,10 @@ rec {
|
||||
# writeDash "example" ''
|
||||
# echo hello world
|
||||
# ''
|
||||
writeDash = makeScriptWriter {
|
||||
interpreter = "${lib.getExe pkgs.dash}";
|
||||
};
|
||||
writeDash = makeScriptWriter { interpreter = "${lib.getExe pkgs.dash}"; };
|
||||
|
||||
# Like writeScriptBin but the first line is a shebang to dash
|
||||
writeDashBin = name:
|
||||
writeDash "/bin/${name}";
|
||||
writeDashBin = name: writeDash "/bin/${name}";
|
||||
|
||||
# Like writeScript but the first line is a shebang to fish
|
||||
#
|
||||
@ -165,12 +179,11 @@ rec {
|
||||
# ''
|
||||
writeFish = makeScriptWriter {
|
||||
interpreter = "${lib.getExe pkgs.fish} --no-config";
|
||||
check = "${lib.getExe pkgs.fish} --no-config --no-execute"; # syntax check only
|
||||
check = "${lib.getExe pkgs.fish} --no-config --no-execute"; # syntax check only
|
||||
};
|
||||
|
||||
# Like writeScriptBin but the first line is a shebang to fish
|
||||
writeFishBin = name:
|
||||
writeFish "/bin/${name}";
|
||||
writeFishBin = name: writeFish "/bin/${name}";
|
||||
|
||||
# writeHaskell takes a name, an attrset with libraries and haskell version (both optional)
|
||||
# and some haskell source code and returns an executable.
|
||||
@ -181,29 +194,31 @@ rec {
|
||||
#
|
||||
# main = launchMissiles
|
||||
# '';
|
||||
writeHaskell = name: {
|
||||
libraries ? [],
|
||||
ghc ? pkgs.ghc,
|
||||
ghcArgs ? [],
|
||||
threadedRuntime ? true,
|
||||
strip ? true
|
||||
}:
|
||||
writeHaskell =
|
||||
name:
|
||||
{
|
||||
libraries ? [ ],
|
||||
ghc ? pkgs.ghc,
|
||||
ghcArgs ? [ ],
|
||||
threadedRuntime ? true,
|
||||
strip ? true,
|
||||
}:
|
||||
let
|
||||
appendIfNotSet = el: list: if elem el list then list else list ++ [ el ];
|
||||
ghcArgs' = if threadedRuntime then appendIfNotSet "-threaded" ghcArgs else ghcArgs;
|
||||
|
||||
in makeBinWriter {
|
||||
in
|
||||
makeBinWriter {
|
||||
compileScript = ''
|
||||
cp $contentPath tmp.hs
|
||||
${(ghc.withPackages (_: libraries ))}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs
|
||||
${(ghc.withPackages (_: libraries))}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs
|
||||
mv tmp $out
|
||||
'';
|
||||
inherit strip;
|
||||
} name;
|
||||
|
||||
# writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin)
|
||||
writeHaskellBin = name:
|
||||
writeHaskell "/bin/${name}";
|
||||
writeHaskellBin = name: writeHaskell "/bin/${name}";
|
||||
|
||||
# Like writeScript but the first line is a shebang to nu
|
||||
#
|
||||
@ -211,30 +226,30 @@ rec {
|
||||
# writeNu "example" ''
|
||||
# echo hello world
|
||||
# ''
|
||||
writeNu = makeScriptWriter {
|
||||
interpreter = "${lib.getExe pkgs.nushell} --no-config-file";
|
||||
};
|
||||
writeNu = makeScriptWriter { interpreter = "${lib.getExe pkgs.nushell} --no-config-file"; };
|
||||
|
||||
# Like writeScriptBin but the first line is a shebang to nu
|
||||
writeNuBin = name:
|
||||
writeNu "/bin/${name}";
|
||||
writeNuBin = name: writeNu "/bin/${name}";
|
||||
|
||||
# makeRubyWriter takes ruby and compatible rubyPackages and produces ruby script writer,
|
||||
# If any libraries are specified, ruby.withPackages is used as interpreter, otherwise the "bare" ruby is used.
|
||||
makeRubyWriter = ruby: rubyPackages: buildRubyPackages: name: { libraries ? [], ... } @ args:
|
||||
makeScriptWriter (
|
||||
(builtins.removeAttrs args ["libraries"])
|
||||
// {
|
||||
interpreter =
|
||||
if libraries == []
|
||||
then "${ruby}/bin/ruby"
|
||||
else "${(ruby.withPackages (ps: libraries))}/bin/ruby";
|
||||
# Rubocop doesnt seem to like running in this fashion.
|
||||
#check = (writeDash "rubocop.sh" ''
|
||||
# exec ${lib.getExe buildRubyPackages.rubocop} "$1"
|
||||
#'');
|
||||
}
|
||||
) name;
|
||||
makeRubyWriter =
|
||||
ruby: _rubyPackages: _buildRubyPackages: name:
|
||||
{
|
||||
libraries ? [ ],
|
||||
...
|
||||
}@args:
|
||||
makeScriptWriter (
|
||||
(builtins.removeAttrs args [ "libraries" ])
|
||||
// {
|
||||
interpreter =
|
||||
if libraries == [ ] then "${ruby}/bin/ruby" else "${(ruby.withPackages (_ps: libraries))}/bin/ruby";
|
||||
# Rubocop doesnt seem to like running in this fashion.
|
||||
#check = (writeDash "rubocop.sh" ''
|
||||
# exec ${lib.getExe buildRubyPackages.rubocop} "$1"
|
||||
#'');
|
||||
}
|
||||
) name;
|
||||
|
||||
# Like writeScript but the first line is a shebang to ruby
|
||||
#
|
||||
@ -244,26 +259,29 @@ rec {
|
||||
# ''
|
||||
writeRuby = makeRubyWriter pkgs.ruby pkgs.rubyPackages buildPackages.rubyPackages;
|
||||
|
||||
writeRubyBin = name:
|
||||
writeRuby "/bin/${name}";
|
||||
writeRubyBin = name: writeRuby "/bin/${name}";
|
||||
|
||||
# makeLuaWriter takes lua and compatible luaPackages and produces lua script writer,
|
||||
# which validates the script with luacheck at build time. If any libraries are specified,
|
||||
# lua.withPackages is used as interpreter, otherwise the "bare" lua is used.
|
||||
makeLuaWriter = lua: luaPackages: buildLuaPackages: name: { libraries ? [], ... } @ args:
|
||||
makeScriptWriter (
|
||||
(builtins.removeAttrs args ["libraries"])
|
||||
// {
|
||||
interpreter = lua.interpreter;
|
||||
makeLuaWriter =
|
||||
lua: _luaPackages: buildLuaPackages: name:
|
||||
{ ... }@args:
|
||||
makeScriptWriter (
|
||||
(builtins.removeAttrs args [ "libraries" ])
|
||||
// {
|
||||
interpreter = lua.interpreter;
|
||||
# if libraries == []
|
||||
# then lua.interpreter
|
||||
# else (lua.withPackages (ps: libraries)).interpreter
|
||||
# This should support packages! I just cant figure out why some dependency collision happens whenever I try to run this.
|
||||
check = (writeDash "luacheck.sh" ''
|
||||
exec ${buildLuaPackages.luacheck}/bin/luacheck "$1"
|
||||
'');
|
||||
}
|
||||
) name;
|
||||
check = (
|
||||
writeDash "luacheck.sh" ''
|
||||
exec ${buildLuaPackages.luacheck}/bin/luacheck "$1"
|
||||
''
|
||||
);
|
||||
}
|
||||
) name;
|
||||
|
||||
# writeLua takes a name an attributeset with libraries and some lua source code and
|
||||
# returns an executable (should also work with luajit)
|
||||
@ -287,27 +305,27 @@ rec {
|
||||
# ''
|
||||
writeLua = makeLuaWriter pkgs.lua pkgs.luaPackages buildPackages.luaPackages;
|
||||
|
||||
writeLuaBin = name:
|
||||
writeLua "/bin/${name}";
|
||||
writeLuaBin = name: writeLua "/bin/${name}";
|
||||
|
||||
writeRust = name: {
|
||||
writeRust =
|
||||
name:
|
||||
{
|
||||
rustc ? pkgs.rustc,
|
||||
rustcArgs ? [],
|
||||
strip ? true
|
||||
}:
|
||||
let
|
||||
darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ];
|
||||
in
|
||||
rustcArgs ? [ ],
|
||||
strip ? true,
|
||||
}:
|
||||
let
|
||||
darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ];
|
||||
in
|
||||
makeBinWriter {
|
||||
compileScript = ''
|
||||
cp "$contentPath" tmp.rs
|
||||
PATH=${lib.makeBinPath [pkgs.gcc]} ${rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs
|
||||
PATH=${lib.makeBinPath [ pkgs.gcc ]} ${rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs
|
||||
'';
|
||||
inherit strip;
|
||||
} name;
|
||||
|
||||
writeRustBin = name:
|
||||
writeRust "/bin/${name}";
|
||||
writeRustBin = name: writeRust "/bin/${name}";
|
||||
|
||||
# writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and
|
||||
# returns an executable
|
||||
@ -319,23 +337,26 @@ rec {
|
||||
# var result = UglifyJS.minify(code);
|
||||
# console.log(result.code);
|
||||
# ''
|
||||
writeJS = name: { libraries ? [] }: content:
|
||||
let
|
||||
node-env = pkgs.buildEnv {
|
||||
name = "node";
|
||||
paths = libraries;
|
||||
pathsToLink = [
|
||||
"/lib/node_modules"
|
||||
];
|
||||
};
|
||||
in writeDash name ''
|
||||
export NODE_PATH=${node-env}/lib/node_modules
|
||||
exec ${lib.getExe pkgs.nodejs} ${pkgs.writeText "js" content} "$@"
|
||||
'';
|
||||
writeJS =
|
||||
name:
|
||||
{
|
||||
libraries ? [ ],
|
||||
}:
|
||||
content:
|
||||
let
|
||||
node-env = pkgs.buildEnv {
|
||||
name = "node";
|
||||
paths = libraries;
|
||||
pathsToLink = [ "/lib/node_modules" ];
|
||||
};
|
||||
in
|
||||
writeDash name ''
|
||||
export NODE_PATH=${node-env}/lib/node_modules
|
||||
exec ${lib.getExe pkgs.nodejs} ${pkgs.writeText "js" content} "$@"
|
||||
'';
|
||||
|
||||
# writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin)
|
||||
writeJSBin = name:
|
||||
writeJS "/bin/${name}";
|
||||
writeJSBin = name: writeJS "/bin/${name}";
|
||||
|
||||
awkFormatNginx = builtins.toFile "awkFormat-nginx.awk" ''
|
||||
awk -f
|
||||
@ -343,18 +364,22 @@ rec {
|
||||
/\{/{ctx++;idx=1}
|
||||
/\}/{ctx--}
|
||||
{id="";for(i=idx;i<ctx;i++)id=sprintf("%s%s", id, "\t");printf "%s%s\n", id, $0}
|
||||
'';
|
||||
|
||||
writeNginxConfig = name: text: pkgs.runCommandLocal name {
|
||||
inherit text;
|
||||
passAsFile = [ "text" ];
|
||||
nativeBuildInputs = [ gixy ];
|
||||
} /* sh */ ''
|
||||
# nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16
|
||||
awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out
|
||||
gixy $out
|
||||
'';
|
||||
|
||||
writeNginxConfig =
|
||||
name: text:
|
||||
pkgs.runCommandLocal name
|
||||
{
|
||||
inherit text;
|
||||
passAsFile = [ "text" ];
|
||||
nativeBuildInputs = [ gixy ];
|
||||
} # sh
|
||||
''
|
||||
# nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16
|
||||
awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out
|
||||
gixy $out
|
||||
'';
|
||||
|
||||
# writePerl takes a name an attributeset with libraries and some perl sourcecode and
|
||||
# returns an executable
|
||||
#
|
||||
@ -363,42 +388,55 @@ rec {
|
||||
# use boolean;
|
||||
# print "Howdy!\n" if true;
|
||||
# ''
|
||||
writePerl = name: { libraries ? [], ... } @ args:
|
||||
writePerl =
|
||||
name:
|
||||
{
|
||||
libraries ? [ ],
|
||||
...
|
||||
}@args:
|
||||
makeScriptWriter (
|
||||
(builtins.removeAttrs args ["libraries"])
|
||||
(builtins.removeAttrs args [ "libraries" ])
|
||||
// {
|
||||
interpreter = "${lib.getExe (pkgs.perl.withPackages (p: libraries))}";
|
||||
interpreter = "${lib.getExe (pkgs.perl.withPackages (_p: libraries))}";
|
||||
}
|
||||
) name;
|
||||
|
||||
# writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin)
|
||||
writePerlBin = name:
|
||||
writePerl "/bin/${name}";
|
||||
writePerlBin = name: writePerl "/bin/${name}";
|
||||
|
||||
# makePythonWriter takes python and compatible pythonPackages and produces python script writer,
|
||||
# which validates the script with flake8 at build time. If any libraries are specified,
|
||||
# python.withPackages is used as interpreter, otherwise the "bare" python is used.
|
||||
makePythonWriter = python: pythonPackages: buildPythonPackages: name: { libraries ? [], flakeIgnore ? [], ... } @ args:
|
||||
let
|
||||
ignoreAttribute = optionalString (flakeIgnore != []) "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}";
|
||||
in
|
||||
makeScriptWriter
|
||||
(
|
||||
(builtins.removeAttrs args ["libraries" "flakeIgnore"])
|
||||
makePythonWriter =
|
||||
python: pythonPackages: buildPythonPackages: name:
|
||||
{
|
||||
libraries ? [ ],
|
||||
flakeIgnore ? [ ],
|
||||
...
|
||||
}@args:
|
||||
let
|
||||
ignoreAttribute =
|
||||
optionalString (flakeIgnore != [ ])
|
||||
"--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}";
|
||||
in
|
||||
makeScriptWriter (
|
||||
(builtins.removeAttrs args [
|
||||
"libraries"
|
||||
"flakeIgnore"
|
||||
])
|
||||
// {
|
||||
interpreter =
|
||||
if pythonPackages != pkgs.pypy2Packages || pythonPackages != pkgs.pypy3Packages then
|
||||
if libraries == []
|
||||
then python.interpreter
|
||||
else (python.withPackages (ps: libraries)).interpreter
|
||||
else python.interpreter
|
||||
;
|
||||
check = optionalString python.isPy3k (writeDash "pythoncheck.sh" ''
|
||||
exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1"
|
||||
'');
|
||||
if libraries == [ ] then python.interpreter else (python.withPackages (_ps: libraries)).interpreter
|
||||
else
|
||||
python.interpreter;
|
||||
check = optionalString python.isPy3k (
|
||||
writeDash "pythoncheck.sh" ''
|
||||
exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1"
|
||||
''
|
||||
);
|
||||
}
|
||||
)
|
||||
name;
|
||||
) name;
|
||||
|
||||
# writePyPy2 takes a name an attributeset with libraries and some pypy2 sourcecode and
|
||||
# returns an executable
|
||||
@ -415,8 +453,7 @@ rec {
|
||||
writePyPy2 = makePythonWriter pkgs.pypy2 pkgs.pypy2Packages buildPackages.pypy2Packages;
|
||||
|
||||
# writePyPy2Bin takes the same arguments as writePyPy2 but outputs a directory (like writeScriptBin)
|
||||
writePyPy2Bin = name:
|
||||
writePyPy2 "/bin/${name}";
|
||||
writePyPy2Bin = name: writePyPy2 "/bin/${name}";
|
||||
|
||||
# writePython3 takes a name an attributeset with libraries and some python3 sourcecode and
|
||||
# returns an executable
|
||||
@ -433,8 +470,7 @@ rec {
|
||||
writePython3 = makePythonWriter pkgs.python3 pkgs.python3Packages buildPackages.python3Packages;
|
||||
|
||||
# writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin)
|
||||
writePython3Bin = name:
|
||||
writePython3 "/bin/${name}";
|
||||
writePython3Bin = name: writePython3 "/bin/${name}";
|
||||
|
||||
# writePyPy3 takes a name an attributeset with libraries and some pypy3 sourcecode and
|
||||
# returns an executable
|
||||
@ -451,47 +487,61 @@ rec {
|
||||
writePyPy3 = makePythonWriter pkgs.pypy3 pkgs.pypy3Packages buildPackages.pypy3Packages;
|
||||
|
||||
# writePyPy3Bin takes the same arguments as writePyPy3 but outputs a directory (like writeScriptBin)
|
||||
writePyPy3Bin = name:
|
||||
writePyPy3 "/bin/${name}";
|
||||
writePyPy3Bin = name: writePyPy3 "/bin/${name}";
|
||||
|
||||
makeFSharpWriter =
|
||||
{
|
||||
dotnet-sdk ? pkgs.dotnet-sdk,
|
||||
fsi-flags ? "",
|
||||
libraries ? _: [ ],
|
||||
...
|
||||
}@args:
|
||||
nameOrPath:
|
||||
let
|
||||
fname = last (builtins.split "/" nameOrPath);
|
||||
path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx";
|
||||
_nugetDeps = mkNugetDeps {
|
||||
name = "${fname}-nuget-deps";
|
||||
nugetDeps = libraries;
|
||||
};
|
||||
|
||||
makeFSharpWriter = { dotnet-sdk ? pkgs.dotnet-sdk, fsi-flags ? "", libraries ? _: [], ... } @ args: nameOrPath:
|
||||
let
|
||||
fname = last (builtins.split "/" nameOrPath);
|
||||
path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx";
|
||||
_nugetDeps = mkNugetDeps { name = "${fname}-nuget-deps"; nugetDeps = libraries; };
|
||||
nuget-source = mkNugetSource {
|
||||
name = "${fname}-nuget-source";
|
||||
description = "A Nuget source with the dependencies for ${fname}";
|
||||
deps = [ _nugetDeps ];
|
||||
};
|
||||
|
||||
nuget-source = mkNugetSource {
|
||||
name = "${fname}-nuget-source";
|
||||
description = "A Nuget source with the dependencies for ${fname}";
|
||||
deps = [ _nugetDeps ];
|
||||
};
|
||||
fsi = writeBash "fsi" ''
|
||||
export HOME=$NIX_BUILD_TOP/.home
|
||||
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
|
||||
export DOTNET_CLI_TELEMETRY_OPTOUT=1
|
||||
export DOTNET_NOLOGO=1
|
||||
script="$1"; shift
|
||||
${lib.getExe dotnet-sdk} fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script"
|
||||
'';
|
||||
|
||||
fsi = writeBash "fsi" ''
|
||||
export HOME=$NIX_BUILD_TOP/.home
|
||||
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
|
||||
export DOTNET_CLI_TELEMETRY_OPTOUT=1
|
||||
export DOTNET_NOLOGO=1
|
||||
script="$1"; shift
|
||||
${lib.getExe dotnet-sdk} fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script"
|
||||
'';
|
||||
in
|
||||
content:
|
||||
makeScriptWriter
|
||||
(
|
||||
(builtins.removeAttrs args [
|
||||
"dotnet-sdk"
|
||||
"fsi-flags"
|
||||
"libraries"
|
||||
])
|
||||
// {
|
||||
interpreter = fsi;
|
||||
}
|
||||
)
|
||||
path
|
||||
''
|
||||
#i "nuget: ${nuget-source}/lib"
|
||||
${content}
|
||||
exit 0
|
||||
'';
|
||||
|
||||
in content: makeScriptWriter (
|
||||
(builtins.removeAttrs args ["dotnet-sdk" "fsi-flags" "libraries"])
|
||||
// {
|
||||
interpreter = fsi;
|
||||
}
|
||||
) path
|
||||
''
|
||||
#i "nuget: ${nuget-source}/lib"
|
||||
${ content }
|
||||
exit 0
|
||||
'';
|
||||
writeFSharp = makeFSharpWriter { };
|
||||
|
||||
writeFSharp =
|
||||
makeFSharpWriter {};
|
||||
|
||||
writeFSharpBin = name:
|
||||
writeFSharp "/bin/${name}";
|
||||
writeFSharpBin = name: writeFSharp "/bin/${name}";
|
||||
|
||||
}
|
||||
|
@ -1,3 +1,4 @@
|
||||
# shellcheck shell=bash
|
||||
source_up
|
||||
|
||||
watch_file flake-module.nix shell.nix default.nix
|
||||
|
@ -1,3 +1,4 @@
|
||||
# shellcheck shell=bash
|
||||
source_up
|
||||
|
||||
watch_file flake-module.nix shell.nix default.nix
|
||||
|
@ -23,6 +23,7 @@ from . import (
|
||||
machines,
|
||||
secrets,
|
||||
state,
|
||||
vars,
|
||||
vms,
|
||||
)
|
||||
from .clan_uri import FlakeId
|
||||
@ -272,6 +273,43 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c
|
||||
)
|
||||
facts.register_parser(parser_facts)
|
||||
|
||||
# like facts but with vars instead of facts
|
||||
parser_vars = subparsers.add_parser(
|
||||
"vars",
|
||||
help="WIP: manage vars",
|
||||
description="WIP: manage vars",
|
||||
epilog=(
|
||||
f"""
|
||||
This subcommand provides an interface to vars of clan machines.
|
||||
Vars are variables that a service can generate.
|
||||
There are public and secret vars.
|
||||
Public vars can be referenced by other machines directly.
|
||||
Public vars can include: ip addresses, public keys.
|
||||
Secret vars can include: passwords, private keys.
|
||||
|
||||
A service is an included clan-module that implements vars generation functionality.
|
||||
For example the zerotier module will generate private and public vars.
|
||||
In this case the public var will be the resulting zerotier-ip of the machine.
|
||||
The secret var will be the zerotier-identity-secret, which is used by zerotier
|
||||
to prove the machine has control of the zerotier-ip.
|
||||
|
||||
Examples:
|
||||
|
||||
$ clan vars generate
|
||||
Will generate vars for all machines.
|
||||
|
||||
$ clan vars generate --service [SERVICE] --regenerate
|
||||
Will regenerate vars, if they are already generated for a specific service.
|
||||
This is especially useful for resetting certain passwords while leaving the rest
|
||||
of the vars for a machine in place.
|
||||
|
||||
For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")}
|
||||
"""
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
vars.register_parser(parser_vars)
|
||||
|
||||
parser_machine = subparsers.add_parser(
|
||||
"machines",
|
||||
help="manage machines and their configuration",
|
||||
|
@ -81,11 +81,11 @@ def cast(value: Any, input_type: Any, opt_description: str) -> Any:
|
||||
else:
|
||||
raise ClanError(f"Invalid value {value} for boolean")
|
||||
# handle lists
|
||||
elif get_origin(input_type) == list:
|
||||
elif get_origin(input_type) is list:
|
||||
subtype = input_type.__args__[0]
|
||||
return [cast([x], subtype, opt_description) for x in value]
|
||||
# handle dicts
|
||||
elif get_origin(input_type) == dict:
|
||||
elif get_origin(input_type) is dict:
|
||||
if not isinstance(value, dict):
|
||||
raise ClanError(
|
||||
f"Cannot set {opt_description} directly. Specify a suboption like {opt_description}.<name>"
|
||||
|
@ -69,12 +69,26 @@ class Machine:
|
||||
def public_facts_module(self) -> str:
|
||||
return self.deployment["facts"]["publicModule"]
|
||||
|
||||
@property
|
||||
def secret_vars_module(self) -> str:
|
||||
return self.deployment["vars"]["secretModule"]
|
||||
|
||||
@property
|
||||
def public_vars_module(self) -> str:
|
||||
return self.deployment["vars"]["publicModule"]
|
||||
|
||||
@property
|
||||
def facts_data(self) -> dict[str, dict[str, Any]]:
|
||||
if self.deployment["facts"]["services"]:
|
||||
return self.deployment["facts"]["services"]
|
||||
return {}
|
||||
|
||||
@property
|
||||
def vars_generators(self) -> dict[str, dict[str, Any]]:
|
||||
if self.deployment["vars"]["generators"]:
|
||||
return self.deployment["vars"]["generators"]
|
||||
return {}
|
||||
|
||||
@property
|
||||
def secrets_upload_directory(self) -> str:
|
||||
return self.deployment["facts"]["secretUploadDirectory"]
|
||||
|
132
pkgs/clan-cli/clan_cli/vars/__init__.py
Normal file
132
pkgs/clan-cli/clan_cli/vars/__init__.py
Normal file
@ -0,0 +1,132 @@
|
||||
# !/usr/bin/env python3
|
||||
import argparse
|
||||
|
||||
from ..hyperlink import help_hyperlink
|
||||
from .check import register_check_parser
|
||||
from .generate import register_generate_parser
|
||||
from .list import register_list_parser
|
||||
from .upload import register_upload_parser
|
||||
|
||||
|
||||
# takes a (sub)parser and configures it
|
||||
def register_parser(parser: argparse.ArgumentParser) -> None:
|
||||
subparser = parser.add_subparsers(
|
||||
title="command",
|
||||
description="the command to run",
|
||||
help="the command to run",
|
||||
required=True,
|
||||
)
|
||||
|
||||
check_parser = subparser.add_parser(
|
||||
"check",
|
||||
help="check if facts are up to date",
|
||||
epilog=(
|
||||
f"""
|
||||
This subcommand allows checking if all facts are up to date.
|
||||
|
||||
Examples:
|
||||
|
||||
$ clan facts check [MACHINE]
|
||||
Will check facts for the specified machine.
|
||||
|
||||
|
||||
For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")}
|
||||
"""
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
register_check_parser(check_parser)
|
||||
|
||||
list_parser = subparser.add_parser(
|
||||
"list",
|
||||
help="list all facts",
|
||||
epilog=(
|
||||
f"""
|
||||
This subcommand allows listing all public facts for a specific machine.
|
||||
|
||||
The resulting list will be a json string with the name of the fact as its key
|
||||
and the fact itself as it's value.
|
||||
|
||||
This is how an example output might look like:
|
||||
```
|
||||
\u007b
|
||||
"[FACT_NAME]": "[FACT]"
|
||||
\u007d
|
||||
```
|
||||
|
||||
Examples:
|
||||
|
||||
$ clan facts list [MACHINE]
|
||||
Will list facts for the specified machine.
|
||||
|
||||
|
||||
For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")}
|
||||
"""
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
register_list_parser(list_parser)
|
||||
|
||||
parser_generate = subparser.add_parser(
|
||||
"generate",
|
||||
help="generate public and secret facts for machines",
|
||||
epilog=(
|
||||
f"""
|
||||
This subcommand allows control of the generation of facts.
|
||||
Often this function will be invoked automatically on deploying machines,
|
||||
but there are situations the user may want to have more granular control,
|
||||
especially for the regeneration of certain services.
|
||||
|
||||
A service is an included clan-module that implements facts generation functionality.
|
||||
For example the zerotier module will generate private and public facts.
|
||||
In this case the public fact will be the resulting zerotier-ip of the machine.
|
||||
The secret fact will be the zerotier-identity-secret, which is used by zerotier
|
||||
to prove the machine has control of the zerotier-ip.
|
||||
|
||||
|
||||
Examples:
|
||||
|
||||
$ clan facts generate
|
||||
Will generate facts for all machines.
|
||||
|
||||
$ clan facts generate [MACHINE]
|
||||
Will generate facts for the specified machine.
|
||||
|
||||
$ clan facts generate [MACHINE] --service [SERVICE]
|
||||
Will generate facts for the specified machine for the specified service.
|
||||
|
||||
$ clan facts generate --service [SERVICE] --regenerate
|
||||
Will regenerate facts, if they are already generated for a specific service.
|
||||
This is especially useful for resetting certain passwords while leaving the rest
|
||||
of the facts for a machine in place.
|
||||
|
||||
For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")}
|
||||
"""
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
register_generate_parser(parser_generate)
|
||||
|
||||
parser_upload = subparser.add_parser(
|
||||
"upload",
|
||||
help="upload secrets for machines",
|
||||
epilog=(
|
||||
f"""
|
||||
This subcommand allows uploading secrets to remote machines.
|
||||
|
||||
If using sops as a secret backend it will upload the private key to the machine.
|
||||
If using password store it uploads all the secrets you manage to the machine.
|
||||
|
||||
The default backend is sops.
|
||||
|
||||
Examples:
|
||||
|
||||
$ clan facts upload [MACHINE]
|
||||
Will upload secrets to a specific machine.
|
||||
|
||||
For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")}
|
||||
"""
|
||||
),
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
register_upload_parser(parser_upload)
|
64
pkgs/clan-cli/clan_cli/vars/check.py
Normal file
64
pkgs/clan-cli/clan_cli/vars/check.py
Normal file
@ -0,0 +1,64 @@
|
||||
import argparse
|
||||
import importlib
|
||||
import logging
|
||||
|
||||
from ..completions import add_dynamic_completer, complete_machines
|
||||
from ..machines.machines import Machine
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_secrets(machine: Machine, generator_name: None | str = None) -> bool:
|
||||
secret_vars_module = importlib.import_module(machine.secret_vars_module)
|
||||
secret_vars_store = secret_vars_module.SecretStore(machine=machine)
|
||||
public_vars_module = importlib.import_module(machine.public_vars_module)
|
||||
public_vars_store = public_vars_module.FactStore(machine=machine)
|
||||
|
||||
missing_secret_vars = []
|
||||
missing_public_vars = []
|
||||
if generator_name:
|
||||
services = [generator_name]
|
||||
else:
|
||||
services = list(machine.vars_generators.keys())
|
||||
for generator_name in services:
|
||||
for name, file in machine.vars_generators[generator_name]["files"].items():
|
||||
if file["secret"] and not secret_vars_store.exists(generator_name, name):
|
||||
log.info(
|
||||
f"Secret fact '{name}' for service '{generator_name}' in machine {machine.name} is missing."
|
||||
)
|
||||
missing_secret_vars.append((generator_name, name))
|
||||
if not file["secret"] and not public_vars_store.exists(
|
||||
generator_name, name
|
||||
):
|
||||
log.info(
|
||||
f"Public fact '{name}' for service '{generator_name}' in machine {machine.name} is missing."
|
||||
)
|
||||
missing_public_vars.append((generator_name, name))
|
||||
|
||||
log.debug(f"missing_secret_vars: {missing_secret_vars}")
|
||||
log.debug(f"missing_public_vars: {missing_public_vars}")
|
||||
if missing_secret_vars or missing_public_vars:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def check_command(args: argparse.Namespace) -> None:
|
||||
machine = Machine(
|
||||
name=args.machine,
|
||||
flake=args.flake,
|
||||
)
|
||||
check_secrets(machine, generator_name=args.service)
|
||||
|
||||
|
||||
def register_check_parser(parser: argparse.ArgumentParser) -> None:
|
||||
machines_parser = parser.add_argument(
|
||||
"machine",
|
||||
help="The machine to check secrets for",
|
||||
)
|
||||
add_dynamic_completer(machines_parser, complete_machines)
|
||||
|
||||
parser.add_argument(
|
||||
"--service",
|
||||
help="the service to check",
|
||||
)
|
||||
parser.set_defaults(func=check_command)
|
240
pkgs/clan-cli/clan_cli/vars/generate.py
Normal file
240
pkgs/clan-cli/clan_cli/vars/generate.py
Normal file
@ -0,0 +1,240 @@
|
||||
import argparse
|
||||
import importlib
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from clan_cli.cmd import run
|
||||
|
||||
from ..completions import (
|
||||
add_dynamic_completer,
|
||||
complete_machines,
|
||||
complete_services_for_machine,
|
||||
)
|
||||
from ..errors import ClanError
|
||||
from ..git import commit_files
|
||||
from ..machines.inventory import get_all_machines, get_selected_machines
|
||||
from ..machines.machines import Machine
|
||||
from ..nix import nix_shell
|
||||
from .check import check_secrets
|
||||
from .public_modules import FactStoreBase
|
||||
from .secret_modules import SecretStoreBase
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
|
||||
"""
|
||||
Read multi-line input from stdin.
|
||||
"""
|
||||
print(prompt, flush=True)
|
||||
proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True)
|
||||
log.info("Input received. Processing...")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]:
|
||||
# fmt: off
|
||||
return nix_shell(
|
||||
[
|
||||
"nixpkgs#bash",
|
||||
"nixpkgs#bubblewrap",
|
||||
],
|
||||
[
|
||||
"bwrap",
|
||||
"--ro-bind", "/nix/store", "/nix/store",
|
||||
"--tmpfs", "/usr/lib/systemd",
|
||||
"--dev", "/dev",
|
||||
"--bind", str(generator_dir), str(generator_dir),
|
||||
"--unshare-all",
|
||||
"--unshare-user",
|
||||
"--uid", "1000",
|
||||
"--",
|
||||
"bash", "-c", generator
|
||||
],
|
||||
)
|
||||
# fmt: on
|
||||
|
||||
|
||||
def execute_generator(
|
||||
machine: Machine,
|
||||
generator_name: str,
|
||||
regenerate: bool,
|
||||
secret_vars_store: SecretStoreBase,
|
||||
public_vars_store: FactStoreBase,
|
||||
tmpdir: Path,
|
||||
prompt: Callable[[str], str],
|
||||
) -> bool:
|
||||
generator_dir = tmpdir / generator_name
|
||||
# check if all secrets exist and generate them if at least one is missing
|
||||
needs_regeneration = not check_secrets(machine, generator_name=generator_name)
|
||||
log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}")
|
||||
if not (needs_regeneration or regenerate):
|
||||
return False
|
||||
if not isinstance(machine.flake, Path):
|
||||
msg = f"flake is not a Path: {machine.flake}"
|
||||
msg += "fact/secret generation is only supported for local flakes"
|
||||
|
||||
env = os.environ.copy()
|
||||
generator_dir.mkdir(parents=True)
|
||||
env["out"] = str(generator_dir)
|
||||
# compatibility for old outputs.nix users
|
||||
generator = machine.vars_generators[generator_name]["finalScript"]
|
||||
# if machine.vars_data[generator_name]["generator"]["prompt"]:
|
||||
# prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"])
|
||||
# env["prompt_value"] = prompt_value
|
||||
if sys.platform == "linux":
|
||||
cmd = bubblewrap_cmd(generator, generator_dir)
|
||||
else:
|
||||
cmd = ["bash", "-c", generator]
|
||||
run(
|
||||
cmd,
|
||||
env=env,
|
||||
)
|
||||
files_to_commit = []
|
||||
# store secrets
|
||||
files = machine.vars_generators[generator_name]["files"]
|
||||
for file_name, file in files.items():
|
||||
groups = file.get("groups", [])
|
||||
|
||||
secret_file = generator_dir / file_name
|
||||
if not secret_file.is_file():
|
||||
msg = f"did not generate a file for '{file_name}' when running the following command:\n"
|
||||
msg += generator
|
||||
raise ClanError(msg)
|
||||
if file["secret"]:
|
||||
file_path = secret_vars_store.set(
|
||||
generator_name, file_name, secret_file.read_bytes(), groups
|
||||
)
|
||||
else:
|
||||
file_path = public_vars_store.set(
|
||||
generator_name, file_name, secret_file.read_bytes()
|
||||
)
|
||||
if file_path:
|
||||
files_to_commit.append(file_path)
|
||||
commit_files(
|
||||
files_to_commit,
|
||||
machine.flake_dir,
|
||||
f"Update facts/secrets for service {generator_name} in machine {machine.name}",
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def prompt_func(text: str) -> str:
|
||||
print(f"{text}: ")
|
||||
return read_multiline_input()
|
||||
|
||||
|
||||
def _generate_vars_for_machine(
|
||||
machine: Machine,
|
||||
generator_name: str | None,
|
||||
regenerate: bool,
|
||||
tmpdir: Path,
|
||||
prompt: Callable[[str], str] = prompt_func,
|
||||
) -> bool:
|
||||
local_temp = tmpdir / machine.name
|
||||
local_temp.mkdir()
|
||||
secret_vars_module = importlib.import_module(machine.secret_vars_module)
|
||||
secret_vars_store = secret_vars_module.SecretStore(machine=machine)
|
||||
|
||||
public_vars_module = importlib.import_module(machine.public_vars_module)
|
||||
public_vars_store = public_vars_module.FactStore(machine=machine)
|
||||
|
||||
machine_updated = False
|
||||
|
||||
if generator_name and generator_name not in machine.vars_generators:
|
||||
generators = list(machine.vars_generators.keys())
|
||||
raise ClanError(
|
||||
f"Could not find generator with name: {generator_name}. The following generators are available: {generators}"
|
||||
)
|
||||
|
||||
if generator_name:
|
||||
machine_generator_facts = {
|
||||
generator_name: machine.vars_generators[generator_name]
|
||||
}
|
||||
else:
|
||||
machine_generator_facts = machine.vars_generators
|
||||
|
||||
for generator_name in machine_generator_facts:
|
||||
machine_updated |= execute_generator(
|
||||
machine=machine,
|
||||
generator_name=generator_name,
|
||||
regenerate=regenerate,
|
||||
secret_vars_store=secret_vars_store,
|
||||
public_vars_store=public_vars_store,
|
||||
tmpdir=local_temp,
|
||||
prompt=prompt,
|
||||
)
|
||||
if machine_updated:
|
||||
# flush caches to make sure the new secrets are available in evaluation
|
||||
machine.flush_caches()
|
||||
return machine_updated
|
||||
|
||||
|
||||
def generate_vars(
|
||||
machines: list[Machine],
|
||||
generator_name: str | None,
|
||||
regenerate: bool,
|
||||
prompt: Callable[[str], str] = prompt_func,
|
||||
) -> bool:
|
||||
was_regenerated = False
|
||||
with TemporaryDirectory() as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
|
||||
for machine in machines:
|
||||
errors = 0
|
||||
try:
|
||||
was_regenerated |= _generate_vars_for_machine(
|
||||
machine, generator_name, regenerate, tmpdir, prompt
|
||||
)
|
||||
except Exception as exc:
|
||||
log.error(f"Failed to generate facts for {machine.name}: {exc}")
|
||||
errors += 1
|
||||
if errors > 0:
|
||||
raise ClanError(
|
||||
f"Failed to generate facts for {errors} hosts. Check the logs above"
|
||||
)
|
||||
|
||||
if not was_regenerated:
|
||||
print("All secrets and facts are already up to date")
|
||||
return was_regenerated
|
||||
|
||||
|
||||
def generate_command(args: argparse.Namespace) -> None:
|
||||
if len(args.machines) == 0:
|
||||
machines = get_all_machines(args.flake, args.option)
|
||||
else:
|
||||
machines = get_selected_machines(args.flake, args.option, args.machines)
|
||||
generate_vars(machines, args.service, args.regenerate)
|
||||
|
||||
|
||||
def register_generate_parser(parser: argparse.ArgumentParser) -> None:
|
||||
machines_parser = parser.add_argument(
|
||||
"machines",
|
||||
type=str,
|
||||
help="machine to generate facts for. if empty, generate facts for all machines",
|
||||
nargs="*",
|
||||
default=[],
|
||||
)
|
||||
add_dynamic_completer(machines_parser, complete_machines)
|
||||
|
||||
service_parser = parser.add_argument(
|
||||
"--service",
|
||||
type=str,
|
||||
help="service to generate facts for, if empty, generate facts for every service",
|
||||
default=None,
|
||||
)
|
||||
add_dynamic_completer(service_parser, complete_services_for_machine)
|
||||
|
||||
parser.add_argument(
|
||||
"--regenerate",
|
||||
type=bool,
|
||||
action=argparse.BooleanOptionalAction,
|
||||
help="whether to regenerate facts for the specified machine",
|
||||
default=None,
|
||||
)
|
||||
parser.set_defaults(func=generate_command)
|
47
pkgs/clan-cli/clan_cli/vars/list.py
Normal file
47
pkgs/clan-cli/clan_cli/vars/list.py
Normal file
@ -0,0 +1,47 @@
|
||||
import argparse
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
|
||||
from ..completions import add_dynamic_completer, complete_machines
|
||||
from ..machines.machines import Machine
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# TODO get also secret facts
|
||||
def get_all_facts(machine: Machine) -> dict:
|
||||
public_facts_module = importlib.import_module(machine.public_facts_module)
|
||||
public_facts_store = public_facts_module.FactStore(machine=machine)
|
||||
|
||||
# for service in machine.secrets_data:
|
||||
# facts[service] = {}
|
||||
# for fact in machine.secrets_data[service]["facts"]:
|
||||
# fact_content = fact_store.get(service, fact)
|
||||
# if fact_content:
|
||||
# facts[service][fact] = fact_content.decode()
|
||||
# else:
|
||||
# log.error(f"Fact {fact} for service {service} is missing")
|
||||
return public_facts_store.get_all()
|
||||
|
||||
|
||||
def get_command(args: argparse.Namespace) -> None:
|
||||
machine = Machine(name=args.machine, flake=args.flake)
|
||||
|
||||
# the raw_facts are bytestrings making them not json serializable
|
||||
raw_facts = get_all_facts(machine)
|
||||
facts = dict()
|
||||
for key in raw_facts["TODO"]:
|
||||
facts[key] = raw_facts["TODO"][key].decode("utf8")
|
||||
|
||||
print(json.dumps(facts, indent=4))
|
||||
|
||||
|
||||
def register_list_parser(parser: argparse.ArgumentParser) -> None:
|
||||
machines_parser = parser.add_argument(
|
||||
"machine",
|
||||
help="The machine to print facts for",
|
||||
)
|
||||
add_dynamic_completer(machines_parser, complete_machines)
|
||||
|
||||
parser.set_defaults(func=get_command)
|
28
pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py
Normal file
28
pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py
Normal file
@ -0,0 +1,28 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.machines.machines import Machine
|
||||
|
||||
|
||||
class FactStoreBase(ABC):
|
||||
@abstractmethod
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def set(self, service: str, name: str, value: bytes) -> Path | None:
|
||||
pass
|
||||
|
||||
# get a single fact
|
||||
@abstractmethod
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
pass
|
||||
|
||||
# get all facts
|
||||
@abstractmethod
|
||||
def get_all(self) -> dict[str, dict[str, bytes]]:
|
||||
pass
|
64
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
Normal file
64
pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py
Normal file
@ -0,0 +1,64 @@
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.errors import ClanError
|
||||
from clan_cli.machines.machines import Machine
|
||||
|
||||
from . import FactStoreBase
|
||||
|
||||
|
||||
class FactStore(FactStoreBase):
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
self.machine = machine
|
||||
self.works_remotely = False
|
||||
|
||||
def set(self, generator_name: str, name: str, value: bytes) -> Path | None:
|
||||
if self.machine.flake.is_local():
|
||||
fact_path = (
|
||||
self.machine.flake.path
|
||||
/ "machines"
|
||||
/ self.machine.name
|
||||
/ "vars"
|
||||
/ generator_name
|
||||
/ name
|
||||
)
|
||||
fact_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fact_path.touch()
|
||||
fact_path.write_bytes(value)
|
||||
return fact_path
|
||||
else:
|
||||
raise ClanError(
|
||||
f"in_flake fact storage is only supported for local flakes: {self.machine.flake}"
|
||||
)
|
||||
|
||||
def exists(self, generator_name: str, name: str) -> bool:
|
||||
fact_path = (
|
||||
self.machine.flake_dir
|
||||
/ "machines"
|
||||
/ self.machine.name
|
||||
/ "vars"
|
||||
/ generator_name
|
||||
/ name
|
||||
)
|
||||
return fact_path.exists()
|
||||
|
||||
# get a single fact
|
||||
def get(self, generator_name: str, name: str) -> bytes:
|
||||
fact_path = (
|
||||
self.machine.flake_dir
|
||||
/ "machines"
|
||||
/ self.machine.name
|
||||
/ "vars"
|
||||
/ generator_name
|
||||
/ name
|
||||
)
|
||||
return fact_path.read_bytes()
|
||||
|
||||
# get all public vars
|
||||
def get_all(self) -> dict[str, dict[str, bytes]]:
|
||||
facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars"
|
||||
facts: dict[str, dict[str, bytes]] = {}
|
||||
facts["TODO"] = {}
|
||||
if facts_folder.exists():
|
||||
for fact_path in facts_folder.iterdir():
|
||||
facts["TODO"][fact_path.name] = fact_path.read_bytes()
|
||||
return facts
|
46
pkgs/clan-cli/clan_cli/vars/public_modules/vm.py
Normal file
46
pkgs/clan-cli/clan_cli/vars/public_modules/vm.py
Normal file
@ -0,0 +1,46 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.dirs import vm_state_dir
|
||||
from clan_cli.errors import ClanError
|
||||
from clan_cli.machines.machines import Machine
|
||||
|
||||
from . import FactStoreBase
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FactStore(FactStoreBase):
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
self.machine = machine
|
||||
self.works_remotely = False
|
||||
self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts"
|
||||
log.debug(f"FactStore initialized with dir {self.dir}")
|
||||
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
fact_path = self.dir / service / name
|
||||
return fact_path.exists()
|
||||
|
||||
def set(self, service: str, name: str, value: bytes) -> Path | None:
|
||||
fact_path = self.dir / service / name
|
||||
fact_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fact_path.write_bytes(value)
|
||||
return None
|
||||
|
||||
# get a single fact
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
fact_path = self.dir / service / name
|
||||
if fact_path.exists():
|
||||
return fact_path.read_bytes()
|
||||
raise ClanError(f"Fact {name} for service {service} not found")
|
||||
|
||||
# get all facts
|
||||
def get_all(self) -> dict[str, dict[str, bytes]]:
|
||||
facts: dict[str, dict[str, bytes]] = {}
|
||||
if self.dir.exists():
|
||||
for service in self.dir.iterdir():
|
||||
facts[service.name] = {}
|
||||
for fact in service.iterdir():
|
||||
facts[service.name][fact.name] = fact.read_bytes()
|
||||
|
||||
return facts
|
31
pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py
Normal file
31
pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py
Normal file
@ -0,0 +1,31 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.machines.machines import Machine
|
||||
|
||||
|
||||
class SecretStoreBase(ABC):
|
||||
@abstractmethod
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def set(
|
||||
self, service: str, name: str, value: bytes, groups: list[str]
|
||||
) -> Path | None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
pass
|
||||
|
||||
def update_check(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def upload(self, output_dir: Path) -> None:
|
||||
pass
|
117
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
Normal file
117
pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py
Normal file
@ -0,0 +1,117 @@
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.machines.machines import Machine
|
||||
from clan_cli.nix import nix_shell
|
||||
|
||||
from . import SecretStoreBase
|
||||
|
||||
|
||||
class SecretStore(SecretStoreBase):
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
self.machine = machine
|
||||
|
||||
def set(
|
||||
self, service: str, name: str, value: bytes, groups: list[str]
|
||||
) -> Path | None:
|
||||
subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#pass"],
|
||||
["pass", "insert", "-m", f"machines/{self.machine.name}/{name}"],
|
||||
),
|
||||
input=value,
|
||||
check=True,
|
||||
)
|
||||
return None # we manage the files outside of the git repo
|
||||
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
return subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#pass"],
|
||||
["pass", "show", f"machines/{self.machine.name}/{name}"],
|
||||
),
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout
|
||||
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
password_store = os.environ.get(
|
||||
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
|
||||
)
|
||||
secret_path = Path(password_store) / f"machines/{self.machine.name}/{name}.gpg"
|
||||
return secret_path.exists()
|
||||
|
||||
def generate_hash(self) -> bytes:
|
||||
password_store = os.environ.get(
|
||||
"PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store"
|
||||
)
|
||||
hashes = []
|
||||
hashes.append(
|
||||
subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#git"],
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
password_store,
|
||||
"log",
|
||||
"-1",
|
||||
"--format=%H",
|
||||
f"machines/{self.machine.name}",
|
||||
],
|
||||
),
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout.strip()
|
||||
)
|
||||
for symlink in Path(password_store).glob(f"machines/{self.machine.name}/**/*"):
|
||||
if symlink.is_symlink():
|
||||
hashes.append(
|
||||
subprocess.run(
|
||||
nix_shell(
|
||||
["nixpkgs#git"],
|
||||
[
|
||||
"git",
|
||||
"-C",
|
||||
password_store,
|
||||
"log",
|
||||
"-1",
|
||||
"--format=%H",
|
||||
str(symlink),
|
||||
],
|
||||
),
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout.strip()
|
||||
)
|
||||
|
||||
# we sort the hashes to make sure that the order is always the same
|
||||
hashes.sort()
|
||||
return b"\n".join(hashes)
|
||||
|
||||
# FIXME: add this when we switch to python3.12
|
||||
# @override
|
||||
def update_check(self) -> bool:
|
||||
local_hash = self.generate_hash()
|
||||
remote_hash = self.machine.target_host.run(
|
||||
# TODO get the path to the secrets from the machine
|
||||
["cat", f"{self.machine.secrets_upload_directory}/.pass_info"],
|
||||
check=False,
|
||||
stdout=subprocess.PIPE,
|
||||
).stdout.strip()
|
||||
|
||||
if not remote_hash:
|
||||
print("remote hash is empty")
|
||||
return False
|
||||
|
||||
return local_hash.decode() == remote_hash
|
||||
|
||||
def upload(self, output_dir: Path) -> None:
|
||||
for service in self.machine.facts_data:
|
||||
for secret in self.machine.facts_data[service]["secret"]:
|
||||
if isinstance(secret, dict):
|
||||
secret_name = secret["name"]
|
||||
else:
|
||||
# TODO: drop old format soon
|
||||
secret_name = secret
|
||||
(output_dir / secret_name).write_bytes(self.get(service, secret_name))
|
||||
(output_dir / ".pass_info").write_bytes(self.generate_hash())
|
69
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
Normal file
69
pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py
Normal file
@ -0,0 +1,69 @@
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.machines.machines import Machine
|
||||
from clan_cli.secrets.folders import sops_secrets_folder
|
||||
from clan_cli.secrets.machines import add_machine, has_machine
|
||||
from clan_cli.secrets.secrets import decrypt_secret, encrypt_secret, has_secret
|
||||
from clan_cli.secrets.sops import generate_private_key
|
||||
|
||||
from . import SecretStoreBase
|
||||
|
||||
|
||||
class SecretStore(SecretStoreBase):
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
self.machine = machine
|
||||
|
||||
# no need to generate keys if we don't manage secrets
|
||||
if not hasattr(self.machine, "vars_data") or not self.machine.vars_generators:
|
||||
return
|
||||
for generator in self.machine.vars_generators.values():
|
||||
if "files" in generator:
|
||||
for file in generator["files"].values():
|
||||
if file["secret"]:
|
||||
return
|
||||
|
||||
if has_machine(self.machine.flake_dir, self.machine.name):
|
||||
return
|
||||
priv_key, pub_key = generate_private_key()
|
||||
encrypt_secret(
|
||||
self.machine.flake_dir,
|
||||
sops_secrets_folder(self.machine.flake_dir)
|
||||
/ f"{self.machine.name}-age.key",
|
||||
priv_key,
|
||||
)
|
||||
add_machine(self.machine.flake_dir, self.machine.name, pub_key, False)
|
||||
|
||||
def set(
|
||||
self, generator_name: str, name: str, value: bytes, groups: list[str]
|
||||
) -> Path | None:
|
||||
path = (
|
||||
sops_secrets_folder(self.machine.flake_dir)
|
||||
/ f"{self.machine.name}-{generator_name}-{name}"
|
||||
)
|
||||
encrypt_secret(
|
||||
self.machine.flake_dir,
|
||||
path,
|
||||
value,
|
||||
add_machines=[self.machine.name],
|
||||
add_groups=groups,
|
||||
)
|
||||
return path
|
||||
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
return decrypt_secret(
|
||||
self.machine.flake_dir, f"{self.machine.name}-{name}"
|
||||
).encode("utf-8")
|
||||
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
return has_secret(
|
||||
self.machine.flake_dir,
|
||||
f"{self.machine.name}-{name}",
|
||||
)
|
||||
|
||||
def upload(self, output_dir: Path) -> None:
|
||||
key_name = f"{self.machine.name}-age.key"
|
||||
if not has_secret(self.machine.flake_dir, key_name):
|
||||
# skip uploading the secret, not managed by us
|
||||
return
|
||||
key = decrypt_secret(self.machine.flake_dir, key_name)
|
||||
(output_dir / "key.txt").write_text(key)
|
35
pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py
Normal file
35
pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py
Normal file
@ -0,0 +1,35 @@
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from clan_cli.dirs import vm_state_dir
|
||||
from clan_cli.machines.machines import Machine
|
||||
|
||||
from . import SecretStoreBase
|
||||
|
||||
|
||||
class SecretStore(SecretStoreBase):
|
||||
def __init__(self, machine: Machine) -> None:
|
||||
self.machine = machine
|
||||
self.dir = vm_state_dir(str(machine.flake), machine.name) / "secrets"
|
||||
self.dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def set(
|
||||
self, service: str, name: str, value: bytes, groups: list[str]
|
||||
) -> Path | None:
|
||||
secret_file = self.dir / service / name
|
||||
secret_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
secret_file.write_bytes(value)
|
||||
return None # we manage the files outside of the git repo
|
||||
|
||||
def get(self, service: str, name: str) -> bytes:
|
||||
secret_file = self.dir / service / name
|
||||
return secret_file.read_bytes()
|
||||
|
||||
def exists(self, service: str, name: str) -> bool:
|
||||
return (self.dir / service / name).exists()
|
||||
|
||||
def upload(self, output_dir: Path) -> None:
|
||||
if os.path.exists(output_dir):
|
||||
shutil.rmtree(output_dir)
|
||||
shutil.copytree(self.dir, output_dir)
|
58
pkgs/clan-cli/clan_cli/vars/upload.py
Normal file
58
pkgs/clan-cli/clan_cli/vars/upload.py
Normal file
@ -0,0 +1,58 @@
|
||||
import argparse
|
||||
import importlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from ..cmd import Log, run
|
||||
from ..completions import add_dynamic_completer, complete_machines
|
||||
from ..machines.machines import Machine
|
||||
from ..nix import nix_shell
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def upload_secrets(machine: Machine) -> None:
|
||||
secret_facts_module = importlib.import_module(machine.secret_facts_module)
|
||||
secret_facts_store = secret_facts_module.SecretStore(machine=machine)
|
||||
|
||||
if secret_facts_store.update_check():
|
||||
log.info("Secrets already up to date")
|
||||
return
|
||||
with TemporaryDirectory() as tempdir:
|
||||
secret_facts_store.upload(Path(tempdir))
|
||||
host = machine.target_host
|
||||
|
||||
ssh_cmd = host.ssh_cmd()
|
||||
run(
|
||||
nix_shell(
|
||||
["nixpkgs#rsync"],
|
||||
[
|
||||
"rsync",
|
||||
"-e",
|
||||
" ".join(["ssh"] + ssh_cmd[2:]),
|
||||
"-az",
|
||||
"--delete",
|
||||
"--chown=root:root",
|
||||
"--chmod=D700,F600",
|
||||
f"{tempdir!s}/",
|
||||
f"{host.user}@{host.host}:{machine.secrets_upload_directory}/",
|
||||
],
|
||||
),
|
||||
log=Log.BOTH,
|
||||
)
|
||||
|
||||
|
||||
def upload_command(args: argparse.Namespace) -> None:
|
||||
machine = Machine(name=args.machine, flake=args.flake)
|
||||
upload_secrets(machine)
|
||||
|
||||
|
||||
def register_upload_parser(parser: argparse.ArgumentParser) -> None:
|
||||
machines_parser = parser.add_argument(
|
||||
"machine",
|
||||
help="The machine to upload secrets to",
|
||||
)
|
||||
add_dynamic_completer(machines_parser, complete_machines)
|
||||
|
||||
parser.set_defaults(func=upload_command)
|
@ -254,6 +254,9 @@ def collect_commands() -> list[Category]:
|
||||
if isinstance(action, argparse._SubParsersAction):
|
||||
subparsers: dict[str, argparse.ArgumentParser] = action.choices
|
||||
for name, subparser in subparsers.items():
|
||||
if str(subparser.description).startswith("WIP"):
|
||||
print(f"Excluded {name} from documentation as it is marked as WIP")
|
||||
continue
|
||||
(_options, _positionals, _subcommands) = get_subcommands(
|
||||
subparser, to=result, level=2, prefix=[name]
|
||||
)
|
||||
|
@ -58,6 +58,7 @@
|
||||
python docs.py reference
|
||||
mkdir -p $out
|
||||
cp -r out/* $out
|
||||
ls -lah $out
|
||||
'';
|
||||
};
|
||||
clan-ts-api = pkgs.stdenv.mkDerivation {
|
||||
|
@ -108,7 +108,7 @@ def test_type_from_schema_path_simple() -> None:
|
||||
schema = dict(
|
||||
type="boolean",
|
||||
)
|
||||
assert parsing.type_from_schema_path(schema, []) == bool
|
||||
assert parsing.type_from_schema_path(schema, []) is bool
|
||||
|
||||
|
||||
def test_type_from_schema_path_nested() -> None:
|
||||
@ -125,8 +125,8 @@ def test_type_from_schema_path_nested() -> None:
|
||||
age=dict(type="integer"),
|
||||
),
|
||||
)
|
||||
assert parsing.type_from_schema_path(schema, ["age"]) == int
|
||||
assert parsing.type_from_schema_path(schema, ["name", "first"]) == str
|
||||
assert parsing.type_from_schema_path(schema, ["age"]) is int
|
||||
assert parsing.type_from_schema_path(schema, ["name", "first"]) is str
|
||||
|
||||
|
||||
def test_type_from_schema_path_dynamic_attrs() -> None:
|
||||
@ -140,16 +140,16 @@ def test_type_from_schema_path_dynamic_attrs() -> None:
|
||||
),
|
||||
),
|
||||
)
|
||||
assert parsing.type_from_schema_path(schema, ["age"]) == int
|
||||
assert parsing.type_from_schema_path(schema, ["users", "foo"]) == str
|
||||
assert parsing.type_from_schema_path(schema, ["age"]) is int
|
||||
assert parsing.type_from_schema_path(schema, ["users", "foo"]) is str
|
||||
|
||||
|
||||
def test_map_type() -> None:
|
||||
with pytest.raises(ClanError):
|
||||
config.map_type("foo")
|
||||
assert config.map_type("string") == str
|
||||
assert config.map_type("integer") == int
|
||||
assert config.map_type("boolean") == bool
|
||||
assert config.map_type("string") is str
|
||||
assert config.map_type("integer") is int
|
||||
assert config.map_type("boolean") is bool
|
||||
assert config.map_type("attribute set of string") == dict[str, str]
|
||||
assert config.map_type("attribute set of integer") == dict[str, int]
|
||||
assert config.map_type("null or string") == str | None
|
||||
|
49
pkgs/clan-cli/tests/test_vars.py
Normal file
49
pkgs/clan-cli/tests/test_vars.py
Normal file
@ -0,0 +1,49 @@
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from fixtures_flakes import generate_flake
|
||||
from helpers.cli import Cli
|
||||
from root import CLAN_CORE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.impure
|
||||
def test_generate_secret(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
temporary_home: Path,
|
||||
# age_keys: list["KeyPair"],
|
||||
) -> None:
|
||||
flake = generate_flake(
|
||||
temporary_home,
|
||||
flake_template=CLAN_CORE / "templates" / "minimal",
|
||||
machine_configs=dict(
|
||||
my_machine=dict(
|
||||
clan=dict(
|
||||
core=dict(
|
||||
vars=dict(
|
||||
generators=dict(
|
||||
my_generator=dict(
|
||||
files=dict(
|
||||
my_secret=dict(
|
||||
secret=False,
|
||||
)
|
||||
),
|
||||
script="echo hello > $out/my_secret",
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
)
|
||||
monkeypatch.chdir(flake.path)
|
||||
cli = Cli()
|
||||
cmd = ["vars", "generate", "--flake", str(flake.path), "my_machine"]
|
||||
cli.run(cmd)
|
||||
assert (
|
||||
flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret"
|
||||
).is_file()
|
@ -19,29 +19,6 @@ let
|
||||
};
|
||||
};
|
||||
};
|
||||
installerModule =
|
||||
{ config, modulesPath, ... }:
|
||||
{
|
||||
imports = [
|
||||
wifiModule
|
||||
self.nixosModules.installer
|
||||
self.inputs.nixos-generators.nixosModules.all-formats
|
||||
(modulesPath + "/installer/cd-dvd/iso-image.nix")
|
||||
];
|
||||
|
||||
isoImage.squashfsCompression = "zstd";
|
||||
|
||||
system.stateVersion = config.system.nixos.version;
|
||||
nixpkgs.pkgs = self.inputs.nixpkgs.legacyPackages.x86_64-linux;
|
||||
};
|
||||
|
||||
installerSystem = lib.nixosSystem {
|
||||
modules = [
|
||||
self.inputs.disko.nixosModules.default
|
||||
installerModule
|
||||
{ disko.memSize = 4096; } # FIXME: otherwise the image builder goes OOM
|
||||
];
|
||||
};
|
||||
|
||||
flashInstallerModule =
|
||||
{ config, ... }:
|
||||
@ -98,14 +75,6 @@ let
|
||||
in
|
||||
{
|
||||
clan = {
|
||||
# To build a generic installer image (without ssh pubkeys),
|
||||
# use the following command:
|
||||
# $ nix build .#iso-installer
|
||||
machines.iso-installer = {
|
||||
imports = [ installerModule ];
|
||||
fileSystems."/".device = lib.mkDefault "/dev/null";
|
||||
};
|
||||
|
||||
# To directly flash the installer to a disk, use the following command:
|
||||
# $ clan flash flash-installer --disk main /dev/sdX --yes
|
||||
# This will include your ssh public keys in the installer.
|
||||
@ -114,7 +83,4 @@ in
|
||||
boot.loader.grub.enable = lib.mkDefault true;
|
||||
};
|
||||
};
|
||||
flake.packages.x86_64-linux.iso-installer = installerSystem.config.formats.iso;
|
||||
flake.apps.x86_64-linux.install-vm.program = installerSystem.config.formats.vm.outPath;
|
||||
flake.apps.x86_64-linux.install-vm-nogui.program = installerSystem.config.formats.vm-nogui.outPath;
|
||||
}
|
||||
|
@ -1,3 +1,4 @@
|
||||
# shellcheck shell=bash
|
||||
source_up
|
||||
|
||||
watch_file flake-module.nix default.nix
|
||||
|
Loading…
Reference in New Issue
Block a user