Merge branch 'main' into DavHau-dave
All checks were successful
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.12-qemu" Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.12-mypy" Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-minimal-inventory-machine Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.template-minimal Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-inventory-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.module-clan-vars-eval Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
checks / checks-impure (pull_request) Successful in 4m37s

This commit is contained in:
Mic92 2024-07-09 09:33:11 +00:00
commit 3e6c41593d
51 changed files with 1930 additions and 748 deletions

3
.envrc
View File

@ -1,3 +1,4 @@
# shellcheck shell=bash
if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then
source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4="
fi
@ -5,7 +6,7 @@ fi
watch_file .direnv/selected-shell
if [ -e .direnv/selected-shell ]; then
use flake .#$(cat .direnv/selected-shell)
use flake ".#$(cat .direnv/selected-shell)"
else
use flake
fi

View File

@ -1,100 +0,0 @@
From bc199a27f23b0fcf175b116f7cf606c0d22b422a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 11 Jun 2024 11:40:47 +0200
Subject: [PATCH 1/2] register_new_matrix_user: add password-file flag
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
getpass in python expects stdin to be a tty, hence we cannot just pipe
into register_new_matrix_user. --password-file instead works better and
it would also allow the use of stdin if /dev/stdin is passed.
Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
---
changelog.d/17294.feature | 2 ++
debian/register_new_matrix_user.ronn | 9 +++++++--
synapse/_scripts/register_new_matrix_user.py | 20 +++++++++++++++-----
3 files changed, 24 insertions(+), 7 deletions(-)
create mode 100644 changelog.d/17294.feature
diff --git a/changelog.d/17294.feature b/changelog.d/17294.feature
new file mode 100644
index 000000000..33aac7b0b
--- /dev/null
+++ b/changelog.d/17294.feature
@@ -0,0 +1,2 @@
+`register_new_matrix_user` now supports a --password-file flag, which
+is useful for scripting.
diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn
index 0410b1f4c..d99e9215a 100644
--- a/debian/register_new_matrix_user.ronn
+++ b/debian/register_new_matrix_user.ronn
@@ -31,8 +31,13 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
Local part of the new user. Will prompt if omitted.
* `-p`, `--password`:
- New password for user. Will prompt if omitted. Supplying the password
- on the command line is not recommended. Use the STDIN instead.
+ New password for user. Will prompt if this option and `--password-file` are omitted.
+ Supplying the password on the command line is not recommended.
+ Use `--password-file` if possible.
+
+ * `--password-file`:
+ File containing the new password for user. If set, overrides `--password`.
+ This is a more secure alternative to specifying the password on the command line.
* `-a`, `--admin`:
Register new user as an admin. Will prompt if omitted.
diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
index 77a7129ee..972b35e2d 100644
--- a/synapse/_scripts/register_new_matrix_user.py
+++ b/synapse/_scripts/register_new_matrix_user.py
@@ -173,11 +173,18 @@ def main() -> None:
default=None,
help="Local part of the new user. Will prompt if omitted.",
)
- parser.add_argument(
+ password_group = parser.add_mutually_exclusive_group()
+ password_group.add_argument(
"-p",
"--password",
default=None,
- help="New password for user. Will prompt if omitted.",
+ help="New password for user. Will prompt for a password if "
+ "this flag and `--password-file` are both omitted.",
+ )
+ password_group.add_argument(
+ "--password-file",
+ default=None,
+ help="File containing the new password for user. If set, will override `--password`.",
)
parser.add_argument(
"-t",
@@ -247,6 +254,11 @@ def main() -> None:
print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr)
sys.exit(1)
+ if args.password_file:
+ password = _read_file(args.password_file, "password-file").strip()
+ else:
+ password = args.password
+
if args.server_url:
server_url = args.server_url
elif config is not None:
@@ -269,9 +281,7 @@ def main() -> None:
if args.admin or args.no_admin:
admin = args.admin
- register_new_user(
- args.user, args.password, server_url, secret, admin, args.user_type
- )
+ register_new_user(args.user, password, server_url, secret, admin, args.user_type)
def _read_file(file_path: Any, config_path: str) -> str:
--
2.44.1

View File

@ -1,94 +0,0 @@
From 1789416df425d22693b0055a6688d8686e0ee4a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Thu, 13 Jun 2024 14:38:19 +0200
Subject: [PATCH 2/2] register-new-matrix-user: add a flag to ignore already
existing users
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This allows to register users in a more declarative and stateless way.
Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
---
synapse/_scripts/register_new_matrix_user.py | 22 ++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
index 972b35e2d..233e7267d 100644
--- a/synapse/_scripts/register_new_matrix_user.py
+++ b/synapse/_scripts/register_new_matrix_user.py
@@ -52,6 +52,7 @@ def request_registration(
user_type: Optional[str] = None,
_print: Callable[[str], None] = print,
exit: Callable[[int], None] = sys.exit,
+ exists_ok: bool = False,
) -> None:
url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),)
@@ -97,6 +98,10 @@ def request_registration(
r = requests.post(url, json=data)
if r.status_code != 200:
+ response = r.json()
+ if exists_ok and response["errcode"] == "M_USER_IN_USE":
+ _print("User already exists. Skipping.")
+ return
_print("ERROR! Received %d %s" % (r.status_code, r.reason))
if 400 <= r.status_code < 500:
try:
@@ -115,6 +120,7 @@ def register_new_user(
shared_secret: str,
admin: Optional[bool],
user_type: Optional[str],
+ exists_ok: bool = False,
) -> None:
if not user:
try:
@@ -154,7 +160,13 @@ def register_new_user(
admin = False
request_registration(
- user, password, server_location, shared_secret, bool(admin), user_type
+ user,
+ password,
+ server_location,
+ shared_secret,
+ bool(admin),
+ user_type,
+ exists_ok=exists_ok,
)
@@ -173,6 +185,11 @@ def main() -> None:
default=None,
help="Local part of the new user. Will prompt if omitted.",
)
+ parser.add_argument(
+ "--exists-ok",
+ action="store_true",
+ help="Do not fail if user already exists.",
+ )
password_group = parser.add_mutually_exclusive_group()
password_group.add_argument(
"-p",
@@ -192,6 +209,7 @@ def main() -> None:
default=None,
help="User type as specified in synapse.api.constants.UserTypes",
)
+
admin_group = parser.add_mutually_exclusive_group()
admin_group.add_argument(
"-a",
@@ -281,7 +299,7 @@ def main() -> None:
if args.admin or args.no_admin:
admin = args.admin
- register_new_user(args.user, password, server_url, secret, admin, args.user_type)
+ register_new_user(args.user, password, server_url, secret, admin, args.user_type, exists_ok=args.exists_ok)
def _read_file(file_path: Any, config_path: str) -> str:
--
2.44.1

View File

@ -17,19 +17,8 @@ let
ln -s $out/config.json $out/config.${nginx-vhost}.json
'';
# FIXME: This was taken from upstream. Drop this when our patch is upstream
synapseCfg = config.services.matrix-synapse;
wantedExtras =
synapseCfg.extras
++ lib.optional (synapseCfg.settings ? oidc_providers) "oidc"
++ lib.optional (synapseCfg.settings ? jwt_config) "jwt"
++ lib.optional (synapseCfg.settings ? saml2_config) "saml2"
++ lib.optional (synapseCfg.settings ? redis) "redis"
++ lib.optional (synapseCfg.settings ? sentry) "sentry"
++ lib.optional (synapseCfg.settings ? user_directory) "user-search"
++ lib.optional (synapseCfg.settings.url_preview_enabled) "url-preview"
++ lib.optional (synapseCfg.settings.database.name == "psycopg2") "postgres";
in
# FIXME: This was taken from upstream. Drop this when our patch is upstream
{
options.services.matrix-synapse.package = lib.mkOption { readOnly = false; };
options.clan.matrix-synapse = {
@ -78,21 +67,6 @@ in
];
config = {
services.matrix-synapse = {
package = lib.mkForce (
pkgs.matrix-synapse.override {
matrix-synapse-unwrapped = pkgs.matrix-synapse.unwrapped.overrideAttrs (_old: {
doInstallCheck = false; # too slow, nixpkgs maintainer already run this.
patches = [
# see: https://github.com/element-hq/synapse/pull/17304
./0001-register_new_matrix_user-add-password-file-flag.patch
./0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch
];
});
extras = wantedExtras;
plugins = synapseCfg.plugins;
}
);
enable = true;
settings = {
server_name = cfg.domain;

View File

@ -30,6 +30,16 @@ in
default = [ config.clan.core.machineName ];
description = "Hosts that should be excluded";
};
networkIps = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Extra zerotier network Ips that should be accepted";
};
networkIds = lib.mkOption {
type = lib.types.listOf lib.types.str;
default = [ ];
description = "Extra zerotier network Ids that should be accepted";
};
};
config.systemd.services.zerotier-static-peers-autoaccept =
@ -56,6 +66,7 @@ in
lib.nameValuePair (builtins.readFile fullPath) [ machine ]
) filteredMachines
);
allHostIPs = config.clan.zerotier-static-peers.networkIps ++ hosts;
in
lib.mkIf (config.clan.core.networking.zerotier.controller.enable) {
wantedBy = [ "multi-user.target" ];
@ -65,7 +76,10 @@ in
#!/bin/sh
${lib.concatMapStringsSep "\n" (host: ''
${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host}
'') hosts}
'') allHostIPs}
${lib.concatMapStringsSep "\n" (host: ''
${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow ${host}
'') config.clan.zerotier-static-peers.networkIds}
'';
};

View File

@ -1,6 +1,8 @@
# shellcheck shell=bash
source_up
watch_file $(find ./nix -name "*.nix" -printf '%p ')
mapfile -d '' -t nix_files < <(find ./nix -name "*.nix" -print0)
watch_file "${nix_files[@]}"
# Because we depend on nixpkgs sources, uploading to builders takes a long time
use flake .#docs --builders ''

View File

@ -14,6 +14,7 @@ markdown_extensions:
- attr_list
- footnotes
- md_in_html
- def_list
- meta
- plantuml_markdown
- pymdownx.emoji:
@ -49,6 +50,8 @@ nav:
- Mesh VPN: getting-started/mesh-vpn.md
- Backup & Restore: getting-started/backups.md
- Flake-parts: getting-started/flake-parts.md
- Concepts:
- Configuration: concepts/configuration.md
- Reference:
- Clan Modules:
- reference/clanModules/borgbackup-static.md

View File

@ -26,6 +26,7 @@ pkgs.stdenv.mkDerivation {
mkdocs-material
mkdocs-rss-plugin
mkdocs-macros
filelock # FIXME: this should be already provided by mkdocs-rss-plugin
]);
configurePhase = ''
mkdir -p ./site/reference/cli

View File

@ -38,7 +38,7 @@
patchShebangs --build $out
ruff format --check --diff $out
ruff --line-length 88 $out
ruff check --line-length 88 $out
mypy --strict $out
'';

View File

@ -0,0 +1,114 @@
# Configuration
## Introduction
When managing machine configuration this can be done through many possible ways.
Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming).
clan currently offers the following methods to configure machines:
!!! Success "Recommended for nix people"
- flake.nix (i.e. via `buildClan`)
- `machine` argument
- `inventory` argument
- machines/`machine_name`/configuration.nix (`autoincluded` if it exists)
???+ Note "Used by CLI & UI"
- inventory.json
- machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists)
!!! Warning "Deprecated"
machines/`machine_name`/settings.json
## BuildClan
The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services.
### Inputs
`directory`
: The directory containing the machines subdirectory
`machines`
: Allows to include machine-specific modules i.e. machines.${name} = { ... }
`meta`
: An optional set
: `{ name :: string, icon :: string, description :: string }`
`inventory`
: Service set for easily configuring distributed services, such as backups
: For more details see [Inventory](#inventory)
`specialArgs`
: Extra arguments to pass to nixosSystem i.e. useful to make self available
`pkgsForSystem`
: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system.
This improves performance, but all nipxkgs.* options will be ignored.
`(string -> pkgs )`
## Inventory
`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries.
The following is the specification of the inventory in `cuelang`
```cue
{
meta: {
// A name of the clan (primarily shown by the UI)
name: string
// A description of the clan
description?: string
// The icon path
icon?: string
}
// A map of services
services: [string]: [string]: {
// Required meta fields
meta: {
name: string,
icon?: string
description?: string,
},
// Machines are added via the avilable roles
// Membership depends only on this field
roles: [string]: {
machines: [...string],
tags: [...string],
}
machines?: {
[string]: {
config?: {
...
}
}
},
// Global Configuration for the service
// Applied to all machines.
config?: {
// Schema depends on the module.
// It declares the interface how the service can be configured.
...
}
}
// A map of machines, extends the machines of `buildClan`
machines: [string]: {
name: string,
description?: string,
icon?: string
tags: [...string]
system: string
}
}
```

View File

@ -7,11 +7,11 @@
]
},
"locked": {
"lastModified": 1718846788,
"narHash": "sha256-9dtXYtEkmXoUJV+PGLqscqF7qTn4AIhAKpFWRFU2NYs=",
"lastModified": 1720056646,
"narHash": "sha256-BymcV4HWtx2VFuabDCM4/nEJcfivCx0S02wUCz11mAY=",
"owner": "nix-community",
"repo": "disko",
"rev": "e1174d991944a01eaaa04bc59c6281edca4c0e6e",
"rev": "64679cd7f318c9b6595902b47d4585b1d51d5f9e",
"type": "github"
},
"original": {
@ -27,11 +27,11 @@
]
},
"locked": {
"lastModified": 1717285511,
"narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
"lastModified": 1719994518,
"narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
"rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
"type": "github"
},
"original": {
@ -40,29 +40,6 @@
"type": "github"
}
},
"nixos-generators": {
"inputs": {
"nixlib": [
"nixpkgs"
],
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1718025593,
"narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=",
"owner": "nix-community",
"repo": "nixos-generators",
"rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nixos-generators",
"type": "github"
}
},
"nixos-images": {
"inputs": {
"nixos-stable": [],
@ -71,11 +48,11 @@
]
},
"locked": {
"lastModified": 1718845599,
"narHash": "sha256-HbQ0iKohKJC5grC95HNjLxGPdgsc/BJgoENDYNbzkLo=",
"lastModified": 1720055024,
"narHash": "sha256-c5rsiI1R7tnCDpcgfsa7ouSdn6wpctbme9TUp53CFyU=",
"owner": "nix-community",
"repo": "nixos-images",
"rev": "c1e6a5f7b08f1c9993de1cfc5f15f838bf783b88",
"rev": "f8650460d37d9d1820a93ebb7f0db5b6c3621946",
"type": "github"
},
"original": {
@ -86,11 +63,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1719451888,
"narHash": "sha256-Ky0sgEEJMcBmNEJztY6KcVn+6bq74EKM7pd1CR1wnPQ=",
"lastModified": 1720340162,
"narHash": "sha256-iVLH0Ygtw/Iw9Q1cFFX7OhNnoPbc7/ZWW6J3c0zbiZw=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "3664857c48feacb35770c00abfdc671e55849be5",
"rev": "60a94e515488e335bd5bce096431d490486915e3",
"type": "github"
},
"original": {
@ -104,7 +81,6 @@
"inputs": {
"disko": "disko",
"flake-parts": "flake-parts",
"nixos-generators": "nixos-generators",
"nixos-images": "nixos-images",
"nixpkgs": "nixpkgs",
"sops-nix": "sops-nix",
@ -119,11 +95,11 @@
"nixpkgs-stable": []
},
"locked": {
"lastModified": 1719111739,
"narHash": "sha256-kr2QzRrplzlCP87ddayCZQS+dhGW98kw2zy7+jUXtF4=",
"lastModified": 1720321395,
"narHash": "sha256-kcI8q9Nh8/CSj0ygfWq1DLckHl8IHhFarL8ie6g7OEk=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "5e2e9421e9ed2b918be0a441c4535cfa45e04811",
"rev": "c184aca4db5d71c3db0c8cbfcaaec337a5d065ea",
"type": "github"
},
"original": {
@ -139,11 +115,11 @@
]
},
"locked": {
"lastModified": 1718522839,
"narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=",
"lastModified": 1720436211,
"narHash": "sha256-/cKXod0oGLl+vH4bKBZnTV3qxrw4jgOLnyQ8KXey5J8=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81",
"rev": "6fc8bded78715cdd43a3278a14ded226eb3a239e",
"type": "github"
},
"original": {

View File

@ -13,9 +13,6 @@
sops-nix.url = "github:Mic92/sops-nix";
sops-nix.inputs.nixpkgs.follows = "nixpkgs";
sops-nix.inputs.nixpkgs-stable.follows = "";
nixos-generators.url = "github:nix-community/nixos-generators";
nixos-generators.inputs.nixpkgs.follows = "nixpkgs";
nixos-generators.inputs.nixlib.follows = "nixpkgs";
nixos-images.url = "github:nix-community/nixos-images";
nixos-images.inputs.nixos-unstable.follows = "nixpkgs";
# unused input

View File

@ -1,4 +1,4 @@
{ lib, inputs, ... }:
{ inputs, ... }:
{
imports = [ inputs.treefmt-nix.flakeModule ];
perSystem =
@ -8,43 +8,19 @@
treefmt.programs.shellcheck.enable = true;
treefmt.programs.mypy.enable = true;
treefmt.programs.nixfmt.enable = true;
treefmt.programs.nixfmt.package = pkgs.nixfmt-rfc-style;
treefmt.programs.deadnix.enable = true;
treefmt.programs.mypy.directories = {
"pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
"pkgs/clan-app".extraPythonPackages =
# clan-app currently only exists on linux
(self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies;
};
treefmt.programs.ruff.check = true;
treefmt.programs.ruff.format = true;
treefmt.settings.formatter.nix = {
command = "sh";
options = [
"-eucx"
''
# First deadnix
${lib.getExe pkgs.deadnix} --edit "$@"
# Then nixpkgs-fmt
${lib.getExe pkgs.nixfmt-rfc-style} "$@"
''
"--" # this argument is ignored by bash
];
includes = [ "*.nix" ];
excludes = [
# Was copied from nixpkgs. Keep diff minimal to simplify upstreaming.
"pkgs/builders/script-writers.nix"
];
};
treefmt.settings.formatter.python = {
command = "sh";
options = [
"-eucx"
''
${lib.getExe pkgs.ruff} check --fix "$@"
${lib.getExe pkgs.ruff} format "$@"
''
"--" # this argument is ignored by bash
];
includes = [ "*.py" ];
};
# FIXME: currently broken in CI
#treefmt.settings.formatter.vale =
# let

View File

@ -1,4 +1,7 @@
{
"meta": {
"name": "Minimal inventory"
},
"machines": {
"minimal-inventory-machine": {
"name": "foo",

View File

@ -152,6 +152,13 @@ let
in
(machineImports settings)
++ [
{
# Autoinclude configuration.nix and hardware-configuration.nix
imports = builtins.filter (p: builtins.pathExists p) [
"${directory}/machines/${name}/configuration.nix"
"${directory}/machines/${name}/hardware-configuration.nix"
];
}
settings
clan-core.nixosModules.clanCore
extraConfig

View File

@ -1,3 +1,4 @@
# shellcheck shell=bash
source_up
watch_file flake-module.nix

View File

@ -5,6 +5,7 @@ package schema
description?: string,
icon?: string
tags: [...string]
system?: string
}
#role: string

View File

@ -1,5 +1,26 @@
{ lib, ... }:
{
lib,
config,
pkgs,
...
}:
let
inherit (lib.types) submoduleWith;
submodule =
module:
submoduleWith {
specialArgs.pkgs = pkgs;
modules = [ module ];
};
in
{
imports = [
./public/in_repo.nix
# ./public/vm.nix
# ./secret/password-store.nix
./secret/sops.nix
# ./secret/vm.nix
];
options.clan.core.vars = lib.mkOption {
visible = false;
description = ''
@ -11,6 +32,20 @@
- generate secrets like private keys automatically when they are needed
- output multiple values like private and public keys simultaneously
'';
type = lib.types.submoduleWith { modules = [ ./interface.nix ]; };
type = submodule { imports = [ ./interface.nix ]; };
};
config.system.clan.deployment.data = {
vars = {
generators = lib.flip lib.mapAttrs config.clan.core.vars.generators (
_name: generator: {
inherit (generator) finalScript;
files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; });
}
);
inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule;
};
inherit (config.clan.networking) targetHost buildHost;
inherit (config.clan.deployment) requireExplicitUpdate;
};
}

View File

@ -54,21 +54,6 @@ in
};
};
# Ensure that generators.imports works
# This allows importing generators from third party projects without providing
# them access to other settings.
test_generator_modules =
let
generator_module = {
my-generator.files.password = { };
};
config = eval { generators.imports = [ generator_module ]; };
in
{
expr = config.generators ? my-generator;
expected = true;
};
# script can be text
test_script_text =
let

View File

@ -1,8 +1,12 @@
{ lib, ... }:
{
lib,
config,
pkgs,
...
}:
let
inherit (lib) mkOption;
inherit (lib.types)
anything
attrsOf
bool
either
@ -14,30 +18,27 @@ let
submoduleWith
;
# the original types.submodule has strange behavior
submodule = module: submoduleWith { modules = [ module ]; };
submodule =
module:
submoduleWith {
specialArgs.pkgs = pkgs;
modules = [ module ];
};
options = lib.mapAttrs (_: mkOption);
subOptions = opts: submodule { options = options opts; };
in
{
options = options {
settings = {
options = {
settings = import ./settings-opts.nix { inherit lib; };
generators = lib.mkOption {
description = ''
Settings for the generated variables.
A set of generators that can be used to generate files.
Generators are scripts that produce files based on the values of other generators and user input.
Each generator is expected to produce a set of files under a directory.
'';
type = submodule {
freeformType = anything;
imports = [ ./settings.nix ];
};
};
generators = {
default = {
imports = [
# implementation of the generator
./generator.nix
];
};
type = submodule {
freeformType = attrsOf (subOptions {
default = { };
type = attrsOf (submodule {
imports = [ ./generator.nix ];
options = options {
dependencies = {
description = ''
A list of other generators that this generator depends on.
@ -52,32 +53,45 @@ in
A set of files to generate.
The generator 'script' is expected to produce exactly these files under $out.
'';
type = attrsOf (subOptions {
secret = {
description = ''
Whether the file should be treated as a secret.
'';
type = bool;
default = true;
};
path = {
description = ''
The path to the file containing the content of the generated value.
This will be set automatically
'';
type = str;
readOnly = true;
};
value = {
description = ''
The content of the generated value.
Only available if the file is not secret.
'';
type = str;
default = throw "Cannot access value of secret file";
defaultText = "Throws error because the value of a secret file is not accessible";
};
});
type = attrsOf (
submodule (file: {
imports = [ config.settings.fileModule ];
options = options {
name = {
type = lib.types.str;
description = ''
name of the public fact
'';
readOnly = true;
default = file.config._module.args.name;
};
secret = {
description = ''
Whether the file should be treated as a secret.
'';
type = bool;
default = true;
};
path = {
description = ''
The path to the file containing the content of the generated value.
This will be set automatically
'';
type = str;
readOnly = true;
};
value = {
description = ''
The content of the generated value.
Only available if the file is not secret.
'';
type = str;
default = throw "Cannot access value of secret file";
defaultText = "Throws error because the value of a secret file is not accessible";
};
};
})
);
};
prompts = {
description = ''
@ -85,28 +99,30 @@ in
Prompts are available to the generator script as files.
For example, a prompt named 'prompt1' will be available via $prompts/prompt1
'';
type = attrsOf (subOptions {
description = {
description = ''
The description of the prompted value
'';
type = str;
example = "SSH private key";
};
type = {
description = ''
The input type of the prompt.
The following types are available:
- hidden: A hidden text (e.g. password)
- line: A single line of text
- multiline: A multiline text
'';
type = enum [
"hidden"
"line"
"multiline"
];
default = "line";
type = attrsOf (submodule {
options = {
description = {
description = ''
The description of the prompted value
'';
type = str;
example = "SSH private key";
};
type = {
description = ''
The input type of the prompt.
The following types are available:
- hidden: A hidden text (e.g. password)
- line: A single line of text
- multiline: A multiline text
'';
type = enum [
"hidden"
"line"
"multiline"
];
default = "line";
};
};
});
};
@ -140,8 +156,8 @@ in
internal = true;
visible = false;
};
});
};
};
});
};
};
}

View File

@ -0,0 +1,12 @@
{ config, lib, ... }:
{
config.clan.core.vars.settings =
lib.mkIf (config.clan.core.vars.settings.publicStore == "in_repo")
{
publicModule = "clan_cli.vars.public_modules.in_repo";
fileModule = file: {
path =
config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}";
};
};
}

View File

@ -0,0 +1,61 @@
{
config,
lib,
pkgs,
...
}:
let
secretsDir = config.clan.core.clanDir + "/sops/secrets";
groupsDir = config.clan.core.clanDir + "/sops/groups";
# My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation?
containsSymlink =
path:
builtins.pathExists path
&& (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink");
containsMachine =
parent: name: type:
type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clan.core.machineName}";
containsMachineOrGroups =
name: type:
(containsMachine secretsDir name type)
|| lib.any (
group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}"
) groups;
filterDir =
filter: dir:
lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir));
groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir);
secrets = filterDir containsMachineOrGroups secretsDir;
in
{
config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
# Before we generate a secret we cannot know the path yet, so we need to set it to an empty string
fileModule = file: {
path =
lib.mkIf file.secret
config.sops.secrets.${"${config.clan.core.machineName}-${file.config.name}"}.path
or "/no-such-path";
};
secretModule = "clan_cli.vars.secret_modules.sops";
secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix";
};
config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") {
secrets = builtins.mapAttrs (name: _: {
sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret";
format = "binary";
}) secrets;
# To get proper error messages about missing secrets we need a dummy secret file that is always present
defaultSopsFile = lib.mkIf config.sops.validateSopsFiles (
lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" ""))
);
age.keyFile = lib.mkIf (builtins.pathExists (
config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret"
)) (lib.mkDefault "/var/lib/sops-nix/key.txt");
};
}

View File

@ -0,0 +1,71 @@
{ lib, ... }:
{
secretStore = lib.mkOption {
type = lib.types.enum [
"sops"
"password-store"
"vm"
"custom"
];
default = "sops";
description = ''
method to store secret facts
custom can be used to define a custom secret fact store.
'';
};
secretModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the secret module
'';
};
secretUploadDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where secrets are uploaded into, This is backend specific.
'';
};
fileModule = lib.mkOption {
type = lib.types.deferredModule;
internal = true;
description = ''
A module to be imported in every vars.files.<name> submodule.
Used by backends to define the `path` attribute.
'';
default = { };
};
publicStore = lib.mkOption {
type = lib.types.enum [
"in_repo"
"vm"
"custom"
];
default = "in_repo";
description = ''
method to store public facts.
custom can be used to define a custom public fact store.
'';
};
publicModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the public module
'';
};
publicDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where public facts are stored.
'';
};
}

View File

@ -1,72 +0,0 @@
{ lib, ... }:
{
options = {
secretStore = lib.mkOption {
type = lib.types.enum [
"sops"
"password-store"
"vm"
"custom"
];
default = "sops";
description = ''
method to store secret facts
custom can be used to define a custom secret fact store.
'';
};
secretModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the secret module
'';
};
secretUploadDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where secrets are uploaded into, This is backend specific.
'';
};
secretPathFunction = lib.mkOption {
type = lib.types.raw;
description = ''
The function to use to generate the path for a secret.
The default function will use the path attribute of the secret.
The function will be called with the secret submodule as an argument.
'';
};
publicStore = lib.mkOption {
type = lib.types.enum [
"in_repo"
"vm"
"custom"
];
default = "in_repo";
description = ''
method to store public facts.
custom can be used to define a custom public fact store.
'';
};
publicModule = lib.mkOption {
type = lib.types.str;
internal = true;
description = ''
the python import path to the public module
'';
};
publicDirectory = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
The directory where public facts are stored.
'';
};
};
}

View File

@ -27,107 +27,127 @@ rec {
# Examples:
# writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; }
# makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world"
makeScriptWriter = { interpreter, check ? "", makeWrapperArgs ? [], }: nameOrPath: content:
assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
makeScriptWriter =
{
interpreter,
check ? "",
makeWrapperArgs ? [ ],
}:
nameOrPath: content:
assert lib.or (types.path.check nameOrPath) (
builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null
);
assert lib.or (types.path.check content) (types.str.check content);
let
name = last (builtins.split "/" nameOrPath);
in
pkgs.runCommandLocal name (
{
inherit makeWrapperArgs;
nativeBuildInputs = [
makeWrapper
];
}
// lib.optionalAttrs (nameOrPath == "/bin/${name}") {
meta.mainProgram = name;
}
// (
if (types.str.check content) then {
inherit content interpreter;
passAsFile = [ "content" ];
} else {
inherit interpreter;
contentPath = content;
pkgs.runCommandLocal name
(
{
inherit makeWrapperArgs;
nativeBuildInputs = [ makeWrapper ];
}
// lib.optionalAttrs (nameOrPath == "/bin/${name}") { meta.mainProgram = name; }
// (
if (types.str.check content) then
{
inherit content interpreter;
passAsFile = [ "content" ];
}
else
{
inherit interpreter;
contentPath = content;
}
)
)
)
''
# On darwin a script cannot be used as an interpreter in a shebang but
# there doesn't seem to be a limit to the size of shebang and multiple
# arguments to the interpreter are allowed.
if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
then
wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
# Get first word from the line (note: xargs echo remove leading spaces)
wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
if isScript $wrapperInterpreter
''
# On darwin a script cannot be used as an interpreter in a shebang but
# there doesn't seem to be a limit to the size of shebang and multiple
# arguments to the interpreter are allowed.
if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
then
echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
exit 1
wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
# Get first word from the line (note: xargs echo remove leading spaces)
wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
if isScript $wrapperInterpreter
then
echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
exit 1