diff --git a/.envrc b/.envrc index 18a7568f..fda7fa8f 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,4 @@ +# shellcheck shell=bash if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4=" fi @@ -5,7 +6,7 @@ fi watch_file .direnv/selected-shell if [ -e .direnv/selected-shell ]; then - use flake .#$(cat .direnv/selected-shell) + use flake ".#$(cat .direnv/selected-shell)" else use flake fi diff --git a/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch b/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch deleted file mode 100644 index fb9e2eb2..00000000 --- a/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch +++ /dev/null @@ -1,100 +0,0 @@ -From bc199a27f23b0fcf175b116f7cf606c0d22b422a Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= -Date: Tue, 11 Jun 2024 11:40:47 +0200 -Subject: [PATCH 1/2] register_new_matrix_user: add password-file flag -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -getpass in python expects stdin to be a tty, hence we cannot just pipe -into register_new_matrix_user. --password-file instead works better and -it would also allow the use of stdin if /dev/stdin is passed. - -Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> -Signed-off-by: Jörg Thalheim ---- - changelog.d/17294.feature | 2 ++ - debian/register_new_matrix_user.ronn | 9 +++++++-- - synapse/_scripts/register_new_matrix_user.py | 20 +++++++++++++++----- - 3 files changed, 24 insertions(+), 7 deletions(-) - create mode 100644 changelog.d/17294.feature - -diff --git a/changelog.d/17294.feature b/changelog.d/17294.feature -new file mode 100644 -index 000000000..33aac7b0b ---- /dev/null -+++ b/changelog.d/17294.feature -@@ -0,0 +1,2 @@ -+`register_new_matrix_user` now supports a --password-file flag, which -+is useful for scripting. -diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn -index 0410b1f4c..d99e9215a 100644 ---- a/debian/register_new_matrix_user.ronn -+++ b/debian/register_new_matrix_user.ronn -@@ -31,8 +31,13 @@ A sample YAML file accepted by `register_new_matrix_user` is described below: - Local part of the new user. Will prompt if omitted. - - * `-p`, `--password`: -- New password for user. Will prompt if omitted. Supplying the password -- on the command line is not recommended. Use the STDIN instead. -+ New password for user. Will prompt if this option and `--password-file` are omitted. -+ Supplying the password on the command line is not recommended. -+ Use `--password-file` if possible. -+ -+ * `--password-file`: -+ File containing the new password for user. If set, overrides `--password`. -+ This is a more secure alternative to specifying the password on the command line. - - * `-a`, `--admin`: - Register new user as an admin. Will prompt if omitted. -diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py -index 77a7129ee..972b35e2d 100644 ---- a/synapse/_scripts/register_new_matrix_user.py -+++ b/synapse/_scripts/register_new_matrix_user.py -@@ -173,11 +173,18 @@ def main() -> None: - default=None, - help="Local part of the new user. Will prompt if omitted.", - ) -- parser.add_argument( -+ password_group = parser.add_mutually_exclusive_group() -+ password_group.add_argument( - "-p", - "--password", - default=None, -- help="New password for user. Will prompt if omitted.", -+ help="New password for user. Will prompt for a password if " -+ "this flag and `--password-file` are both omitted.", -+ ) -+ password_group.add_argument( -+ "--password-file", -+ default=None, -+ help="File containing the new password for user. If set, will override `--password`.", - ) - parser.add_argument( - "-t", -@@ -247,6 +254,11 @@ def main() -> None: - print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr) - sys.exit(1) - -+ if args.password_file: -+ password = _read_file(args.password_file, "password-file").strip() -+ else: -+ password = args.password -+ - if args.server_url: - server_url = args.server_url - elif config is not None: -@@ -269,9 +281,7 @@ def main() -> None: - if args.admin or args.no_admin: - admin = args.admin - -- register_new_user( -- args.user, args.password, server_url, secret, admin, args.user_type -- ) -+ register_new_user(args.user, password, server_url, secret, admin, args.user_type) - - - def _read_file(file_path: Any, config_path: str) -> str: --- -2.44.1 - diff --git a/clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch b/clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch deleted file mode 100644 index df02ad52..00000000 --- a/clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch +++ /dev/null @@ -1,94 +0,0 @@ -From 1789416df425d22693b0055a6688d8686e0ee4a1 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= -Date: Thu, 13 Jun 2024 14:38:19 +0200 -Subject: [PATCH 2/2] register-new-matrix-user: add a flag to ignore already - existing users -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This allows to register users in a more declarative and stateless way. - -Signed-off-by: Jörg Thalheim ---- - synapse/_scripts/register_new_matrix_user.py | 22 ++++++++++++++++++-- - 1 file changed, 20 insertions(+), 2 deletions(-) - -diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py -index 972b35e2d..233e7267d 100644 ---- a/synapse/_scripts/register_new_matrix_user.py -+++ b/synapse/_scripts/register_new_matrix_user.py -@@ -52,6 +52,7 @@ def request_registration( - user_type: Optional[str] = None, - _print: Callable[[str], None] = print, - exit: Callable[[int], None] = sys.exit, -+ exists_ok: bool = False, - ) -> None: - url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),) - -@@ -97,6 +98,10 @@ def request_registration( - r = requests.post(url, json=data) - - if r.status_code != 200: -+ response = r.json() -+ if exists_ok and response["errcode"] == "M_USER_IN_USE": -+ _print("User already exists. Skipping.") -+ return - _print("ERROR! Received %d %s" % (r.status_code, r.reason)) - if 400 <= r.status_code < 500: - try: -@@ -115,6 +120,7 @@ def register_new_user( - shared_secret: str, - admin: Optional[bool], - user_type: Optional[str], -+ exists_ok: bool = False, - ) -> None: - if not user: - try: -@@ -154,7 +160,13 @@ def register_new_user( - admin = False - - request_registration( -- user, password, server_location, shared_secret, bool(admin), user_type -+ user, -+ password, -+ server_location, -+ shared_secret, -+ bool(admin), -+ user_type, -+ exists_ok=exists_ok, - ) - - -@@ -173,6 +185,11 @@ def main() -> None: - default=None, - help="Local part of the new user. Will prompt if omitted.", - ) -+ parser.add_argument( -+ "--exists-ok", -+ action="store_true", -+ help="Do not fail if user already exists.", -+ ) - password_group = parser.add_mutually_exclusive_group() - password_group.add_argument( - "-p", -@@ -192,6 +209,7 @@ def main() -> None: - default=None, - help="User type as specified in synapse.api.constants.UserTypes", - ) -+ - admin_group = parser.add_mutually_exclusive_group() - admin_group.add_argument( - "-a", -@@ -281,7 +299,7 @@ def main() -> None: - if args.admin or args.no_admin: - admin = args.admin - -- register_new_user(args.user, password, server_url, secret, admin, args.user_type) -+ register_new_user(args.user, password, server_url, secret, admin, args.user_type, exists_ok=args.exists_ok) - - - def _read_file(file_path: Any, config_path: str) -> str: --- -2.44.1 - diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix index 29245b1d..399813e1 100644 --- a/clanModules/matrix-synapse/default.nix +++ b/clanModules/matrix-synapse/default.nix @@ -17,19 +17,8 @@ let ln -s $out/config.json $out/config.${nginx-vhost}.json ''; - # FIXME: This was taken from upstream. Drop this when our patch is upstream - synapseCfg = config.services.matrix-synapse; - wantedExtras = - synapseCfg.extras - ++ lib.optional (synapseCfg.settings ? oidc_providers) "oidc" - ++ lib.optional (synapseCfg.settings ? jwt_config) "jwt" - ++ lib.optional (synapseCfg.settings ? saml2_config) "saml2" - ++ lib.optional (synapseCfg.settings ? redis) "redis" - ++ lib.optional (synapseCfg.settings ? sentry) "sentry" - ++ lib.optional (synapseCfg.settings ? user_directory) "user-search" - ++ lib.optional (synapseCfg.settings.url_preview_enabled) "url-preview" - ++ lib.optional (synapseCfg.settings.database.name == "psycopg2") "postgres"; in +# FIXME: This was taken from upstream. Drop this when our patch is upstream { options.services.matrix-synapse.package = lib.mkOption { readOnly = false; }; options.clan.matrix-synapse = { @@ -78,21 +67,6 @@ in ]; config = { services.matrix-synapse = { - package = lib.mkForce ( - pkgs.matrix-synapse.override { - matrix-synapse-unwrapped = pkgs.matrix-synapse.unwrapped.overrideAttrs (_old: { - doInstallCheck = false; # too slow, nixpkgs maintainer already run this. - patches = [ - # see: https://github.com/element-hq/synapse/pull/17304 - ./0001-register_new_matrix_user-add-password-file-flag.patch - ./0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch - ]; - }); - extras = wantedExtras; - plugins = synapseCfg.plugins; - } - ); - enable = true; settings = { server_name = cfg.domain; diff --git a/clanModules/zerotier-static-peers/default.nix b/clanModules/zerotier-static-peers/default.nix index a55889d2..c613e42d 100644 --- a/clanModules/zerotier-static-peers/default.nix +++ b/clanModules/zerotier-static-peers/default.nix @@ -30,6 +30,16 @@ in default = [ config.clan.core.machineName ]; description = "Hosts that should be excluded"; }; + networkIps = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra zerotier network Ips that should be accepted"; + }; + networkIds = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = "Extra zerotier network Ids that should be accepted"; + }; }; config.systemd.services.zerotier-static-peers-autoaccept = @@ -56,6 +66,7 @@ in lib.nameValuePair (builtins.readFile fullPath) [ machine ] ) filteredMachines ); + allHostIPs = config.clan.zerotier-static-peers.networkIps ++ hosts; in lib.mkIf (config.clan.core.networking.zerotier.controller.enable) { wantedBy = [ "multi-user.target" ]; @@ -65,7 +76,10 @@ in #!/bin/sh ${lib.concatMapStringsSep "\n" (host: '' ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host} - '') hosts} + '') allHostIPs} + ${lib.concatMapStringsSep "\n" (host: '' + ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow ${host} + '') config.clan.zerotier-static-peers.networkIds} ''; }; diff --git a/docs/.envrc b/docs/.envrc index ffe12138..1eab3d8d 100644 --- a/docs/.envrc +++ b/docs/.envrc @@ -1,6 +1,8 @@ +# shellcheck shell=bash source_up -watch_file $(find ./nix -name "*.nix" -printf '%p ') +mapfile -d '' -t nix_files < <(find ./nix -name "*.nix" -print0) +watch_file "${nix_files[@]}" # Because we depend on nixpkgs sources, uploading to builders takes a long time use flake .#docs --builders '' diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 358092ca..80ff55b8 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -14,6 +14,7 @@ markdown_extensions: - attr_list - footnotes - md_in_html + - def_list - meta - plantuml_markdown - pymdownx.emoji: @@ -49,6 +50,8 @@ nav: - Mesh VPN: getting-started/mesh-vpn.md - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md + - Concepts: + - Configuration: concepts/configuration.md - Reference: - Clan Modules: - reference/clanModules/borgbackup-static.md diff --git a/docs/nix/default.nix b/docs/nix/default.nix index 0e16c705..01dae322 100644 --- a/docs/nix/default.nix +++ b/docs/nix/default.nix @@ -26,6 +26,7 @@ pkgs.stdenv.mkDerivation { mkdocs-material mkdocs-rss-plugin mkdocs-macros + filelock # FIXME: this should be already provided by mkdocs-rss-plugin ]); configurePhase = '' mkdir -p ./site/reference/cli diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index 87385cf7..bd44e94b 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -38,7 +38,7 @@ patchShebangs --build $out ruff format --check --diff $out - ruff --line-length 88 $out + ruff check --line-length 88 $out mypy --strict $out ''; diff --git a/docs/site/concepts/configuration.md b/docs/site/concepts/configuration.md new file mode 100644 index 00000000..b3d97a1b --- /dev/null +++ b/docs/site/concepts/configuration.md @@ -0,0 +1,114 @@ +# Configuration + +## Introduction + +When managing machine configuration this can be done through many possible ways. +Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming). + +clan currently offers the following methods to configure machines: + +!!! Success "Recommended for nix people" + + - flake.nix (i.e. via `buildClan`) + - `machine` argument + - `inventory` argument + + - machines/`machine_name`/configuration.nix (`autoincluded` if it exists) + +???+ Note "Used by CLI & UI" + + - inventory.json + - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists) + + +!!! Warning "Deprecated" + + machines/`machine_name`/settings.json + +## BuildClan + +The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services. + +### Inputs + +`directory` +: The directory containing the machines subdirectory + +`machines` +: Allows to include machine-specific modules i.e. machines.${name} = { ... } + +`meta` +: An optional set + +: `{ name :: string, icon :: string, description :: string }` + +`inventory` +: Service set for easily configuring distributed services, such as backups + +: For more details see [Inventory](#inventory) + +`specialArgs` +: Extra arguments to pass to nixosSystem i.e. useful to make self available + +`pkgsForSystem` +: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system. + This improves performance, but all nipxkgs.* options will be ignored. + `(string -> pkgs )` + +## Inventory + +`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries. + +The following is the specification of the inventory in `cuelang` + +```cue +{ + meta: { + // A name of the clan (primarily shown by the UI) + name: string + // A description of the clan + description?: string + // The icon path + icon?: string + } + + // A map of services + services: [string]: [string]: { + // Required meta fields + meta: { + name: string, + icon?: string + description?: string, + }, + // Machines are added via the avilable roles + // Membership depends only on this field + roles: [string]: { + machines: [...string], + tags: [...string], + } + machines?: { + [string]: { + config?: { + ... + } + } + }, + + // Global Configuration for the service + // Applied to all machines. + config?: { + // Schema depends on the module. + // It declares the interface how the service can be configured. + ... + } + } + // A map of machines, extends the machines of `buildClan` + machines: [string]: { + name: string, + description?: string, + icon?: string + tags: [...string] + system: string + } +} +``` diff --git a/flake.lock b/flake.lock index 0b9fe0ad..7151ac9c 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1718846788, - "narHash": "sha256-9dtXYtEkmXoUJV+PGLqscqF7qTn4AIhAKpFWRFU2NYs=", + "lastModified": 1720056646, + "narHash": "sha256-BymcV4HWtx2VFuabDCM4/nEJcfivCx0S02wUCz11mAY=", "owner": "nix-community", "repo": "disko", - "rev": "e1174d991944a01eaaa04bc59c6281edca4c0e6e", + "rev": "64679cd7f318c9b6595902b47d4585b1d51d5f9e", "type": "github" }, "original": { @@ -27,11 +27,11 @@ ] }, "locked": { - "lastModified": 1717285511, - "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -40,29 +40,6 @@ "type": "github" } }, - "nixos-generators": { - "inputs": { - "nixlib": [ - "nixpkgs" - ], - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1718025593, - "narHash": "sha256-WZ1gdKq/9u1Ns/oXuNsDm+W0salonVA0VY1amw8urJ4=", - "owner": "nix-community", - "repo": "nixos-generators", - "rev": "35c20ba421dfa5059e20e0ef2343c875372bdcf3", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nixos-generators", - "type": "github" - } - }, "nixos-images": { "inputs": { "nixos-stable": [], @@ -71,11 +48,11 @@ ] }, "locked": { - "lastModified": 1718845599, - "narHash": "sha256-HbQ0iKohKJC5grC95HNjLxGPdgsc/BJgoENDYNbzkLo=", + "lastModified": 1720055024, + "narHash": "sha256-c5rsiI1R7tnCDpcgfsa7ouSdn6wpctbme9TUp53CFyU=", "owner": "nix-community", "repo": "nixos-images", - "rev": "c1e6a5f7b08f1c9993de1cfc5f15f838bf783b88", + "rev": "f8650460d37d9d1820a93ebb7f0db5b6c3621946", "type": "github" }, "original": { @@ -86,11 +63,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1719451888, - "narHash": "sha256-Ky0sgEEJMcBmNEJztY6KcVn+6bq74EKM7pd1CR1wnPQ=", + "lastModified": 1720340162, + "narHash": "sha256-iVLH0Ygtw/Iw9Q1cFFX7OhNnoPbc7/ZWW6J3c0zbiZw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3664857c48feacb35770c00abfdc671e55849be5", + "rev": "60a94e515488e335bd5bce096431d490486915e3", "type": "github" }, "original": { @@ -104,7 +81,6 @@ "inputs": { "disko": "disko", "flake-parts": "flake-parts", - "nixos-generators": "nixos-generators", "nixos-images": "nixos-images", "nixpkgs": "nixpkgs", "sops-nix": "sops-nix", @@ -119,11 +95,11 @@ "nixpkgs-stable": [] }, "locked": { - "lastModified": 1719111739, - "narHash": "sha256-kr2QzRrplzlCP87ddayCZQS+dhGW98kw2zy7+jUXtF4=", + "lastModified": 1720321395, + "narHash": "sha256-kcI8q9Nh8/CSj0ygfWq1DLckHl8IHhFarL8ie6g7OEk=", "owner": "Mic92", "repo": "sops-nix", - "rev": "5e2e9421e9ed2b918be0a441c4535cfa45e04811", + "rev": "c184aca4db5d71c3db0c8cbfcaaec337a5d065ea", "type": "github" }, "original": { @@ -139,11 +115,11 @@ ] }, "locked": { - "lastModified": 1718522839, - "narHash": "sha256-ULzoKzEaBOiLRtjeY3YoGFJMwWSKRYOic6VNw2UyTls=", + "lastModified": 1720436211, + "narHash": "sha256-/cKXod0oGLl+vH4bKBZnTV3qxrw4jgOLnyQ8KXey5J8=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "68eb1dc333ce82d0ab0c0357363ea17c31ea1f81", + "rev": "6fc8bded78715cdd43a3278a14ded226eb3a239e", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1b2fd745..3b09a906 100644 --- a/flake.nix +++ b/flake.nix @@ -13,9 +13,6 @@ sops-nix.url = "github:Mic92/sops-nix"; sops-nix.inputs.nixpkgs.follows = "nixpkgs"; sops-nix.inputs.nixpkgs-stable.follows = ""; - nixos-generators.url = "github:nix-community/nixos-generators"; - nixos-generators.inputs.nixpkgs.follows = "nixpkgs"; - nixos-generators.inputs.nixlib.follows = "nixpkgs"; nixos-images.url = "github:nix-community/nixos-images"; nixos-images.inputs.nixos-unstable.follows = "nixpkgs"; # unused input diff --git a/formatter.nix b/formatter.nix index 60a934f7..1cdc3365 100644 --- a/formatter.nix +++ b/formatter.nix @@ -1,4 +1,4 @@ -{ lib, inputs, ... }: +{ inputs, ... }: { imports = [ inputs.treefmt-nix.flakeModule ]; perSystem = @@ -8,43 +8,19 @@ treefmt.programs.shellcheck.enable = true; treefmt.programs.mypy.enable = true; + treefmt.programs.nixfmt.enable = true; + treefmt.programs.nixfmt.package = pkgs.nixfmt-rfc-style; + treefmt.programs.deadnix.enable = true; + treefmt.programs.mypy.directories = { "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies; "pkgs/clan-app".extraPythonPackages = # clan-app currently only exists on linux (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies; }; + treefmt.programs.ruff.check = true; + treefmt.programs.ruff.format = true; - treefmt.settings.formatter.nix = { - command = "sh"; - options = [ - "-eucx" - '' - # First deadnix - ${lib.getExe pkgs.deadnix} --edit "$@" - # Then nixpkgs-fmt - ${lib.getExe pkgs.nixfmt-rfc-style} "$@" - '' - "--" # this argument is ignored by bash - ]; - includes = [ "*.nix" ]; - excludes = [ - # Was copied from nixpkgs. Keep diff minimal to simplify upstreaming. - "pkgs/builders/script-writers.nix" - ]; - }; - treefmt.settings.formatter.python = { - command = "sh"; - options = [ - "-eucx" - '' - ${lib.getExe pkgs.ruff} check --fix "$@" - ${lib.getExe pkgs.ruff} format "$@" - '' - "--" # this argument is ignored by bash - ]; - includes = [ "*.py" ]; - }; # FIXME: currently broken in CI #treefmt.settings.formatter.vale = # let diff --git a/inventory.json b/inventory.json index bd9d2570..207630e5 100644 --- a/inventory.json +++ b/inventory.json @@ -1,4 +1,7 @@ { + "meta": { + "name": "Minimal inventory" + }, "machines": { "minimal-inventory-machine": { "name": "foo", diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index d0bb8e6a..13169700 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -152,6 +152,13 @@ let in (machineImports settings) ++ [ + { + # Autoinclude configuration.nix and hardware-configuration.nix + imports = builtins.filter (p: builtins.pathExists p) [ + "${directory}/machines/${name}/configuration.nix" + "${directory}/machines/${name}/hardware-configuration.nix" + ]; + } settings clan-core.nixosModules.clanCore extraConfig diff --git a/lib/inventory/.envrc b/lib/inventory/.envrc index 8d1c6842..22103077 100644 --- a/lib/inventory/.envrc +++ b/lib/inventory/.envrc @@ -1,3 +1,4 @@ +# shellcheck shell=bash source_up watch_file flake-module.nix diff --git a/lib/inventory/spec/schema/schema.cue b/lib/inventory/spec/schema/schema.cue index 428fbe9b..eded84e2 100644 --- a/lib/inventory/spec/schema/schema.cue +++ b/lib/inventory/spec/schema/schema.cue @@ -5,6 +5,7 @@ package schema description?: string, icon?: string tags: [...string] + system?: string } #role: string diff --git a/nixosModules/clanCore/vars/default.nix b/nixosModules/clanCore/vars/default.nix index 38209e87..34ee569b 100644 --- a/nixosModules/clanCore/vars/default.nix +++ b/nixosModules/clanCore/vars/default.nix @@ -1,5 +1,26 @@ -{ lib, ... }: { + lib, + config, + pkgs, + ... +}: +let + inherit (lib.types) submoduleWith; + submodule = + module: + submoduleWith { + specialArgs.pkgs = pkgs; + modules = [ module ]; + }; +in +{ + imports = [ + ./public/in_repo.nix + # ./public/vm.nix + # ./secret/password-store.nix + ./secret/sops.nix + # ./secret/vm.nix + ]; options.clan.core.vars = lib.mkOption { visible = false; description = '' @@ -11,6 +32,20 @@ - generate secrets like private keys automatically when they are needed - output multiple values like private and public keys simultaneously ''; - type = lib.types.submoduleWith { modules = [ ./interface.nix ]; }; + type = submodule { imports = [ ./interface.nix ]; }; + }; + + config.system.clan.deployment.data = { + vars = { + generators = lib.flip lib.mapAttrs config.clan.core.vars.generators ( + _name: generator: { + inherit (generator) finalScript; + files = lib.flip lib.mapAttrs generator.files (_name: file: { inherit (file) secret; }); + } + ); + inherit (config.clan.core.vars.settings) secretUploadDirectory secretModule publicModule; + }; + inherit (config.clan.networking) targetHost buildHost; + inherit (config.clan.deployment) requireExplicitUpdate; }; } diff --git a/nixosModules/clanCore/vars/eval-tests/default.nix b/nixosModules/clanCore/vars/eval-tests/default.nix index b9713915..40bd31f5 100644 --- a/nixosModules/clanCore/vars/eval-tests/default.nix +++ b/nixosModules/clanCore/vars/eval-tests/default.nix @@ -54,21 +54,6 @@ in }; }; - # Ensure that generators.imports works - # This allows importing generators from third party projects without providing - # them access to other settings. - test_generator_modules = - let - generator_module = { - my-generator.files.password = { }; - }; - config = eval { generators.imports = [ generator_module ]; }; - in - { - expr = config.generators ? my-generator; - expected = true; - }; - # script can be text test_script_text = let diff --git a/nixosModules/clanCore/vars/interface.nix b/nixosModules/clanCore/vars/interface.nix index 6e551faf..39f0df0b 100644 --- a/nixosModules/clanCore/vars/interface.nix +++ b/nixosModules/clanCore/vars/interface.nix @@ -1,8 +1,12 @@ -{ lib, ... }: +{ + lib, + config, + pkgs, + ... +}: let inherit (lib) mkOption; inherit (lib.types) - anything attrsOf bool either @@ -14,30 +18,27 @@ let submoduleWith ; # the original types.submodule has strange behavior - submodule = module: submoduleWith { modules = [ module ]; }; + submodule = + module: + submoduleWith { + specialArgs.pkgs = pkgs; + modules = [ module ]; + }; options = lib.mapAttrs (_: mkOption); - subOptions = opts: submodule { options = options opts; }; in { - options = options { - settings = { + options = { + settings = import ./settings-opts.nix { inherit lib; }; + generators = lib.mkOption { description = '' - Settings for the generated variables. + A set of generators that can be used to generate files. + Generators are scripts that produce files based on the values of other generators and user input. + Each generator is expected to produce a set of files under a directory. ''; - type = submodule { - freeformType = anything; - imports = [ ./settings.nix ]; - }; - }; - generators = { - default = { - imports = [ - # implementation of the generator - ./generator.nix - ]; - }; - type = submodule { - freeformType = attrsOf (subOptions { + default = { }; + type = attrsOf (submodule { + imports = [ ./generator.nix ]; + options = options { dependencies = { description = '' A list of other generators that this generator depends on. @@ -52,32 +53,45 @@ in A set of files to generate. The generator 'script' is expected to produce exactly these files under $out. ''; - type = attrsOf (subOptions { - secret = { - description = '' - Whether the file should be treated as a secret. - ''; - type = bool; - default = true; - }; - path = { - description = '' - The path to the file containing the content of the generated value. - This will be set automatically - ''; - type = str; - readOnly = true; - }; - value = { - description = '' - The content of the generated value. - Only available if the file is not secret. - ''; - type = str; - default = throw "Cannot access value of secret file"; - defaultText = "Throws error because the value of a secret file is not accessible"; - }; - }); + type = attrsOf ( + submodule (file: { + imports = [ config.settings.fileModule ]; + options = options { + name = { + type = lib.types.str; + description = '' + name of the public fact + ''; + readOnly = true; + default = file.config._module.args.name; + }; + secret = { + description = '' + Whether the file should be treated as a secret. + ''; + type = bool; + default = true; + }; + path = { + description = '' + The path to the file containing the content of the generated value. + This will be set automatically + ''; + type = str; + readOnly = true; + }; + value = { + description = '' + The content of the generated value. + Only available if the file is not secret. + ''; + type = str; + default = throw "Cannot access value of secret file"; + defaultText = "Throws error because the value of a secret file is not accessible"; + }; + }; + }) + ); }; prompts = { description = '' @@ -85,28 +99,30 @@ in Prompts are available to the generator script as files. For example, a prompt named 'prompt1' will be available via $prompts/prompt1 ''; - type = attrsOf (subOptions { - description = { - description = '' - The description of the prompted value - ''; - type = str; - example = "SSH private key"; - }; - type = { - description = '' - The input type of the prompt. - The following types are available: - - hidden: A hidden text (e.g. password) - - line: A single line of text - - multiline: A multiline text - ''; - type = enum [ - "hidden" - "line" - "multiline" - ]; - default = "line"; + type = attrsOf (submodule { + options = { + description = { + description = '' + The description of the prompted value + ''; + type = str; + example = "SSH private key"; + }; + type = { + description = '' + The input type of the prompt. + The following types are available: + - hidden: A hidden text (e.g. password) + - line: A single line of text + - multiline: A multiline text + ''; + type = enum [ + "hidden" + "line" + "multiline" + ]; + default = "line"; + }; }; }); }; @@ -140,8 +156,8 @@ in internal = true; visible = false; }; - }); - }; + }; + }); }; }; } diff --git a/nixosModules/clanCore/vars/public/in_repo.nix b/nixosModules/clanCore/vars/public/in_repo.nix new file mode 100644 index 00000000..ba7a6aa4 --- /dev/null +++ b/nixosModules/clanCore/vars/public/in_repo.nix @@ -0,0 +1,12 @@ +{ config, lib, ... }: +{ + config.clan.core.vars.settings = + lib.mkIf (config.clan.core.vars.settings.publicStore == "in_repo") + { + publicModule = "clan_cli.vars.public_modules.in_repo"; + fileModule = file: { + path = + config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/vars/${file.config.name}"; + }; + }; +} diff --git a/nixosModules/clanCore/vars/secret/sops.nix b/nixosModules/clanCore/vars/secret/sops.nix new file mode 100644 index 00000000..bbf9cbc5 --- /dev/null +++ b/nixosModules/clanCore/vars/secret/sops.nix @@ -0,0 +1,61 @@ +{ + config, + lib, + pkgs, + ... +}: +let + secretsDir = config.clan.core.clanDir + "/sops/secrets"; + groupsDir = config.clan.core.clanDir + "/sops/groups"; + + # My symlink is in the nixos module detected as a directory also it works in the repl. Is this because of pure evaluation? + containsSymlink = + path: + builtins.pathExists path + && (builtins.readFileType path == "directory" || builtins.readFileType path == "symlink"); + + containsMachine = + parent: name: type: + type == "directory" && containsSymlink "${parent}/${name}/machines/${config.clan.core.machineName}"; + + containsMachineOrGroups = + name: type: + (containsMachine secretsDir name type) + || lib.any ( + group: type == "directory" && containsSymlink "${secretsDir}/${name}/groups/${group}" + ) groups; + + filterDir = + filter: dir: + lib.optionalAttrs (builtins.pathExists dir) (lib.filterAttrs filter (builtins.readDir dir)); + + groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir); + secrets = filterDir containsMachineOrGroups secretsDir; +in +{ + config.clan.core.vars.settings = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") { + # Before we generate a secret we cannot know the path yet, so we need to set it to an empty string + fileModule = file: { + path = + lib.mkIf file.secret + config.sops.secrets.${"${config.clan.core.machineName}-${file.config.name}"}.path + or "/no-such-path"; + }; + secretModule = "clan_cli.vars.secret_modules.sops"; + secretUploadDirectory = lib.mkDefault "/var/lib/sops-nix"; + }; + + config.sops = lib.mkIf (config.clan.core.vars.settings.secretStore == "sops") { + secrets = builtins.mapAttrs (name: _: { + sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret"; + format = "binary"; + }) secrets; + # To get proper error messages about missing secrets we need a dummy secret file that is always present + defaultSopsFile = lib.mkIf config.sops.validateSopsFiles ( + lib.mkDefault (builtins.toString (pkgs.writeText "dummy.yaml" "")) + ); + age.keyFile = lib.mkIf (builtins.pathExists ( + config.clan.core.clanDir + "/sops/secrets/${config.clan.core.machineName}-age.key/secret" + )) (lib.mkDefault "/var/lib/sops-nix/key.txt"); + }; +} diff --git a/nixosModules/clanCore/vars/settings-opts.nix b/nixosModules/clanCore/vars/settings-opts.nix new file mode 100644 index 00000000..cd31adf9 --- /dev/null +++ b/nixosModules/clanCore/vars/settings-opts.nix @@ -0,0 +1,71 @@ +{ lib, ... }: +{ + secretStore = lib.mkOption { + type = lib.types.enum [ + "sops" + "password-store" + "vm" + "custom" + ]; + default = "sops"; + description = '' + method to store secret facts + custom can be used to define a custom secret fact store. + ''; + }; + + secretModule = lib.mkOption { + type = lib.types.str; + internal = true; + description = '' + the python import path to the secret module + ''; + }; + + secretUploadDirectory = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The directory where secrets are uploaded into, This is backend specific. + ''; + }; + + fileModule = lib.mkOption { + type = lib.types.deferredModule; + internal = true; + description = '' + A module to be imported in every vars.files. submodule. + Used by backends to define the `path` attribute. + ''; + default = { }; + }; + + publicStore = lib.mkOption { + type = lib.types.enum [ + "in_repo" + "vm" + "custom" + ]; + default = "in_repo"; + description = '' + method to store public facts. + custom can be used to define a custom public fact store. + ''; + }; + + publicModule = lib.mkOption { + type = lib.types.str; + internal = true; + description = '' + the python import path to the public module + ''; + }; + + publicDirectory = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + The directory where public facts are stored. + ''; + }; +} diff --git a/nixosModules/clanCore/vars/settings.nix b/nixosModules/clanCore/vars/settings.nix deleted file mode 100644 index dd3e8f7c..00000000 --- a/nixosModules/clanCore/vars/settings.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ lib, ... }: -{ - options = { - secretStore = lib.mkOption { - type = lib.types.enum [ - "sops" - "password-store" - "vm" - "custom" - ]; - default = "sops"; - description = '' - method to store secret facts - custom can be used to define a custom secret fact store. - ''; - }; - - secretModule = lib.mkOption { - type = lib.types.str; - internal = true; - description = '' - the python import path to the secret module - ''; - }; - - secretUploadDirectory = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = '' - The directory where secrets are uploaded into, This is backend specific. - ''; - }; - - secretPathFunction = lib.mkOption { - type = lib.types.raw; - description = '' - The function to use to generate the path for a secret. - The default function will use the path attribute of the secret. - The function will be called with the secret submodule as an argument. - ''; - }; - - publicStore = lib.mkOption { - type = lib.types.enum [ - "in_repo" - "vm" - "custom" - ]; - default = "in_repo"; - description = '' - method to store public facts. - custom can be used to define a custom public fact store. - ''; - }; - - publicModule = lib.mkOption { - type = lib.types.str; - internal = true; - description = '' - the python import path to the public module - ''; - }; - - publicDirectory = lib.mkOption { - type = lib.types.nullOr lib.types.path; - default = null; - description = '' - The directory where public facts are stored. - ''; - }; - }; -} diff --git a/pkgs/builders/script-writers.nix b/pkgs/builders/script-writers.nix index b8182bdd..8bbaacdc 100644 --- a/pkgs/builders/script-writers.nix +++ b/pkgs/builders/script-writers.nix @@ -27,107 +27,127 @@ rec { # Examples: # writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; } # makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world" - makeScriptWriter = { interpreter, check ? "", makeWrapperArgs ? [], }: nameOrPath: content: - assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); + makeScriptWriter = + { + interpreter, + check ? "", + makeWrapperArgs ? [ ], + }: + nameOrPath: content: + assert lib.or (types.path.check nameOrPath) ( + builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null + ); assert lib.or (types.path.check content) (types.str.check content); let name = last (builtins.split "/" nameOrPath); in - pkgs.runCommandLocal name ( - { - inherit makeWrapperArgs; - nativeBuildInputs = [ - makeWrapper - ]; - } - // lib.optionalAttrs (nameOrPath == "/bin/${name}") { - meta.mainProgram = name; - } - // ( - if (types.str.check content) then { - inherit content interpreter; - passAsFile = [ "content" ]; - } else { - inherit interpreter; - contentPath = content; + pkgs.runCommandLocal name + ( + { + inherit makeWrapperArgs; + nativeBuildInputs = [ makeWrapper ]; } + // lib.optionalAttrs (nameOrPath == "/bin/${name}") { meta.mainProgram = name; } + // ( + if (types.str.check content) then + { + inherit content interpreter; + passAsFile = [ "content" ]; + } + else + { + inherit interpreter; + contentPath = content; + } + ) ) - ) - '' - # On darwin a script cannot be used as an interpreter in a shebang but - # there doesn't seem to be a limit to the size of shebang and multiple - # arguments to the interpreter are allowed. - if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter - then - wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3) - # Get first word from the line (note: xargs echo remove leading spaces) - wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1) - - if isScript $wrapperInterpreter + '' + # On darwin a script cannot be used as an interpreter in a shebang but + # there doesn't seem to be a limit to the size of shebang and multiple + # arguments to the interpreter are allowed. + if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter then - echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported." - exit 1 + wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3) + # Get first word from the line (note: xargs echo remove leading spaces) + wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1) + + if isScript $wrapperInterpreter + then + echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported." + exit 1 + fi + + # This should work as long as wrapperInterpreter is a shell, which is + # the case for programs wrapped with makeWrapper, like + # python3.withPackages etc. + interpreterLine="$wrapperInterpreterLine $interpreter" + else + interpreterLine=$interpreter fi - # This should work as long as wrapperInterpreter is a shell, which is - # the case for programs wrapped with makeWrapper, like - # python3.withPackages etc. - interpreterLine="$wrapperInterpreterLine $interpreter" - else - interpreterLine=$interpreter - fi + echo "#! $interpreterLine" > $out + cat "$contentPath" >> $out + ${optionalString (check != "") '' + ${check} $out + ''} + chmod +x $out - echo "#! $interpreterLine" > $out - cat "$contentPath" >> $out - ${optionalString (check != "") '' - ${check} $out - ''} - chmod +x $out - - # Relocate executable if path was specified instead of name. - # Only in this case wrapProgram is applied, as it wouldn't work with a - # single executable file under $out. - ${optionalString (types.path.check nameOrPath) '' - mv $out tmp - mkdir -p $out/$(dirname "${nameOrPath}") - mv tmp $out/${nameOrPath} - wrapProgram $out/${nameOrPath} ''${makeWrapperArgs[@]} - ''} - ''; + # Relocate executable if path was specified instead of name. + # Only in this case wrapProgram is applied, as it wouldn't work with a + # single executable file under $out. + ${optionalString (types.path.check nameOrPath) '' + mv $out tmp + mkdir -p $out/$(dirname "${nameOrPath}") + mv tmp $out/${nameOrPath} + wrapProgram $out/${nameOrPath} ''${makeWrapperArgs[@]} + ''} + ''; # Base implementation for compiled executables. # Takes a compile script, which in turn takes the name as an argument. # # Examples: # writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; } - makeBinWriter = { compileScript, strip ? true }: nameOrPath: content: - assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null); + makeBinWriter = + { + compileScript, + strip ? true, + }: + nameOrPath: content: + assert lib.or (types.path.check nameOrPath) ( + builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null + ); assert lib.or (types.path.check content) (types.str.check content); let name = last (builtins.split "/" nameOrPath); in - pkgs.runCommand name ((if (types.str.check content) then { - inherit content; - passAsFile = [ "content" ]; - } else { - contentPath = content; - }) // lib.optionalAttrs (nameOrPath == "/bin/${name}") { - meta.mainProgram = name; - }) '' - ${compileScript} - ${lib.optionalString strip - "${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"} - # Sometimes binaries produced for darwin (e. g. by GHC) won't be valid - # mach-o executables from the get-go, but need to be corrected somehow - # which is done by fixupPhase. - ${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"} - ${optionalString (types.path.check nameOrPath) '' - mv $out tmp - mkdir -p $out/$(dirname "${nameOrPath}") - mv tmp $out/${nameOrPath} - ''} - ''; + pkgs.runCommand name + ( + ( + if (types.str.check content) then + { + inherit content; + passAsFile = [ "content" ]; + } + else + { contentPath = content; } + ) + // lib.optionalAttrs (nameOrPath == "/bin/${name}") { meta.mainProgram = name; } + ) + '' + ${compileScript} + ${lib.optionalString strip "${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"} + # Sometimes binaries produced for darwin (e. g. by GHC) won't be valid + # mach-o executables from the get-go, but need to be corrected somehow + # which is done by fixupPhase. + ${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"} + ${optionalString (types.path.check nameOrPath) '' + mv $out tmp + mkdir -p $out/$(dirname "${nameOrPath}") + mv tmp $out/${nameOrPath} + ''} + ''; # Like writeScript but the first line is a shebang to bash # @@ -135,13 +155,10 @@ rec { # writeBash "example" '' # echo hello world # '' - writeBash = makeScriptWriter { - interpreter = "${lib.getExe pkgs.bash}"; - }; + writeBash = makeScriptWriter { interpreter = "${lib.getExe pkgs.bash}"; }; # Like writeScriptBin but the first line is a shebang to bash - writeBashBin = name: - writeBash "/bin/${name}"; + writeBashBin = name: writeBash "/bin/${name}"; # Like writeScript but the first line is a shebang to dash # @@ -149,13 +166,10 @@ rec { # writeDash "example" '' # echo hello world # '' - writeDash = makeScriptWriter { - interpreter = "${lib.getExe pkgs.dash}"; - }; + writeDash = makeScriptWriter { interpreter = "${lib.getExe pkgs.dash}"; }; # Like writeScriptBin but the first line is a shebang to dash - writeDashBin = name: - writeDash "/bin/${name}"; + writeDashBin = name: writeDash "/bin/${name}"; # Like writeScript but the first line is a shebang to fish # @@ -165,12 +179,11 @@ rec { # '' writeFish = makeScriptWriter { interpreter = "${lib.getExe pkgs.fish} --no-config"; - check = "${lib.getExe pkgs.fish} --no-config --no-execute"; # syntax check only + check = "${lib.getExe pkgs.fish} --no-config --no-execute"; # syntax check only }; # Like writeScriptBin but the first line is a shebang to fish - writeFishBin = name: - writeFish "/bin/${name}"; + writeFishBin = name: writeFish "/bin/${name}"; # writeHaskell takes a name, an attrset with libraries and haskell version (both optional) # and some haskell source code and returns an executable. @@ -181,29 +194,31 @@ rec { # # main = launchMissiles # ''; - writeHaskell = name: { - libraries ? [], - ghc ? pkgs.ghc, - ghcArgs ? [], - threadedRuntime ? true, - strip ? true - }: + writeHaskell = + name: + { + libraries ? [ ], + ghc ? pkgs.ghc, + ghcArgs ? [ ], + threadedRuntime ? true, + strip ? true, + }: let appendIfNotSet = el: list: if elem el list then list else list ++ [ el ]; ghcArgs' = if threadedRuntime then appendIfNotSet "-threaded" ghcArgs else ghcArgs; - in makeBinWriter { + in + makeBinWriter { compileScript = '' cp $contentPath tmp.hs - ${(ghc.withPackages (_: libraries ))}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs + ${(ghc.withPackages (_: libraries))}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs mv tmp $out ''; inherit strip; } name; # writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin) - writeHaskellBin = name: - writeHaskell "/bin/${name}"; + writeHaskellBin = name: writeHaskell "/bin/${name}"; # Like writeScript but the first line is a shebang to nu # @@ -211,30 +226,30 @@ rec { # writeNu "example" '' # echo hello world # '' - writeNu = makeScriptWriter { - interpreter = "${lib.getExe pkgs.nushell} --no-config-file"; - }; + writeNu = makeScriptWriter { interpreter = "${lib.getExe pkgs.nushell} --no-config-file"; }; # Like writeScriptBin but the first line is a shebang to nu - writeNuBin = name: - writeNu "/bin/${name}"; + writeNuBin = name: writeNu "/bin/${name}"; # makeRubyWriter takes ruby and compatible rubyPackages and produces ruby script writer, # If any libraries are specified, ruby.withPackages is used as interpreter, otherwise the "bare" ruby is used. - makeRubyWriter = ruby: rubyPackages: buildRubyPackages: name: { libraries ? [], ... } @ args: - makeScriptWriter ( - (builtins.removeAttrs args ["libraries"]) - // { - interpreter = - if libraries == [] - then "${ruby}/bin/ruby" - else "${(ruby.withPackages (ps: libraries))}/bin/ruby"; - # Rubocop doesnt seem to like running in this fashion. - #check = (writeDash "rubocop.sh" '' - # exec ${lib.getExe buildRubyPackages.rubocop} "$1" - #''); - } - ) name; + makeRubyWriter = + ruby: _rubyPackages: _buildRubyPackages: name: + { + libraries ? [ ], + ... + }@args: + makeScriptWriter ( + (builtins.removeAttrs args [ "libraries" ]) + // { + interpreter = + if libraries == [ ] then "${ruby}/bin/ruby" else "${(ruby.withPackages (_ps: libraries))}/bin/ruby"; + # Rubocop doesnt seem to like running in this fashion. + #check = (writeDash "rubocop.sh" '' + # exec ${lib.getExe buildRubyPackages.rubocop} "$1" + #''); + } + ) name; # Like writeScript but the first line is a shebang to ruby # @@ -244,26 +259,29 @@ rec { # '' writeRuby = makeRubyWriter pkgs.ruby pkgs.rubyPackages buildPackages.rubyPackages; - writeRubyBin = name: - writeRuby "/bin/${name}"; + writeRubyBin = name: writeRuby "/bin/${name}"; # makeLuaWriter takes lua and compatible luaPackages and produces lua script writer, # which validates the script with luacheck at build time. If any libraries are specified, # lua.withPackages is used as interpreter, otherwise the "bare" lua is used. - makeLuaWriter = lua: luaPackages: buildLuaPackages: name: { libraries ? [], ... } @ args: - makeScriptWriter ( - (builtins.removeAttrs args ["libraries"]) - // { - interpreter = lua.interpreter; + makeLuaWriter = + lua: _luaPackages: buildLuaPackages: name: + { ... }@args: + makeScriptWriter ( + (builtins.removeAttrs args [ "libraries" ]) + // { + interpreter = lua.interpreter; # if libraries == [] # then lua.interpreter # else (lua.withPackages (ps: libraries)).interpreter # This should support packages! I just cant figure out why some dependency collision happens whenever I try to run this. - check = (writeDash "luacheck.sh" '' - exec ${buildLuaPackages.luacheck}/bin/luacheck "$1" - ''); - } - ) name; + check = ( + writeDash "luacheck.sh" '' + exec ${buildLuaPackages.luacheck}/bin/luacheck "$1" + '' + ); + } + ) name; # writeLua takes a name an attributeset with libraries and some lua source code and # returns an executable (should also work with luajit) @@ -287,27 +305,27 @@ rec { # '' writeLua = makeLuaWriter pkgs.lua pkgs.luaPackages buildPackages.luaPackages; - writeLuaBin = name: - writeLua "/bin/${name}"; + writeLuaBin = name: writeLua "/bin/${name}"; - writeRust = name: { + writeRust = + name: + { rustc ? pkgs.rustc, - rustcArgs ? [], - strip ? true - }: - let - darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ]; - in + rustcArgs ? [ ], + strip ? true, + }: + let + darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ]; + in makeBinWriter { compileScript = '' cp "$contentPath" tmp.rs - PATH=${lib.makeBinPath [pkgs.gcc]} ${rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs + PATH=${lib.makeBinPath [ pkgs.gcc ]} ${rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs ''; inherit strip; } name; - writeRustBin = name: - writeRust "/bin/${name}"; + writeRustBin = name: writeRust "/bin/${name}"; # writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and # returns an executable @@ -319,23 +337,26 @@ rec { # var result = UglifyJS.minify(code); # console.log(result.code); # '' - writeJS = name: { libraries ? [] }: content: - let - node-env = pkgs.buildEnv { - name = "node"; - paths = libraries; - pathsToLink = [ - "/lib/node_modules" - ]; - }; - in writeDash name '' - export NODE_PATH=${node-env}/lib/node_modules - exec ${lib.getExe pkgs.nodejs} ${pkgs.writeText "js" content} "$@" - ''; + writeJS = + name: + { + libraries ? [ ], + }: + content: + let + node-env = pkgs.buildEnv { + name = "node"; + paths = libraries; + pathsToLink = [ "/lib/node_modules" ]; + }; + in + writeDash name '' + export NODE_PATH=${node-env}/lib/node_modules + exec ${lib.getExe pkgs.nodejs} ${pkgs.writeText "js" content} "$@" + ''; # writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin) - writeJSBin = name: - writeJS "/bin/${name}"; + writeJSBin = name: writeJS "/bin/${name}"; awkFormatNginx = builtins.toFile "awkFormat-nginx.awk" '' awk -f @@ -343,18 +364,22 @@ rec { /\{/{ctx++;idx=1} /\}/{ctx--} {id="";for(i=idx;i $out - gixy $out ''; + writeNginxConfig = + name: text: + pkgs.runCommandLocal name + { + inherit text; + passAsFile = [ "text" ]; + nativeBuildInputs = [ gixy ]; + } # sh + '' + # nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16 + awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out + gixy $out + ''; + # writePerl takes a name an attributeset with libraries and some perl sourcecode and # returns an executable # @@ -363,42 +388,55 @@ rec { # use boolean; # print "Howdy!\n" if true; # '' - writePerl = name: { libraries ? [], ... } @ args: + writePerl = + name: + { + libraries ? [ ], + ... + }@args: makeScriptWriter ( - (builtins.removeAttrs args ["libraries"]) + (builtins.removeAttrs args [ "libraries" ]) // { - interpreter = "${lib.getExe (pkgs.perl.withPackages (p: libraries))}"; + interpreter = "${lib.getExe (pkgs.perl.withPackages (_p: libraries))}"; } ) name; # writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin) - writePerlBin = name: - writePerl "/bin/${name}"; + writePerlBin = name: writePerl "/bin/${name}"; # makePythonWriter takes python and compatible pythonPackages and produces python script writer, # which validates the script with flake8 at build time. If any libraries are specified, # python.withPackages is used as interpreter, otherwise the "bare" python is used. - makePythonWriter = python: pythonPackages: buildPythonPackages: name: { libraries ? [], flakeIgnore ? [], ... } @ args: - let - ignoreAttribute = optionalString (flakeIgnore != []) "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}"; - in - makeScriptWriter - ( - (builtins.removeAttrs args ["libraries" "flakeIgnore"]) + makePythonWriter = + python: pythonPackages: buildPythonPackages: name: + { + libraries ? [ ], + flakeIgnore ? [ ], + ... + }@args: + let + ignoreAttribute = + optionalString (flakeIgnore != [ ]) + "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}"; + in + makeScriptWriter ( + (builtins.removeAttrs args [ + "libraries" + "flakeIgnore" + ]) // { interpreter = if pythonPackages != pkgs.pypy2Packages || pythonPackages != pkgs.pypy3Packages then - if libraries == [] - then python.interpreter - else (python.withPackages (ps: libraries)).interpreter - else python.interpreter - ; - check = optionalString python.isPy3k (writeDash "pythoncheck.sh" '' - exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1" - ''); + if libraries == [ ] then python.interpreter else (python.withPackages (_ps: libraries)).interpreter + else + python.interpreter; + check = optionalString python.isPy3k ( + writeDash "pythoncheck.sh" '' + exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1" + '' + ); } - ) - name; + ) name; # writePyPy2 takes a name an attributeset with libraries and some pypy2 sourcecode and # returns an executable @@ -415,8 +453,7 @@ rec { writePyPy2 = makePythonWriter pkgs.pypy2 pkgs.pypy2Packages buildPackages.pypy2Packages; # writePyPy2Bin takes the same arguments as writePyPy2 but outputs a directory (like writeScriptBin) - writePyPy2Bin = name: - writePyPy2 "/bin/${name}"; + writePyPy2Bin = name: writePyPy2 "/bin/${name}"; # writePython3 takes a name an attributeset with libraries and some python3 sourcecode and # returns an executable @@ -433,8 +470,7 @@ rec { writePython3 = makePythonWriter pkgs.python3 pkgs.python3Packages buildPackages.python3Packages; # writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin) - writePython3Bin = name: - writePython3 "/bin/${name}"; + writePython3Bin = name: writePython3 "/bin/${name}"; # writePyPy3 takes a name an attributeset with libraries and some pypy3 sourcecode and # returns an executable @@ -451,47 +487,61 @@ rec { writePyPy3 = makePythonWriter pkgs.pypy3 pkgs.pypy3Packages buildPackages.pypy3Packages; # writePyPy3Bin takes the same arguments as writePyPy3 but outputs a directory (like writeScriptBin) - writePyPy3Bin = name: - writePyPy3 "/bin/${name}"; + writePyPy3Bin = name: writePyPy3 "/bin/${name}"; + makeFSharpWriter = + { + dotnet-sdk ? pkgs.dotnet-sdk, + fsi-flags ? "", + libraries ? _: [ ], + ... + }@args: + nameOrPath: + let + fname = last (builtins.split "/" nameOrPath); + path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx"; + _nugetDeps = mkNugetDeps { + name = "${fname}-nuget-deps"; + nugetDeps = libraries; + }; - makeFSharpWriter = { dotnet-sdk ? pkgs.dotnet-sdk, fsi-flags ? "", libraries ? _: [], ... } @ args: nameOrPath: - let - fname = last (builtins.split "/" nameOrPath); - path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx"; - _nugetDeps = mkNugetDeps { name = "${fname}-nuget-deps"; nugetDeps = libraries; }; + nuget-source = mkNugetSource { + name = "${fname}-nuget-source"; + description = "A Nuget source with the dependencies for ${fname}"; + deps = [ _nugetDeps ]; + }; - nuget-source = mkNugetSource { - name = "${fname}-nuget-source"; - description = "A Nuget source with the dependencies for ${fname}"; - deps = [ _nugetDeps ]; - }; + fsi = writeBash "fsi" '' + export HOME=$NIX_BUILD_TOP/.home + export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + export DOTNET_CLI_TELEMETRY_OPTOUT=1 + export DOTNET_NOLOGO=1 + script="$1"; shift + ${lib.getExe dotnet-sdk} fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script" + ''; - fsi = writeBash "fsi" '' - export HOME=$NIX_BUILD_TOP/.home - export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - export DOTNET_CLI_TELEMETRY_OPTOUT=1 - export DOTNET_NOLOGO=1 - script="$1"; shift - ${lib.getExe dotnet-sdk} fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script" - ''; + in + content: + makeScriptWriter + ( + (builtins.removeAttrs args [ + "dotnet-sdk" + "fsi-flags" + "libraries" + ]) + // { + interpreter = fsi; + } + ) + path + '' + #i "nuget: ${nuget-source}/lib" + ${content} + exit 0 + ''; - in content: makeScriptWriter ( - (builtins.removeAttrs args ["dotnet-sdk" "fsi-flags" "libraries"]) - // { - interpreter = fsi; - } - ) path - '' - #i "nuget: ${nuget-source}/lib" - ${ content } - exit 0 - ''; + writeFSharp = makeFSharpWriter { }; - writeFSharp = - makeFSharpWriter {}; - - writeFSharpBin = name: - writeFSharp "/bin/${name}"; + writeFSharpBin = name: writeFSharp "/bin/${name}"; } diff --git a/pkgs/clan-app/.envrc b/pkgs/clan-app/.envrc index 77e11659..9e53834a 100644 --- a/pkgs/clan-app/.envrc +++ b/pkgs/clan-app/.envrc @@ -1,3 +1,4 @@ +# shellcheck shell=bash source_up watch_file flake-module.nix shell.nix default.nix diff --git a/pkgs/clan-app/default.nix b/pkgs/clan-app/default.nix index ac352d39..893ccf45 100644 --- a/pkgs/clan-app/default.nix +++ b/pkgs/clan-app/default.nix @@ -6,7 +6,7 @@ pygobject3, wrapGAppsHook, gtk4, - gnome, + adwaita-icon-theme, pygobject-stubs, gobject-introspection, clan-cli, @@ -39,7 +39,7 @@ let gtk4 libadwaita webkitgtk_6_0 - gnome.adwaita-icon-theme + adwaita-icon-theme ]; # Deps including python packages from the local project diff --git a/pkgs/clan-cli/.envrc b/pkgs/clan-cli/.envrc index 24d9f311..85cfcabd 100644 --- a/pkgs/clan-cli/.envrc +++ b/pkgs/clan-cli/.envrc @@ -1,3 +1,4 @@ +# shellcheck shell=bash source_up watch_file flake-module.nix shell.nix default.nix diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 503a519d..29acb1e5 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -23,6 +23,7 @@ from . import ( machines, secrets, state, + vars, vms, ) from .clan_uri import FlakeId @@ -272,6 +273,43 @@ For more detailed information, visit: {help_hyperlink("secrets", "https://docs.c ) facts.register_parser(parser_facts) + # like facts but with vars instead of facts + parser_vars = subparsers.add_parser( + "vars", + help="WIP: manage vars", + description="WIP: manage vars", + epilog=( + f""" +This subcommand provides an interface to vars of clan machines. +Vars are variables that a service can generate. +There are public and secret vars. +Public vars can be referenced by other machines directly. +Public vars can include: ip addresses, public keys. +Secret vars can include: passwords, private keys. + +A service is an included clan-module that implements vars generation functionality. +For example the zerotier module will generate private and public vars. +In this case the public var will be the resulting zerotier-ip of the machine. +The secret var will be the zerotier-identity-secret, which is used by zerotier +to prove the machine has control of the zerotier-ip. + +Examples: + + $ clan vars generate + Will generate vars for all machines. + + $ clan vars generate --service [SERVICE] --regenerate + Will regenerate vars, if they are already generated for a specific service. + This is especially useful for resetting certain passwords while leaving the rest + of the vars for a machine in place. + +For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")} + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + vars.register_parser(parser_vars) + parser_machine = subparsers.add_parser( "machines", help="manage machines and their configuration", diff --git a/pkgs/clan-cli/clan_cli/api/util.py b/pkgs/clan-cli/clan_cli/api/util.py index c93c0520..505fab54 100644 --- a/pkgs/clan-cli/clan_cli/api/util.py +++ b/pkgs/clan-cli/clan_cli/api/util.py @@ -76,6 +76,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> properties = { f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}", type_map) for f in fields + if not f.name.startswith("_") } required = set() @@ -127,7 +128,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> if origin is None: # Non-generic user-defined or built-in type # TODO: handle custom types - raise JSchemaTypeError("Unhandled Type: ", origin) + raise JSchemaTypeError(f"{scope} Unhandled Type: ", origin) elif origin is Literal: # Handle Literal values for enums in JSON Schema @@ -172,7 +173,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> new_map.update(inspect_dataclass_fields(t)) return type_to_dict(origin, scope, new_map) - raise JSchemaTypeError(f"Error api type not yet supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error api type not yet supported {t!s}") elif isinstance(t, type): if t is str: @@ -187,7 +188,7 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> return {"type": "object"} if t is Any: raise JSchemaTypeError( - f"Usage of the Any type is not supported for API functions. In: {scope}" + f"{scope} - Usage of the Any type is not supported for API functions. In: {scope}" ) if t is pathlib.Path: return { @@ -196,13 +197,13 @@ def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> } if t is dict: raise JSchemaTypeError( - "Error: generic dict type not supported. Use dict[str, Any] instead" + f"{scope} - Generic 'dict' type not supported. Use dict[str, Any] or any more expressive type." ) # Optional[T] gets internally transformed Union[T,NoneType] if t is NoneType: return {"type": "null"} - raise JSchemaTypeError(f"Error primitive type not supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error primitive type not supported {t!s}") else: - raise JSchemaTypeError(f"Error type not supported {t!s}") + raise JSchemaTypeError(f"{scope} - Error type not supported {t!s}") diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index d1a7ce51..1085bc31 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -81,11 +81,11 @@ def cast(value: Any, input_type: Any, opt_description: str) -> Any: else: raise ClanError(f"Invalid value {value} for boolean") # handle lists - elif get_origin(input_type) == list: + elif get_origin(input_type) is list: subtype = input_type.__args__[0] return [cast([x], subtype, opt_description) for x in value] # handle dicts - elif get_origin(input_type) == dict: + elif get_origin(input_type) is dict: if not isinstance(value, dict): raise ClanError( f"Cannot set {opt_description} directly. Specify a suboption like {opt_description}." diff --git a/pkgs/clan-cli/clan_cli/machines/machines.py b/pkgs/clan-cli/clan_cli/machines/machines.py index 36eda175..b8be7d8d 100644 --- a/pkgs/clan-cli/clan_cli/machines/machines.py +++ b/pkgs/clan-cli/clan_cli/machines/machines.py @@ -20,7 +20,7 @@ class Machine: name: str flake: FlakeId nix_options: list[str] = field(default_factory=list) - cached_deployment: None | dict = None + cached_deployment: None | dict[str, Any] = None _eval_cache: dict[str, str] = field(default_factory=dict) _build_cache: dict[str, Path] = field(default_factory=dict) @@ -69,12 +69,26 @@ class Machine: def public_facts_module(self) -> str: return self.deployment["facts"]["publicModule"] + @property + def secret_vars_module(self) -> str: + return self.deployment["vars"]["secretModule"] + + @property + def public_vars_module(self) -> str: + return self.deployment["vars"]["publicModule"] + @property def facts_data(self) -> dict[str, dict[str, Any]]: if self.deployment["facts"]["services"]: return self.deployment["facts"]["services"] return {} + @property + def vars_generators(self) -> dict[str, dict[str, Any]]: + if self.deployment["vars"]["generators"]: + return self.deployment["vars"]["generators"] + return {} + @property def secrets_upload_directory(self) -> str: return self.deployment["facts"]["secretUploadDirectory"] diff --git a/pkgs/clan-cli/clan_cli/vars/__init__.py b/pkgs/clan-cli/clan_cli/vars/__init__.py new file mode 100644 index 00000000..4a344f4f --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/__init__.py @@ -0,0 +1,132 @@ +# !/usr/bin/env python3 +import argparse + +from ..hyperlink import help_hyperlink +from .check import register_check_parser +from .generate import register_generate_parser +from .list import register_list_parser +from .upload import register_upload_parser + + +# takes a (sub)parser and configures it +def register_parser(parser: argparse.ArgumentParser) -> None: + subparser = parser.add_subparsers( + title="command", + description="the command to run", + help="the command to run", + required=True, + ) + + check_parser = subparser.add_parser( + "check", + help="check if facts are up to date", + epilog=( + f""" +This subcommand allows checking if all facts are up to date. + +Examples: + + $ clan facts check [MACHINE] + Will check facts for the specified machine. + + +For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")} + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + register_check_parser(check_parser) + + list_parser = subparser.add_parser( + "list", + help="list all facts", + epilog=( + f""" +This subcommand allows listing all public facts for a specific machine. + +The resulting list will be a json string with the name of the fact as its key +and the fact itself as it's value. + +This is how an example output might look like: +``` +\u007b +"[FACT_NAME]": "[FACT]" +\u007d +``` + +Examples: + + $ clan facts list [MACHINE] + Will list facts for the specified machine. + + +For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")} + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + register_list_parser(list_parser) + + parser_generate = subparser.add_parser( + "generate", + help="generate public and secret facts for machines", + epilog=( + f""" +This subcommand allows control of the generation of facts. +Often this function will be invoked automatically on deploying machines, +but there are situations the user may want to have more granular control, +especially for the regeneration of certain services. + +A service is an included clan-module that implements facts generation functionality. +For example the zerotier module will generate private and public facts. +In this case the public fact will be the resulting zerotier-ip of the machine. +The secret fact will be the zerotier-identity-secret, which is used by zerotier +to prove the machine has control of the zerotier-ip. + + +Examples: + + $ clan facts generate + Will generate facts for all machines. + + $ clan facts generate [MACHINE] + Will generate facts for the specified machine. + + $ clan facts generate [MACHINE] --service [SERVICE] + Will generate facts for the specified machine for the specified service. + + $ clan facts generate --service [SERVICE] --regenerate + Will regenerate facts, if they are already generated for a specific service. + This is especially useful for resetting certain passwords while leaving the rest + of the facts for a machine in place. + +For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")} + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + register_generate_parser(parser_generate) + + parser_upload = subparser.add_parser( + "upload", + help="upload secrets for machines", + epilog=( + f""" +This subcommand allows uploading secrets to remote machines. + +If using sops as a secret backend it will upload the private key to the machine. +If using password store it uploads all the secrets you manage to the machine. + +The default backend is sops. + +Examples: + + $ clan facts upload [MACHINE] + Will upload secrets to a specific machine. + +For more detailed information, visit: {help_hyperlink("secrets", "https://docs.clan.lol/getting-started/secrets")} + """ + ), + formatter_class=argparse.RawTextHelpFormatter, + ) + register_upload_parser(parser_upload) diff --git a/pkgs/clan-cli/clan_cli/vars/check.py b/pkgs/clan-cli/clan_cli/vars/check.py new file mode 100644 index 00000000..2f2c7fa3 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/check.py @@ -0,0 +1,64 @@ +import argparse +import importlib +import logging + +from ..completions import add_dynamic_completer, complete_machines +from ..machines.machines import Machine + +log = logging.getLogger(__name__) + + +def check_secrets(machine: Machine, generator_name: None | str = None) -> bool: + secret_vars_module = importlib.import_module(machine.secret_vars_module) + secret_vars_store = secret_vars_module.SecretStore(machine=machine) + public_vars_module = importlib.import_module(machine.public_vars_module) + public_vars_store = public_vars_module.FactStore(machine=machine) + + missing_secret_vars = [] + missing_public_vars = [] + if generator_name: + services = [generator_name] + else: + services = list(machine.vars_generators.keys()) + for generator_name in services: + for name, file in machine.vars_generators[generator_name]["files"].items(): + if file["secret"] and not secret_vars_store.exists(generator_name, name): + log.info( + f"Secret fact '{name}' for service '{generator_name}' in machine {machine.name} is missing." + ) + missing_secret_vars.append((generator_name, name)) + if not file["secret"] and not public_vars_store.exists( + generator_name, name + ): + log.info( + f"Public fact '{name}' for service '{generator_name}' in machine {machine.name} is missing." + ) + missing_public_vars.append((generator_name, name)) + + log.debug(f"missing_secret_vars: {missing_secret_vars}") + log.debug(f"missing_public_vars: {missing_public_vars}") + if missing_secret_vars or missing_public_vars: + return False + return True + + +def check_command(args: argparse.Namespace) -> None: + machine = Machine( + name=args.machine, + flake=args.flake, + ) + check_secrets(machine, generator_name=args.service) + + +def register_check_parser(parser: argparse.ArgumentParser) -> None: + machines_parser = parser.add_argument( + "machine", + help="The machine to check secrets for", + ) + add_dynamic_completer(machines_parser, complete_machines) + + parser.add_argument( + "--service", + help="the service to check", + ) + parser.set_defaults(func=check_command) diff --git a/pkgs/clan-cli/clan_cli/vars/generate.py b/pkgs/clan-cli/clan_cli/vars/generate.py new file mode 100644 index 00000000..9a54abb6 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/generate.py @@ -0,0 +1,240 @@ +import argparse +import importlib +import logging +import os +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from tempfile import TemporaryDirectory + +from clan_cli.cmd import run + +from ..completions import ( + add_dynamic_completer, + complete_machines, + complete_services_for_machine, +) +from ..errors import ClanError +from ..git import commit_files +from ..machines.inventory import get_all_machines, get_selected_machines +from ..machines.machines import Machine +from ..nix import nix_shell +from .check import check_secrets +from .public_modules import FactStoreBase +from .secret_modules import SecretStoreBase + +log = logging.getLogger(__name__) + + +def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str: + """ + Read multi-line input from stdin. + """ + print(prompt, flush=True) + proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True) + log.info("Input received. Processing...") + return proc.stdout + + +def bubblewrap_cmd(generator: str, generator_dir: Path) -> list[str]: + # fmt: off + return nix_shell( + [ + "nixpkgs#bash", + "nixpkgs#bubblewrap", + ], + [ + "bwrap", + "--ro-bind", "/nix/store", "/nix/store", + "--tmpfs", "/usr/lib/systemd", + "--dev", "/dev", + "--bind", str(generator_dir), str(generator_dir), + "--unshare-all", + "--unshare-user", + "--uid", "1000", + "--", + "bash", "-c", generator + ], + ) + # fmt: on + + +def execute_generator( + machine: Machine, + generator_name: str, + regenerate: bool, + secret_vars_store: SecretStoreBase, + public_vars_store: FactStoreBase, + tmpdir: Path, + prompt: Callable[[str], str], +) -> bool: + generator_dir = tmpdir / generator_name + # check if all secrets exist and generate them if at least one is missing + needs_regeneration = not check_secrets(machine, generator_name=generator_name) + log.debug(f"{generator_name} needs_regeneration: {needs_regeneration}") + if not (needs_regeneration or regenerate): + return False + if not isinstance(machine.flake, Path): + msg = f"flake is not a Path: {machine.flake}" + msg += "fact/secret generation is only supported for local flakes" + + env = os.environ.copy() + generator_dir.mkdir(parents=True) + env["out"] = str(generator_dir) + # compatibility for old outputs.nix users + generator = machine.vars_generators[generator_name]["finalScript"] + # if machine.vars_data[generator_name]["generator"]["prompt"]: + # prompt_value = prompt(machine.vars_data[generator_name]["generator"]["prompt"]) + # env["prompt_value"] = prompt_value + if sys.platform == "linux": + cmd = bubblewrap_cmd(generator, generator_dir) + else: + cmd = ["bash", "-c", generator] + run( + cmd, + env=env, + ) + files_to_commit = [] + # store secrets + files = machine.vars_generators[generator_name]["files"] + for file_name, file in files.items(): + groups = file.get("groups", []) + + secret_file = generator_dir / file_name + if not secret_file.is_file(): + msg = f"did not generate a file for '{file_name}' when running the following command:\n" + msg += generator + raise ClanError(msg) + if file["secret"]: + file_path = secret_vars_store.set( + generator_name, file_name, secret_file.read_bytes(), groups + ) + else: + file_path = public_vars_store.set( + generator_name, file_name, secret_file.read_bytes() + ) + if file_path: + files_to_commit.append(file_path) + commit_files( + files_to_commit, + machine.flake_dir, + f"Update facts/secrets for service {generator_name} in machine {machine.name}", + ) + return True + + +def prompt_func(text: str) -> str: + print(f"{text}: ") + return read_multiline_input() + + +def _generate_vars_for_machine( + machine: Machine, + generator_name: str | None, + regenerate: bool, + tmpdir: Path, + prompt: Callable[[str], str] = prompt_func, +) -> bool: + local_temp = tmpdir / machine.name + local_temp.mkdir() + secret_vars_module = importlib.import_module(machine.secret_vars_module) + secret_vars_store = secret_vars_module.SecretStore(machine=machine) + + public_vars_module = importlib.import_module(machine.public_vars_module) + public_vars_store = public_vars_module.FactStore(machine=machine) + + machine_updated = False + + if generator_name and generator_name not in machine.vars_generators: + generators = list(machine.vars_generators.keys()) + raise ClanError( + f"Could not find generator with name: {generator_name}. The following generators are available: {generators}" + ) + + if generator_name: + machine_generator_facts = { + generator_name: machine.vars_generators[generator_name] + } + else: + machine_generator_facts = machine.vars_generators + + for generator_name in machine_generator_facts: + machine_updated |= execute_generator( + machine=machine, + generator_name=generator_name, + regenerate=regenerate, + secret_vars_store=secret_vars_store, + public_vars_store=public_vars_store, + tmpdir=local_temp, + prompt=prompt, + ) + if machine_updated: + # flush caches to make sure the new secrets are available in evaluation + machine.flush_caches() + return machine_updated + + +def generate_vars( + machines: list[Machine], + generator_name: str | None, + regenerate: bool, + prompt: Callable[[str], str] = prompt_func, +) -> bool: + was_regenerated = False + with TemporaryDirectory() as tmp: + tmpdir = Path(tmp) + + for machine in machines: + errors = 0 + try: + was_regenerated |= _generate_vars_for_machine( + machine, generator_name, regenerate, tmpdir, prompt + ) + except Exception as exc: + log.error(f"Failed to generate facts for {machine.name}: {exc}") + errors += 1 + if errors > 0: + raise ClanError( + f"Failed to generate facts for {errors} hosts. Check the logs above" + ) + + if not was_regenerated: + print("All secrets and facts are already up to date") + return was_regenerated + + +def generate_command(args: argparse.Namespace) -> None: + if len(args.machines) == 0: + machines = get_all_machines(args.flake, args.option) + else: + machines = get_selected_machines(args.flake, args.option, args.machines) + generate_vars(machines, args.service, args.regenerate) + + +def register_generate_parser(parser: argparse.ArgumentParser) -> None: + machines_parser = parser.add_argument( + "machines", + type=str, + help="machine to generate facts for. if empty, generate facts for all machines", + nargs="*", + default=[], + ) + add_dynamic_completer(machines_parser, complete_machines) + + service_parser = parser.add_argument( + "--service", + type=str, + help="service to generate facts for, if empty, generate facts for every service", + default=None, + ) + add_dynamic_completer(service_parser, complete_services_for_machine) + + parser.add_argument( + "--regenerate", + type=bool, + action=argparse.BooleanOptionalAction, + help="whether to regenerate facts for the specified machine", + default=None, + ) + parser.set_defaults(func=generate_command) diff --git a/pkgs/clan-cli/clan_cli/vars/list.py b/pkgs/clan-cli/clan_cli/vars/list.py new file mode 100644 index 00000000..0ec2fcdc --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/list.py @@ -0,0 +1,47 @@ +import argparse +import importlib +import json +import logging + +from ..completions import add_dynamic_completer, complete_machines +from ..machines.machines import Machine + +log = logging.getLogger(__name__) + + +# TODO get also secret facts +def get_all_facts(machine: Machine) -> dict: + public_facts_module = importlib.import_module(machine.public_facts_module) + public_facts_store = public_facts_module.FactStore(machine=machine) + + # for service in machine.secrets_data: + # facts[service] = {} + # for fact in machine.secrets_data[service]["facts"]: + # fact_content = fact_store.get(service, fact) + # if fact_content: + # facts[service][fact] = fact_content.decode() + # else: + # log.error(f"Fact {fact} for service {service} is missing") + return public_facts_store.get_all() + + +def get_command(args: argparse.Namespace) -> None: + machine = Machine(name=args.machine, flake=args.flake) + + # the raw_facts are bytestrings making them not json serializable + raw_facts = get_all_facts(machine) + facts = dict() + for key in raw_facts["TODO"]: + facts[key] = raw_facts["TODO"][key].decode("utf8") + + print(json.dumps(facts, indent=4)) + + +def register_list_parser(parser: argparse.ArgumentParser) -> None: + machines_parser = parser.add_argument( + "machine", + help="The machine to print facts for", + ) + add_dynamic_completer(machines_parser, complete_machines) + + parser.set_defaults(func=get_command) diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py b/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py new file mode 100644 index 00000000..a53ba10c --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/__init__.py @@ -0,0 +1,28 @@ +from abc import ABC, abstractmethod +from pathlib import Path + +from clan_cli.machines.machines import Machine + + +class FactStoreBase(ABC): + @abstractmethod + def __init__(self, machine: Machine) -> None: + pass + + @abstractmethod + def exists(self, service: str, name: str) -> bool: + pass + + @abstractmethod + def set(self, service: str, name: str, value: bytes) -> Path | None: + pass + + # get a single fact + @abstractmethod + def get(self, service: str, name: str) -> bytes: + pass + + # get all facts + @abstractmethod + def get_all(self) -> dict[str, dict[str, bytes]]: + pass diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py new file mode 100644 index 00000000..c8c2c892 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/in_repo.py @@ -0,0 +1,64 @@ +from pathlib import Path + +from clan_cli.errors import ClanError +from clan_cli.machines.machines import Machine + +from . import FactStoreBase + + +class FactStore(FactStoreBase): + def __init__(self, machine: Machine) -> None: + self.machine = machine + self.works_remotely = False + + def set(self, generator_name: str, name: str, value: bytes) -> Path | None: + if self.machine.flake.is_local(): + fact_path = ( + self.machine.flake.path + / "machines" + / self.machine.name + / "vars" + / generator_name + / name + ) + fact_path.parent.mkdir(parents=True, exist_ok=True) + fact_path.touch() + fact_path.write_bytes(value) + return fact_path + else: + raise ClanError( + f"in_flake fact storage is only supported for local flakes: {self.machine.flake}" + ) + + def exists(self, generator_name: str, name: str) -> bool: + fact_path = ( + self.machine.flake_dir + / "machines" + / self.machine.name + / "vars" + / generator_name + / name + ) + return fact_path.exists() + + # get a single fact + def get(self, generator_name: str, name: str) -> bytes: + fact_path = ( + self.machine.flake_dir + / "machines" + / self.machine.name + / "vars" + / generator_name + / name + ) + return fact_path.read_bytes() + + # get all public vars + def get_all(self) -> dict[str, dict[str, bytes]]: + facts_folder = self.machine.flake_dir / "machines" / self.machine.name / "vars" + facts: dict[str, dict[str, bytes]] = {} + facts["TODO"] = {} + if facts_folder.exists(): + for fact_path in facts_folder.iterdir(): + facts["TODO"][fact_path.name] = fact_path.read_bytes() + return facts diff --git a/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py new file mode 100644 index 00000000..10e0c6b7 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/public_modules/vm.py @@ -0,0 +1,46 @@ +import logging +from pathlib import Path + +from clan_cli.dirs import vm_state_dir +from clan_cli.errors import ClanError +from clan_cli.machines.machines import Machine + +from . import FactStoreBase + +log = logging.getLogger(__name__) + + +class FactStore(FactStoreBase): + def __init__(self, machine: Machine) -> None: + self.machine = machine + self.works_remotely = False + self.dir = vm_state_dir(str(machine.flake), machine.name) / "facts" + log.debug(f"FactStore initialized with dir {self.dir}") + + def exists(self, service: str, name: str) -> bool: + fact_path = self.dir / service / name + return fact_path.exists() + + def set(self, service: str, name: str, value: bytes) -> Path | None: + fact_path = self.dir / service / name + fact_path.parent.mkdir(parents=True, exist_ok=True) + fact_path.write_bytes(value) + return None + + # get a single fact + def get(self, service: str, name: str) -> bytes: + fact_path = self.dir / service / name + if fact_path.exists(): + return fact_path.read_bytes() + raise ClanError(f"Fact {name} for service {service} not found") + + # get all facts + def get_all(self) -> dict[str, dict[str, bytes]]: + facts: dict[str, dict[str, bytes]] = {} + if self.dir.exists(): + for service in self.dir.iterdir(): + facts[service.name] = {} + for fact in service.iterdir(): + facts[service.name][fact.name] = fact.read_bytes() + + return facts diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py new file mode 100644 index 00000000..5e26009c --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/__init__.py @@ -0,0 +1,31 @@ +from abc import ABC, abstractmethod +from pathlib import Path + +from clan_cli.machines.machines import Machine + + +class SecretStoreBase(ABC): + @abstractmethod + def __init__(self, machine: Machine) -> None: + pass + + @abstractmethod + def set( + self, service: str, name: str, value: bytes, groups: list[str] + ) -> Path | None: + pass + + @abstractmethod + def get(self, service: str, name: str) -> bytes: + pass + + @abstractmethod + def exists(self, service: str, name: str) -> bool: + pass + + def update_check(self) -> bool: + return False + + @abstractmethod + def upload(self, output_dir: Path) -> None: + pass diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py new file mode 100644 index 00000000..c804456f --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/password_store.py @@ -0,0 +1,117 @@ +import os +import subprocess +from pathlib import Path + +from clan_cli.machines.machines import Machine +from clan_cli.nix import nix_shell + +from . import SecretStoreBase + + +class SecretStore(SecretStoreBase): + def __init__(self, machine: Machine) -> None: + self.machine = machine + + def set( + self, service: str, name: str, value: bytes, groups: list[str] + ) -> Path | None: + subprocess.run( + nix_shell( + ["nixpkgs#pass"], + ["pass", "insert", "-m", f"machines/{self.machine.name}/{name}"], + ), + input=value, + check=True, + ) + return None # we manage the files outside of the git repo + + def get(self, service: str, name: str) -> bytes: + return subprocess.run( + nix_shell( + ["nixpkgs#pass"], + ["pass", "show", f"machines/{self.machine.name}/{name}"], + ), + check=True, + stdout=subprocess.PIPE, + ).stdout + + def exists(self, service: str, name: str) -> bool: + password_store = os.environ.get( + "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" + ) + secret_path = Path(password_store) / f"machines/{self.machine.name}/{name}.gpg" + return secret_path.exists() + + def generate_hash(self) -> bytes: + password_store = os.environ.get( + "PASSWORD_STORE_DIR", f"{os.environ['HOME']}/.password-store" + ) + hashes = [] + hashes.append( + subprocess.run( + nix_shell( + ["nixpkgs#git"], + [ + "git", + "-C", + password_store, + "log", + "-1", + "--format=%H", + f"machines/{self.machine.name}", + ], + ), + stdout=subprocess.PIPE, + ).stdout.strip() + ) + for symlink in Path(password_store).glob(f"machines/{self.machine.name}/**/*"): + if symlink.is_symlink(): + hashes.append( + subprocess.run( + nix_shell( + ["nixpkgs#git"], + [ + "git", + "-C", + password_store, + "log", + "-1", + "--format=%H", + str(symlink), + ], + ), + stdout=subprocess.PIPE, + ).stdout.strip() + ) + + # we sort the hashes to make sure that the order is always the same + hashes.sort() + return b"\n".join(hashes) + + # FIXME: add this when we switch to python3.12 + # @override + def update_check(self) -> bool: + local_hash = self.generate_hash() + remote_hash = self.machine.target_host.run( + # TODO get the path to the secrets from the machine + ["cat", f"{self.machine.secrets_upload_directory}/.pass_info"], + check=False, + stdout=subprocess.PIPE, + ).stdout.strip() + + if not remote_hash: + print("remote hash is empty") + return False + + return local_hash.decode() == remote_hash + + def upload(self, output_dir: Path) -> None: + for service in self.machine.facts_data: + for secret in self.machine.facts_data[service]["secret"]: + if isinstance(secret, dict): + secret_name = secret["name"] + else: + # TODO: drop old format soon + secret_name = secret + (output_dir / secret_name).write_bytes(self.get(service, secret_name)) + (output_dir / ".pass_info").write_bytes(self.generate_hash()) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py new file mode 100644 index 00000000..9bc6bcbc --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/sops.py @@ -0,0 +1,69 @@ +from pathlib import Path + +from clan_cli.machines.machines import Machine +from clan_cli.secrets.folders import sops_secrets_folder +from clan_cli.secrets.machines import add_machine, has_machine +from clan_cli.secrets.secrets import decrypt_secret, encrypt_secret, has_secret +from clan_cli.secrets.sops import generate_private_key + +from . import SecretStoreBase + + +class SecretStore(SecretStoreBase): + def __init__(self, machine: Machine) -> None: + self.machine = machine + + # no need to generate keys if we don't manage secrets + if not hasattr(self.machine, "vars_data") or not self.machine.vars_generators: + return + for generator in self.machine.vars_generators.values(): + if "files" in generator: + for file in generator["files"].values(): + if file["secret"]: + return + + if has_machine(self.machine.flake_dir, self.machine.name): + return + priv_key, pub_key = generate_private_key() + encrypt_secret( + self.machine.flake_dir, + sops_secrets_folder(self.machine.flake_dir) + / f"{self.machine.name}-age.key", + priv_key, + ) + add_machine(self.machine.flake_dir, self.machine.name, pub_key, False) + + def set( + self, generator_name: str, name: str, value: bytes, groups: list[str] + ) -> Path | None: + path = ( + sops_secrets_folder(self.machine.flake_dir) + / f"{self.machine.name}-{generator_name}-{name}" + ) + encrypt_secret( + self.machine.flake_dir, + path, + value, + add_machines=[self.machine.name], + add_groups=groups, + ) + return path + + def get(self, service: str, name: str) -> bytes: + return decrypt_secret( + self.machine.flake_dir, f"{self.machine.name}-{name}" + ).encode("utf-8") + + def exists(self, service: str, name: str) -> bool: + return has_secret( + self.machine.flake_dir, + f"{self.machine.name}-{name}", + ) + + def upload(self, output_dir: Path) -> None: + key_name = f"{self.machine.name}-age.key" + if not has_secret(self.machine.flake_dir, key_name): + # skip uploading the secret, not managed by us + return + key = decrypt_secret(self.machine.flake_dir, key_name) + (output_dir / "key.txt").write_text(key) diff --git a/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py new file mode 100644 index 00000000..fc3ea3cd --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/secret_modules/vm.py @@ -0,0 +1,35 @@ +import os +import shutil +from pathlib import Path + +from clan_cli.dirs import vm_state_dir +from clan_cli.machines.machines import Machine + +from . import SecretStoreBase + + +class SecretStore(SecretStoreBase): + def __init__(self, machine: Machine) -> None: + self.machine = machine + self.dir = vm_state_dir(str(machine.flake), machine.name) / "secrets" + self.dir.mkdir(parents=True, exist_ok=True) + + def set( + self, service: str, name: str, value: bytes, groups: list[str] + ) -> Path | None: + secret_file = self.dir / service / name + secret_file.parent.mkdir(parents=True, exist_ok=True) + secret_file.write_bytes(value) + return None # we manage the files outside of the git repo + + def get(self, service: str, name: str) -> bytes: + secret_file = self.dir / service / name + return secret_file.read_bytes() + + def exists(self, service: str, name: str) -> bool: + return (self.dir / service / name).exists() + + def upload(self, output_dir: Path) -> None: + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + shutil.copytree(self.dir, output_dir) diff --git a/pkgs/clan-cli/clan_cli/vars/upload.py b/pkgs/clan-cli/clan_cli/vars/upload.py new file mode 100644 index 00000000..6cb6bf8b --- /dev/null +++ b/pkgs/clan-cli/clan_cli/vars/upload.py @@ -0,0 +1,58 @@ +import argparse +import importlib +import logging +from pathlib import Path +from tempfile import TemporaryDirectory + +from ..cmd import Log, run +from ..completions import add_dynamic_completer, complete_machines +from ..machines.machines import Machine +from ..nix import nix_shell + +log = logging.getLogger(__name__) + + +def upload_secrets(machine: Machine) -> None: + secret_facts_module = importlib.import_module(machine.secret_facts_module) + secret_facts_store = secret_facts_module.SecretStore(machine=machine) + + if secret_facts_store.update_check(): + log.info("Secrets already up to date") + return + with TemporaryDirectory() as tempdir: + secret_facts_store.upload(Path(tempdir)) + host = machine.target_host + + ssh_cmd = host.ssh_cmd() + run( + nix_shell( + ["nixpkgs#rsync"], + [ + "rsync", + "-e", + " ".join(["ssh"] + ssh_cmd[2:]), + "-az", + "--delete", + "--chown=root:root", + "--chmod=D700,F600", + f"{tempdir!s}/", + f"{host.user}@{host.host}:{machine.secrets_upload_directory}/", + ], + ), + log=Log.BOTH, + ) + + +def upload_command(args: argparse.Namespace) -> None: + machine = Machine(name=args.machine, flake=args.flake) + upload_secrets(machine) + + +def register_upload_parser(parser: argparse.ArgumentParser) -> None: + machines_parser = parser.add_argument( + "machine", + help="The machine to upload secrets to", + ) + add_dynamic_completer(machines_parser, complete_machines) + + parser.set_defaults(func=upload_command) diff --git a/pkgs/clan-cli/docs.py b/pkgs/clan-cli/docs.py index f2a01dd5..e1bcf0ba 100644 --- a/pkgs/clan-cli/docs.py +++ b/pkgs/clan-cli/docs.py @@ -254,6 +254,9 @@ def collect_commands() -> list[Category]: if isinstance(action, argparse._SubParsersAction): subparsers: dict[str, argparse.ArgumentParser] = action.choices for name, subparser in subparsers.items(): + if str(subparser.description).startswith("WIP"): + print(f"Excluded {name} from documentation as it is marked as WIP") + continue (_options, _positionals, _subcommands) = get_subcommands( subparser, to=result, level=2, prefix=[name] ) diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index adc586ab..c41143a3 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -58,6 +58,7 @@ python docs.py reference mkdir -p $out cp -r out/* $out + ls -lah $out ''; }; clan-ts-api = pkgs.stdenv.mkDerivation { diff --git a/pkgs/clan-cli/tests/test_api_dataclass_compat.py b/pkgs/clan-cli/tests/test_api_dataclass_compat.py new file mode 100644 index 00000000..d54f4e54 --- /dev/null +++ b/pkgs/clan-cli/tests/test_api_dataclass_compat.py @@ -0,0 +1,142 @@ +import ast +import importlib.util +import os +import sys +from dataclasses import is_dataclass +from pathlib import Path + +from clan_cli.api.util import JSchemaTypeError, type_to_dict +from clan_cli.errors import ClanError + + +def find_dataclasses_in_directory( + directory: Path, exclude_paths: list[str] = [] +) -> list[tuple[str, str]]: + """ + Find all dataclass classes in all Python files within a nested directory. + + Args: + directory (str): The root directory to start searching from. + + Returns: + List[Tuple[str, str]]: A list of tuples containing the file path and the dataclass name. + """ + dataclass_files = [] + + excludes = [os.path.join(directory, d) for d in exclude_paths] + + for root, _, files in os.walk(directory, topdown=False): + for file in files: + if not file.endswith(".py"): + continue + + file_path = os.path.join(root, file) + + if file_path in excludes: + print(f"Skipping dataclass check for file: {file_path}") + continue + + with open(file_path, encoding="utf-8") as f: + try: + tree = ast.parse(f.read(), filename=file_path) + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + for deco in node.decorator_list: + if ( + isinstance(deco, ast.Name) + and deco.id == "dataclass" + ): + dataclass_files.append((file_path, node.name)) + elif ( + isinstance(deco, ast.Call) + and isinstance(deco.func, ast.Name) + and deco.func.id == "dataclass" + ): + dataclass_files.append((file_path, node.name)) + except (SyntaxError, UnicodeDecodeError) as e: + print(f"Error parsing {file_path}: {e}") + + return dataclass_files + + +def load_dataclass_from_file( + file_path: str, class_name: str, root_dir: str +) -> type | None: + """ + Load a dataclass from a given file path. + + Args: + file_path (str): Path to the file. + class_name (str): Name of the class to load. + + Returns: + List[Type]: The dataclass type if found, else an empty list. + """ + module_name = ( + os.path.relpath(file_path, root_dir).replace(os.path.sep, ".").rstrip(".py") + ) + try: + sys.path.insert(0, root_dir) + spec = importlib.util.spec_from_file_location(module_name, file_path) + if not spec: + raise ClanError(f"Could not load spec from file: {file_path}") + + module = importlib.util.module_from_spec(spec) + if not module: + raise ClanError(f"Could not create module: {file_path}") + + if not spec.loader: + raise ClanError(f"Could not load loader from spec: {spec}") + + spec.loader.exec_module(module) + + finally: + sys.path.pop(0) + dataclass_type = getattr(module, class_name, None) + + if dataclass_type and is_dataclass(dataclass_type): + return dataclass_type + + raise ClanError(f"Could not load dataclass {class_name} from file: {file_path}") + + +def test_all_dataclasses() -> None: + """ + This Test ensures that all dataclasses are compatible with the API. + + It will load all dataclasses from the clan_cli directory and + generate a JSON schema for each of them. + + It will fail if any dataclass cannot be converted to JSON schema. + This means the dataclass in its current form is not compatible with the API. + """ + + # Excludes: + # - API includes Type Generic wrappers, that are not known in the init file. + excludes = ["api/__init__.py"] + + cli_path = Path("clan_cli").resolve() + dataclasses = find_dataclasses_in_directory(cli_path, excludes) + + for file, dataclass in dataclasses: + print(f"checking dataclass {dataclass} in file: {file}") + try: + dclass = load_dataclass_from_file(file, dataclass, str(cli_path.parent)) + type_to_dict(dclass) + except JSchemaTypeError as e: + print(f"Error loading dataclass {dataclass} from {file}: {e}") + raise ClanError( + f""" +-------------------------------------------------------------------------------- +Error converting dataclass 'class {dataclass}()' from {file} + +Details: + {e} + +Help: +- Converting public fields to PRIVATE by prefixing them with underscore ('_') +- Ensure all private fields are initialized the API wont provide initial values for them. +-------------------------------------------------------------------------------- +""", + location=__file__, + ) diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 508f361a..64424d35 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -108,7 +108,7 @@ def test_type_from_schema_path_simple() -> None: schema = dict( type="boolean", ) - assert parsing.type_from_schema_path(schema, []) == bool + assert parsing.type_from_schema_path(schema, []) is bool def test_type_from_schema_path_nested() -> None: @@ -125,8 +125,8 @@ def test_type_from_schema_path_nested() -> None: age=dict(type="integer"), ), ) - assert parsing.type_from_schema_path(schema, ["age"]) == int - assert parsing.type_from_schema_path(schema, ["name", "first"]) == str + assert parsing.type_from_schema_path(schema, ["age"]) is int + assert parsing.type_from_schema_path(schema, ["name", "first"]) is str def test_type_from_schema_path_dynamic_attrs() -> None: @@ -140,16 +140,16 @@ def test_type_from_schema_path_dynamic_attrs() -> None: ), ), ) - assert parsing.type_from_schema_path(schema, ["age"]) == int - assert parsing.type_from_schema_path(schema, ["users", "foo"]) == str + assert parsing.type_from_schema_path(schema, ["age"]) is int + assert parsing.type_from_schema_path(schema, ["users", "foo"]) is str def test_map_type() -> None: with pytest.raises(ClanError): config.map_type("foo") - assert config.map_type("string") == str - assert config.map_type("integer") == int - assert config.map_type("boolean") == bool + assert config.map_type("string") is str + assert config.map_type("integer") is int + assert config.map_type("boolean") is bool assert config.map_type("attribute set of string") == dict[str, str] assert config.map_type("attribute set of integer") == dict[str, int] assert config.map_type("null or string") == str | None diff --git a/pkgs/clan-cli/tests/test_vars.py b/pkgs/clan-cli/tests/test_vars.py new file mode 100644 index 00000000..04a141b9 --- /dev/null +++ b/pkgs/clan-cli/tests/test_vars.py @@ -0,0 +1,49 @@ +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from fixtures_flakes import generate_flake +from helpers.cli import Cli +from root import CLAN_CORE + +if TYPE_CHECKING: + pass + + +@pytest.mark.impure +def test_generate_secret( + monkeypatch: pytest.MonkeyPatch, + temporary_home: Path, + # age_keys: list["KeyPair"], +) -> None: + flake = generate_flake( + temporary_home, + flake_template=CLAN_CORE / "templates" / "minimal", + machine_configs=dict( + my_machine=dict( + clan=dict( + core=dict( + vars=dict( + generators=dict( + my_generator=dict( + files=dict( + my_secret=dict( + secret=False, + ) + ), + script="echo hello > $out/my_secret", + ) + ) + ) + ) + ) + ) + ), + ) + monkeypatch.chdir(flake.path) + cli = Cli() + cmd = ["vars", "generate", "--flake", str(flake.path), "my_machine"] + cli.run(cmd) + assert ( + flake.path / "machines" / "my_machine" / "vars" / "my_generator" / "my_secret" + ).is_file() diff --git a/pkgs/installer/flake-module.nix b/pkgs/installer/flake-module.nix index f6243289..bfd4dc55 100644 --- a/pkgs/installer/flake-module.nix +++ b/pkgs/installer/flake-module.nix @@ -19,29 +19,6 @@ let }; }; }; - installerModule = - { config, modulesPath, ... }: - { - imports = [ - wifiModule - self.nixosModules.installer - self.inputs.nixos-generators.nixosModules.all-formats - (modulesPath + "/installer/cd-dvd/iso-image.nix") - ]; - - isoImage.squashfsCompression = "zstd"; - - system.stateVersion = config.system.nixos.version; - nixpkgs.pkgs = self.inputs.nixpkgs.legacyPackages.x86_64-linux; - }; - - installerSystem = lib.nixosSystem { - modules = [ - self.inputs.disko.nixosModules.default - installerModule - { disko.memSize = 4096; } # FIXME: otherwise the image builder goes OOM - ]; - }; flashInstallerModule = { config, ... }: @@ -98,14 +75,6 @@ let in { clan = { - # To build a generic installer image (without ssh pubkeys), - # use the following command: - # $ nix build .#iso-installer - machines.iso-installer = { - imports = [ installerModule ]; - fileSystems."/".device = lib.mkDefault "/dev/null"; - }; - # To directly flash the installer to a disk, use the following command: # $ clan flash flash-installer --disk main /dev/sdX --yes # This will include your ssh public keys in the installer. @@ -114,7 +83,4 @@ in boot.loader.grub.enable = lib.mkDefault true; }; }; - flake.packages.x86_64-linux.iso-installer = installerSystem.config.formats.iso; - flake.apps.x86_64-linux.install-vm.program = installerSystem.config.formats.vm.outPath; - flake.apps.x86_64-linux.install-vm-nogui.program = installerSystem.config.formats.vm-nogui.outPath; } diff --git a/pkgs/webview-ui/.envrc b/pkgs/webview-ui/.envrc index f0f92bed..8e53fd24 100644 --- a/pkgs/webview-ui/.envrc +++ b/pkgs/webview-ui/.envrc @@ -1,3 +1,4 @@ +# shellcheck shell=bash source_up watch_file flake-module.nix default.nix