From 9b0e2a87e869b1632ff8e73f1662f4bc9307924b Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Tue, 25 Jun 2024 21:17:01 +0200 Subject: [PATCH] ClanModules: Add docs and api to retrieve metadata --- checks/installation/flake-module.nix | 2 +- clanModules/borgbackup-static/README.md | 7 +- clanModules/borgbackup/README.md | 9 +- clanModules/borgbackup/default.nix | 87 +++++----- clanModules/deltachat/README.md | 3 +- clanModules/disk-layouts/README.md | 2 - clanModules/ergochat/README.md | 3 +- clanModules/flake-module.nix | 3 - clanModules/localbackup/README.md | 3 +- clanModules/localsend/README.md | 3 +- clanModules/matrix-synapse/README.md | 3 +- clanModules/moonlight/README.md | 3 +- clanModules/postgresql/README.md | 3 +- clanModules/root-password/README.md | 3 +- clanModules/sshd/README.md | 3 +- clanModules/static-hosts/README.md | 3 +- clanModules/sunshine/README.md | 3 +- clanModules/syncthing-static-peers/README.md | 3 +- clanModules/syncthing/README.md | 3 +- clanModules/thelounge/README.md | 3 +- clanModules/trusted-nix-caches/README.md | 3 +- clanModules/user-password/README.md | 3 +- clanModules/xfce/README.md | 3 +- clanModules/zerotier-static-peers/README.md | 3 +- clanModules/zt-tcp-relay/README.md | 3 +- docs/mkdocs.yml | 1 - docs/nix/flake-module.nix | 32 ++-- docs/nix/get-module-docs.nix | 11 -- docs/nix/scripts/renderOptions.py | 63 ++++---- flakeModules/clan.nix | 2 + lib/build-clan/default.nix | 3 + lib/default.nix | 3 +- lib/description.nix | 58 +++---- .../disk-layouts/default.nix | 0 pkgs/clan-cli/clan_cli/__init__.py | 13 +- pkgs/clan-cli/clan_cli/api/modules.py | 150 ++++++++++++++++++ 36 files changed, 342 insertions(+), 161 deletions(-) delete mode 100644 clanModules/disk-layouts/README.md rename {clanModules => nixosModules}/disk-layouts/default.nix (100%) create mode 100644 pkgs/clan-cli/clan_cli/api/modules.py diff --git a/checks/installation/flake-module.nix b/checks/installation/flake-module.nix index c81b9c06..2b87b9ee 100644 --- a/checks/installation/flake-module.nix +++ b/checks/installation/flake-module.nix @@ -12,7 +12,7 @@ { lib, modulesPath, ... }: { imports = [ - self.clanModules.disk-layouts + "${self}/nixosModules/disk-layouts" (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests (modulesPath + "/profiles/qemu-guest.nix") ]; diff --git a/clanModules/borgbackup-static/README.md b/clanModules/borgbackup-static/README.md index 2bb91e76..2e52cfef 100644 --- a/clanModules/borgbackup-static/README.md +++ b/clanModules/borgbackup-static/README.md @@ -1,10 +1,11 @@ -Statically configure borgbackup with sane defaults. --- -This module implements the `borgbackup` backend and implements sane defaults +description = "Statically configure borgbackup with sane defaults." +--- +This module implements the `borgbackup` backend and implements sane defaults for backup management through `borgbackup` for members of the clan. Configure target machines where the backups should be sent to through `targets`. Configure machines that should be backuped either through `includeMachines` -which will exclusively add the included machines to be backuped, or through +which will exclusively add the included machines to be backuped, or through `excludeMachines`, which will add every machine except the excluded machine to the backup. diff --git a/clanModules/borgbackup/README.md b/clanModules/borgbackup/README.md index b639786d..db9a447f 100644 --- a/clanModules/borgbackup/README.md +++ b/clanModules/borgbackup/README.md @@ -1,2 +1,7 @@ -Efficient, deduplicating backup program with optional compression and secure encryption. ---- \ No newline at end of file +--- +description = "Efficient, deduplicating backup program with optional compression and secure encryption." +categories = ["backup"] +--- +Long explanations1 +Long explanations2 +Long explanations3 diff --git a/clanModules/borgbackup/default.nix b/clanModules/borgbackup/default.nix index b46624be..fbc05db2 100644 --- a/clanModules/borgbackup/default.nix +++ b/clanModules/borgbackup/default.nix @@ -27,54 +27,51 @@ let exit 1 fi ''; - - # Each .nix file in the roles directory is a role - # TODO: Helper function to set available roles within module meta. - roles = - if builtins.pathExists ./roles then - lib.pipe ./roles [ - builtins.readDir - (lib.filterAttrs (_n: v: v == "regular")) - lib.attrNames - (map (fileName: lib.removeSuffix ".nix" fileName)) - ] - else - null; - - # TODO: make this an interface of every module - # Maybe load from readme.md - metaInfoOption = lib.mkOption { - readOnly = true; - description = '' - Meta is used to retrieve information about this module. - - - `availableRoles` is a list of roles that can be assigned via the inventory. - - `category` is used to group services in the clan marketplace. - - `description` is a short description of the service for the clan marketplace. - ''; - default = { - description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption."; - availableRoles = roles; - category = "backup"; - }; - type = lib.types.submodule { - options = { - description = lib.mkOption { type = lib.types.str; }; - availableRoles = lib.mkOption { type = lib.types.nullOr (lib.types.listOf lib.types.str); }; - category = lib.mkOption { - description = "A category for the service. This is used to group services in the clan ui"; - type = lib.types.enum [ - "backup" - "network" - ]; - }; - }; - }; - }; in +# Each .nix file in the roles directory is a role +# TODO: Helper function to set available roles within module meta. +# roles = +# if builtins.pathExists ./roles then +# lib.pipe ./roles [ +# builtins.readDir +# (lib.filterAttrs (_n: v: v == "regular")) +# lib.attrNames +# (map (fileName: lib.removeSuffix ".nix" fileName)) +# ] +# else +# null; +# TODO: make this an interface of every module +# Maybe load from readme.md +# metaInfoOption = lib.mkOption { +# readOnly = true; +# description = '' +# Meta is used to retrieve information about this module. +# - `availableRoles` is a list of roles that can be assigned via the inventory. +# - `category` is used to group services in the clan marketplace. +# - `description` is a short description of the service for the clan marketplace. +# ''; +# default = { +# description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption."; +# availableRoles = roles; +# category = "backup"; +# }; +# type = lib.types.submodule { +# options = { +# description = lib.mkOption { type = lib.types.str; }; +# availableRoles = lib.mkOption { type = lib.types.nullOr (lib.types.listOf lib.types.str); }; +# category = lib.mkOption { +# description = "A category for the service. This is used to group services in the clan ui"; +# type = lib.types.enum [ +# "backup" +# "network" +# ]; +# }; +# }; +# }; +# }; { - options.clan.borgbackup.meta = metaInfoOption; + # options.clan.borgbackup.meta = metaInfoOption; options.clan.borgbackup.destinations = lib.mkOption { type = lib.types.attrsOf ( diff --git a/clanModules/deltachat/README.md b/clanModules/deltachat/README.md index 3b5583fa..f38d692d 100644 --- a/clanModules/deltachat/README.md +++ b/clanModules/deltachat/README.md @@ -1,4 +1,5 @@ -Email-based instant messaging for Desktop. +--- +description = "Email-based instant messaging for Desktop." --- !!! warning "Under construction" diff --git a/clanModules/disk-layouts/README.md b/clanModules/disk-layouts/README.md deleted file mode 100644 index d88ecb7a..00000000 --- a/clanModules/disk-layouts/README.md +++ /dev/null @@ -1,2 +0,0 @@ -Automatically format a disk drive on clan installation ---- diff --git a/clanModules/ergochat/README.md b/clanModules/ergochat/README.md index ff4d1497..403896f2 100644 --- a/clanModules/ergochat/README.md +++ b/clanModules/ergochat/README.md @@ -1,2 +1,3 @@ -A modern IRC server +--- +description = "A modern IRC server" --- diff --git a/clanModules/flake-module.nix b/clanModules/flake-module.nix index 928b8e5b..a7adac70 100644 --- a/clanModules/flake-module.nix +++ b/clanModules/flake-module.nix @@ -1,9 +1,6 @@ { ... }: { flake.clanModules = { - disk-layouts = { - imports = [ ./disk-layouts ]; - }; borgbackup = ./borgbackup; borgbackup-static = ./borgbackup-static; deltachat = ./deltachat; diff --git a/clanModules/localbackup/README.md b/clanModules/localbackup/README.md index 47eea811..1beeae75 100644 --- a/clanModules/localbackup/README.md +++ b/clanModules/localbackup/README.md @@ -1,2 +1,3 @@ -Automatically backups current machine to local directory. +--- +description = "Automatically backups current machine to local directory." --- diff --git a/clanModules/localsend/README.md b/clanModules/localsend/README.md index 7fd4d9dc..ea7b6f2d 100644 --- a/clanModules/localsend/README.md +++ b/clanModules/localsend/README.md @@ -1,2 +1,3 @@ -Securely sharing files and messages over a local network without internet connectivity. +--- +description = "Securely sharing files and messages over a local network without internet connectivity." --- diff --git a/clanModules/matrix-synapse/README.md b/clanModules/matrix-synapse/README.md index e53000b2..70343a4e 100644 --- a/clanModules/matrix-synapse/README.md +++ b/clanModules/matrix-synapse/README.md @@ -1,2 +1,3 @@ -A federated messaging server with end-to-end encryption. +--- +description = "A federated messaging server with end-to-end encryption." --- diff --git a/clanModules/moonlight/README.md b/clanModules/moonlight/README.md index 9abdc53f..f9c291a7 100644 --- a/clanModules/moonlight/README.md +++ b/clanModules/moonlight/README.md @@ -1,2 +1,3 @@ -A desktop streaming client optimized for remote gaming and synchronized movie viewing. +--- +description = "A desktop streaming client optimized for remote gaming and synchronized movie viewing." --- diff --git a/clanModules/postgresql/README.md b/clanModules/postgresql/README.md index 76f5b0f8..86108a33 100644 --- a/clanModules/postgresql/README.md +++ b/clanModules/postgresql/README.md @@ -1,2 +1,3 @@ -A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance. +--- +description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance." --- diff --git a/clanModules/root-password/README.md b/clanModules/root-password/README.md index a5a6d485..982426c7 100644 --- a/clanModules/root-password/README.md +++ b/clanModules/root-password/README.md @@ -1,4 +1,5 @@ -Automatically generates and configures a password for the root user. +--- +description = "Automatically generates and configures a password for the root user." --- After the system was installed/deployed the following command can be used to display the root-password: diff --git a/clanModules/sshd/README.md b/clanModules/sshd/README.md index 57c1c333..34e6ab28 100644 --- a/clanModules/sshd/README.md +++ b/clanModules/sshd/README.md @@ -1,2 +1,3 @@ -Enables secure remote access to the machine over ssh +--- +description = "Enables secure remote access to the machine over ssh" --- diff --git a/clanModules/static-hosts/README.md b/clanModules/static-hosts/README.md index 30192f64..92a38220 100644 --- a/clanModules/static-hosts/README.md +++ b/clanModules/static-hosts/README.md @@ -1,2 +1,3 @@ -Statically configure the host names of machines based on their respective zerotier-ip. +--- +description = "Statically configure the host names of machines based on their respective zerotier-ip." --- diff --git a/clanModules/sunshine/README.md b/clanModules/sunshine/README.md index 11dbf9b0..b1445c26 100644 --- a/clanModules/sunshine/README.md +++ b/clanModules/sunshine/README.md @@ -1,2 +1,3 @@ -A desktop streaming server optimized for remote gaming and synchronized movie viewing. +--- +description = "A desktop streaming server optimized for remote gaming and synchronized movie viewing." --- diff --git a/clanModules/syncthing-static-peers/README.md b/clanModules/syncthing-static-peers/README.md index 6d79cf60..63d78a1d 100644 --- a/clanModules/syncthing-static-peers/README.md +++ b/clanModules/syncthing-static-peers/README.md @@ -1,2 +1,3 @@ -Statically configure syncthing peers through clan +--- +description = "Statically configure syncthing peers through clan" --- diff --git a/clanModules/syncthing/README.md b/clanModules/syncthing/README.md index 1bffe740..bd0ffdc3 100644 --- a/clanModules/syncthing/README.md +++ b/clanModules/syncthing/README.md @@ -1,4 +1,5 @@ -A secure, file synchronization app for devices over networks, offering a private alternative to cloud services. +--- +description = "A secure, file synchronization app for devices over networks, offering a private alternative to cloud services." --- ## Usage diff --git a/clanModules/thelounge/README.md b/clanModules/thelounge/README.md index 71714692..fb929bcf 100644 --- a/clanModules/thelounge/README.md +++ b/clanModules/thelounge/README.md @@ -1,2 +1,3 @@ -Modern web IRC client +--- +description = "Modern web IRC client" --- diff --git a/clanModules/trusted-nix-caches/README.md b/clanModules/trusted-nix-caches/README.md index ffd52fb4..ab388a66 100644 --- a/clanModules/trusted-nix-caches/README.md +++ b/clanModules/trusted-nix-caches/README.md @@ -1,2 +1,3 @@ -This module sets the `clan.lol` and `nix-community` cache up as a trusted cache. +--- +description = "This module sets the `clan.lol` and `nix-community` cache up as a trusted cache." ---- diff --git a/clanModules/user-password/README.md b/clanModules/user-password/README.md index 97db50ce..fb361fbf 100644 --- a/clanModules/user-password/README.md +++ b/clanModules/user-password/README.md @@ -1,4 +1,5 @@ -Automatically generates and configures a password for the specified user account. +--- +description = "Automatically generates and configures a password for the specified user account." --- If setting the option prompt to true, the user will be prompted to type in their desired password. diff --git a/clanModules/xfce/README.md b/clanModules/xfce/README.md index 85e35918..38ae6e46 100644 --- a/clanModules/xfce/README.md +++ b/clanModules/xfce/README.md @@ -1,2 +1,3 @@ -A lightweight desktop manager +--- +description = "A lightweight desktop manager" --- diff --git a/clanModules/zerotier-static-peers/README.md b/clanModules/zerotier-static-peers/README.md index cf480f41..a5430153 100644 --- a/clanModules/zerotier-static-peers/README.md +++ b/clanModules/zerotier-static-peers/README.md @@ -1,4 +1,5 @@ -Statically configure the `zerotier` peers of a clan network. +--- +description = "Statically configure the `zerotier` peers of a clan network." --- Statically configure the `zerotier` peers of a clan network. diff --git a/clanModules/zt-tcp-relay/README.md b/clanModules/zt-tcp-relay/README.md index 2101ad33..ca42f562 100644 --- a/clanModules/zt-tcp-relay/README.md +++ b/clanModules/zt-tcp-relay/README.md @@ -1,2 +1,3 @@ -Enable ZeroTier VPN over TCP for networks where UDP is blocked. +--- +description = "Enable ZeroTier VPN over TCP for networks where UDP is blocked." --- diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index daaf98e9..9be53df1 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -54,7 +54,6 @@ nav: - reference/clanModules/borgbackup-static.md - reference/clanModules/borgbackup.md - reference/clanModules/deltachat.md - - reference/clanModules/disk-layouts.md - reference/clanModules/ergochat.md - reference/clanModules/localbackup.md - reference/clanModules/localsend.md diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index 8370c0a2..10019b5c 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -18,8 +18,8 @@ }; clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules); - clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes); - clanModulesMeta = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesMeta); + # clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes); + # clanModulesMeta = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesMeta); # Simply evaluated options (JSON) renderOptions = @@ -30,6 +30,7 @@ nativeBuildInputs = [ pkgs.python3 pkgs.mypy + self'.packages.clan-cli ]; } '' @@ -50,18 +51,25 @@ sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc="; }; - module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } '' - export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json - # A file that contains the links to all clanModule docs - export CLAN_MODULES=${clanModulesFileInfo} - export CLAN_MODULES_READMES=${clanModulesReadmes} - export CLAN_MODULES_META=${clanModulesMeta} + module-docs = + pkgs.runCommand "rendered" + { + nativeBuildInputs = [ + pkgs.python3 + self'.packages.clan-cli + ]; + } + '' + export CLAN_CORE_PATH=${self} + export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json + # A file that contains the links to all clanModule docs + export CLAN_MODULES=${clanModulesFileInfo} - mkdir $out + mkdir $out - # The python script will place mkDocs files in the output directory - python3 ${renderOptions} - ''; + # The python script will place mkDocs files in the output directory + python3 ${renderOptions} + ''; in { devShells.docs = pkgs.callPackage ./shell.nix { diff --git a/docs/nix/get-module-docs.nix b/docs/nix/get-module-docs.nix index b1d2ed9d..97c4225f 100644 --- a/docs/nix/get-module-docs.nix +++ b/docs/nix/get-module-docs.nix @@ -3,7 +3,6 @@ pkgs, clanCore, clanModules, - self, }: let allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [ @@ -39,20 +38,10 @@ let name: module: (evalDocs ((getOptionsWithoutCore [ module ]).clan.${name} or { })).optionsJSON ) clanModules; - clanModulesReadmes = builtins.mapAttrs ( - module_name: _module: self.lib.modules.getReadme module_name - ) clanModules; - - clanModulesMeta = builtins.mapAttrs ( - module_name: _module: - (self.lib.evalClanModules [ module_name ]).config.clan.${module_name}.meta or { } - ) clanModules; - # clanCore docs clanCoreDocs = (evalDocs (getOptions [ ]).clan.core).optionsJSON; in { - inherit clanModulesReadmes clanModulesMeta; clanCore = clanCoreDocs; clanModules = clanModulesDocs; } diff --git a/docs/nix/scripts/renderOptions.py b/docs/nix/scripts/renderOptions.py index 909d6108..5984c6a2 100644 --- a/docs/nix/scripts/renderOptions.py +++ b/docs/nix/scripts/renderOptions.py @@ -28,12 +28,12 @@ import os from pathlib import Path from typing import Any -# Get environment variables -CLAN_CORE = os.getenv("CLAN_CORE") -CLAN_MODULES = os.environ.get("CLAN_MODULES") -CLAN_MODULES_READMES = os.environ.get("CLAN_MODULES_READMES") -CLAN_MODULES_META = os.environ.get("CLAN_MODULES_META") +from clan_cli.api.modules import Frontmatter, extract_frontmatter, get_roles +# Get environment variables +CLAN_CORE_PATH = os.getenv("CLAN_CORE_PATH") +CLAN_CORE_DOCS = os.getenv("CLAN_CORE_DOCS") +CLAN_MODULES = os.environ.get("CLAN_MODULES") OUT = os.environ.get("out") @@ -124,7 +124,7 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str: def module_header(module_name: str) -> str: - return f"# {module_name}\n" + return f"# {module_name}\n\n" def module_usage(module_name: str) -> str: @@ -150,9 +150,9 @@ options_head = "\n## Module Options\n" def produce_clan_core_docs() -> None: - if not CLAN_CORE: + if not CLAN_CORE_DOCS: raise ValueError( - f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}" + f"Environment variables are not set correctly: $CLAN_CORE_DOCS={CLAN_CORE_DOCS}" ) if not OUT: @@ -160,7 +160,7 @@ def produce_clan_core_docs() -> None: # A mapping of output file to content core_outputs: dict[str, str] = {} - with open(CLAN_CORE) as f: + with open(CLAN_CORE_DOCS) as f: options: dict[str, dict[str, Any]] = json.load(f) module_name = "clan-core" for option_name, info in options.items(): @@ -192,13 +192,11 @@ def produce_clan_core_docs() -> None: of.write(output) -def render_meta(meta: dict[str, Any], module_name: str) -> str: - roles = meta.get("availableRoles", None) - +def render_roles(roles: list[str] | None, module_name: str) -> str: if roles: roles_list = "\n".join([f" - `{r}`" for r in roles]) return f""" -???+ tip "Inventory (WIP)" +???+ tip "Inventory usage" Predefined roles: @@ -226,14 +224,10 @@ def produce_clan_modules_docs() -> None: raise ValueError( f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}" ) - if not CLAN_MODULES_READMES: - raise ValueError( - f"Environment variables are not set correctly: $CLAN_MODULES_READMES={CLAN_MODULES_READMES}" - ) - if not CLAN_MODULES_META: + if not CLAN_CORE_PATH: raise ValueError( - f"Environment variables are not set correctly: $CLAN_MODULES_META={CLAN_MODULES_META}" + f"Environment variables are not set correctly: $CLAN_CORE_PATH={CLAN_CORE_PATH}" ) if not OUT: @@ -242,27 +236,36 @@ def produce_clan_modules_docs() -> None: with open(CLAN_MODULES) as f: links: dict[str, str] = json.load(f) - with open(CLAN_MODULES_READMES) as readme: - readme_map: dict[str, str] = json.load(readme) + # with open(CLAN_MODULES_READMES) as readme: + # readme_map: dict[str, str] = json.load(readme) - with open(CLAN_MODULES_META) as f: - meta_map: dict[str, Any] = json.load(f) - print(meta_map) + # with open(CLAN_MODULES_META) as f: + # meta_map: dict[str, Any] = json.load(f) + # print(meta_map) # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... } for module_name, options_file in links.items(): + readme_file = Path(CLAN_CORE_PATH) / "clanModules" / module_name / "README.md" + print(module_name, readme_file) + with open(readme_file) as f: + readme = f.read() + frontmatter: Frontmatter + frontmatter, readme_content = extract_frontmatter(readme, str(readme_file)) + print(frontmatter, readme_content) + with open(Path(options_file) / "share/doc/nixos/options.json") as f: options: dict[str, dict[str, Any]] = json.load(f) print(f"Rendering options for {module_name}...") output = module_header(module_name) - if readme_map.get(module_name, None): - output += f"{readme_map[module_name]}\n" + if frontmatter.description: + output += f"**{frontmatter.description}**\n\n" + output += f"{readme_content}\n" - # Add meta information: - # - Inventory implementation status - if meta_map.get(module_name, None): - output += render_meta(meta_map.get(module_name, {}), module_name) + # get_roles(str) -> list[str] | None + roles = get_roles(str(Path(CLAN_CORE_PATH) / "clanModules" / module_name)) + if roles: + output += render_roles(roles, module_name) output += module_usage(module_name) diff --git a/flakeModules/clan.nix b/flakeModules/clan.nix index b1a1247a..a1421935 100644 --- a/flakeModules/clan.nix +++ b/flakeModules/clan.nix @@ -93,6 +93,8 @@ in inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; inventoryFile = lib.mkOption { type = lib.types.unspecified; }; + clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; }; + source = lib.mkOption { type = lib.types.path; }; meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; }; machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); }; diff --git a/lib/build-clan/default.nix b/lib/build-clan/default.nix index 452f3839..2f3d3436 100644 --- a/lib/build-clan/default.nix +++ b/lib/build-clan/default.nix @@ -212,6 +212,9 @@ builtins.deepSeq deprecationWarnings { inherit nixosConfigurations; clanInternals = { + inherit (clan-core) clanModules; + source = "${clan-core}"; + meta = mergedInventory.meta; inventory = mergedInventory; diff --git a/lib/default.nix b/lib/default.nix index 086fcce8..30b68b5c 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -8,6 +8,7 @@ evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; }; inventory = import ./inventory { inherit lib clan-core; }; jsonschema = import ./jsonschema { inherit lib; }; - modules = import ./description.nix { inherit clan-core lib; }; + # TODO: migrate to also use toml frontmatter + # modules = import ./description.nix { inherit clan-core lib; }; buildClan = import ./build-clan { inherit clan-core lib nixpkgs; }; } diff --git a/lib/description.nix b/lib/description.nix index dc285730..917836f0 100644 --- a/lib/description.nix +++ b/lib/description.nix @@ -1,33 +1,33 @@ -{ lib, clan-core, ... }: +{ ... }: rec { - getReadme = - modulename: - let - readme = "${clan-core}/clanModules/${modulename}/README.md"; - readmeContents = - if (builtins.pathExists readme) then - (builtins.readFile readme) - else - throw "No README.md found for module ${modulename}"; - in - readmeContents; + # getReadme = + # modulename: + # let + # readme = "${clan-core}/clanModules/${modulename}/README.md"; + # readmeContents = + # if (builtins.pathExists readme) then + # (builtins.readFile readme) + # else + # throw "No README.md found for module ${modulename}"; + # in + # readmeContents; - getShortDescription = - modulename: - let - content = (getReadme modulename); - parts = lib.splitString "---" content; - description = builtins.head parts; - number_of_newlines = builtins.length (lib.splitString "\n" description); - in - if (builtins.length parts) > 1 then - if number_of_newlines > 4 then - throw "Short description in README.md for module ${modulename} is too long. Max 3 newlines." - else if number_of_newlines <= 1 then - throw "Missing short description in README.md for module ${modulename}." - else - description - else - throw "Short description delimiter `---` not found in README.md for module ${modulename}"; + # getShortDescription = + # modulename: + # let + # content = (getReadme modulename); + # parts = lib.splitString "---" content; + # description = builtins.head parts; + # number_of_newlines = builtins.length (lib.splitString "\n" description); + # in + # if (builtins.length parts) > 1 then + # if number_of_newlines > 4 then + # throw "Short description in README.md for module ${modulename} is too long. Max 3 newlines." + # else if number_of_newlines <= 1 then + # throw "Missing short description in README.md for module ${modulename}." + # else + # description + # else + # throw "Short description delimiter `---` not found in README.md for module ${modulename}"; } diff --git a/clanModules/disk-layouts/default.nix b/nixosModules/disk-layouts/default.nix similarity index 100% rename from clanModules/disk-layouts/default.nix rename to nixosModules/disk-layouts/default.nix diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 4f997bec..7b42a8e2 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -5,11 +5,12 @@ from pathlib import Path from types import ModuleType # These imports are unused, but necessary for @API.register to run once. -from clan_cli.api import directory, mdns_discovery +from clan_cli.api import directory, mdns_discovery, modules from clan_cli.arg_actions import AppendOptionAction from clan_cli.clan import show -__all__ = ["directory", "mdns_discovery"] +# API endpoints that are not used in the cli. +__all__ = ["directory", "mdns_discovery", "modules"] from . import ( backups, @@ -121,6 +122,14 @@ Note: The meta results from clan/meta.json and manual flake arguments. It may no ) show_parser.set_defaults(func=show.show_command) + modules_parser = subparsers.add_parser("modules", help="Show modules") + modules_parser.add_argument( + "module_name", + help="name of the module", + type=str, + ) + modules_parser.set_defaults(func=modules.command) + parser_backups = subparsers.add_parser( "backups", help="manage backups of clan machines", diff --git a/pkgs/clan-cli/clan_cli/api/modules.py b/pkgs/clan-cli/clan_cli/api/modules.py new file mode 100644 index 00000000..c8dec842 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/api/modules.py @@ -0,0 +1,150 @@ +import argparse +import json +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path + +from clan_cli.cmd import run_no_stdout +from clan_cli.errors import ClanCmdError, ClanError +from clan_cli.nix import nix_eval + +from . import API + + +@dataclass +class Frontmatter: + description: str + categories: list[str] | None = None + + +def extract_frontmatter(readme_content: str, err_scope: str) -> tuple[Frontmatter, str]: + """ + Extracts TOML frontmatter from a README file content. + + Parameters: + - readme_content (str): The content of the README file as a string. + + Returns: + - str: The extracted frontmatter as a string. + - str: The content of the README file without the frontmatter. + + Raises: + - ValueError: If the README does not contain valid frontmatter. + """ + # Pattern to match YAML frontmatter enclosed by triple-dashed lines + frontmatter_pattern = r"^---\s+(.*?)\s+---\s?+(.*)$" + + # Search for the frontmatter using the pattern + match = re.search(frontmatter_pattern, readme_content, re.DOTALL) + + # If a match is found, return the frontmatter content + match = re.search(frontmatter_pattern, readme_content, re.DOTALL) + + # If a match is found, parse the TOML frontmatter and return both parts + if match: + frontmatter_raw, remaining_content = match.groups() + try: + # Parse the TOML frontmatter + frontmatter_parsed = tomllib.loads(frontmatter_raw) + except tomllib.TOMLDecodeError as e: + raise ClanError( + f"Error parsing TOML frontmatter: {e}", + description=f"Invalid TOML frontmatter. {err_scope}", + location="extract_frontmatter", + ) + + return Frontmatter(**frontmatter_parsed), remaining_content + + # If no frontmatter is found, raise an error + raise ClanError( + "Invalid README: Frontmatter not found.", + location="extract_frontmatter", + description=f"{err_scope} does not contain valid frontmatter.", + ) + + +def get_roles(module_path: str) -> None | list[str]: + roles_dir = Path(module_path) / "roles" + if not roles_dir.exists() or not roles_dir.is_dir(): + return None + + return [ + role.stem # filename without .nix extension + for role in roles_dir.iterdir() + if role.is_file() and role.suffix == ".nix" + ] + + +@dataclass +class ModuleInfo: + description: str + categories: list[str] | None + roles: list[str] | None + + +def get_modules(base_path: str) -> dict[str, str]: + cmd = nix_eval( + [ + f"{base_path}#clanInternals.clanModules", + "--json", + ] + ) + try: + proc = run_no_stdout(cmd) + res = proc.stdout.strip() + except ClanCmdError: + raise ClanError( + "clanInternals might not have clanModules attributes", + location=f"list_modules {base_path}", + description="Evaluation failed on clanInternals.clanModules attribute", + ) + modules: dict[str, str] = json.loads(res) + return modules + + +@API.register +def list_modules(base_path: str) -> list[str]: + """ + Show information about a module + """ + modules = get_modules(base_path) + return [m for m in modules.keys()] + + +@API.register +def show_module_info(base_path: str, module_name: str) -> ModuleInfo: + """ + Show information about a module + """ + modules = get_modules(base_path) + module_path = modules.get(module_name, None) + if not module_path: + raise ClanError( + "Module not found", + location=f"show_module_info {module_name}", + description="Module does not exist", + ) + module_readme = Path(module_path) / "README.md" + if not module_readme.exists(): + raise ClanError( + "Module not found", + location=f"show_module_info {module_name}", + description="Module does not exist or doesn't have any README.md file", + ) + with open(module_readme) as f: + readme = f.read() + frontmatter, readme_content = extract_frontmatter( + readme, f"{module_path}/README.md" + ) + + return ModuleInfo( + description=frontmatter.description, + categories=frontmatter.categories, + roles=get_roles(module_path), + ) + + +def command(args: argparse.Namespace) -> None: + res = show_module_info(args.flake, args.module_name) + print(res)