1
0
forked from clan/clan-core

ClanModules: Add docs and api to retrieve metadata

This commit is contained in:
Johannes Kirschbauer 2024-06-25 21:17:01 +02:00
parent 4022c13b31
commit 9b0e2a87e8
Signed by: hsjobeki
SSH Key Fingerprint: SHA256:vX3utDqig7Ph5L0JPv87ZTPb/w7cMzREKVZzzLFg9qU
36 changed files with 342 additions and 161 deletions

View File

@ -12,7 +12,7 @@
{ lib, modulesPath, ... }:
{
imports = [
self.clanModules.disk-layouts
"${self}/nixosModules/disk-layouts"
(modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests
(modulesPath + "/profiles/qemu-guest.nix")
];

View File

@ -1,10 +1,11 @@
Statically configure borgbackup with sane defaults.
---
This module implements the `borgbackup` backend and implements sane defaults
description = "Statically configure borgbackup with sane defaults."
---
This module implements the `borgbackup` backend and implements sane defaults
for backup management through `borgbackup` for members of the clan.
Configure target machines where the backups should be sent to through `targets`.
Configure machines that should be backuped either through `includeMachines`
which will exclusively add the included machines to be backuped, or through
which will exclusively add the included machines to be backuped, or through
`excludeMachines`, which will add every machine except the excluded machine to the backup.

View File

@ -1,2 +1,7 @@
Efficient, deduplicating backup program with optional compression and secure encryption.
---
---
description = "Efficient, deduplicating backup program with optional compression and secure encryption."
categories = ["backup"]
---
Long explanations1
Long explanations2
Long explanations3

View File

@ -27,54 +27,51 @@ let
exit 1
fi
'';
# Each .nix file in the roles directory is a role
# TODO: Helper function to set available roles within module meta.
roles =
if builtins.pathExists ./roles then
lib.pipe ./roles [
builtins.readDir
(lib.filterAttrs (_n: v: v == "regular"))
lib.attrNames
(map (fileName: lib.removeSuffix ".nix" fileName))
]
else
null;
# TODO: make this an interface of every module
# Maybe load from readme.md
metaInfoOption = lib.mkOption {
readOnly = true;
description = ''
Meta is used to retrieve information about this module.
- `availableRoles` is a list of roles that can be assigned via the inventory.
- `category` is used to group services in the clan marketplace.
- `description` is a short description of the service for the clan marketplace.
'';
default = {
description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption.";
availableRoles = roles;
category = "backup";
};
type = lib.types.submodule {
options = {
description = lib.mkOption { type = lib.types.str; };
availableRoles = lib.mkOption { type = lib.types.nullOr (lib.types.listOf lib.types.str); };
category = lib.mkOption {
description = "A category for the service. This is used to group services in the clan ui";
type = lib.types.enum [
"backup"
"network"
];
};
};
};
};
in
# Each .nix file in the roles directory is a role
# TODO: Helper function to set available roles within module meta.
# roles =
# if builtins.pathExists ./roles then
# lib.pipe ./roles [
# builtins.readDir
# (lib.filterAttrs (_n: v: v == "regular"))
# lib.attrNames
# (map (fileName: lib.removeSuffix ".nix" fileName))
# ]
# else
# null;
# TODO: make this an interface of every module
# Maybe load from readme.md
# metaInfoOption = lib.mkOption {
# readOnly = true;
# description = ''
# Meta is used to retrieve information about this module.
# - `availableRoles` is a list of roles that can be assigned via the inventory.
# - `category` is used to group services in the clan marketplace.
# - `description` is a short description of the service for the clan marketplace.
# '';
# default = {
# description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption.";
# availableRoles = roles;
# category = "backup";
# };
# type = lib.types.submodule {
# options = {
# description = lib.mkOption { type = lib.types.str; };
# availableRoles = lib.mkOption { type = lib.types.nullOr (lib.types.listOf lib.types.str); };
# category = lib.mkOption {
# description = "A category for the service. This is used to group services in the clan ui";
# type = lib.types.enum [
# "backup"
# "network"
# ];
# };
# };
# };
# };
{
options.clan.borgbackup.meta = metaInfoOption;
# options.clan.borgbackup.meta = metaInfoOption;
options.clan.borgbackup.destinations = lib.mkOption {
type = lib.types.attrsOf (

View File

@ -1,4 +1,5 @@
Email-based instant messaging for Desktop.
---
description = "Email-based instant messaging for Desktop."
---
!!! warning "Under construction"

View File

@ -1,2 +0,0 @@
Automatically format a disk drive on clan installation
---

View File

@ -1,2 +1,3 @@
A modern IRC server
---
description = "A modern IRC server"
---

View File

@ -1,9 +1,6 @@
{ ... }:
{
flake.clanModules = {
disk-layouts = {
imports = [ ./disk-layouts ];
};
borgbackup = ./borgbackup;
borgbackup-static = ./borgbackup-static;
deltachat = ./deltachat;

View File

@ -1,2 +1,3 @@
Automatically backups current machine to local directory.
---
description = "Automatically backups current machine to local directory."
---

View File

@ -1,2 +1,3 @@
Securely sharing files and messages over a local network without internet connectivity.
---
description = "Securely sharing files and messages over a local network without internet connectivity."
---

View File

@ -1,2 +1,3 @@
A federated messaging server with end-to-end encryption.
---
description = "A federated messaging server with end-to-end encryption."
---

View File

@ -1,2 +1,3 @@
A desktop streaming client optimized for remote gaming and synchronized movie viewing.
---
description = "A desktop streaming client optimized for remote gaming and synchronized movie viewing."
---

View File

@ -1,2 +1,3 @@
A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
---
description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance."
---

View File

@ -1,4 +1,5 @@
Automatically generates and configures a password for the root user.
---
description = "Automatically generates and configures a password for the root user."
---
After the system was installed/deployed the following command can be used to display the root-password:

View File

@ -1,2 +1,3 @@
Enables secure remote access to the machine over ssh
---
description = "Enables secure remote access to the machine over ssh"
---

View File

@ -1,2 +1,3 @@
Statically configure the host names of machines based on their respective zerotier-ip.
---
description = "Statically configure the host names of machines based on their respective zerotier-ip."
---

View File

@ -1,2 +1,3 @@
A desktop streaming server optimized for remote gaming and synchronized movie viewing.
---
description = "A desktop streaming server optimized for remote gaming and synchronized movie viewing."
---

View File

@ -1,2 +1,3 @@
Statically configure syncthing peers through clan
---
description = "Statically configure syncthing peers through clan"
---

View File

@ -1,4 +1,5 @@
A secure, file synchronization app for devices over networks, offering a private alternative to cloud services.
---
description = "A secure, file synchronization app for devices over networks, offering a private alternative to cloud services."
---
## Usage

View File

@ -1,2 +1,3 @@
Modern web IRC client
---
description = "Modern web IRC client"
---

View File

@ -1,2 +1,3 @@
This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.
---
description = "This module sets the `clan.lol` and `nix-community` cache up as a trusted cache."
----

View File

@ -1,4 +1,5 @@
Automatically generates and configures a password for the specified user account.
---
description = "Automatically generates and configures a password for the specified user account."
---
If setting the option prompt to true, the user will be prompted to type in their desired password.

View File

@ -1,2 +1,3 @@
A lightweight desktop manager
---
description = "A lightweight desktop manager"
---

View File

@ -1,4 +1,5 @@
Statically configure the `zerotier` peers of a clan network.
---
description = "Statically configure the `zerotier` peers of a clan network."
---
Statically configure the `zerotier` peers of a clan network.

View File

@ -1,2 +1,3 @@
Enable ZeroTier VPN over TCP for networks where UDP is blocked.
---
description = "Enable ZeroTier VPN over TCP for networks where UDP is blocked."
---

View File

@ -54,7 +54,6 @@ nav:
- reference/clanModules/borgbackup-static.md
- reference/clanModules/borgbackup.md
- reference/clanModules/deltachat.md
- reference/clanModules/disk-layouts.md
- reference/clanModules/ergochat.md
- reference/clanModules/localbackup.md
- reference/clanModules/localsend.md

View File

@ -18,8 +18,8 @@
};
clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules);
clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes);
clanModulesMeta = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesMeta);
# clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes);
# clanModulesMeta = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesMeta);
# Simply evaluated options (JSON)
renderOptions =
@ -30,6 +30,7 @@
nativeBuildInputs = [
pkgs.python3
pkgs.mypy
self'.packages.clan-cli
];
}
''
@ -50,18 +51,25 @@
sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc=";
};
module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json
# A file that contains the links to all clanModule docs
export CLAN_MODULES=${clanModulesFileInfo}
export CLAN_MODULES_READMES=${clanModulesReadmes}
export CLAN_MODULES_META=${clanModulesMeta}
module-docs =
pkgs.runCommand "rendered"
{
nativeBuildInputs = [
pkgs.python3
self'.packages.clan-cli
];
}
''
export CLAN_CORE_PATH=${self}
export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json
# A file that contains the links to all clanModule docs
export CLAN_MODULES=${clanModulesFileInfo}
mkdir $out
mkdir $out
# The python script will place mkDocs files in the output directory
python3 ${renderOptions}
'';
# The python script will place mkDocs files in the output directory
python3 ${renderOptions}
'';
in
{
devShells.docs = pkgs.callPackage ./shell.nix {

View File

@ -3,7 +3,6 @@
pkgs,
clanCore,
clanModules,
self,
}:
let
allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [
@ -39,20 +38,10 @@ let
name: module: (evalDocs ((getOptionsWithoutCore [ module ]).clan.${name} or { })).optionsJSON
) clanModules;
clanModulesReadmes = builtins.mapAttrs (
module_name: _module: self.lib.modules.getReadme module_name
) clanModules;
clanModulesMeta = builtins.mapAttrs (
module_name: _module:
(self.lib.evalClanModules [ module_name ]).config.clan.${module_name}.meta or { }
) clanModules;
# clanCore docs
clanCoreDocs = (evalDocs (getOptions [ ]).clan.core).optionsJSON;
in
{
inherit clanModulesReadmes clanModulesMeta;
clanCore = clanCoreDocs;
clanModules = clanModulesDocs;
}

View File

@ -28,12 +28,12 @@ import os
from pathlib import Path
from typing import Any
# Get environment variables
CLAN_CORE = os.getenv("CLAN_CORE")
CLAN_MODULES = os.environ.get("CLAN_MODULES")
CLAN_MODULES_READMES = os.environ.get("CLAN_MODULES_READMES")
CLAN_MODULES_META = os.environ.get("CLAN_MODULES_META")
from clan_cli.api.modules import Frontmatter, extract_frontmatter, get_roles
# Get environment variables
CLAN_CORE_PATH = os.getenv("CLAN_CORE_PATH")
CLAN_CORE_DOCS = os.getenv("CLAN_CORE_DOCS")
CLAN_MODULES = os.environ.get("CLAN_MODULES")
OUT = os.environ.get("out")
@ -124,7 +124,7 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
def module_header(module_name: str) -> str:
return f"# {module_name}\n"
return f"# {module_name}\n\n"
def module_usage(module_name: str) -> str:
@ -150,9 +150,9 @@ options_head = "\n## Module Options\n"
def produce_clan_core_docs() -> None:
if not CLAN_CORE:
if not CLAN_CORE_DOCS:
raise ValueError(
f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}"
f"Environment variables are not set correctly: $CLAN_CORE_DOCS={CLAN_CORE_DOCS}"
)
if not OUT:
@ -160,7 +160,7 @@ def produce_clan_core_docs() -> None:
# A mapping of output file to content
core_outputs: dict[str, str] = {}
with open(CLAN_CORE) as f:
with open(CLAN_CORE_DOCS) as f:
options: dict[str, dict[str, Any]] = json.load(f)
module_name = "clan-core"
for option_name, info in options.items():
@ -192,13 +192,11 @@ def produce_clan_core_docs() -> None:
of.write(output)
def render_meta(meta: dict[str, Any], module_name: str) -> str:
roles = meta.get("availableRoles", None)
def render_roles(roles: list[str] | None, module_name: str) -> str:
if roles:
roles_list = "\n".join([f" - `{r}`" for r in roles])
return f"""
???+ tip "Inventory (WIP)"
???+ tip "Inventory usage"
Predefined roles:
@ -226,14 +224,10 @@ def produce_clan_modules_docs() -> None:
raise ValueError(
f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}"
)
if not CLAN_MODULES_READMES:
raise ValueError(
f"Environment variables are not set correctly: $CLAN_MODULES_READMES={CLAN_MODULES_READMES}"
)
if not CLAN_MODULES_META:
if not CLAN_CORE_PATH:
raise ValueError(
f"Environment variables are not set correctly: $CLAN_MODULES_META={CLAN_MODULES_META}"
f"Environment variables are not set correctly: $CLAN_CORE_PATH={CLAN_CORE_PATH}"
)
if not OUT:
@ -242,27 +236,36 @@ def produce_clan_modules_docs() -> None:
with open(CLAN_MODULES) as f:
links: dict[str, str] = json.load(f)
with open(CLAN_MODULES_READMES) as readme:
readme_map: dict[str, str] = json.load(readme)
# with open(CLAN_MODULES_READMES) as readme:
# readme_map: dict[str, str] = json.load(readme)
with open(CLAN_MODULES_META) as f:
meta_map: dict[str, Any] = json.load(f)
print(meta_map)
# with open(CLAN_MODULES_META) as f:
# meta_map: dict[str, Any] = json.load(f)
# print(meta_map)
# {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... }
for module_name, options_file in links.items():
readme_file = Path(CLAN_CORE_PATH) / "clanModules" / module_name / "README.md"
print(module_name, readme_file)
with open(readme_file) as f:
readme = f.read()
frontmatter: Frontmatter
frontmatter, readme_content = extract_frontmatter(readme, str(readme_file))
print(frontmatter, readme_content)
with open(Path(options_file) / "share/doc/nixos/options.json") as f:
options: dict[str, dict[str, Any]] = json.load(f)
print(f"Rendering options for {module_name}...")
output = module_header(module_name)
if readme_map.get(module_name, None):
output += f"{readme_map[module_name]}\n"
if frontmatter.description:
output += f"**{frontmatter.description}**\n\n"
output += f"{readme_content}\n"
# Add meta information:
# - Inventory implementation status
if meta_map.get(module_name, None):
output += render_meta(meta_map.get(module_name, {}), module_name)
# get_roles(str) -> list[str] | None
roles = get_roles(str(Path(CLAN_CORE_PATH) / "clanModules" / module_name))
if roles:
output += render_roles(roles, module_name)
output += module_usage(module_name)

View File

@ -93,6 +93,8 @@ in
inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
inventoryFile = lib.mkOption { type = lib.types.unspecified; };
clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; };
source = lib.mkOption { type = lib.types.path; };
meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };

View File

@ -212,6 +212,9 @@ builtins.deepSeq deprecationWarnings {
inherit nixosConfigurations;
clanInternals = {
inherit (clan-core) clanModules;
source = "${clan-core}";
meta = mergedInventory.meta;
inventory = mergedInventory;

View File

@ -8,6 +8,7 @@
evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; };
inventory = import ./inventory { inherit lib clan-core; };
jsonschema = import ./jsonschema { inherit lib; };
modules = import ./description.nix { inherit clan-core lib; };
# TODO: migrate to also use toml frontmatter
# modules = import ./description.nix { inherit clan-core lib; };
buildClan = import ./build-clan { inherit clan-core lib nixpkgs; };
}

View File

@ -1,33 +1,33 @@
{ lib, clan-core, ... }:
{ ... }:
rec {
getReadme =
modulename:
let
readme = "${clan-core}/clanModules/${modulename}/README.md";
readmeContents =
if (builtins.pathExists readme) then
(builtins.readFile readme)
else
throw "No README.md found for module ${modulename}";
in
readmeContents;
# getReadme =
# modulename:
# let
# readme = "${clan-core}/clanModules/${modulename}/README.md";
# readmeContents =
# if (builtins.pathExists readme) then
# (builtins.readFile readme)
# else
# throw "No README.md found for module ${modulename}";
# in
# readmeContents;
getShortDescription =
modulename:
let
content = (getReadme modulename);
parts = lib.splitString "---" content;
description = builtins.head parts;
number_of_newlines = builtins.length (lib.splitString "\n" description);
in
if (builtins.length parts) > 1 then
if number_of_newlines > 4 then
throw "Short description in README.md for module ${modulename} is too long. Max 3 newlines."
else if number_of_newlines <= 1 then
throw "Missing short description in README.md for module ${modulename}."
else
description
else
throw "Short description delimiter `---` not found in README.md for module ${modulename}";
# getShortDescription =
# modulename:
# let
# content = (getReadme modulename);
# parts = lib.splitString "---" content;
# description = builtins.head parts;
# number_of_newlines = builtins.length (lib.splitString "\n" description);
# in
# if (builtins.length parts) > 1 then
# if number_of_newlines > 4 then
# throw "Short description in README.md for module ${modulename} is too long. Max 3 newlines."
# else if number_of_newlines <= 1 then
# throw "Missing short description in README.md for module ${modulename}."
# else
# description
# else
# throw "Short description delimiter `---` not found in README.md for module ${modulename}";
}

View File

@ -5,11 +5,12 @@ from pathlib import Path
from types import ModuleType
# These imports are unused, but necessary for @API.register to run once.
from clan_cli.api import directory, mdns_discovery
from clan_cli.api import directory, mdns_discovery, modules
from clan_cli.arg_actions import AppendOptionAction
from clan_cli.clan import show
__all__ = ["directory", "mdns_discovery"]
# API endpoints that are not used in the cli.
__all__ = ["directory", "mdns_discovery", "modules"]
from . import (
backups,
@ -121,6 +122,14 @@ Note: The meta results from clan/meta.json and manual flake arguments. It may no
)
show_parser.set_defaults(func=show.show_command)
modules_parser = subparsers.add_parser("modules", help="Show modules")
modules_parser.add_argument(
"module_name",
help="name of the module",
type=str,
)
modules_parser.set_defaults(func=modules.command)
parser_backups = subparsers.add_parser(
"backups",
help="manage backups of clan machines",

View File

@ -0,0 +1,150 @@
import argparse
import json
import re
import tomllib
from dataclasses import dataclass
from pathlib import Path
from clan_cli.cmd import run_no_stdout
from clan_cli.errors import ClanCmdError, ClanError
from clan_cli.nix import nix_eval
from . import API
@dataclass
class Frontmatter:
description: str
categories: list[str] | None = None
def extract_frontmatter(readme_content: str, err_scope: str) -> tuple[Frontmatter, str]:
"""
Extracts TOML frontmatter from a README file content.
Parameters:
- readme_content (str): The content of the README file as a string.
Returns:
- str: The extracted frontmatter as a string.
- str: The content of the README file without the frontmatter.
Raises:
- ValueError: If the README does not contain valid frontmatter.
"""
# Pattern to match YAML frontmatter enclosed by triple-dashed lines
frontmatter_pattern = r"^---\s+(.*?)\s+---\s?+(.*)$"
# Search for the frontmatter using the pattern
match = re.search(frontmatter_pattern, readme_content, re.DOTALL)
# If a match is found, return the frontmatter content
match = re.search(frontmatter_pattern, readme_content, re.DOTALL)
# If a match is found, parse the TOML frontmatter and return both parts
if match:
frontmatter_raw, remaining_content = match.groups()
try:
# Parse the TOML frontmatter
frontmatter_parsed = tomllib.loads(frontmatter_raw)
except tomllib.TOMLDecodeError as e:
raise ClanError(
f"Error parsing TOML frontmatter: {e}",
description=f"Invalid TOML frontmatter. {err_scope}",
location="extract_frontmatter",
)
return Frontmatter(**frontmatter_parsed), remaining_content
# If no frontmatter is found, raise an error
raise ClanError(
"Invalid README: Frontmatter not found.",
location="extract_frontmatter",
description=f"{err_scope} does not contain valid frontmatter.",
)
def get_roles(module_path: str) -> None | list[str]:
roles_dir = Path(module_path) / "roles"
if not roles_dir.exists() or not roles_dir.is_dir():
return None
return [
role.stem # filename without .nix extension
for role in roles_dir.iterdir()
if role.is_file() and role.suffix == ".nix"
]
@dataclass
class ModuleInfo:
description: str
categories: list[str] | None
roles: list[str] | None
def get_modules(base_path: str) -> dict[str, str]:
cmd = nix_eval(
[
f"{base_path}#clanInternals.clanModules",
"--json",
]
)
try:
proc = run_no_stdout(cmd)
res = proc.stdout.strip()
except ClanCmdError:
raise ClanError(
"clanInternals might not have clanModules attributes",
location=f"list_modules {base_path}",
description="Evaluation failed on clanInternals.clanModules attribute",
)
modules: dict[str, str] = json.loads(res)
return modules
@API.register
def list_modules(base_path: str) -> list[str]:
"""
Show information about a module
"""
modules = get_modules(base_path)
return [m for m in modules.keys()]
@API.register
def show_module_info(base_path: str, module_name: str) -> ModuleInfo:
"""
Show information about a module
"""
modules = get_modules(base_path)
module_path = modules.get(module_name, None)
if not module_path:
raise ClanError(
"Module not found",
location=f"show_module_info {module_name}",
description="Module does not exist",
)
module_readme = Path(module_path) / "README.md"
if not module_readme.exists():
raise ClanError(
"Module not found",
location=f"show_module_info {module_name}",
description="Module does not exist or doesn't have any README.md file",
)
with open(module_readme) as f:
readme = f.read()
frontmatter, readme_content = extract_frontmatter(
readme, f"{module_path}/README.md"
)
return ModuleInfo(
description=frontmatter.description,
categories=frontmatter.categories,
roles=get_roles(module_path),
)
def command(args: argparse.Namespace) -> None:
res = show_module_info(args.flake, args.module_name)
print(res)