Merge pull request 'docs: add reference index pages' (#1781) from hsjobeki/clan-core:rearrange-docs-front into main

docs/.gitignore

@@ -1,3 +1,6 @@
-/site/reference
+/site/reference/clan-core
+/site/reference/clanModules
+/site/reference/nix-api/inventory.md
+/site/reference/cli
 /site/static/Roboto-Regular.ttf
 /site/static/FiraCode-VF.ttf

docs/mkdocs.yml

@@ -48,11 +48,13 @@ nav:
       - Mesh VPN: getting-started/mesh-vpn.md
       - Backup & Restore: getting-started/backups.md
       - Flake-parts: getting-started/flake-parts.md
-  - Concepts:
-      - Configuration: concepts/configuration.md
-      - Inventory: concepts/inventory.md
+  - Guides:
+      - guides/index.md
+      - Inventory: guides/inventory.md
   - Reference:
+      - reference/index.md
       - Clan Modules:
+          - reference/clanModules/index.md
           - reference/clanModules/borgbackup-static.md
           - reference/clanModules/borgbackup.md
           - reference/clanModules/deltachat.md
@@ -77,13 +79,13 @@ nav:
           - reference/clanModules/zerotier-static-peers.md
           - reference/clanModules/zt-tcp-relay.md
       - CLI:
+          - reference/cli/index.md
           - reference/cli/backups.md
           - reference/cli/config.md
           - reference/cli/facts.md
           - reference/cli/flakes.md
           - reference/cli/flash.md
           - reference/cli/history.md
-          - reference/cli/index.md
           - reference/cli/machines.md
           - reference/cli/secrets.md
           - reference/cli/show.md
@@ -98,6 +100,10 @@ nav:
           - reference/clan-core/state.md
           - reference/clan-core/deployment.md
           - reference/clan-core/networking.md
+      - Nix API:
+          - reference/nix-api/index.md
+          - buildClan: reference/nix-api/buildclan.md
+          - Inventory: reference/nix-api/inventory.md
   - Contributing: contributing/contributing.md
   - Blog:
       - blog/index.md

docs/nix/default.nix

@@ -2,6 +2,7 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  inventory-api-docs,
   asciinema-player-js,
   asciinema-player-css,
   roboto,
@@ -32,6 +33,7 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+    cp -af ${inventory-api-docs} ./site/reference/nix-api/inventory.md
 
     mkdir -p ./site/static/asciinema-player
     ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js

docs/nix/flake-module.nix

@@ -73,7 +73,7 @@
     in
     {
       devShells.docs = pkgs.callPackage ./shell.nix {
-        inherit (self'.packages) docs clan-cli-docs;
+        inherit (self'.packages) docs clan-cli-docs inventory-api-docs;
         inherit
           asciinema-player-js
           asciinema-player-css
@@ -83,7 +83,7 @@
       };
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
-          inherit (self'.packages) clan-cli-docs;
+          inherit (self'.packages) clan-cli-docs inventory-api-docs;
           inherit (inputs) nixpkgs;
           inherit module-docs;
           inherit asciinema-player-js;

docs/nix/scripts/renderOptions.py

@@ -220,6 +220,11 @@ def render_roles(roles: list[str] | None, module_name: str) -> str:
     return ""
 
 
+clan_modules_descr = """Clan modules are [NixOS modules](https://wiki.nixos.org/wiki/NixOS_modules) which have been enhanced with additional features provided by Clan, with certain option types restricted to enable configuration through a graphical interface.
+
+"""
+
+
 def produce_clan_modules_docs() -> None:
     if not CLAN_MODULES:
         raise ValueError(
@@ -245,6 +250,12 @@ def produce_clan_modules_docs() -> None:
     #     print(meta_map)
 
     # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... }
+
+    modules_index = "# Modules Overview\n\n"
+    modules_index += clan_modules_descr
+    modules_index += "## Overview\n\n"
+    modules_index += '<div class="grid cards" markdown>\n\n'
+
     for module_name, options_file in links.items():
         readme_file = Path(CLAN_CORE_PATH) / "clanModules" / module_name / "README.md"
         print(module_name, readme_file)
@@ -254,6 +265,8 @@ def produce_clan_modules_docs() -> None:
             frontmatter, readme_content = extract_frontmatter(readme, str(readme_file))
             print(frontmatter, readme_content)
 
+        modules_index += build_option_card(module_name, frontmatter)
+
         with open(Path(options_file) / "share/doc/nixos/options.json") as f:
             options: dict[str, dict[str, Any]] = json.load(f)
             print(f"Rendering options for {module_name}...")
@@ -282,6 +295,39 @@ def produce_clan_modules_docs() -> None:
             with open(outfile, "w") as of:
                 of.write(output)
 
+    modules_index += "</div>"
+    modules_index += "\n"
+    modules_outfile = Path(OUT) / "clanModules/index.md"
+
+    with open(modules_outfile, "w") as of:
+        of.write(modules_index)
+
+
+def build_option_card(module_name: str, frontmatter: Frontmatter) -> str:
+    """
+    Build the overview index card for each reference target option.
+    """
+
+    def indent_all(text: str, indent_size: int = 4) -> str:
+        """
+        Indent all lines in a string.
+        """
+        indent = " " * indent_size
+        lines = text.split("\n")
+        indented_text = indent + ("\n" + indent).join(lines)
+        return indented_text
+
+    def to_md_li(module_name: str, frontmatter: Frontmatter) -> str:
+        md_li = (
+            f"""-   **[{module_name}](./{"-".join(module_name.split(" "))}.md)**\n\n"""
+        )
+        md_li += f"""{indent_all("---", 4)}\n\n"""
+        fmd = f"\n{frontmatter.description.strip()}" if frontmatter.description else ""
+        md_li += f"""{indent_all(fmd, 4)}"""
+        return md_li
+
+    return f"{to_md_li(module_name, frontmatter)}\n\n"
+
 
 if __name__ == "__main__":
     produce_clan_core_docs()

docs/nix/shell.nix

@@ -3,6 +3,7 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  inventory-api-docs,
   asciinema-player-js,
   asciinema-player-css,
   roboto,
@@ -19,6 +20,8 @@ pkgs.mkShell {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+    cp -af ${inventory-api-docs} ./site/reference/nix-api/inventory.md
+
     chmod +w ./site/reference/*
 
     echo "Generated API documentation in './site/reference/' "

docs/site/concepts/configuration.md

@@ -1,56 +0,0 @@
-# Configuration
-
-## Introduction
-
-When managing machine configuration this can be done through many possible ways.
-Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming).
-
-clan currently offers the following methods to configure machines:
-
-!!! Success "Recommended for nix people"
-
-    - flake.nix (i.e. via `buildClan`)
-        - `machine` argument
-        - `inventory` argument
-
-    - machines/`machine_name`/configuration.nix (`autoincluded` if it exists)
-
-???+ Note "Used by CLI & UI"
-
-    - inventory.json
-    - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists)
-
-
-!!! Warning "Deprecated"
-
-    machines/`machine_name`/settings.json
-
-## BuildClan
-
-The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services.
-
-### Inputs
-
-`directory`
-: The directory containing the machines subdirectory
-
-`machines`
-: Allows to include machine-specific modules i.e. machines.${name} = { ... }
-
-`meta`
-: An optional set
-
-: `{ name :: string, icon :: string, description :: string }`
-
-`inventory`
-: Service set for easily configuring distributed services, such as backups
-
-: For more details see [Inventory](./inventory.md)
-
-`specialArgs`
-: Extra arguments to pass to nixosSystem i.e. useful to make self available
-
-`pkgsForSystem`
-: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system.
-  This improves performance, but all nipxkgs.* options will be ignored.
-  `(string -> pkgs )`

docs/site/getting-started/configure.md

@@ -1,5 +1,31 @@
 # Configuration - How to configure clan with your own machines
 
+Managing machine configurations can be done in the following ways:
+
+- writing `nix` expressions in a `flake.nix` file,
+- placing `autoincluded` files into your machine directory,
+- configuring everything in a simple UI (upcoming).
+
+Clan currently offers the following methods to configure machines:
+
+!!! Success "Recommended for nix people"
+
+    - flake.nix (i.e. via `buildClan`)
+        - `machine` argument
+        - `inventory` argument
+
+    - machines/`machine_name`/configuration.nix (`autoincluded` if it exists)
+
+???+ Note "Used by CLI & UI"
+
+    - inventory.json
+    - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists)
+
+
+!!! Warning "Deprecated"
+
+    machines/`machine_name`/settings.json
+
 ## Global configuration
 
 In the `flake.nix` file:

docs/site/guides/index.md

@@ -0,0 +1,5 @@
+# Guides
+
+Detailed guides into the following subtopics:
+
+- [Inventory](./inventory.md): Configuring Services across machine boundaries

docs/site/guides/inventory.md

@@ -2,31 +2,35 @@
 
 `Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries.
 
-## Meta
+See [Inventory API Documentation](../reference/nix-api/inventory.md)
+
+This guide will walk you through setting up a backup-service, where the inventory becomes useful.
+
+## Prerequisites Meta (optional)
 
 Metadata about the clan, will be displayed upfront in the upcomming clan-app, make sure to choose a unique name.
 
+Make sure to set `name` either via `inventory.meta` OR via `clan.meta`.
+
 ```{.nix hl_lines="3-8"}
 buildClan {
     inventory = {
         meta = {
-            # The following options are available
-            # name:        string # Required, name of the clan.
-            # description: null | string
-            # icon:        null | string
+            name = "Superclan"
+            description = "Awesome backups and family stuff"
         };
     };
 }
 ```
 
-## Machines
+## How to add machines
 
-Machines and a small pieve of their configuration can be added via `inventory.machines`.
+Machines can be added via `inventory.machines` OR via `buildClan` directly.
 
 !!! Note
     It doesn't matter where the machine gets introduced to buildClan - All delarations are valid, duplications are merged.
 
-    However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nixos configs.
+    However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nix code or nixos configs.
 
 In the following example `backup_server` is one machine - it may specify parts of its configuration in different places.
 
@@ -78,9 +82,10 @@ A module can be added to one or multiple machines via `Roles`. clan's `Role` int
 Each service can still be customized and configured according to the modules options.
 
 - Per instance configuration via `services.<serviceName>.<instanceName>.config`
+- Per role configuration via `services.<serviceName>.<instanceName>.roles.<roleName>.config`
 - Per machine configuration via `services.<serviceName>.<instanceName>.machines.<machineName>.config`
 
-### Configuration Examples
+### Setting up the Backup Service
 
 !!! Example "Borgbackup Example"
 
@@ -112,49 +117,33 @@ Each service can still be customized and configured according to the modules opt
     }
     ```
 
-!!! Example "Packages Example"
+### Scalabling the Backup
 
-    This example shows how to add `pkgs.firefox` via the inventory interface.
+The inventory allows machines to set **Tags**
 
-    ```{.nix hl_lines="8-11"}
-    buildClan {
-        inventory = {
-            machines = {
-                "sara" = {};
-                "jon" = {};
-            };
-            services = {
-                packages.set_1 = {
-                    roles.default.machines = [ "jon" "sara" ];
-                    # Packages is a configuration option of the "packages" clanModule
-                    config.packages = ["firefox"];
-                };
-            };
-        };
-    }
-    ```
-
-### Tags
-
-It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "default"`
+It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "client"`
 
 !!! Example "Tags Example"
 
-    ```{.nix hl_lines="5 8 13"}
+    ```{.nix hl_lines="9 12 17"}
     buildClan {
         inventory = {
             machines = {
-                "sara" = {
-                    tags = ["browsing"];
+                "backup_server" = {
+                    # Don't include any nixos config here
+                    # See inventory.Machines for available options
                 };
                 "jon" = {
-                    tags = ["browsing"];
+                    tags = [ "backup" ];
+                };
+                "sara" = {
+                    tags = [ "backup" ];
                 };
             };
             services = {
-                packages.set_1 = {
-                    roles.default.tags = [ "browsing" ];
-                    config.packages = ["firefox"];
+                borgbackup.instance_1 = {
+                    roles.client.tags = [ "backup" ];
+                    roles.server.machines = [ "backup_server" ];
                 };
             };
         };
@@ -164,9 +153,8 @@ It is possible to add services to multiple machines via tags. The service instan
 ### Multiple Service Instances
 
 !!! danger "Important"
-    Not all modules support multiple instances yet.
-
-Some modules have support for adding multiple instances of the same service in different roles or configurations.
+    Not all modules implement support for multiple instances yet.
+    Multiple instance usage could create complexity, refer to each modules documentation, for intended usage.
 
 !!! Example
 
@@ -194,9 +182,11 @@ Some modules have support for adding multiple instances of the same service in d
     }
     ```
 
-### Schema specification
+### API specification
 
-The complete schema specification can be retrieved via:
+**The complete schema specification is available [here](../reference/nix-api/inventory.md)**
+
+Or it can build anytime via:
 
 ```sh
 nix build git+https://git.clan.lol/clan/clan-core#inventory-schema

docs/site/reference/index.md

@@ -0,0 +1,8 @@
+# Documentation
+
+This section of the site contains information about the following topics:
+
+- How to use the [Clan CLI](./cli/index.md)
+- Available services and application [modules](./clanModules/index.md)
+- [Configuration options](./clan-core/index.md) controlling the essential features
+- Descriptions of the [Nix interfaces](./nix-api/index.md) for defining a Clan

docs/site/reference/nix-api/buildclan.md

@@ -0,0 +1,29 @@
+# buildClan
+
+The core [function](https://git.clan.lol/clan/clan-core/src/branch/main/lib/build-clan/default.nix) that produces a Clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services.
+
+## Inputs
+
+`directory`
+: The directory containing the machines subdirectory
+
+`machines`
+: Allows to include machine-specific modules i.e. machines.${name} = { ... }
+
+`meta`
+: An optional set
+
+: `{ name :: string, icon :: string, description :: string }`
+
+`inventory`
+: Service set for easily configuring distributed services, such as backups
+
+: For more details see [Inventory](./inventory.md)
+
+`specialArgs`
+: Extra arguments to pass to nixosSystem i.e. useful to make self available
+
+`pkgsForSystem`
+: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system.
+  This improves performance, but all nipxkgs.* options will be ignored.
+  `(string -> pkgs )`

docs/site/reference/nix-api/index.md

@@ -0,0 +1,6 @@
+# Nix API Overview
+
+There are two top-level components of the Nix API, which together allow for the declarative definition of a Clan:
+
+- the [Inventory](./inventory.md), a structure representing the machines, services, custom configurations, and other data that constitute a Clan, and
+- the [`buildClan`](./buildclan.md) function, which constructs a Clan from an Inventory definition.

docs/site/reference/nix-api/inventory.md

@@ -0,0 +1,185 @@
+# Inventory API
+
+*Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries.
+
+The following is a specification of the inventory in [cuelang](https://cuelang.org/) format.
+
+```cue
+package compose
+
+#Root: {
+	@jsonschema(schema="http://json-schema.org/draft-07/schema#")
+	machines?: [string]: {
+		deploy?: {
+			// Configuration for the deployment of the machine
+			targetHost: null | string
+		}
+		description: null | string
+		icon:        null | string
+		name:        string
+		system:      null | string
+		tags: [...string]
+	}
+	meta?: {
+		description: null | string
+		icon:        null | string
+		name:        string
+	}
+	services?: {
+		borgbackup?: [string]: {
+			// borgbackup-config
+			config?: {
+				// destinations where the machine should be backuped to
+				destinations?: {
+					[string]: {
+						// the name of the backup job
+						name: string
+
+						// the borgbackup repository to backup to
+						repo: string
+					}
+				}
+			} | *{
+				...
+			}
+			machines?: [string]: {
+				// borgbackup-config
+				config?: {
+					// destinations where the machine should be backuped to
+					destinations?: {
+						[string]: {
+							// the name of the backup job
+							name: string
+
+							// the borgbackup repository to backup to
+							repo: string
+						}
+					}
+				} | *{
+					...
+				}
+				imports: [...string]
+			}
+			meta?: {
+				description: null | string
+				icon:        null | string
+				name:        string
+			}
+			roles?: {
+				client?: {
+					// borgbackup-config
+					config?: {
+						// destinations where the machine should be backuped to
+						destinations?: {
+							[string]: {
+								// the name of the backup job
+								name: string
+
+								// the borgbackup repository to backup to
+								repo: string
+							}
+						}
+					} | *{
+						...
+					}
+					imports: [...string]
+					machines: [...string]
+					tags: [...string]
+				}
+				server?: {
+					// borgbackup-config
+					config?: {
+						// destinations where the machine should be backuped to
+						destinations?: {
+							[string]: {
+								// the name of the backup job
+								name: string
+
+								// the borgbackup repository to backup to
+								repo: string
+							}
+						}
+					} | *{
+						...
+					}
+					imports: [...string]
+					machines: [...string]
+					tags: [...string]
+				}
+			}
+		}
+		packages?: [string]: {
+			// packages-config
+			config?: {
+				// The packages to install on the machine
+				packages: [...string]
+			} | *{
+				...
+			}
+			machines?: [string]: {
+				// packages-config
+				config?: {
+					// The packages to install on the machine
+					packages: [...string]
+				} | *{
+					...
+				}
+				imports: [...string]
+			}
+			meta?: {
+				description: null | string
+				icon:        null | string
+				name:        string
+			}
+			roles?: default?: {
+				// packages-config
+				config?: {
+					// The packages to install on the machine
+					packages: [...string]
+				} | *{
+					...
+				}
+				imports: [...string]
+				machines: [...string]
+				tags: [...string]
+			}
+		}
+		"single-disk"?: [string]: {
+			// single-disk-config
+			config?: {
+				// The primary disk device to install the system on
+				device: null | string
+			} | *{
+				...
+			}
+			machines?: [string]: {
+				// single-disk-config
+				config?: {
+					// The primary disk device to install the system on
+					device: null | string
+				} | *{
+					...
+				}
+				imports: [...string]
+			}
+			meta?: {
+				description: null | string
+				icon:        null | string
+				name:        string
+			}
+			roles?: default?: {
+				// single-disk-config
+				config?: {
+					// The primary disk device to install the system on
+					device: null | string
+				} | *{
+					...
+				}
+				imports: [...string]
+				machines: [...string]
+				tags: [...string]
+			}
+		}
+	}
+}
+```

lib/inventory/flake-module.nix

@@ -46,6 +46,29 @@ in
       };
 
       # Inventory schema with concrete module implementations
+      packages.inventory-api-docs = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema";
+        buildInputs = [ ];
+        src = ./.;
+        buildPhase = ''
+          cat <<EOF > "$out"
+          # Inventory API
+
+          *Inventory* is an abstract service layer for consistently configuring distributed services across machine boundaries.
+
+          The following is a specification of the inventory in [cuelang](https://cuelang.org/) format.
+
+          \`\`\`cue
+          EOF
+
+          cat ${self'.packages.inventory-schema-pretty}/schema.cue >> $out
+
+          cat <<EOF >> $out
+          \`\`\`
+          EOF
+        '';
+      };
+
       packages.inventory-schema = pkgs.stdenv.mkDerivation {
         name = "inventory-schema";
         buildInputs = [ pkgs.cue ];