diff --git a/docs/.envrc b/docs/.envrc index 293e9919..277af678 100644 --- a/docs/.envrc +++ b/docs/.envrc @@ -1,6 +1,6 @@ source_up -watch_file nix/flake-module.nix nix/shell.nix nix/default.nix +watch_file $(find ./nix -name "*.nix" -printf '"%p" ') # Because we depend on nixpkgs sources, uploading to builders takes a long time use flake .#docs --builders '' diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..db65b3a9 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +/site/reference \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f2456015..800dc41f 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -10,6 +10,11 @@ validation: unrecognized_links: warn markdown_extensions: + - attr_list + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.tasklist: custom_checkbox: true - pymdownx.superfences @@ -40,6 +45,28 @@ nav: - Backup & Restore: getting-started/backups.md - Flake-parts: getting-started/flake-parts.md - Templates: templates/index.md + - Reference: + - ClanCore: reference/clan-core.md + - ClanModules: + - reference/borgbackup.md + - reference/deltachat.md + - reference/diskLayouts.md + - reference/ergochat.md + - reference/graphical.md + - reference/localbackup.md + - reference/localsend.md + - reference/matrix-synapse.md + - reference/moonlight.md + - reference/root-password.md + - reference/sshd.md + - reference/sunshine.md + - reference/syncthing.md + - reference/thelounge.md + - reference/vm-user.md + - reference/waypipe.md + - reference/xfce-vm.md + - reference/xfce.md + - reference/zt-tcp-relay.md - Contributing: contributing/contributing.md docs_dir: site diff --git a/docs/modules/default.nix b/docs/modules/default.nix deleted file mode 100644 index 4b8c5168..00000000 --- a/docs/modules/default.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ - self, - lib, - inputs, - ... -}: -{ - imports = [ ./zola-pages.nix ]; - - perSystem = - { pkgs, ... }: - let - - allNixosModules = (import "${inputs.nixpkgs}/nixos/modules/module-list.nix") ++ [ - "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix" - { nixpkgs.hostPlatform = "x86_64-linux"; } - ]; - - clanCoreNixosModules = [ - self.nixosModules.clanCore - { clanCore.clanDir = ./.; } - ] ++ allNixosModules; - - # TODO: optimally we would not have to evaluate all nixos modules for every page - # but some of our module options secretly depend on nixos modules. - # We would have to get rid of these implicit dependencies and make them explicit - clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; }; - - # using extendModules here instead of re-evaluating nixos every time - # improves eval performance slightly (10%) - options = modules: (clanCoreNixos.extendModules { inherit modules; }).options; - - docs = - options: - pkgs.nixosOptionsDoc { - options = options; - warningsAreErrors = false; - # transform each option so that the declaration link points to git.clan.lol - # and not to the /nix/store - transformOptions = - opt: - opt - // { - declarations = lib.forEach opt.declarations ( - decl: - if lib.hasPrefix "${self}" decl then - let - subpath = lib.removePrefix "${self}" decl; - in - { - url = "https://git.clan.lol/clan/clan-core/src/branch/main/" + subpath; - name = subpath; - } - else - decl - ); - }; - }; - - outputsFor = name: docs: { packages."docs-md-${name}" = docs.optionsCommonMark; }; - - clanModulesPages = lib.flip lib.mapAttrsToList self.clanModules ( - name: module: outputsFor "module-${name}" (docs ((options [ module ]).clan.${name} or { })) - ); - in - { - imports = clanModulesPages ++ [ - # renders all clanCore options in a single page - (outputsFor "core-options" (docs (options [ ]).clanCore)) - ]; - }; -} diff --git a/docs/modules/zola-pages.nix b/docs/modules/zola-pages.nix deleted file mode 100644 index 7be72b0e..00000000 --- a/docs/modules/zola-pages.nix +++ /dev/null @@ -1,88 +0,0 @@ -{ - perSystem = - { - lib, - pkgs, - self', - ... - }: - let - - getMdPages = - prefix: - let - mdDocs' = lib.filterAttrs (name: _: lib.hasPrefix prefix name) self'.packages; - mdDocs = lib.mapAttrs' (name: pkg: lib.nameValuePair (lib.removePrefix prefix name) pkg) mdDocs'; - in - if mdDocs != { } then - mdDocs - else - throw '' - Error: no markdown files found in clan-core.packages' with prefix "${prefix}" - ''; - - makeZolaIndexMd = - title: weight: - pkgs.writeText "_index.md" '' - +++ - title = "${title}" - template = "docs/section.html" - weight = ${toString weight} - sort_by = "title" - draft = false - +++ - ''; - - makeZolaPages = - { - sectionTitle, - files, - makeIntro ? _name: "", - weight ? 9999, - }: - pkgs.runCommand "zola-pages-clan-core" { } '' - mkdir $out - # create new section via _index.md - cp ${makeZolaIndexMd sectionTitle weight} $out/_index.md - # generate zola compatible md files for each nixos options md - ${lib.concatStringsSep "\n" ( - lib.flip lib.mapAttrsToList files ( - name: md: '' - # generate header for zola with title, template, weight - title="${name}" - echo -e "+++\ntitle = \"$title\"\ntemplate = \"docs/page.html\"\n+++" > "$out/${name}.md" - cat <> "$out/${name}.md" - ${makeIntro name} - EOF - # append everything from the nixpkgs generated md file - cat "${md}" >> "$out/${name}.md" - '' - ) - )} - ''; - in - { - packages.docs-zola-pages-core = makeZolaPages { - sectionTitle = "cLAN Core Reference"; - files = getMdPages "docs-md-core-"; - weight = 20; - }; - - packages.docs-zola-pages-modules = makeZolaPages { - sectionTitle = "cLAN Modules Reference"; - files = getMdPages "docs-md-module-"; - weight = 25; - makeIntro = name: '' - To use this module, import it like this: - - \`\`\`nix - {config, lib, inputs, ...}: { - imports = [inputs.clan-core.clanModules.${name}]; - # ... - } - \`\`\` - - ''; - }; - }; -} diff --git a/docs/nix/default.nix b/docs/nix/default.nix index 2be84c34..f3f16ca9 100644 --- a/docs/nix/default.nix +++ b/docs/nix/default.nix @@ -1,4 +1,4 @@ -{ pkgs, ... }: +{ pkgs, module-docs, ... }: pkgs.stdenv.mkDerivation { name = "clan-documentation"; @@ -10,6 +10,11 @@ pkgs.stdenv.mkDerivation { mkdocs mkdocs-material ]); + configurePhase = '' + mkdir -p ./site/reference + cp -af ${module-docs}/* ./site/reference/ + + ''; buildPhase = '' mkdocs build --strict diff --git a/docs/nix/flake-module.nix b/docs/nix/flake-module.nix index 13e5990a..67a8b505 100644 --- a/docs/nix/flake-module.nix +++ b/docs/nix/flake-module.nix @@ -1,4 +1,4 @@ -{ inputs, ... }: +{ inputs, self, ... }: { perSystem = { @@ -7,11 +7,61 @@ pkgs, ... }: + let + # Simply evaluated options (JSON) + # { clanCore = «derivation JSON»; clanModules = { ${name} = «derivation JSON» }; } + jsonDocs = import ./get-module-docs.nix { + inherit (inputs) nixpkgs; + inherit pkgs; + inherit (self.nixosModules) clanCore; + inherit (self) clanModules; + }; + + clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules); + + # Simply evaluated options (JSON) + renderOptions = + pkgs.runCommand "renderOptions.py" + { + # TODO: ruff does not splice properly in nativeBuildInputs + depsBuildBuild = [ pkgs.ruff ]; + nativeBuildInputs = [ + pkgs.python3 + pkgs.mypy + ]; + } + '' + install ${./scripts/renderOptions.py} $out + patchShebangs --build $out + + ruff format --check --diff $out + ruff --line-length 88 $out + mypy --strict $out + ''; + + module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } '' + export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json + # A file that contains the links to all clanModule docs + export CLAN_MODULES=${clanModulesFileInfo} + + mkdir $out + + # The python script will place mkDocs files in the output directory + python3 ${renderOptions} + ''; + in { - devShells.docs = pkgs.callPackage ./shell.nix { inherit (self'.packages) docs; }; + devShells.docs = pkgs.callPackage ./shell.nix { + inherit (self'.packages) docs; + inherit module-docs; + }; packages = { - docs = pkgs.python3.pkgs.callPackage ./default.nix { inherit (inputs) nixpkgs; }; + docs = pkgs.python3.pkgs.callPackage ./default.nix { + inherit (inputs) nixpkgs; + inherit module-docs; + }; deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; }; + inherit module-docs; }; }; } diff --git a/docs/nix/get-module-docs.nix b/docs/nix/get-module-docs.nix new file mode 100644 index 00000000..1bb9d333 --- /dev/null +++ b/docs/nix/get-module-docs.nix @@ -0,0 +1,45 @@ +{ + nixpkgs, + pkgs, + clanCore, + clanModules, +}: +let + allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [ + "${nixpkgs}/nixos/modules/misc/assertions.nix" + { nixpkgs.hostPlatform = "x86_64-linux"; } + ]; + + clanCoreNixosModules = [ + clanCore + { clanCore.clanDir = ./.; } + ] ++ allNixosModules; + + # TODO: optimally we would not have to evaluate all nixos modules for every page + # but some of our module options secretly depend on nixos modules. + # We would have to get rid of these implicit dependencies and make them explicit + clanCoreNixos = pkgs.nixos { imports = clanCoreNixosModules; }; + + # using extendModules here instead of re-evaluating nixos every time + # improves eval performance slightly (10%) + getOptions = modules: (clanCoreNixos.extendModules { inherit modules; }).options; + + evalDocs = + options: + pkgs.nixosOptionsDoc { + options = options; + warningsAreErrors = false; + }; + + # clanModules docs + clanModulesDocs = builtins.mapAttrs ( + name: module: (evalDocs ((getOptions [ module ]).clan.${name} or { })).optionsJSON + ) clanModules; + + # clanCore docs + clanCoreDocs = (evalDocs (getOptions [ ]).clanCore).optionsJSON; +in +{ + clanCore = clanCoreDocs; + clanModules = clanModulesDocs; +} diff --git a/docs/nix/scripts/renderOptions.py b/docs/nix/scripts/renderOptions.py new file mode 100644 index 00000000..b44b7860 --- /dev/null +++ b/docs/nix/scripts/renderOptions.py @@ -0,0 +1,159 @@ +# Options are available in the following format: +# https://github.com/nixos/nixpkgs/blob/master/nixos/lib/make-options-doc/default.nix +# +# ```json +# { +# ... +# "fileSystems..options": { +# "declarations": ["nixos/modules/tasks/filesystems.nix"], +# "default": { +# "_type": "literalExpression", +# "text": "[\n \"defaults\"\n]" +# }, +# "description": "Options used to mount the file system.", +# "example": { +# "_type": "literalExpression", +# "text": "[\n \"data=journal\"\n]" +# }, +# "loc": ["fileSystems", "", "options"], +# "readOnly": false, +# "type": "non-empty (list of string (with check: non-empty))" +# "relatedPackages": "- [`pkgs.tmux`](\n https://search.nixos.org/packages?show=tmux&sort=relevance&query=tmux\n )\n", +# } +# } +# ``` + +import json +import os +from pathlib import Path +from typing import Any + +# Get environment variables +CLAN_CORE = os.getenv("CLAN_CORE") +CLAN_MODULES = os.environ.get("CLAN_MODULES") +OUT = os.environ.get("out") + + +def sanitize(text: str) -> str: + return text.replace(">", "\\>") + + +def replace_store_path(text: str) -> Path: + res = text + if text.startswith("/nix/store/"): + res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str( + Path(*Path(text).parts[4:]) + ) + return Path(res) + + +def render_option(name: str, option: dict[str, Any]) -> str: + read_only = option.get("readOnly") + + res = f""" +## {sanitize(name)} +{"Readonly" if read_only else ""} +{option.get("description", "No description available.")} + +**Type**: `{option["type"]}` + +""" + if option.get("default"): + res += f""" +**Default**: + +```nix +{option["default"]["text"] if option.get("default") else "No default set."} +``` + """ + example = option.get("example", {}).get("text") + if example: + res += f""" + +??? example + + ```nix + {example} + ``` +""" + if option.get("relatedPackages"): + res += f""" +### Related Packages + +{option["relatedPackages"]} +""" + + decls = option.get("declarations", []) + source_path = replace_store_path(decls[0]) + res += f""" +:simple-git: [{source_path.name}]({source_path}) +""" + res += "\n" + + return res + + +def module_header(module_name: str) -> str: + return f"""# {module_name} + +To use this module, import it like this: + +```nix + {{config, lib, inputs, ...}}: {{ + imports = [ inputs.clan-core.clanModules.{module_name} ]; + # ... + }} +``` +""" + + +def produce_clan_core_docs() -> None: + if not CLAN_CORE: + raise ValueError( + f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}" + ) + + if not OUT: + raise ValueError(f"Environment variables are not set correctly: $out={OUT}") + + with open(CLAN_CORE) as f: + options: dict[str, dict[str, Any]] = json.load(f) + module_name = "clan-core" + output = module_header(module_name) + for option_name, info in options.items(): + output += render_option(option_name, info) + + outfile = Path(OUT) / f"{module_name}.md" + with open(outfile, "w") as of: + of.write(output) + + +def produce_clan_modules_docs() -> None: + if not CLAN_MODULES: + raise ValueError( + f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}" + ) + + if not OUT: + raise ValueError(f"Environment variables are not set correctly: $out={OUT}") + + with open(CLAN_MODULES) as f: + links: dict[str, str] = json.load(f) + + # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... } + for module_name, options_file in links.items(): + with open(Path(options_file) / "share/doc/nixos/options.json") as f: + options: dict[str, dict[str, Any]] = json.load(f) + print(f"Rendering options for {module_name}...") + output = module_header(module_name) + for option_name, info in options.items(): + output += render_option(option_name, info) + + outfile = Path(OUT) / f"{module_name}.md" + with open(outfile, "w") as of: + of.write(output) + + +if __name__ == "__main__": + produce_clan_core_docs() + produce_clan_modules_docs() diff --git a/docs/nix/shell.nix b/docs/nix/shell.nix index a4728b7f..49658f30 100644 --- a/docs/nix/shell.nix +++ b/docs/nix/shell.nix @@ -1 +1,16 @@ -{ docs, pkgs, ... }: pkgs.mkShell { inputsFrom = [ docs ]; } +{ + docs, + pkgs, + module-docs, + ... +}: +pkgs.mkShell { + inputsFrom = [ docs ]; + shellHook = '' + mkdir -p ./site/reference + cp -af ${module-docs}/* ./site/reference/ + chmod +w ./site/reference/* + + echo "Generated API documentation in './site/reference/' " + ''; +}