Merge pull request 'zerotier-static-peers: use correct exclusion source' (#1548) from kenji/clan-core:modules/fix/static into main

Reviewed-on: clan/clan-core#1548

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contributing to cLAN
+# Contributing to Clan
 
 ## Live-reloading documentation
 

README.md

@@ -1,6 +1,6 @@
 # Clan Core Repository
 
-Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the cLAN project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
+Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
 ## Why Clan?
 

checks/backups/flake-module.nix

@@ -145,14 +145,14 @@
             machine.succeed("echo testing > /var/test-backups/somefile")
 
             # create
-            machine.succeed("clan --debug --flake ${self} backups create test-backup")
+            machine.succeed("clan backups create --debug --flake ${self} test-backup")
             machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2")
             machine.succeed("test -f /run/mount-external-disk")
             machine.succeed("test -f /run/unmount-external-disk")
 
             # list
             backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
-            out = machine.succeed("clan --debug --flake ${self} backups list test-backup").strip()
+            out = machine.succeed("clan backups list --debug --flake ${self} test-backup").strip()
             print(out)
             assert backup_id in out, f"backup {backup_id} not found in {out}"
             localbackup_id = "hdd::/mnt/external-disk/snapshot.0"
@@ -160,14 +160,14 @@
 
             ## borgbackup restore
             machine.succeed("rm -f /var/test-backups/somefile")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
+            machine.succeed(f"clan backups restore --debug --flake ${self} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
 
             ## localbackup restore
             machine.succeed("rm -f /var/test-backups/somefile /var/test-service/{pre,post}-restore-command")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup localbackup '{localbackup_id}' >&2")
+            machine.succeed(f"clan backups restore --debug --flake ${self} test-backup localbackup '{localbackup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")

checks/flash/flake-module.nix

@@ -1,33 +1,50 @@
-{ ... }:
+{ self, ... }:
 {
   perSystem =
-    { ... }:
     {
-      # checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
-      #   flash = (import ../lib/test-base.nix) {
-      #     name = "flash";
-      #     nodes.target = {
-      #       virtualisation.emptyDiskImages = [ 4096 ];
-      #       virtualisation.memorySize = 3000;
-      #       environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
-      #       environment.etc."install-closure".source = "${closureInfo}/store-paths"; 
+      nodes,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      dependencies = [
+        pkgs.disko
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.toplevel
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.diskoScript
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.diskoScript.drvPath
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.clan.deployment.file
+      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+    in
+    {
+      # Currently disabled...
+      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
+        flash = (import ../lib/test-base.nix) {
+          name = "flash";
+          nodes.target = {
+            virtualisation.emptyDiskImages = [ 4096 ];
+            virtualisation.memorySize = 3000;
+            environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+            environment.etc."install-closure".source = "${closureInfo}/store-paths";
 
-      #       nix.settings = {
-      #         substituters = lib.mkForce [ ];
-      #         hashed-mirrors = null;
-      #         connect-timeout = lib.mkForce 3;
-      #         flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-      #         experimental-features = [
-      #           "nix-command"
-      #           "flakes"
-      #         ];
-      #       };
-      #     };
-      #     testScript = ''
-      #       start_all()
-      #       machine.succeed("clan --debug --flake ${../..} flash --yes --disk main /dev/vdb test_install_machine")
-      #     '';
-      #   } { inherit pkgs self; };
-      # };
+            nix.settings = {
+              substituters = lib.mkForce [ ];
+              hashed-mirrors = null;
+              connect-timeout = lib.mkForce 3;
+              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+              experimental-features = [
+                "nix-command"
+                "flakes"
+              ];
+            };
+          };
+          testScript = ''
+            start_all()
+
+            machine.succeed("clan flash --debug --flake ${../..} --yes --disk main /dev/vdb test_install_machine")
+          '';
+        } { inherit pkgs self; };
+      };
     };
 }

checks/installation/flake-module.nix

@@ -2,8 +2,8 @@
 {
   clan.machines.test_install_machine = {
     clan.networking.targetHost = "test_install_machine";
-    fileSystems."/".device = lib.mkDefault "/dev/null";
-    boot.loader.grub.device = lib.mkDefault "/dev/null";
+    fileSystems."/".device = lib.mkDefault "/dev/vdb";
+    boot.loader.grub.device = lib.mkDefault "/dev/vdb";
 
     imports = [ self.nixosModules.test_install_machine ];
   };
@@ -98,7 +98,7 @@
             client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519")
             client.wait_until_succeeds("ssh -o StrictHostKeyChecking=accept-new -v root@target hostname")
 
-            client.succeed("clan --debug --flake ${../..} machines install --yes test_install_machine root@target >&2")
+            client.succeed("clan machines install --debug --flake ${../..} --yes test_install_machine root@target >&2")
             try:
               target.shutdown()
             except BrokenPipeError:

checks/lib/container-test.nix

@@ -10,6 +10,7 @@ in
     hostPkgs = pkgs;
     # speed-up evaluation
     defaults = {
+      nix.package = pkgs.nixVersions.latest;
       documentation.enable = lib.mkDefault false;
       boot.isContainer = true;
 

checks/lib/test-base.nix

@@ -10,6 +10,7 @@ in
   defaults = {
     documentation.enable = lib.mkDefault false;
     nix.settings.min-free = 0;
+    nix.package = pkgs.nixVersions.latest;
   };
 
   # to accept external dependencies such as disko

clanModules/localsend/default.nix

@@ -9,7 +9,7 @@
   # - cli frontend: https://github.com/localsend/localsend/issues/11
   # - ipv6 support: https://github.com/localsend/localsend/issues/549
   options.clan.localsend = {
-    enable = lib.mkEnableOption (lib.mdDoc "enable the localsend module");
+    enable = lib.mkEnableOption "enable the localsend module";
     defaultLocation = lib.mkOption {
       type = lib.types.str;
       description = "The default download location";

clanModules/static-hosts/default.nix

@@ -3,9 +3,15 @@
   options.clan.static-hosts = {
     excludeHosts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = [ config.clanCore.machineName ];
+      default =
+        if config.clan.static-hosts.topLevelDomain != "" then [ ] else [ config.clanCore.machineName ];
       description = "Hosts that should be excluded";
     };
+    topLevelDomain = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = "Top level domain to reach hosts";
+    };
   };
 
   config.networking.hosts =
@@ -24,7 +30,15 @@
         let
           path = zerotierIpMachinePath machine;
         in
-        if builtins.pathExists path then lib.nameValuePair (builtins.readFile path) [ machine ] else null
+        if builtins.pathExists path then
+          lib.nameValuePair (builtins.readFile path) (
+            if (config.clan.static-hosts.topLevelDomain == "") then
+              [ machine ]
+            else
+              [ "${machine}.${config.clan.static-hosts.topLevelDomain}" ]
+          )
+        else
+          null
       ) filteredMachines
     );
 }

clanModules/zerotier-static-peers/default.nix

@@ -20,7 +20,7 @@ let
     if builtins.pathExists fullPath then builtins.readFile fullPath else null
   ) machines;
   networkIds = lib.filter (machine: machine != null) networkIdsUnchecked;
-  networkId = builtins.elemAt networkIds 0;
+  networkId = if builtins.length networkIds == 0 then null else builtins.elemAt networkIds 0;
 in
 #TODO:trace on multiple found network-ids
 #TODO:trace on no single found networkId
@@ -38,7 +38,7 @@ in
       machines = builtins.readDir machineDir;
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
       filteredMachines = lib.filterAttrs (
-        name: _: !(lib.elem name config.clan.static-hosts.excludeHosts)
+        name: _: !(lib.elem name config.clan.zerotier-static-peers.excludeHosts)
       ) machines;
       hosts = lib.mapAttrsToList (host: _: host) (
         lib.mapAttrs' (

devShell.nix

@@ -1,3 +1,4 @@
+{ ... }:
 {
   perSystem =
     {
@@ -26,7 +27,8 @@
         packages = [
           select-shell
           pkgs.tea
-          pkgs.nix
+          # Better error messages than nix 2.18
+          pkgs.nixVersions.latest
           self'.packages.tea-create-pr
           self'.packages.merge-after-ci
           self'.packages.pending-reviews
@@ -34,9 +36,6 @@
           config.treefmt.build.wrapper
         ];
         shellHook = ''
-          # no longer used
-          rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
-
           echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
         '';
       };

docs/.gitignore

@@ -1 +1,3 @@
-/site/reference
+/site/reference
+/site/static/Roboto-Regular.ttf
+/site/static/FiraCode-VF.ttf

docs/CONTRIBUTING.md

@@ -15,92 +15,124 @@ Let's get your development environment up and running:
 
 1. **Install Nix Package Manager**:
 
-   - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
-     ```bash
-     curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
-     ```
+      - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
+        ```bash
+        curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
+        ```
 
 2. **Install direnv**:
 
-   - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command:
-     ```bash
-     curl -sfL https://direnv.net/install.sh | bash
-     ```
+      - To automatically setup a devshell on entering the directory
+        ```bash
+        nix profile install nixpkgs#nix-direnv-flakes
+        ```
 
 3. **Add direnv to your shell**:
 
-   - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
-     You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
+      - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
+        You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
 
-   ```bash
-   echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
-   ```
-
-4. **Clone the Repository and Navigate**:
-
-   - Clone this repository and navigate to it.
-
-5. **Allow .envrc**:
-
-   - When you enter the directory, you'll receive an error message like this:
-     ```bash
-     direnv: error .envrc is blocked. Run `direnv allow` to approve its content
-     ```
-   - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
-
-# Setting Up Your Git Workflow
-
-Let's set up your Git workflow to collaborate effectively:
-
-1. **Register Your Gitea Account Locally**:
-
-   - Execute the following command to add your Gitea account locally:
-     ```bash
-     tea login add
-     ```
-   - Fill out the prompt as follows:
-     - URL of Gitea instance: `https://git.clan.lol`
-     - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171`
-     - Do you have an access token? No
-     - Username: YourUsername
-     - Password: YourPassword
-     - Set Optional settings: No
-
-2. **Git Workflow**:
-
-   1. Add your changes to Git using `git add <file1> <file2>`.
-   2. Run `nix fmt` to lint your files.
-   3. Commit your changes with a descriptive message: `git commit -a -m "My descriptive commit message"`.
-   4. Make sure your branch has the latest changes from upstream by executing:
       ```bash
-      git fetch && git rebase origin/main --autostash
+      echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
       ```
-   5. Use `git status` to check for merge conflicts.
-   6. If conflicts exist, resolve them. Here's a tutorial for resolving conflicts in [VSCode](https://code.visualstudio.com/docs/sourcecontrol/overview#_merge-conflicts).
-   7. After resolving conflicts, execute `git merge --continue` and repeat step 5 until there are no conflicts.
 
-3. **Create a Pull Request**:
+4. **Create a Gitea Account**:
+      - Register an account on https://git.clan.lol
+      - Fork the [clan-core](https://git.clan.lol/clan/clan-core) repository
+      - Clone the repository and navigate to it
+      - Add a new remote called upstream:
+         ```bash
+         git remote add upstream gitea@git.clan.lol:clan/clan-core.git
+         ```
 
-   - To automatically open a pull request that gets merged if all tests pass, execute:
-     ```bash
-     merge-after-ci
-     ```
+5. **Register Your Gitea Account Locally**:
 
-4. **Review Your Pull Request**:
+      - Execute the following command to add your Gitea account locally:
+        ```bash
+        tea login add
+        ```
+      - Fill out the prompt as follows:
+        - URL of Gitea instance: `https://git.clan.lol`
+        - Name of new Login [git.clan.lol]:
+        - Do you have an access token? No
+        - Username: YourUsername
+        - Password: YourPassword
+        - Set Optional settings: No
 
-   - Visit https://git.clan.lol and go to the project page. Check under "Pull Requests" for any issues with your pull request.
 
-5. **Push Your Changes**:
-   - If there are issues, fix them and redo step 2. Afterward, execute:
-     ```bash
-     git push origin HEAD:YourUsername-main
-     ```
-   - This will directly push to your open pull request.
+6. **Allow .envrc**:
+
+      - When you enter the directory, you'll receive an error message like this:
+        ```bash
+        direnv: error .envrc is blocked. Run `direnv allow` to approve its content
+        ```
+      - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
+
+7. **(Optional) Install Git Hooks**:
+      - To syntax check your code you can run:
+         ```bash
+         nix fmt
+         ```
+      - To make this automatic install the git hooks
+         ```bash
+         ./scripts/pre-commit
+         ```
+
+8. **Open a Pull Request**:
+      - To automatically open up a pull request you can use our tool called:
+      ```
+      merge-after-ci --reviewers Mic92 Lassulus Qubasa
+      ```
 
 # Debugging
 
 Here are some methods for debugging and testing the clan-cli:
 
+## See all possible packages and tests
+
+To quickly show all possible packages and tests execute:
+
+```bash
+nix flake show --system no-eval
+```
+
+Under `checks` you will find all tests that are executed in our CI. Under `packages` you find all our projects.
+
+```
+git+file:///home/lhebendanz/Projects/clan-core
+├───apps
+│   └───x86_64-linux
+│       ├───install-vm: app
+│       └───install-vm-nogui: app
+├───checks
+│   └───x86_64-linux
+│       ├───borgbackup omitted (use '--all-systems' to show)
+│       ├───check-for-breakpoints omitted (use '--all-systems' to show)
+│       ├───clan-dep-age omitted (use '--all-systems' to show)
+│       ├───clan-dep-bash omitted (use '--all-systems' to show)
+│       ├───clan-dep-e2fsprogs omitted (use '--all-systems' to show)
+│       ├───clan-dep-fakeroot omitted (use '--all-systems' to show)
+│       ├───clan-dep-git omitted (use '--all-systems' to show)
+│       ├───clan-dep-nix omitted (use '--all-systems' to show)
+│       ├───clan-dep-openssh omitted (use '--all-systems' to show)
+│       ├───"clan-dep-python3.11-mypy" omitted (use '--all-systems' to show)
+├───packages
+│   └───x86_64-linux
+│       ├───clan-cli omitted (use '--all-systems' to show)
+│       ├───clan-cli-docs omitted (use '--all-systems' to show)
+│       ├───clan-ts-api omitted (use '--all-systems' to show)
+│       ├───clan-vm-manager omitted (use '--all-systems' to show)
+│       ├───default omitted (use '--all-systems' to show)
+│       ├───deploy-docs omitted (use '--all-systems' to show)
+│       ├───docs omitted (use '--all-systems' to show)
+│       ├───editor omitted (use '--all-systems' to show)
+└───templates
+    ├───default: template: Initialize a new clan flake
+    └───new-clan: template: Initialize a new clan flake
+```
+
+You can execute every test separately by following the tree path `nix build .#checks.x86_64-linux.clan-pytest` for example.
+
 ## Test Locally in Devshell with Breakpoints
 
 To test the cli locally in a development environment and set breakpoints for debugging, follow these steps:
@@ -150,12 +182,14 @@ If you need to inspect the Nix sandbox while running tests, follow these steps:
 2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example:
 
    ```bash
-   cntr exec -w your_sandbox_name
    psgrep -a -x your_python_process_name
+   cntr attach <container id, container name or process id>
    ```
 
+Or you can also use the [nix breakpoint hook](https://nixos.org/manual/nixpkgs/stable/#breakpointhook)
+
 
 # Standards
 
-Every new module name should be in kebab-case.
-Every fact definition, where possible should be in kebab-case.
+- Every new module name should be in kebab-case.
+- Every fact definition, where possible should be in kebab-case.

docs/_internal/use-cases/01-self-host.md

@@ -20,11 +20,11 @@ There are several reasons for choosing to self-host. These can include:
 
 Alice wants to self-host a mumble server for her family.
 
-- She visits to the cLAN website, and follows the instructions on how to install cLAN-OS on her server.
-- Alice logs into a terminal on her server via SSH (alternatively uses cLAN GUI app)
-- Using the cLAN CLI or GUI tool, alice creates a new private network for her family (VPN)
-- Alice now browses a list of curated cLAN modules and finds a module for mumble.
-- She adds this module to her network using the cLAN tool.
+- She visits to the Clan website, and follows the instructions on how to install Clan-OS on her server.
+- Alice logs into a terminal on her server via SSH (alternatively uses Clan GUI app)
+- Using the Clan CLI or GUI tool, alice creates a new private network for her family (VPN)
+- Alice now browses a list of curated Clan modules and finds a module for mumble.
+- She adds this module to her network using the Clan tool.
 - After that, she uses the clan tool to invite her family members to her network
 - Other family members join the private network via the invitation.
 - By accepting the invitation, other members automatically install all required software to interact with the network on their machine.
@@ -33,7 +33,7 @@ Alice wants to self-host a mumble server for her family.
 
 Alice wants to add a photos app to her private network
 
-- She uses the clan CLI or GUI tool to manage her existing private cLAN family network
+- She uses the clan CLI or GUI tool to manage her existing private Clan family network
 - She discovers a module for photoprism, and adds it to her server using the tool
 - Other members who are already part of her network, will receive a notification that an update is required to their environment
 - After accepting, all new software and services to interact with the new photoprism service will be installed automatically.

docs/_internal/use-cases/02-join-network.md

@@ -1,4 +1,4 @@
-# Joining a cLAN network
+# Joining a Clan network
 
 ## General Description
 
@@ -8,13 +8,13 @@ Joining a self-hosted infrastructure involves connecting to a network, server, o
 
 ### Story 1: Joining a private network
 
-Alice' son Bob has never heard of cLAN, but receives an invitation URL from Alice who already set up private cLAN network for her family.
+Alice' son Bob has never heard of Clan, but receives an invitation URL from Alice who already set up private Clan network for her family.
 
-Bob opens the invitation link and lands on the cLAN website. He quickly learns about what cLAN is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
+Bob opens the invitation link and lands on the Clan website. He quickly learns about what Clan is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
 
-Bob decides to join the network and follows the instructions to install the cLAN tool on his computer.
+Bob decides to join the network and follows the instructions to install the Clan tool on his computer.
 
-Feeding the invitation link to the cLAN tool, bob registers his machine with the network.
+Feeding the invitation link to the Clan tool, bob registers his machine with the network.
 
 All programs required to interact with the network will be installed and configured automatically and securely.
 
@@ -22,7 +22,7 @@ Optionally, bob can customize the configuration of these programs through a simp
 
 ### Story 2: Receiving breaking changes
 
-The cLAN family network which Bob is part of received an update.
+The Clan family network which Bob is part of received an update.
 
 The existing photo sharing service has been removed and replaced with another alternative service. The new photo sharing service requires a different client app to view and upload photos.
 
@@ -30,7 +30,7 @@ Bob accepts the update. Now his environment will be updated. The old client soft
 
 Because Bob has customized the previous photo viewing app, he is notified that this customization is no longer valid, as the software has been removed (deprecation message).l
 
-Optionally, Bob can now customize the new photo viewing software through his cLAN configuration app or via a config file.
+Optionally, Bob can now customize the new photo viewing software through his Clan configuration app or via a config file.
 
 ## Challenges
 

docs/_internal/use-cases/03-maintaining-clan-modules.md

@@ -1,10 +1,10 @@
-# cLAN module maintaining
+# Clan module maintaining
 
 ## General Description
 
-cLAN modules are pieces of software that can be used by admins to build a private or public infrastructure.
+Clan modules are pieces of software that can be used by admins to build a private or public infrastructure.
 
-cLAN modules should have the following properties:
+Clan modules should have the following properties:
 
 1. Documented: It should be clear what the module does and how to use it.
 1. Self contained: A module should be usable as is. If it requires any other software or settings, those should be delivered with the module itself.

docs/main.py

@@ -16,15 +16,26 @@ def define_env(env: Any) -> None:
     @env.macro
     def asciinema(name: str) -> str:
         return f"""<div id="{name}">
-            <script src="{asciinema_dir}/asciinema-player.min.js"></script>
             <script>
-                AsciinemaPlayer.create('{video_dir + name}',
-                document.getElementById("{name}"), {{
-                    loop: true,
-                    autoPlay: true,
-                    controls: false,
-                    speed: 1.5,
-                    theme: "solarized-light"
-                }});
+                // Function to load the script and then create the Asciinema player
+                function loadAsciinemaPlayer() {{
+                    var script = document.createElement('script');
+                    script.src = "{asciinema_dir}/asciinema-player.min.js";
+                    script.onload = function() {{
+                        AsciinemaPlayer.create('{video_dir + name}', document.getElementById("{name}"), {{
+                            loop: true,
+                            autoPlay: true,
+                            controls: false,
+                            speed: 1.5,
+                            theme: "solarized-light"
+                    }});
+                    }};
+                    document.head.appendChild(script);
+                }}
+
+                // Load the Asciinema player script
+                loadAsciinemaPlayer();
             </script>
+
+            <link rel="stylesheet" type="text/css" href="{asciinema_dir}/asciinema-player.css" />
         </div>"""

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Clan Docs
+site_name: Clan Documentation
 site_url: https://docs.clan.lol
 repo_url: https://git.clan.lol/clan/clan-core/
 repo_name: clan-core
@@ -28,6 +28,7 @@ markdown_extensions:
   - pymdownx.highlight:
       use_pygments: true
       anchor_linenums: true
+  - pymdownx.keys
   - toc:
       title: On this page
 
@@ -38,7 +39,7 @@ exclude_docs: |
 
 nav:
   - Blog:
-    - blog/index.md
+      - blog/index.md
   - Getting started:
       - index.md
       - Installer: getting-started/installer.md
@@ -93,8 +94,9 @@ docs_dir: site
 site_dir: out
 
 theme:
-  logo: static/clan-white.png
-  favicon: static/clan-dark.png
+  font: false
+  logo: https://clan.lol/static/logo/clan-white.png
+  favicon: https://clan.lol/static/dark-favicon/128x128.png
   name: material
   features:
     - navigation.instant
@@ -103,9 +105,8 @@ theme:
     - content.code.copy
     - content.tabs.link
   icon:
-    repo: fontawesome/brands/git
-  font:
-    code: Roboto Mono
+    repo: fontawesome/brands/git-alt
+  custom_dir: overrides
 
   palette:
     # Palette toggle for light mode
@@ -127,8 +128,7 @@ theme:
         name: Switch to light mode
 
 extra_css:
-  - static/asciinema-player/custom-theme.css
-  - static/asciinema-player/asciinema-player.css
+  - static/extra.css
 
 extra:
   social:
@@ -141,7 +141,6 @@ extra:
     - icon: fontawesome/solid/rss
       link: /feed_rss_created.xml
 
-
 plugins:
   - search
   - blog

docs/nix/default.nix

@@ -2,6 +2,10 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
+  roboto,
+  fira-code,
   ...
 }:
 let
@@ -27,6 +31,14 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+
+    mkdir -p ./site/static/asciinema-player
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
+
+    # Link to fonts
+    ln -snf ${roboto}/share/fonts/truetype/Roboto-Regular.ttf ./site/static/
+    ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
   '';
 
   buildPhase = ''

docs/nix/flake-module.nix

@@ -40,6 +40,15 @@
             mypy --strict $out
           '';
 
+      asciinema-player-js = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.min.js";
+        sha256 = "sha256-Ymco/+FinDr5YOrV72ehclpp4amrczjo5EU3jfr/zxs=";
+      };
+      asciinema-player-css = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.css";
+        sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc=";
+      };
+
       module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
         export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
         # A file that contains the links to all clanModule docs
@@ -56,12 +65,16 @@
       devShells.docs = pkgs.callPackage ./shell.nix {
         inherit (self'.packages) docs clan-cli-docs;
         inherit module-docs;
+        inherit asciinema-player-js;
+        inherit asciinema-player-css;
       };
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
           inherit (self'.packages) clan-cli-docs;
           inherit (inputs) nixpkgs;
           inherit module-docs;
+          inherit asciinema-player-js;
+          inherit asciinema-player-css;
         };
         deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
         inherit module-docs;

docs/nix/scripts/renderOptions.py

@@ -40,13 +40,14 @@ def sanitize(text: str) -> str:
     return text.replace(">", "\\>")
 
 
-def replace_store_path(text: str) -> Path:
+def replace_store_path(text: str) -> tuple[str, str]:
     res = text
     if text.startswith("/nix/store/"):
         res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
             Path(*Path(text).parts[4:])
         )
-    return Path(res)
+    name = Path(res).name
+    return (res, name)
 
 
 def render_option_header(name: str) -> str:
@@ -108,9 +109,10 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
 """
 
     decls = option.get("declarations", [])
-    source_path = replace_store_path(decls[0])
+    source_path, name = replace_store_path(decls[0])
+    print(source_path, name)
     res += f"""
-:simple-git: [{source_path.name}]({source_path})
+:simple-git: [{name}]({source_path})
 """
     res += "\n"
 
@@ -160,7 +162,7 @@ def produce_clan_core_docs() -> None:
         for option_name, info in options.items():
             outfile = f"{module_name}/index.md"
 
-            # Create seperate files for nested options
+            # Create separate files for nested options
             if len(option_name.split(".")) <= 2:
                 # i.e. clan-core.clanDir
                 output = core_outputs.get(

docs/nix/shell.nix

@@ -3,6 +3,10 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
+  roboto,
+  fira-code,
   ...
 }:
 pkgs.mkShell {
@@ -14,5 +18,14 @@ pkgs.mkShell {
     chmod +w ./site/reference/*
 
     echo "Generated API documentation in './site/reference/' "
+
+    mkdir -p ./site/static/asciinema-player
+
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
+
+    # Link to fonts
+    ln -snf ${roboto}/share/fonts/truetype/Roboto-Regular.ttf ./site/static/
+    ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
   '';
 }

docs/overrides/main.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block extrahead %}
+<meta property="og:title" content="Clan - Documentation, Blog & Getting Started Guide" />
+<meta property="og:description" content="Documentation for Clan. The peer-to-peer machine deployment framework." />
+<meta property="og:image" content="https://clan.lol/static/dark-favicon/128x128.png" />
+<meta property="og:url" content="https://docs.clan.lol" />
+<meta property="og:type" content="website" />
+<meta property="og:site_name" content="Clan" />
+<meta property="og:locale" content="en_US" />
+
+{% endblock %}

docs/site/blog/.authors.yml

@@ -1,21 +1,26 @@
 authors:
+  DavHau:
+    name: "DavHau"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/davhau.jpg"
+    url: "https://DavHau.com"
   Lassulus:
     name: "Lassulus"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/621759?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/lassulus.jpg"
     url: "https://http://lassul.us/"
   Mic92:
     name: "Mic92"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/96200?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/mic92.jpg"
     url: "https://thalheim.io"
   W:
     name: "W"
     description: "Founder of Clan"
-    avatar: "/static/w_profile.webp"
+    avatar: "https://clan.lol/static/profiles/w_profile.webp"
     url: ""
   Qubasa:
     name: "Qubasa"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/22085373?v=4"
-    url: "https://github.com/Qubasa"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/qubasa.png"
+    url: "https://github.com/Qubasa"

docs/site/blog/posts/jsonschema.md

@@ -0,0 +1,194 @@
+---
+title: "Dev Report: Introducing the NixOS to JSON Schema Converter"
+description: "Discover our new library designed to extract JSON schema interfaces from NixOS modules, streamlining frontend development"
+authors:
+  - DavHau
+date: 2024-05-25
+slug: jsonschema-converter
+---
+
+## Overview
+
+We’ve developed a new library designed to extract interfaces from NixOS modules and convert them into JSON schemas, paving the way for effortless GUI generation. This blog post outlines the motivations behind this development, demonstrates the capabilities of the library, and guides you through leveraging it to create GUIs seamlessly.
+
+## Motivation
+
+In recent months, our team has been exploring various graphical user interfaces (GUIs) to streamline NixOS machine configuration. While our opinionated Clan modules simplify NixOS configurations, there's a need to configure these modules from diverse frontends, such as:
+
+- Command-line interfaces (CLIs)
+- Web-based UIs
+- Desktop applications
+- Mobile applications
+- Large Language Models (LLMs)
+
+Given this need, a universal format like JSON is a natural choice. It is already possible as of now, to import json based NixOS configurations, as illustrated below:
+
+`configuration.json`:
+```json
+{ "networking": { "hostName": "my-machine" } }
+```
+
+This configuration can be then imported inside a classic NixOS config:
+```nix
+{config, lib, pkgs, ...}: {
+  imports = [
+    (lib.importJSON ./configuration.json)
+  ];
+}
+```
+
+This straightforward approach allows us to build a frontend that generates JSON, enabling the configuration of NixOS machines. But, two critical questions arise:
+
+1. How does the frontend learn about existing configuration options?
+2. How can it verify user input without running Nix?
+
+Introducing [JSON schema](https://json-schema.org/), a widely supported standard that defines interfaces in JSON and validates input against them.
+
+Example schema for `networking.hostName`:
+```json
+{
+  "type": "object",
+  "properties": {
+    "networking": {
+      "type": "object",
+      "properties": {
+        "hostName": {
+          "type": "string",
+          "pattern": "^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$"
+        }
+      }
+    }
+  }
+}
+```
+
+## Client-Side Input Validation
+
+Validating input against JSON schemas is both efficient and well-supported across numerous programming languages. Using JSON schema validators, you can accurately check configurations like our `configuration.json`.
+
+Validation example:
+
+```shell
+$ nix-shell -p check-jsonschema
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[SUCCESS]===(./configuration.json)===
+```
+
+In case of invalid input, schema validators provide explicit error messages:
+
+```shell
+$ echo '{ "networking": { "hostName": "my/machine" } }' > configuration.json
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[ValidationError]===(./configuration.json)===
+
+'my/machine' does not match '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$'
+
+Failed validating 'pattern' in schema['properties']['networking']['properties']['hostName']:
+    {'pattern': '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$',
+     'type': 'string'}
+
+On instance['networking']['hostName']:
+    'my/machine'
+```
+
+## Automatic GUI Generation
+
+Certain libraries facilitate straightforward GUI generation from JSON schemas. For instance, the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) auto-generates a form for any given schema.
+
+## NixOS Module to JSON Schema Converter
+
+To enable the development of responsive frontends, our library allows the extraction of interfaces from NixOS modules to JSON schemas. Open-sourced for community collaboration, this library supports building sophisticated user interfaces for NixOS.
+
+Here’s a preview of our library's functions exposed through the [clan-core](https://git.clan.lol/clan/clan-core) flake:
+
+- `lib.jsonschema.parseModule` - Generates a schema for a NixOS module.
+- `lib.jsonschema.parseOption` - Generates a schema for a single NixOS option.
+- `lib.jsonschema.parseOptions` - Generates a schema from an attrset of NixOS options.
+
+Example:
+`module.nix`:
+```nix
+{lib, config, pkgs, ...}: {
+  # a simple service with two options
+  options.services.example-web-service = {
+    enable = lib.mkEnableOption "Example web service";
+    port = lib.mkOption {
+      type = lib.types.int;
+      description = "Port used to serve the content";
+    };
+  };
+}
+```
+
+Converted, using the `parseModule` function:
+```shell
+$ cd clan-core
+$ nix eval --json --impure --expr \
+  '(import ./lib/jsonschema {}).parseModule ./module.nix' | jq | head
+{
+  "properties": {
+    "services": {
+      "properties": {
+        "example-web-service": {
+          "properties": {
+            "enable": {
+              "default": false,
+              "description": "Whether to enable Example web service.",
+              "examples": [
+...
+```
+
+This utility can also generate interfaces for existing NixOS modules or options.
+
+## GUI for NGINX in Under a Minute
+
+Creating a prototype GUI for the NGINX module using our library and [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) can be done quickly:
+
+1. Export all NGINX options into a JSON schema using a Nix expression:
+
+```nix
+# export.nix
+let
+  pkgs = import <nixpkgs> {};
+  clan-core = builtins.getFlake "git+https://git.clan.lol/clan/clan-core";
+  options = (pkgs.nixos {}).options.services.nginx;
+in
+  clan-core.lib.jsonschema.parseOption options
+```
+
+2. Write the schema into a file:
+```shell
+$ nix eval --json -f ./export.nix | jq > nginx.json
+```
+
+3. Open the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/), select `Blank` and paste the `nginx.json` contents.
+
+This provides a quick look at a potential GUI (screenshot is cropped).
+
+![Image title](https://clan.lol/static/blog-post-jsonschema/nginx-gui.jpg)
+
+## Limitations
+
+### Laziness
+
+JSON schema mandates the declaration of all required fields upfront, which might be configured implicitly or remain unused. For instance, `services.nginx.virtualHosts.<name>.sslCertificate` must be specified even if SSL isn’t enabled.
+
+### Limited Types
+
+Certain NixOS module types, like `types.functionTo` and `types.package`, do not map straightforwardly to JSON. For full compatibility, adjustments to NixOS modules might be necessary, such as substituting `listOf package` with `listOf str`.
+
+### Parsing NixOS Modules
+
+Currently, our converter relies on the `options` attribute of evaluated NixOS modules, extracting information from the `type.name` attribute, which is suboptimal. Enhanced introspection capabilities within the NixOS module system would be beneficial.
+
+## Future Prospects
+
+We hope these experiments inspire the community, encourage contributions and further development in this space. Share your ideas and contributions through our issue tracker or matrix channel!
+
+## Links
+
+- [Comments on NixOS Discourse](https://discourse.nixos.org/t/introducing-the-nixos-to-json-schema-converter/45948)
+- [Source Code of the JSON Schema Library](https://git.clan.lol/clan/clan-core/src/branch/main/lib/jsonschema)
+- [Our Issue Tracker](https://git.clan.lol/clan/clan-core/issues)
+- [Our Matrix Channel](https://matrix.to/#/#clan:lassul.us)
+- [react-jsonschema-form Playground](https://rjsf-team.github.io/react-jsonschema-form/)

docs/site/drafts/deployment-framework.md

@@ -0,0 +1,63 @@
+---
+title: "Git Based Machine Deployment with Clan-Core"
+description: ""
+authors:
+  - Qubasa
+date: 2024-05-25
+---
+## Revolutionizing Server Management
+
+In the world of server management, countless tools claim to offer seamless deployment of multiple machines. Yet, many fall short, leaving server admins and self-hosting enthusiasts grappling with complexity. Enter the Clan-Core Framework—a groundbreaking all in one solution designed to transform decentralized self-hosting into an effortless and scalable endeavor.
+
+### The Power of Clan-Core
+
+Imagine having the power to manage your servers with unparalleled ease, scaling your IT infrastructure like never before. Clan-Core empowers you to do just that. At its core, Clan-Core leverages a single Git repository to define everything about your machines. This central repository utilizes Nix or JSON files to specify configurations, including disk formatting, ensuring a streamlined and unified approach.
+
+### Simplified Deployment Process
+
+With Clan-Core, the cumbersome task of bootstrapping a specific ISO is a thing of the past. All you need is SSH access to your Linux server. Clan-Core allows you to overwrite any existing Linux distribution live over SSH, eliminating time-consuming setup processes. This capability means you can deploy updates or new configurations swiftly and efficiently, maximizing uptime and minimizing hassle.
+
+### Secure and Efficient Secret Management
+
+Security is paramount in server management, and Clan-Core takes it seriously. Passwords and other sensitive information are encrypted within the Git repository, automatically decrypted during deployment. This not only ensures the safety of your secrets but also simplifies their management. Clan-Core supports sharing secrets with other admins, fostering collaboration and maintaining reproducibillity and security without sacrificing convenience.
+
+### Services as Apps
+
+Setting up a service can be quite difficult. Many server adjustments need to be made, from setting up a database to adjusting webserver configurations and generating the correct private keys. However, Clan-Core aims to make setting up a service as easy as installing an application. Through Clan-Core's Module system, everything down to secrets can be automatically set up. This transforms the often daunting task of service setup into a smooth, automated process, making it accessible to all.
+
+### Decentralized Mesh VPN
+
+Building on these features is a self-configuring decentralized mesh VPN that interconnects all your machines into a private darknet. This ensures that sensitive services, which might have too much attack surface to be hosted on the public internet, can still be made available privately without the need to worry about potential system compromise. By creating a secure, private network, Clan-Core offers an additional layer of protection for your most critical services.
+
+### Decentralized Domain Name System
+
+Current DNS implementations are distributed but not truly decentralized. For Clan-Core, we implemented our own truly decentralized DNS module. This module uses simple flooding and caching algorithms to discover available domains inside the darknet. This approach ensures that your internal domain name system is robust, reliable, and independent of external control, enhancing the resilience and security of your infrastructure.
+
+
+### A New Era of Decentralized Self-Hosting
+
+Clan-Core is more than just a tool; it's a paradigm shift in server management. By consolidating machine definitions, secrets and network configuration, into a single, secure repository, it transforms how you manage and scale your infrastructure. Whether you're a seasoned server admin or a self-hosting enthusiast, Clan-Core offers a powerful, user-friendly solution to take your capabilities to the next level.
+
+
+### Key Features of Clan-Core:
+
+- **Unified Git Repository**: All machine configurations and secrets stored in a single repository.
+- **Live Overwrites**: Deploy configurations over existing Linux distributions via SSH.
+- **Automated Service Setup**: Easily set up services with Clan-Core's Module system.
+- **Decentralized Mesh VPN**: Securely interconnect all machines into a private darknet.
+- **Decentralized DNS**: Robust, independent DNS using flooding and caching algorithms.
+- **Automated Secret Management**: Encrypted secrets that are automatically decrypted during deployment.
+- **Collaboration Support**: Share secrets securely with other admins.
+
+
+## Clan-Cores Future
+
+Our vision for Clan-Core extends far beyond being just another deployment tool. Clan-Core is a framework we've developed to achieve something much greater. We want to put the "personal" back into "personal computing." Our goal is for everyday users to fully customize their phones or laptops and create truly private spaces for friends and family.
+
+Our first major step is to develop a Graphical User Interface (GUI) that makes configuring all this possible. Initial tests have shown that AI can be leveraged as an alternative to traditional GUIs. This paves the way for a future where people can simply talk to their computers, and they will configure themselves according to the users' wishes. 
+
+By adopting Clan, you're not just embracing a tool—you're joining a movement towards a more efficient, secure, and scalable approach to server management. Join us and revolutionize your IT infrastructure today.
+
+
+
+

docs/site/getting-started/installer.md

@@ -42,13 +42,22 @@ sudo umount /dev/sdb1
 === "**Linux OS**"
     ### Step 2. Flash Custom Installer
 
-    Using clan flash enables the inclusion of ssh public keys and disables ssh password authentication.
-    It also includes the language and keymap currently used into the installer image.
+    Using clan flash enables the inclusion of ssh public keys.
+    It also allows to set language and keymap currently in the installer image.
 
     ```bash
-    clan --flake git+https://git.clan.lol/clan/clan-core flash flash-installer --disk main /dev/sd<X>
+    clan flash --flake git+https://git.clan.lol/clan/clan-core \
+      --ssh-pubkey $HOME/.ssh/id_ed25519.pub \
+      --keymap en \
+      --language en \
+      --disk main /dev/sd<X> \
+      flash-installer
     ```
 
+    The `--ssh-pubkey`, `--language` and `--keymap` are optional.
+    Replace `$HOME/.ssh/id_ed25519.pub` with a path to your SSH public key.
+    If you do not have an ssh key yet, you can generate one with `ssh-keygen -t ed25519` command.
+
     !!! Danger "Specifying the wrong device can lead to unrecoverable data loss."
 
         The `clan flash` utility will erase the disk. Make sure to specify the correct device
@@ -148,10 +157,10 @@ Connected network     FRITZ!Box (Your router device)
 IPv4 address          192.168.188.50 (Your new local ip)
 ```
 
-Press `ctrl-d` to exit `IWD`.
+Press ++ctrl+d++ to exit `IWD`.
 
 !!! Important
-    Press `ctrl-d` **again** to update the displayed QR code and connection information.
+    Press ++ctrl+d++ **again** to update the displayed QR code and connection information.
 
 You're all set up
 

docs/site/getting-started/secrets.md

@@ -4,10 +4,15 @@ Clan enables encryption of secrets (such as passwords & keys) ensuring security
 
 Clan utilizes the [sops](https://github.com/getsops/sops) format and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
 
+This guide will walk you through:
 
-### Create Your Master Keypair
+- **Creating a Keypair for Your User**: Learn how to generate a keypair for $USER to securely control all secrets.
+- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
+- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
 
-To get started, you'll need to create **Your master keypair**.
+## Create Your Admin Keypair
+
+To get started, you'll need to create **Your admin keypair**.
 
 !!! info
     Don't worry — if you've already made one before, this step won't change or overwrite it.
@@ -27,7 +32,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 
 !!! warning
     Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the master key to be unlocked.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
 
 !!! note
     It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
@@ -35,7 +40,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 ### Add Your Public Key
 
 ```bash
-clan secrets users add <your_username> <your_public_key>
+clan secrets users add $USER <your_public_key>
 ```
 
 It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
@@ -101,17 +106,20 @@ In your nixos configuration you can get a path to secrets like this `config.sops
 
 ### Assigning Access
 
-By default, secrets are encrypted for your key. To specify which users and machines can access a secret:
+When using `clan secrets set <secret>` without arguments, secrets are encrypted for the key of the user named like your current $USER.
 
-```bash
-clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
-```
-You can also just add machines/users to existing secrets:
+To add machines/users to an existing secret use:
 
 ```bash
  clan secrets machines add-secret <machine_name> <secret_name>
 ```
 
+Alternatively specify users and machines while creating a secret:
+
+```bash
+clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
+```
+
 ## Advanced
 
 In this section we go into more advanced secret management topics.
@@ -183,11 +191,9 @@ Since our clan secret module will auto-import secrets that are encrypted for a p
 you can now remove `sops.secrets.<secrets> = { };` unless you need to specify more options for the secret like owner/group of the secret file.
 
 
-
 ## Indepth Explanation
 
 
-
 The secrets system conceptually knows two different entities:
 
 - **Machine**: consumes secrets

docs/site/static/asciinema-player/asciinema-player.css

@@ -0,0 +1 @@
+/nix/store/8y5h98wk5p94mv1wyb2c4gkrr7bswd19-asciinema-player.css

docs/site/static/asciinema-player/asciinema-player.min.js

@@ -0,0 +1 @@
+/nix/store/w0i3f9qzn9n6jmfnfgiw5wnab2f9ssdw-asciinema-player.min.js

docs/site/static/asciinema-player/custom-theme.css

@@ -1,45 +0,0 @@
-.asciinema-player-theme-alabaster-auto {
-        --term-color-foreground: #000000; /* Black for foreground text */
-        --term-color-background: #f7f7f7; /* Very light gray for background */
-
-        --term-color-0: #000000; /* Black */
-        --term-color-1: #aa3731; /* Red */
-        --term-color-2: #448c37; /* Green */
-        --term-color-3: #cb9000; /* Yellow */
-        --term-color-4: #325cc0; /* Blue */
-        --term-color-5: #7a3e9d; /* Magenta */
-        --term-color-6: #0083b2; /* Cyan */
-        --term-color-7: #bbbbbb; /* White */
-        --term-color-8: #777777; /* Bright black (gray) */
-        --term-color-9: #f05050; /* Bright red */
-        --term-color-10: #60cb00; /* Bright green */
-        --term-color-11: #ffbc5d; /* Bright yellow */
-        --term-color-12: #007acc; /* Bright blue */
-        --term-color-13: #e64ce6; /* Bright magenta */
-        --term-color-14: #00aacb; /* Bright cyan */
-        --term-color-15: #ffffff; /* Bright white */
-    }
-
-  @media (prefers-color-scheme: dark) {
-    .asciinema-player-theme-solarized-auto {
-      --term-color-foreground: #839496;
-      --term-color-background: #002b36;
-
-      --term-color-0: #073642;
-      --term-color-1: #dc322f;
-      --term-color-2: #859900;
-      --term-color-3: #b58900;
-      --term-color-4: #268bd2;
-      --term-color-5: #d33682;
-      --term-color-6: #2aa198;
-      --term-color-7: #eee8d5;
-      --term-color-8: #002b36;
-      --term-color-9: #cb4b16;
-      --term-color-10: #586e75;
-      --term-color-11: #657b83;
-      --term-color-12: #839496;
-      --term-color-13: #6c71c4;
-      --term-color-14: #93a1a1;
-      --term-color-15: #fdf6e3;
-    }
-  }

docs/site/static/extra.css

@@ -0,0 +1,13 @@
+@font-face {
+    font-family: "Roboto";
+    src: url(./Roboto-Regular.ttf) format('truetype');
+}
+@font-face {
+    font-family: "Fira Code";
+    src: url(./FiraCode-VF.ttf) format('truetype');
+}
+
+:root {
+    --md-text-font: "Roboto"; 
+    --md-code-font: "Fira Code";
+}

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716394172,
-        "narHash": "sha256-B+pNhV8GFeCj9/MoH+qtGqKbgv6fU4hGaw2+NoYYtB0=",
+        "lastModified": 1717177033,
+        "narHash": "sha256-G3CZJafCO8WDy3dyA2EhpUJEmzd5gMJ2IdItAg0Hijw=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "23c63fb09334c3e8958b57e2ddc3870b75b9111d",
+        "rev": "0274af4c92531ebfba4a5bd493251a143bc51f3c",
         "type": "github"
       },
       "original": {
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715865404,
-        "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=",
+        "lastModified": 1717285511,
+        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9",
+        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
         "type": "github"
       },
       "original": {
@@ -57,11 +57,11 @@
     },
     "nixos-2311": {
       "locked": {
-        "lastModified": 1715818734,
-        "narHash": "sha256-WvAJWCwPj/6quKcsgsvQYyZRxV8ho/yUzj0HZQ34DVU=",
+        "lastModified": 1717017538,
+        "narHash": "sha256-S5kltvDDfNQM3xx9XcvzKEOyN2qk8Sa+aSOLqZ+1Ujc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "95742536dc6debb5a8b8b78b27001c38f369f1e7",
+        "rev": "64e468fd2652105710d86cd2ae3e65a5a6d58dec",
         "type": "github"
       },
       "original": {
@@ -79,11 +79,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716123454,
-        "narHash": "sha256-U2o4UPM/UsEyIX2p11+YEQgR9HY3PmjZ2mRl/x5e4xo=",
+        "lastModified": 1716210724,
+        "narHash": "sha256-iqQa3omRcHGpWb1ds75jS9ruA5R39FTmAkeR3J+ve1w=",
         "owner": "nix-community",
         "repo": "nixos-generators",
-        "rev": "a63e0c83dd83fe28cc571b97129e13373436bd82",
+        "rev": "d14b286322c7f4f897ca4b1726ce38cb68596c94",
         "type": "github"
       },
       "original": {
@@ -100,11 +100,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716132123,
-        "narHash": "sha256-rATSWbPaKQfZGaemu0tHL2xfCzVIVwpuTjk+KSBC+k4=",
+        "lastModified": 1717040312,
+        "narHash": "sha256-yI/en4IxuCEClIUpIs3QTyYCCtmSPLOhwLJclfNwdeg=",
         "owner": "nix-community",
         "repo": "nixos-images",
-        "rev": "8c9cab8c44434c12dafc465fbf61a710c5bceb08",
+        "rev": "47bfb55316e105390dd761e0b6e8e0be09462b67",
         "type": "github"
       },
       "original": {
@@ -115,11 +115,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1716127062,
-        "narHash": "sha256-2rk8FqB/iQV2d0vQLs684/Tj5PUHaS1sFwG7fng5vXE=",
+        "lastModified": 1717298511,
+        "narHash": "sha256-9sXuJn/nL+9ImeYtlspTvjt83z1wIgU+9AwfNbnq+tI=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "8a2555763c48e2410054de3f52f7310ce3241ec5",
+        "rev": "6634a0509e9e81e980b129435fbbec518ab246d0",
         "type": "github"
       },
       "original": {
@@ -148,11 +148,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1716087663,
-        "narHash": "sha256-zuSAGlx8Qk0OILGCC2GUyZ58/SJ5R3GZdeUNQ6IS0fQ=",
+        "lastModified": 1717297459,
+        "narHash": "sha256-cZC2f68w5UrJ1f+2NWGV9Gx0dEYmxwomWN2B0lx0QRA=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "0bf1808e70ce80046b0cff821c019df2b19aabf5",
+        "rev": "ab2a43b0d21d1d37d4d5726a892f714eaeb4b075",
         "type": "github"
       },
       "original": {
@@ -168,11 +168,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715940852,
-        "narHash": "sha256-wJqHMg/K6X3JGAE9YLM0LsuKrKb4XiBeVaoeMNlReZg=",
+        "lastModified": 1717278143,
+        "narHash": "sha256-u10aDdYrpiGOLoxzY/mJ9llST9yO8Q7K/UlROoNxzDw=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "2fba33a182602b9d49f0b2440513e5ee091d838b",
+        "rev": "3eb96ca1ae9edf792a8e0963cc92fddfa5a87706",
         "type": "github"
       },
       "original": {

flake.nix

@@ -24,10 +24,14 @@
   };
 
   outputs =
-    inputs@{ flake-parts, ... }:
+    inputs@{ flake-parts, self, ... }:
     flake-parts.lib.mkFlake { inherit inputs; } (
       { ... }:
       {
+        clan = {
+          # meta.name = "clan-core";
+          directory = self;
+        };
         systems = [
           "x86_64-linux"
           "aarch64-linux"

flakeModules/clan.nix

@@ -17,6 +17,33 @@ let
   cfg = config.clan;
 in
 {
+  imports = [
+    # TODO: figure out how to print the deprecation warning
+    # "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix"
+    (lib.mkRenamedOptionModule
+      [
+        "clan"
+        "clanName"
+      ]
+      [
+        "clan"
+        "meta"
+        "name"
+      ]
+    )
+    (lib.mkRenamedOptionModule
+      [
+        "clan"
+        "clanIcon"
+      ]
+      [
+        "clan"
+        "meta"
+        "icon"
+      ]
+    )
+  ];
+
   options.clan = {
     directory = mkOption {
       type = types.path;
@@ -33,15 +60,27 @@ in
       default = { };
       description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
     };
-    clanName = mkOption {
-      type = types.str;
-      description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
-    };
-    clanIcon = mkOption {
-      type = types.nullOr types.path;
-      default = null;
-      description = "A path to an icon to be used for the clan, should be the same for all machines";
+
+    # Checks are performed in 'buildClan'
+    # Not everyone uses flake-parts
+    meta = {
+      name = lib.mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
+      };
+      icon = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = "A path to an icon to be used for the clan in the GUI";
+      };
+      description = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "A short description of the clan";
+      };
     };
+
     pkgsForSystem = mkOption {
       type = types.functionTo types.raw;
       default = _system: null;
@@ -52,6 +91,7 @@ in
     clanInternals = lib.mkOption {
       type = lib.types.submodule {
         options = {
+          meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
           machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
@@ -65,9 +105,8 @@ in
         directory
         specialArgs
         machines
-        clanName
-        clanIcon
         pkgsForSystem
+        meta
         ;
     };
   };

lib/build-clan/default.nix

@@ -7,16 +7,58 @@
   directory, # The directory containing the machines subdirectory
   specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available
   machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... }
-  clanName, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  # DEPRECATED: use meta.name instead
+  clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  # DEPRECATED: use meta.icon instead
   clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
+  meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string
   pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
 # This improves performance, but all nipxkgs.* options will be ignored.
 }:
 let
+  deprecationWarnings = [
+    (lib.warnIf (
+      clanName != null
+    ) "clanName is deprecated, please use meta.name instead. ${clanName}" null)
+    (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null)
+  ];
+
   machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
     builtins.readDir (directory + /machines)
   );
 
+  mergedMeta =
+    let
+      metaFromFile =
+        if (builtins.pathExists "${directory}/clan/meta.json") then
+          let
+            settings = builtins.fromJSON (builtins.readFile "${directory}/clan/meta.json");
+          in
+          settings
+        else
+          { };
+      legacyMeta = lib.filterAttrs (_: v: v != null) {
+        name = clanName;
+        icon = clanIcon;
+      };
+      optionsMeta = lib.filterAttrs (_: v: v != null) meta;
+
+      warnings =
+        builtins.map (
+          name:
+          if
+            metaFromFile.${name} or null != optionsMeta.${name} or null && optionsMeta.${name} or null != null
+          then
+            lib.warn "meta.${name} is set in different places. (exlicit option meta.${name} overrides ${directory}/clan/meta.json)" null
+          else
+            null
+        ) (builtins.attrNames metaFromFile)
+        ++ [ (if (res.name or null == null) then (throw "meta.name should be set") else null) ];
+      res = metaFromFile // legacyMeta // optionsMeta;
+    in
+    # Print out warnings before returning the merged result
+    builtins.deepSeq warnings res;
+
   machineSettings =
     machineName:
     # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
@@ -58,11 +100,15 @@ let
           (machines.${name} or { })
           (
             {
-              networking.hostName = lib.mkDefault name;
-              clanCore.clanName = clanName;
-              clanCore.clanIcon = clanIcon;
+              # Settings
               clanCore.clanDir = directory;
+              # Inherited from clan wide settings
+              clanCore.clanName = meta.name or clanName;
+              clanCore.clanIcon = meta.icon or clanIcon;
+
+              # Machine specific settings
               clanCore.machineName = name;
+              networking.hostName = lib.mkDefault name;
               nixpkgs.hostPlatform = lib.mkDefault system;
 
               # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
@@ -127,10 +173,15 @@ let
     ) supportedSystems
   );
 in
-{
+builtins.deepSeq deprecationWarnings {
   inherit nixosConfigurations;
 
   clanInternals = {
+    # Evaluated clan meta
+    # Merged /clan/meta.json with overrides from buildClan
+    meta = mergedMeta;
+
+    # machine specifics
     machines = configsPerSystem;
     machinesFunc = configsFuncPerSystem;
     all-machines-json = lib.mapAttrs (

nixosModules/clanCore/vm.nix

@@ -122,7 +122,7 @@ in
       cores = lib.mkOption {
         type = lib.types.ints.positive;
         default = 1;
-        description = lib.mdDoc ''
+        description = ''
           Specify the number of cores the guest is permitted to use.
           The number can be higher than the available cores on the
           host system.
@@ -132,7 +132,7 @@ in
       memorySize = lib.mkOption {
         type = lib.types.ints.positive;
         default = 1024;
-        description = lib.mdDoc ''
+        description = ''
           The memory size in megabytes of the virtual machine.
         '';
       };
@@ -140,7 +140,7 @@ in
       graphics = lib.mkOption {
         type = lib.types.bool;
         default = true;
-        description = lib.mdDoc ''
+        description = ''
           Whether to run QEMU with a graphics window, or in nographic mode.
           Serial console will be enabled on both settings, but this will
           change the preferred console.
@@ -150,7 +150,7 @@ in
       waypipe = lib.mkOption {
         type = lib.types.bool;
         default = false;
-        description = lib.mdDoc ''
+        description = ''
           Whether to use waypipe for native wayland passthrough, or not.
         '';
       };

nixosModules/clanCore/zerotier/default.nix

@@ -81,7 +81,7 @@ in
       };
     };
     settings = lib.mkOption {
-      description = lib.mdDoc "override the network config in /var/lib/zerotier/bla/$network.json";
+      description = "override the network config in /var/lib/zerotier/bla/$network.json";
       type = lib.types.submodule { freeformType = (pkgs.formats.json { }).type; };
     };
   };

pkgs/clan-cli/api.py

@@ -1,10 +1,12 @@
+import json
+
 from clan_cli.api import API
 
 
 def main() -> None:
     schema = API.to_json_schema()
     print(
-        f"""export const schema = {schema} as const;
+        f"""export const schema = {json.dumps(schema, indent=2)} as const;
 """
     )
 

pkgs/clan-cli/clan_cli/__init__.py

@@ -51,9 +51,14 @@ class AppendOptionAction(argparse.Action):
         lst.append(values[1])
 
 
-def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
+def flake_path(arg: str) -> str | Path:
+    flake_dir = Path(arg).resolve()
+    if flake_dir.exists() and flake_dir.is_dir():
+        return flake_dir
+    return arg
 
+
+def add_common_flags(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         "--debug",
         help="Enable debug logging",
@@ -70,12 +75,6 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
         default=[],
     )
 
-    def flake_path(arg: str) -> str | Path:
-        flake_dir = Path(arg).resolve()
-        if flake_dir.exists() and flake_dir.is_dir():
-            return flake_dir
-        return arg
-
     parser.add_argument(
         "--flake",
         help="path to the flake where the clan resides in, can be a remote flake or local, can be set through the [CLAN_DIR] environment variable",
@@ -84,51 +83,224 @@ def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
         type=flake_path,
     )
 
+
+def register_common_flags(parser: argparse.ArgumentParser) -> None:
+    has_subparsers = False
+    for action in parser._actions:
+        if isinstance(action, argparse._SubParsersAction):
+            for choice, child_parser in action.choices.items():
+                has_subparsers = True
+                register_common_flags(child_parser)
+    if not has_subparsers:
+        add_common_flags(parser)
+
+
+def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=prog,
+        description="The clan cli tool.",
+        epilog=(
+            """
+Online reference for the clan cli tool: https://docs.clan.lol/reference/cli/
+For more detailed information, visit: https://docs.clan.lol
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     subparsers = parser.add_subparsers()
 
     parser_backups = subparsers.add_parser(
         "backups",
         help="manage backups of clan machines",
         description="manage backups of clan machines",
+        epilog=(
+            """
+This subcommand provides an interface to backups that clan machines expose.
+
+Examples:
+
+  $ clan backups list [MACHINE]
+  List backups for the machine [MACHINE]
+
+  $ clan backups create [MACHINE]
+  Create a backup for the machine [MACHINE].
+
+  $ clan backups restore [MACHINE] [PROVIDER] [NAME]
+  The backup to restore for the machine [MACHINE] with the configured [PROVIDER]
+  with the name [NAME].
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/backups/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     backups.register_parser(parser_backups)
 
     parser_flake = subparsers.add_parser(
-        "flakes", help="create a clan flake inside the current directory"
+        "flakes",
+        help="create a clan flake inside the current directory",
+        description="create a clan flake inside the current directory",
+        epilog=(
+            """
+Examples:
+  $ clan flakes create [DIR]
+  Will create a new clan flake in the specified directory and create it if it
+  doesn't exist yet. The flake will be created from a default template.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
+
     flakes.register_parser(parser_flake)
 
-    parser_config = subparsers.add_parser("config", help="set nixos configuration")
+    parser_config = subparsers.add_parser(
+        "config",
+        help="read a nixos configuration option",
+        description="read a nixos configuration option",
+        epilog=(
+            """
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     config.register_parser(parser_config)
 
-    parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine")
+    parser_ssh = subparsers.add_parser(
+        "ssh",
+        help="ssh to a remote machine",
+        description="ssh to a remote machine",
+        epilog=(
+            """
+This subcommand allows seamless ssh access to the nixos-image builders.
+
+Examples:
+
+  $ clan ssh [ssh_args ...] --json [JSON]
+  Will ssh in to the machine based on the deployment information contained in
+  the json string. [JSON] can either be a json formatted string itself, or point
+  towards a file containing the deployment information
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     ssh_cli.register_parser(parser_ssh)
 
-    parser_secrets = subparsers.add_parser("secrets", help="manage secrets")
+    parser_secrets = subparsers.add_parser(
+        "secrets",
+        help="manage secrets",
+        description="manage secrets",
+        epilog=(
+            """
+This subcommand provides an interface to secret facts.
+
+Examples:
+
+  $ clan secrets list [regex]
+  Will list secrets for all managed machines.
+  It accepts an optional regex, allowing easy filtering of returned secrets.
+
+  $ clan secrets get [SECRET]
+  Will display the content of the specified secret.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     secrets.register_parser(parser_secrets)
 
-    parser_facts = subparsers.add_parser("facts", help="manage facts")
+    parser_facts = subparsers.add_parser(
+        "facts",
+        help="manage facts",
+        description="manage facts",
+        epilog=(
+            """
+
+This subcommand provides an interface to facts of clan machines.
+Facts are artifacts that a service can generate.
+There are public and secret facts.
+Public facts can be referenced by other machines directly.
+Public facts can include: ip addresses, public keys.
+Secret facts can include: passwords, private keys.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     facts.register_parser(parser_facts)
 
     parser_machine = subparsers.add_parser(
-        "machines", help="manage machines and their configuration"
+        "machines",
+        help="manage machines and their configuration",
+        description="manage machines and their configuration",
+        epilog=(
+            """
+This subcommand provides an interface to machines managed by clan.
+
+Examples:
+
+  $ clan machines list
+  List all the machines managed by clan.
+
+  $ clan machines update [MACHINES]
+  Will update the specified machine [MACHINE], if [MACHINE] is omitted, the command
+  will attempt to update every configured machine.
+
+  $ clan machines install [MACHINES] [TARGET_HOST]
+  Will install the specified machine [MACHINE], to the specified [TARGET_HOST].
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     machines.register_parser(parser_machine)
 
-    parser_vms = subparsers.add_parser("vms", help="manage virtual machines")
+    parser_vms = subparsers.add_parser(
+        "vms", help="manage virtual machines", description="manage virtual machines"
+    )
     vms.register_parser(parser_vms)
 
-    parser_history = subparsers.add_parser("history", help="manage history")
+    parser_history = subparsers.add_parser(
+        "history",
+        help="manage history",
+        description="manage history",
+    )
     history.register_parser(parser_history)
 
     parser_flash = subparsers.add_parser(
-        "flash", help="flash machines to usb sticks or into isos"
+        "flash",
+        help="flash machines to usb sticks or into isos",
+        description="flash machines to usb sticks or into isos",
     )
     flash.register_parser(parser_flash)
 
     if argcomplete:
         argcomplete.autocomplete(parser)
 
+    register_common_flags(parser)
+
     return parser
 
 

pkgs/clan-cli/clan_cli/api/__init__.py

@@ -1,18 +1,37 @@
 from collections.abc import Callable
-from typing import Any
+from dataclasses import dataclass
+from typing import Any, Generic, Literal, TypeVar
+
+T = TypeVar("T")
+
+
+ResponseDataType = TypeVar("ResponseDataType")
+
+
+@dataclass
+class ApiError:
+    message: str
+    description: str | None
+    location: list[str] | None
+
+
+@dataclass
+class ApiResponse(Generic[ResponseDataType]):
+    status: Literal["success", "error"]
+    errors: list[ApiError] | None
+    data: ResponseDataType | None
 
 
 class _MethodRegistry:
     def __init__(self) -> None:
-        self._registry: dict[str, Callable] = {}
+        self._registry: dict[str, Callable[[Any], Any]] = {}
 
-    def register(self, fn: Callable) -> Callable:
+    def register(self, fn: Callable[..., T]) -> Callable[..., T]:
         self._registry[fn.__name__] = fn
         return fn
 
-    def to_json_schema(self) -> str:
+    def to_json_schema(self) -> dict[str, Any]:
         # Import only when needed
-        import json
         from typing import get_type_hints
 
         from clan_cli.api.util import type_to_dict
@@ -21,25 +40,51 @@ class _MethodRegistry:
             "$comment": "An object containing API methods. ",
             "type": "object",
             "additionalProperties": False,
-            "required": ["list_machines"],
+            "required": [func_name for func_name in self._registry.keys()],
             "properties": {},
         }
+
         for name, func in self._registry.items():
             hints = get_type_hints(func)
+
             serialized_hints = {
-                "argument" if key != "return" else "return": type_to_dict(
+                key: type_to_dict(
                     value, scope=name + " argument" if key != "return" else "return"
                 )
                 for key, value in hints.items()
             }
+
+            return_type = serialized_hints.pop("return")
+
             api_schema["properties"][name] = {
                 "type": "object",
-                "required": [k for k in serialized_hints.keys()],
+                "required": ["arguments", "return"],
                 "additionalProperties": False,
-                "properties": {**serialized_hints},
+                "properties": {
+                    "return": return_type,
+                    "arguments": {
+                        "type": "object",
+                        "required": [k for k in serialized_hints.keys()],
+                        "additionalProperties": False,
+                        "properties": serialized_hints,
+                    },
+                },
             }
 
-        return json.dumps(api_schema, indent=2)
+        return api_schema
+
+    def get_method_argtype(self, method_name: str, arg_name: str) -> Any:
+        from inspect import signature
+
+        func = self._registry.get(method_name, None)
+        if func:
+            sig = signature(func)
+            param = sig.parameters.get(arg_name)
+            if param:
+                param_class = param.annotation
+                return param_class
+
+        return None
 
 
 API = _MethodRegistry()

pkgs/clan-cli/clan_cli/api/util.py

@@ -42,9 +42,14 @@ def type_to_dict(t: Any, scope: str = "") -> dict:
             return {"type": "array", "items": type_to_dict(t.__args__[0], scope)}
 
         elif issubclass(origin, dict):
-            return {
-                "type": "object",
-            }
+            value_type = t.__args__[1]
+            if value_type is Any:
+                return {"type": "object", "additionalProperties": True}
+            else:
+                return {
+                    "type": "object",
+                    "additionalProperties": type_to_dict(value_type, scope),
+                }
 
         raise BaseException(f"Error api type not yet supported {t!s}")
 

pkgs/clan-cli/clan_cli/backups/create.py

@@ -2,6 +2,7 @@ import argparse
 import json
 import logging
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..errors import ClanError
 from ..machines.machines import Machine
 
@@ -33,13 +34,17 @@ def create_backup(machine: Machine, provider: str | None = None) -> None:
 
 
 def create_command(args: argparse.Namespace) -> None:
+    if args.flake is None:
+        raise ClanError("Could not find clan flake toplevel directory")
     machine = Machine(name=args.machine, flake=args.flake)
     create_backup(machine=machine, provider=args.provider)
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine", type=str, help="machine in the flake to create backups of"
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.add_argument("--provider", type=str, help="backup provider to use")
     parser.set_defaults(func=create_command)

pkgs/clan-cli/clan_cli/backups/list.py

@@ -3,6 +3,7 @@ import json
 import subprocess
 from dataclasses import dataclass
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..errors import ClanError
 from ..machines.machines import Machine
 
@@ -48,6 +49,8 @@ def list_backups(machine: Machine, provider: str | None = None) -> list[Backup]:
 
 
 def list_command(args: argparse.Namespace) -> None:
+    if args.flake is None:
+        raise ClanError("Could not find clan flake toplevel directory")
     machine = Machine(name=args.machine, flake=args.flake)
     backups = list_backups(machine=machine, provider=args.provider)
     for backup in backups:
@@ -55,8 +58,9 @@ def list_command(args: argparse.Namespace) -> None:
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine", type=str, help="machine in the flake to show backups of"
     )
+    add_dynamic_completer(machines_parser, complete_machines)
     parser.add_argument("--provider", type=str, help="backup provider to filter by")
     parser.set_defaults(func=list_command)

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -62,6 +62,8 @@ def restore_backup(
 
 
 def restore_command(args: argparse.Namespace) -> None:
+    if args.flake is None:
+        raise ClanError("Could not find clan flake toplevel directory")
     machine = Machine(name=args.machine, flake=args.flake)
     restore_backup(
         machine=machine,

pkgs/clan-cli/clan_cli/cmd.py

@@ -56,7 +56,7 @@ def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]:
             sys.stderr.buffer.write(ret)
             sys.stderr.flush()
         stderr_buf += ret
-    return stdout_buf.decode("utf-8"), stderr_buf.decode("utf-8")
+    return stdout_buf.decode("utf-8", "replace"), stderr_buf.decode("utf-8", "replace")
 
 
 class TimeTable:
@@ -101,13 +101,19 @@ TIME_TABLE = TimeTable()
 def run(
     cmd: list[str],
     *,
+    input: bytes | None = None,  # noqa: A002
     env: dict[str, str] | None = None,
     cwd: Path = Path.cwd(),
     log: Log = Log.STDERR,
     check: bool = True,
     error_msg: str | None = None,
 ) -> CmdOut:
-    glog.debug(f"$: {shlex.join(cmd)} \nCaller: {get_caller()}")
+    if input:
+        glog.debug(
+            f"""$: echo "{input.decode('utf-8', 'replace')}" | {shlex.join(cmd)} \nCaller: {get_caller()}"""
+        )
+    else:
+        glog.debug(f"$: {shlex.join(cmd)} \nCaller: {get_caller()}")
     tstart = datetime.now()
 
     # Start the subprocess
@@ -120,7 +126,10 @@ def run(
     )
     stdout_buf, stderr_buf = handle_output(process, log)
 
-    rc = process.wait()
+    if input:
+        process.communicate(input)
+    else:
+        process.wait()
     tend = datetime.now()
 
     global TIME_TABLE
@@ -136,7 +145,27 @@ def run(
         msg=error_msg,
     )
 
-    if check and rc != 0:
+    if check and process.returncode != 0:
         raise ClanCmdError(cmd_out)
 
     return cmd_out
+
+
+def run_no_stdout(
+    cmd: list[str],
+    *,
+    env: dict[str, str] | None = None,
+    cwd: Path = Path.cwd(),
+    log: Log = Log.STDERR,
+    check: bool = True,
+    error_msg: str | None = None,
+) -> CmdOut:
+    """
+    Like run, but automatically suppresses stdout, if not in DEBUG log level.
+    If in DEBUG log level the stdout of commands will be shown.
+    """
+    if logging.getLogger(__name__.split(".")[0]).isEnabledFor(logging.DEBUG):
+        return run(cmd, env=env, log=log, check=check, error_msg=error_msg)
+    else:
+        log = Log.NONE
+        return run(cmd, env=env, log=log, check=check, error_msg=error_msg)

pkgs/clan-cli/clan_cli/completions.py

@@ -0,0 +1,157 @@
+import argparse
+import json
+import subprocess
+import threading
+from collections.abc import Callable, Iterable
+from types import ModuleType
+from typing import Any
+
+from .cmd import run
+from .nix import nix_eval
+
+"""
+This module provides dynamic completions.
+The completions should feel fast.
+We target a maximum of 1second on our average machine.
+"""
+
+
+argcomplete: ModuleType | None = None
+try:
+    import argcomplete  # type: ignore[no-redef]
+except ImportError:
+    pass
+
+
+# The default completion timeout for commands
+COMPLETION_TIMEOUT: int = 3
+
+
+def clan_dir(flake: str | None) -> str | None:
+    from .dirs import get_clan_flake_toplevel_or_env
+
+    path_result = get_clan_flake_toplevel_or_env()
+    return str(path_result) if path_result is not None else None
+
+
+def complete_machines(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for machine names configured in the clan.
+    """
+    machines: list[str] = []
+
+    def run_cmd() -> None:
+        try:
+            # In my tests this was consistently faster than:
+            # nix eval .#nixosConfigurations --apply builtins.attrNames
+            cmd = ["nix", "flake", "show", "--system", "no-eval", "--json"]
+            if (clan_dir_result := clan_dir(None)) is not None:
+                cmd.append(clan_dir_result)
+            result = subprocess.run(
+                cmd,
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+
+            data = json.loads(result.stdout)
+            try:
+                machines.extend(data.get("nixosConfigurations").keys())
+            except KeyError:
+                pass
+        except subprocess.CalledProcessError:
+            pass
+
+    thread = threading.Thread(target=run_cmd)
+    thread.start()
+    thread.join(timeout=COMPLETION_TIMEOUT)
+
+    if thread.is_alive():
+        return iter([])
+
+    machines_dict = {name: "machine" for name in machines}
+    return machines_dict
+
+
+def complete_services_for_machine(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for machine facts generation services.
+    """
+    services: list[str] = []
+    # TODO: consolidate, if multiple machines are used
+    machines: list[str] = parsed_args.machines
+
+    def run_cmd() -> None:
+        try:
+            if (clan_dir_result := clan_dir(None)) is not None:
+                flake = clan_dir_result
+            else:
+                flake = "."
+            services_result = json.loads(
+                run(
+                    nix_eval(
+                        flags=[
+                            f"{flake}#nixosConfigurations.{machines[0]}.config.clanCore.facts.services",
+                            "--apply",
+                            "builtins.attrNames",
+                        ],
+                    ),
+                ).stdout.strip()
+            )
+
+            services.extend(services_result)
+        except subprocess.CalledProcessError:
+            pass
+
+    thread = threading.Thread(target=run_cmd)
+    thread.start()
+    thread.join(timeout=COMPLETION_TIMEOUT)
+
+    if thread.is_alive():
+        return iter([])
+
+    services_dict = {name: "service" for name in services}
+    return services_dict
+
+
+def complete_secrets(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for clan secrets
+    """
+    from pathlib import Path
+
+    from .secrets.secrets import ListSecretsOptions, list_secrets
+
+    if (clan_dir_result := clan_dir(None)) is not None:
+        flake = clan_dir_result
+    else:
+        flake = "."
+
+    options = ListSecretsOptions(
+        flake=Path(flake),
+        pattern=None,
+    )
+
+    secrets = list_secrets(options.flake, options.pattern)
+
+    secrets_dict = {name: "secret" for name in secrets}
+    return secrets_dict
+
+
+def add_dynamic_completer(
+    action: argparse.Action,
+    completer: Callable[..., Iterable[str]],
+) -> None:
+    """
+    Add a completion function to an argparse action, this will only be added,
+    if the argcomplete module is loaded.
+    """
+    if argcomplete:
+        # mypy doesn't check this correctly, so we ignore it
+        action.completer = completer  # type: ignore[attr-defined]

pkgs/clan-cli/clan_cli/config/__init__.py

@@ -150,6 +150,15 @@ def read_machine_option_value(
     return out
 
 
+def get_option(args: argparse.Namespace) -> None:
+    print(
+        read_machine_option_value(
+            args.flake, args.machine, args.option, args.show_trace
+        )
+    )
+
+
+# Currently writing is disabled
 def get_or_set_option(args: argparse.Namespace) -> None:
     if args.value == []:
         print(
@@ -307,7 +316,7 @@ def register_parser(
         )
 
     # inject callback function to process the input later
-    parser.set_defaults(func=get_or_set_option)
+    parser.set_defaults(func=get_option)
     parser.add_argument(
         "--machine",
         "-m",
@@ -345,13 +354,6 @@ def register_parser(
         type=str,
     )
 
-    parser.add_argument(
-        "value",
-        # force this arg to be set
-        nargs="*",
-        help="option value to set (if omitted, the current value is printed)",
-    )
-
 
 def main(argv: list[str] | None = None) -> None:
     if argv is None:

pkgs/clan-cli/clan_cli/facts/__init__.py

@@ -16,16 +16,116 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         required=True,
     )
 
-    check_parser = subparser.add_parser("check", help="check if facts are up to date")
+    check_parser = subparser.add_parser(
+        "check",
+        help="check if facts are up to date",
+        epilog=(
+            """
+This subcommand allows checking if all facts are up to date.
+
+Examples:
+
+  $ clan facts check [MACHINE]
+  Will check facts for the specified machine.
+
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_check_parser(check_parser)
 
-    list_parser = subparser.add_parser("list", help="list all facts")
+    list_parser = subparser.add_parser(
+        "list",
+        help="list all facts",
+        epilog=(
+            """
+This subcommand allows listing all public facts for a specific machine.
+
+The resulting list will be a json string with the name of the fact as its key
+and the fact itself as it's value.
+
+This is how an example output might look like:
+```
+{
+"[FACT_NAME]": "[FACT]"
+}
+```
+
+Examples:
+
+  $ clan facts list [MACHINE]
+  Will list facts for the specified machine.
+
+   
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_list_parser(list_parser)
 
     parser_generate = subparser.add_parser(
-        "generate", help="generate secrets for machines if they don't exist yet"
+        "generate",
+        help="generate public and secret facts for machines",
+        epilog=(
+            """
+This subcommand allows control of the generation of facts.
+Often this function will be invoked automatically on deploying machines,
+but there are situations the user may want to have more granular control,
+especially for the regeneration of certain services.
+
+A service is an included clan-module that implements facts generation functionality.
+For example the zerotier module will generate private and public facts.
+In this case the public fact will be the resulting zerotier-ip of the machine.
+The secret fact will be the zerotier-identity-secret, which is used by zerotier
+to prove the machine has control of the zerotier-ip.
+
+
+Examples:
+
+  $ clan facts generate
+  Will generate facts for all machines.
+   
+  $ clan facts generate [MACHINE]
+  Will generate facts for the specified machine.
+
+  $ clan facts generate [MACHINE] --service [SERVICE]
+  Will generate facts for the specified machine for the specified service.
+
+  $ clan facts generate --service [SERVICE] --regenerate
+  Will regenerate facts, if they are already generated for a specific service.
+  This is especially useful for resetting certain passwords while leaving the rest
+  of the facts for a machine in place.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     register_generate_parser(parser_generate)
 
-    parser_upload = subparser.add_parser("upload", help="upload secrets for machines")
+    parser_upload = subparser.add_parser(
+        "upload",
+        help="upload secrets for machines",
+        epilog=(
+            """
+This subcommand allows uploading secrets to remote machines.
+
+If using sops as a secret backend it will upload the private key to the machine.
+If using password store it uploads all the secrets you manage to the machine.
+
+The default backend is sops.
+
+Examples:
+
+  $ clan facts upload [MACHINE]
+  Will upload secrets to a specific machine.
+   
+For more detailed information, visit: https://docs.clan.lol/getting-started/secrets/
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_upload_parser(parser_upload)

pkgs/clan-cli/clan_cli/facts/check.py

@@ -2,6 +2,7 @@ import argparse
 import importlib
 import logging
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 
 log = logging.getLogger(__name__)
@@ -54,10 +55,12 @@ def check_command(args: argparse.Namespace) -> None:
 
 
 def register_check_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         help="The machine to check secrets for",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.add_argument(
         "--service",
         help="the service to check",

pkgs/clan-cli/clan_cli/facts/generate.py

@@ -9,6 +9,11 @@ from tempfile import TemporaryDirectory
 
 from clan_cli.cmd import run
 
+from ..completions import (
+    add_dynamic_completer,
+    complete_machines,
+    complete_services_for_machine,
+)
 from ..errors import ClanError
 from ..git import commit_files
 from ..machines.inventory import get_all_machines, get_selected_machines
@@ -27,12 +32,14 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
     """
     print(prompt, flush=True)
     proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True)
+    log.info("Input received. Processing...")
     return proc.stdout
 
 
 def generate_service_facts(
     machine: Machine,
     service: str,
+    regenerate: bool,
     secret_facts_store: SecretStoreBase,
     public_facts_store: FactStoreBase,
     tmpdir: Path,
@@ -42,7 +49,7 @@ def generate_service_facts(
     # check if all secrets exist and generate them if at least one is missing
     needs_regeneration = not check_secrets(machine, service=service)
     log.debug(f"{service} needs_regeneration: {needs_regeneration}")
-    if not needs_regeneration:
+    if not (needs_regeneration or regenerate):
         return False
     if not isinstance(machine.flake, Path):
         msg = f"flake is not a Path: {machine.flake}"
@@ -134,7 +141,11 @@ def prompt_func(text: str) -> str:
 
 
 def _generate_facts_for_machine(
-    machine: Machine, tmpdir: Path, prompt: Callable[[str], str] = prompt_func
+    machine: Machine,
+    service: str | None,
+    regenerate: bool,
+    tmpdir: Path,
+    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     local_temp = tmpdir / machine.name
     local_temp.mkdir()
@@ -145,10 +156,23 @@ def _generate_facts_for_machine(
     public_facts_store = public_facts_module.FactStore(machine=machine)
 
     machine_updated = False
-    for service in machine.facts_data:
+
+    if service and service not in machine.facts_data:
+        services = list(machine.facts_data.keys())
+        raise ClanError(
+            f"Could not find service with name: {service}. The following services are available: {services}"
+        )
+
+    if service:
+        machine_service_facts = {service: machine.facts_data[service]}
+    else:
+        machine_service_facts = machine.facts_data
+
+    for service in machine_service_facts:
         machine_updated |= generate_service_facts(
             machine=machine,
             service=service,
+            regenerate=regenerate,
             secret_facts_store=secret_facts_store,
             public_facts_store=public_facts_store,
             tmpdir=local_temp,
@@ -161,7 +185,10 @@ def _generate_facts_for_machine(
 
 
 def generate_facts(
-    machines: list[Machine], prompt: Callable[[str], str] = prompt_func
+    machines: list[Machine],
+    service: str | None,
+    regenerate: bool,
+    prompt: Callable[[str], str] = prompt_func,
 ) -> bool:
     was_regenerated = False
     with TemporaryDirectory() as tmp:
@@ -170,7 +197,9 @@ def generate_facts(
         for machine in machines:
             errors = 0
             try:
-                was_regenerated |= _generate_facts_for_machine(machine, tmpdir, prompt)
+                was_regenerated |= _generate_facts_for_machine(
+                    machine, service, regenerate, tmpdir, prompt
+                )
             except Exception as exc:
                 log.error(f"Failed to generate facts for {machine.name}: {exc}")
                 errors += 1
@@ -186,18 +215,35 @@ def generate_facts(
 
 def generate_command(args: argparse.Namespace) -> None:
     if len(args.machines) == 0:
-        machines = get_all_machines(args.flake)
+        machines = get_all_machines(args.flake, args.option)
     else:
-        machines = get_selected_machines(args.flake, args.machines)
-    generate_facts(machines)
+        machines = get_selected_machines(args.flake, args.option, args.machines)
+    generate_facts(machines, args.service, args.regenerate)
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machines",
         type=str,
         help="machine to generate facts for. if empty, generate facts for all machines",
         nargs="*",
         default=[],
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
+    service_parser = parser.add_argument(
+        "--service",
+        type=str,
+        help="service to generate facts for, if empty, generate facts for every service",
+        default=None,
+    )
+    add_dynamic_completer(service_parser, complete_services_for_machine)
+
+    parser.add_argument(
+        "--regenerate",
+        type=bool,
+        action=argparse.BooleanOptionalAction,
+        help="whether to regenerate facts for the specified machine",
+        default=None,
+    )
     parser.set_defaults(func=generate_command)

pkgs/clan-cli/clan_cli/facts/list.py

@@ -3,6 +3,7 @@ import importlib
 import json
 import logging
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 
 log = logging.getLogger(__name__)
@@ -26,12 +27,21 @@ def get_all_facts(machine: Machine) -> dict:
 
 def get_command(args: argparse.Namespace) -> None:
     machine = Machine(name=args.machine, flake=args.flake)
-    print(json.dumps(get_all_facts(machine), indent=4))
+
+    # the raw_facts are bytestrings making them not json serializable
+    raw_facts = get_all_facts(machine)
+    facts = dict()
+    for key in raw_facts["TODO"]:
+        facts[key] = raw_facts["TODO"][key].decode("utf8")
+
+    print(json.dumps(facts, indent=4))
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         help="The machine to print facts for",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.set_defaults(func=get_command)

pkgs/clan-cli/clan_cli/facts/secret_modules/password_store.py

@@ -2,6 +2,7 @@ import os
 import subprocess
 from pathlib import Path
 
+from clan_cli.cmd import Log, run
 from clan_cli.machines.machines import Machine
 from clan_cli.nix import nix_shell
 
@@ -15,25 +16,25 @@ class SecretStore(SecretStoreBase):
     def set(
         self, service: str, name: str, value: bytes, groups: list[str]
     ) -> Path | None:
-        subprocess.run(
+        run(
             nix_shell(
                 ["nixpkgs#pass"],
                 ["pass", "insert", "-m", f"machines/{self.machine.name}/{name}"],
             ),
             input=value,
-            check=True,
+            log=Log.BOTH,
+            error_msg=f"Failed to insert secret {name}",
         )
         return None  # we manage the files outside of the git repo
 
     def get(self, service: str, name: str) -> bytes:
-        return subprocess.run(
+        return run(
             nix_shell(
                 ["nixpkgs#pass"],
                 ["pass", "show", f"machines/{self.machine.name}/{name}"],
             ),
-            check=True,
-            stdout=subprocess.PIPE,
-        ).stdout
+            error_msg=f"Failed to get secret {name}",
+        ).stdout.encode("utf-8")
 
     def exists(self, service: str, name: str) -> bool:
         password_store = os.environ.get(
@@ -48,7 +49,7 @@ class SecretStore(SecretStoreBase):
         )
         hashes = []
         hashes.append(
-            subprocess.run(
+            run(
                 nix_shell(
                     ["nixpkgs#git"],
                     [
@@ -61,13 +62,15 @@ class SecretStore(SecretStoreBase):
                         f"machines/{self.machine.name}",
                     ],
                 ),
-                stdout=subprocess.PIPE,
-            ).stdout.strip()
+                check=False,
+            )
+            .stdout.encode("utf-8")
+            .strip()
         )
         for symlink in Path(password_store).glob(f"machines/{self.machine.name}/**/*"):
             if symlink.is_symlink():
                 hashes.append(
-                    subprocess.run(
+                    run(
                         nix_shell(
                             ["nixpkgs#git"],
                             [
@@ -80,8 +83,10 @@ class SecretStore(SecretStoreBase):
                                 str(symlink),
                             ],
                         ),
-                        stdout=subprocess.PIPE,
-                    ).stdout.strip()
+                        check=False,
+                    )
+                    .stdout.encode("utf-8")
+                    .strip()
                 )
 
         # we sort the hashes to make sure that the order is always the same

pkgs/clan-cli/clan_cli/facts/upload.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..cmd import Log, run
+from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 from ..nix import nix_shell
 
@@ -32,6 +33,8 @@ def upload_secrets(machine: Machine) -> None:
                     " ".join(["ssh"] + ssh_cmd[2:]),
                     "-az",
                     "--delete",
+                    "--chown=root:root",
+                    "--chmod=D700,F600",
                     f"{tempdir!s}/",
                     f"{host.user}@{host.host}:{machine.secrets_upload_directory}/",
                 ],
@@ -46,8 +49,10 @@ def upload_command(args: argparse.Namespace) -> None:
 
 
 def register_upload_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         help="The machine to upload secrets to",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.set_defaults(func=upload_command)

pkgs/clan-cli/clan_cli/flakes/inspect.py

@@ -39,7 +39,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     system = config["system"]
 
     # Check if the machine exists
-    machines = list_machines(False, flake_url)
+    machines = list_machines(flake_url, False)
     if machine_name not in machines:
         raise ClanError(
             f"Machine {machine_name} not found in {flake_url}. Available machines: {', '.join(machines)}"
@@ -53,7 +53,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
         gcroot_icon: Path = machine_gcroot(flake_url=str(flake_url)) / vm.machine_name
         nix_add_to_gcroots(vm.machine_icon, gcroot_icon)
 
-    # Get the cLAN name
+    # Get the Clan name
     cmd = nix_eval(
         [
             f'{flake_url}#clanInternals.machines."{system}"."{machine_name}".config.clanCore.clanName'
@@ -70,7 +70,7 @@ def inspect_flake(flake_url: str | Path, machine_name: str) -> FlakeConfig:
     )
     res = run_cmd(cmd)
 
-    # If the icon is null, no icon is set for this cLAN
+    # If the icon is null, no icon is set for this Clan
     if res == "null":
         icon_path = None
     else:
@@ -113,7 +113,7 @@ def inspect_command(args: argparse.Namespace) -> None:
     res = inspect_flake(
         flake_url=inspect_options.flake, machine_name=inspect_options.machine
     )
-    print("cLAN name:", res.clan_name)
+    print("Clan name:", res.clan_name)
     print("Icon:", res.icon)
     print("Description:", res.description)
     print("Last updated:", res.last_updated)

pkgs/clan-cli/clan_cli/flash.py

@@ -3,7 +3,6 @@ import importlib
 import json
 import logging
 import os
-import re
 import shutil
 import textwrap
 from collections.abc import Sequence
@@ -13,6 +12,7 @@ from tempfile import TemporaryDirectory
 from typing import Any
 
 from .cmd import Log, run
+from .completions import add_dynamic_completer, complete_machines
 from .errors import ClanError
 from .facts.secret_modules import SecretStoreBase
 from .machines.machines import Machine
@@ -21,63 +21,6 @@ from .nix import nix_shell
 log = logging.getLogger(__name__)
 
 
-def list_available_ssh_keys(ssh_dir: Path = Path("~/.ssh").expanduser()) -> list[Path]:
-    """
-    Function to list all available SSH public keys in the default .ssh directory.
-    Returns a list of paths to available public key files.
-    """
-    public_key_patterns = ["*.pub"]
-    available_keys: list[Path] = []
-
-    # Check for public key files
-    for pattern in public_key_patterns:
-        for key_path in ssh_dir.glob(pattern):
-            if key_path.is_file():
-                available_keys.append(key_path)
-
-    return available_keys
-
-
-def read_public_key_contents(public_keys: list[Path]) -> list[str]:
-    """
-    Function to read and return the contents of available SSH public keys.
-    Returns a list containing the contents of each public key.
-    """
-    public_key_contents = []
-
-    for key_path in public_keys:
-        try:
-            with open(key_path.expanduser()) as key_file:
-                public_key_contents.append(key_file.read().strip())
-        except FileNotFoundError:
-            log.error(f"Public key file not found: {key_path}")
-
-    return public_key_contents
-
-
-def get_keymap_and_locale() -> dict[str, str]:
-    locale = "en_US.UTF-8"
-    keymap = "en"
-
-    # Execute the `localectl status` command
-    result = run(["localectl", "status"])
-
-    if result.returncode == 0:
-        output = result.stdout
-
-        # Extract the Keymap (X11 Layout)
-        keymap_match = re.search(r"X11 Layout:\s+(.*)", output)
-        if keymap_match:
-            keymap = keymap_match.group(1)
-
-        # Extract the System Locale (LANG only)
-        locale_match = re.search(r"System Locale:\s+LANG=(.*)", output)
-        if locale_match:
-            locale = locale_match.group(1)
-
-    return {"keymap": keymap, "locale": locale}
-
-
 def flash_machine(
     machine: Machine,
     *,
@@ -85,7 +28,9 @@ def flash_machine(
     disks: dict[str, str],
     system_config: dict[str, Any],
     dry_run: bool,
+    write_efi_boot_entries: bool,
     debug: bool,
+    extra_args: list[str] = [],
 ) -> None:
     secret_facts_module = importlib.import_module(machine.secret_facts_module)
     secret_facts_store: SecretStoreBase = secret_facts_module.SecretStore(
@@ -112,6 +57,8 @@ def flash_machine(
             disko_install.append("sudo")
 
         disko_install.append("disko-install")
+        if write_efi_boot_entries:
+            disko_install.append("--write-efi-boot-entries")
         if dry_run:
             disko_install.append("--dry-run")
         if debug:
@@ -128,6 +75,8 @@ def flash_machine(
                 json.dumps(system_config),
             ]
         )
+        disko_install.extend(["--option", "dry-run", "true"])
+        disko_install.extend(extra_args)
 
         cmd = nix_shell(
             ["nixpkgs#disko"],
@@ -148,6 +97,8 @@ class FlashOptions:
     mode: str
     language: str
     keymap: str
+    write_efi_boot_entries: bool
+    nix_options: list[str]
 
 
 class AppendDiskAction(argparse.Action):
@@ -176,8 +127,10 @@ def flash_command(args: argparse.Namespace) -> None:
         confirm=not args.yes,
         debug=args.debug,
         mode=args.mode,
-        language=args.lang,
+        language=args.language,
         keymap=args.keymap,
+        write_efi_boot_entries=args.write_efi_boot_entries,
+        nix_options=args.option,
     )
 
     machine = Machine(opts.machine, flake=opts.flake)
@@ -191,40 +144,22 @@ def flash_command(args: argparse.Namespace) -> None:
         if ask != "y":
             return
 
-    root_keys = read_public_key_contents(opts.ssh_keys_path)
-    if opts.confirm and not root_keys:
-        msg = "Should we add your SSH public keys to the root user? [y/N] "
-        ask = input(msg)
-        if ask == "y":
-            pubkeys = list_available_ssh_keys()
-            root_keys.extend(read_public_key_contents(pubkeys))
-        else:
-            raise ClanError(
-                "No SSH public keys provided. Use --ssh-pubkey to add keys."
-            )
-    elif not opts.confirm and not root_keys:
-        pubkeys = list_available_ssh_keys()
-        root_keys.extend(read_public_key_contents(pubkeys))
-    # If ssh-pubkeys set, we don't need to ask for confirmation
-    elif opts.confirm and root_keys:
-        pass
-    elif not opts.confirm and root_keys:
-        pass
-    else:
-        raise ClanError("Invalid state")
-
-    localectl = get_keymap_and_locale()
-    extra_config = {
-        "users": {
+    extra_config: dict[str, Any] = {}
+    if opts.ssh_keys_path:
+        root_keys = []
+        for key_path in opts.ssh_keys_path:
+            try:
+                root_keys.append(key_path.read_text())
+            except OSError as e:
+                raise ClanError(f"Cannot read SSH public key file: {key_path}: {e}")
+        extra_config["users"] = {
             "users": {"root": {"openssh": {"authorizedKeys": {"keys": root_keys}}}}
-        },
-        "console": {
-            "keyMap": opts.keymap if opts.keymap else localectl["keymap"],
-        },
-        "i18n": {
-            "defaultLocale": opts.language if opts.language else localectl["locale"],
-        },
-    }
+        }
+    if opts.keymap:
+        extra_config["console"] = {"keyMap": opts.keymap}
+
+    if opts.language:
+        extra_config["i18n"] = {"defaultLocale": opts.language}
 
     flash_machine(
         machine,
@@ -233,15 +168,19 @@ def flash_command(args: argparse.Namespace) -> None:
         system_config=extra_config,
         dry_run=opts.dry_run,
         debug=opts.debug,
+        write_efi_boot_entries=opts.write_efi_boot_entries,
+        extra_args=opts.nix_options,
     )
 
 
 def register_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         type=str,
         help="machine to install",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.add_argument(
         "--disk",
         type=str,
@@ -251,12 +190,14 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         help="device to flash to",
         default={},
     )
-    mode_help = textwrap.dedent("""\
+    mode_help = textwrap.dedent(
+        """\
         Specify the mode of operation. Valid modes are: format, mount."
         Format will format the disk before installing.
         Mount will mount the disk before installing.
         Mount is useful for updating an existing system without losing data.
-        """)
+        """
+    )
     parser.add_argument(
         "--mode",
         type=str,
@@ -272,7 +213,7 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         help="ssh pubkey file to add to the root user. Can be used multiple times",
     )
     parser.add_argument(
-        "--lang",
+        "--language",
         type=str,
         help="system language",
     )
@@ -293,4 +234,16 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         default=False,
         action="store_true",
     )
+    parser.add_argument(
+        "--write-efi-boot-entries",
+        help=textwrap.dedent(
+            """
+          Write EFI boot entries to the NVRAM of the system for the installed system.
+          Specify this option if you plan to boot from this disk on the current machine,
+          but not if you plan to move the disk to another machine.
+        """
+        ).strip(),
+        default=False,
+        action="store_true",
+    )
     parser.set_defaults(func=flash_command)

pkgs/clan-cli/clan_cli/machines/__init__.py

@@ -5,6 +5,7 @@ from .create import register_create_parser
 from .delete import register_delete_parser
 from .install import register_install_parser
 from .list import register_list_parser
+from .show import register_show_parser
 from .update import register_update_parser
 
 
@@ -17,7 +18,26 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         required=True,
     )
 
-    update_parser = subparser.add_parser("update", help="Update a machine")
+    update_parser = subparser.add_parser(
+        "update",
+        help="Update a machine",
+        epilog=(
+            """
+This subcommand provides an interface to update machines managed by clan.
+
+Examples:
+
+  $ clan machines update [MACHINES]
+  Will update the specified machine [MACHINE], if [MACHINE] is omitted, the command
+  will attempt to update every configured machine.
+  To exclude machines being updated `clan.deployment.requireExplicitUpdate = true;`
+  can be set in the machine config.
+
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_update_parser(update_parser)
 
     create_parser = subparser.add_parser("create", help="Create a machine")
@@ -26,9 +46,34 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
     delete_parser = subparser.add_parser("delete", help="Delete a machine")
     register_delete_parser(delete_parser)
 
-    list_parser = subparser.add_parser("list", help="List machines")
+    list_parser = subparser.add_parser(
+        "list",
+        help="List machines",
+        epilog=(
+            """
+This subcommand lists all machines managed by this clan.
+
+Examples:
+
+  $ clan machines list
+  Lists all the machines and their descriptions.
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     register_list_parser(list_parser)
 
+    show_parser = subparser.add_parser(
+        "show",
+        help="Show a machine",
+        epilog=(
+            """
+This subcommand shows the details of a machine managed by this clan like icon, description, etc
+"""
+        ),
+    )
+    register_show_parser(show_parser)
+
     install_parser = subparser.add_parser(
         "install",
         help="Install a machine",
@@ -37,5 +82,23 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
         The target must be a Linux based system reachable via SSH.
         Installing a machine means overwriting the target's disk.
         """,
+        epilog=(
+            """
+This subcommand provides an interface to install machines managed by clan.
+
+Examples:
+
+  $ clan machines install [MACHINE] [TARGET_HOST]
+  Will install the specified machine [MACHINE], to the specified [TARGET_HOST].
+
+  $ clan machines install [MACHINE] --json [JSON]
+  Will install the specified machine [MACHINE] to the host exposed by 
+  the deployment information of the [JSON] deployment string.
+
+For information on how to set up the installer see: https://docs.clan.lol/getting-started/installer/
+For more detailed information, visit: https://docs.clan.lol/getting-started/deploy
+        """
+        ),
+        formatter_class=argparse.RawTextHelpFormatter,
     )
     register_install_parser(install_parser)

pkgs/clan-cli/clan_cli/machines/create.py

@@ -1,13 +1,27 @@
 import argparse
 import logging
+from dataclasses import dataclass
+from pathlib import Path
 
+from clan_cli.api import API
 from clan_cli.config.machine import set_config_for_machine
 
 log = logging.getLogger(__name__)
 
 
+@dataclass
+class MachineCreateRequest:
+    name: str
+    config: dict[str, int]
+
+
+@API.register
+def create_machine(flake_dir: str | Path, machine: MachineCreateRequest) -> None:
+    set_config_for_machine(Path(flake_dir), machine.name, machine.config)
+
+
 def create_command(args: argparse.Namespace) -> None:
-    set_config_for_machine(args.flake, args.machine, dict())
+    create_machine(args.flake, MachineCreateRequest(args.machine, dict()))
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/machines/delete.py

@@ -1,6 +1,7 @@
 import argparse
 import shutil
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..dirs import specific_machine_dir
 from ..errors import ClanError
 
@@ -14,5 +15,7 @@ def delete_command(args: argparse.Namespace) -> None:
 
 
 def register_delete_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument("host", type=str)
+    machines_parser = parser.add_argument("host", type=str)
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.set_defaults(func=delete_command)

pkgs/clan-cli/clan_cli/machines/install.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..cmd import Log, run
+from ..completions import add_dynamic_completer, complete_machines
 from ..facts.generate import generate_facts
 from ..machines.machines import Machine
 from ..nix import nix_shell
@@ -26,6 +27,7 @@ def install_nixos(
     debug: bool = False,
     password: str | None = None,
     no_reboot: bool = False,
+    extra_args: list[str] = [],
 ) -> None:
     secret_facts_module = importlib.import_module(machine.secret_facts_module)
     log.info(f"installing {machine.name}")
@@ -35,7 +37,7 @@ def install_nixos(
     target_host = f"{h.user or 'root'}@{h.host}"
     log.info(f"target host: {target_host}")
 
-    generate_facts([machine])
+    generate_facts([machine], None, False)
 
     with TemporaryDirectory() as tmpdir_:
         tmpdir = Path(tmpdir_)
@@ -56,6 +58,7 @@ def install_nixos(
             f"{machine.flake}#{machine.name}",
             "--extra-files",
             str(tmpdir),
+            *extra_args,
         ]
 
         if no_reboot:
@@ -95,6 +98,7 @@ class InstallOptions:
     debug: bool
     no_reboot: bool
     json_ssh_deploy: dict[str, str] | None
+    nix_options: list[str]
 
 
 def install_command(args: argparse.Namespace) -> None:
@@ -127,6 +131,7 @@ def install_command(args: argparse.Namespace) -> None:
         debug=args.debug,
         no_reboot=args.no_reboot,
         json_ssh_deploy=json_ssh_deploy,
+        nix_options=args.option,
     )
     machine = Machine(opts.machine, flake=opts.flake)
     machine.target_host_address = opts.target_host
@@ -142,6 +147,7 @@ def install_command(args: argparse.Namespace) -> None:
         debug=opts.debug,
         password=password,
         no_reboot=opts.no_reboot,
+        extra_args=opts.nix_options,
     )
 
 
@@ -183,11 +189,14 @@ def register_install_parser(parser: argparse.ArgumentParser) -> None:
         help="do not ask for confirmation",
         default=False,
     )
-    parser.add_argument(
+
+    machines_parser = parser.add_argument(
         "machine",
         type=str,
         help="machine to install",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.add_argument(
         "target_host",
         type=str,

pkgs/clan-cli/clan_cli/machines/inventory.py

@@ -7,7 +7,7 @@ from .machines import Machine
 
 
 # function to speedup eval if we want to evauluate all machines
-def get_all_machines(flake_dir: Path) -> list[Machine]:
+def get_all_machines(flake_dir: Path, nix_options: list[str]) -> list[Machine]:
     config = nix_config()
     system = config["system"]
     json_path = run(
@@ -19,13 +19,20 @@ def get_all_machines(flake_dir: Path) -> list[Machine]:
     machines = []
     for name, machine_data in machines_json.items():
         machines.append(
-            Machine(name=name, flake=flake_dir, deployment_info=machine_data)
+            Machine(
+                name=name,
+                flake=flake_dir,
+                deployment_info=machine_data,
+                nix_options=nix_options,
+            )
         )
     return machines
 
 
-def get_selected_machines(flake_dir: Path, machine_names: list[str]) -> list[Machine]:
+def get_selected_machines(
+    flake_dir: Path, nix_options: list[str], machine_names: list[str]
+) -> list[Machine]:
     machines = []
     for name in machine_names:
-        machines.append(Machine(name=name, flake=flake_dir))
+        machines.append(Machine(name=name, flake=flake_dir, nix_options=nix_options))
     return machines

pkgs/clan-cli/clan_cli/machines/list.py

@@ -5,17 +5,14 @@ from pathlib import Path
 
 from clan_cli.api import API
 
-from ..cmd import Log, run
+from ..cmd import run_no_stdout
 from ..nix import nix_config, nix_eval
 
 log = logging.getLogger(__name__)
 
 
 @API.register
-def list_machines(
-    debug: bool,
-    flake_url: Path | str,
-) -> list[str]:
+def list_machines(flake_url: str | Path, debug: bool) -> list[str]:
     config = nix_config()
     system = config["system"]
     cmd = nix_eval(
@@ -27,18 +24,16 @@ def list_machines(
         ]
     )
 
-    if not debug:
-        proc = run(cmd, log=Log.NONE)
-    else:
-        proc = run(cmd)
+    proc = run_no_stdout(cmd)
 
     res = proc.stdout.strip()
     return json.loads(res)
 
 
 def list_command(args: argparse.Namespace) -> None:
-    for machine in list_machines(args.debug, Path(args.flake)):
-        print(machine)
+    flake_path = Path(args.flake).resolve()
+    for name in list_machines(flake_path, args.debug):
+        print(name)
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:

pkgs/clan-cli/clan_cli/machines/machines.py

@@ -10,7 +10,7 @@ from clan_cli.clan_uri import ClanURI, MachineData
 from clan_cli.dirs import vm_state_dir
 from clan_cli.qemu.qmp import QEMUMonitorProtocol
 
-from ..cmd import run
+from ..cmd import run_no_stdout
 from ..errors import ClanError
 from ..nix import nix_build, nix_config, nix_eval, nix_metadata
 from ..ssh import Host, parse_deployment_address
@@ -41,9 +41,10 @@ class QMPWrapper:
 
 
 class Machine:
-    flake: str | Path
     name: str
+    flake: str | Path
     data: MachineData
+    nix_options: list[str]
     eval_cache: dict[str, str]
     build_cache: dict[str, Path]
     _flake_path: Path | None
@@ -55,6 +56,7 @@ class Machine:
         name: str,
         flake: Path | str,
         deployment_info: dict | None = None,
+        nix_options: list[str] = [],
         machine: MachineData | None = None,
     ) -> None:
         """
@@ -76,6 +78,7 @@ class Machine:
         self.build_cache: dict[str, Path] = {}
         self._flake_path: Path | None = None
         self._deployment_info: None | dict = deployment_info
+        self.nix_options = nix_options
 
         state_dir = vm_state_dir(flake_url=str(self.flake), vm_name=self.data.name)
 
@@ -197,7 +200,7 @@ class Machine:
             config_json.flush()
 
             file_info = json.loads(
-                run(
+                run_no_stdout(
                     nix_eval(
                         [
                             "--impure",
@@ -242,15 +245,15 @@ class Machine:
                 flake = f"path:{self.flake_dir}"
 
             args += [
-                f'{flake}#clanInternals.machines."{system}".{self.data.name}.{attr}',
-                *nix_options,
+                f'{flake}#clanInternals.machines."{system}".{self.data.name}.{attr}'
             ]
+        args += nix_options + self.nix_options
 
         if method == "eval":
-            output = run(nix_eval(args)).stdout.strip()
+            output = run_no_stdout(nix_eval(args)).stdout.strip()
             return output
         elif method == "build":
-            outpath = run(nix_build(args)).stdout.strip()
+            outpath = run_no_stdout(nix_build(args)).stdout.strip()
             return Path(outpath)
         else:
             raise ValueError(f"Unknown method {method}")

pkgs/clan-cli/clan_cli/machines/show.py

@@ -0,0 +1,60 @@
+import argparse
+import dataclasses
+import json
+import logging
+from pathlib import Path
+
+from clan_cli.api import API
+
+from ..cmd import run_no_stdout
+from ..completions import add_dynamic_completer, complete_machines
+from ..nix import nix_config, nix_eval
+from .types import machine_name_type
+
+log = logging.getLogger(__name__)
+
+
+@dataclasses.dataclass
+class MachineInfo:
+    machine_name: str
+    machine_description: str | None
+    machine_icon: str | None
+
+
+@API.register
+def show_machine(flake_url: str | Path, machine_name: str, debug: bool) -> MachineInfo:
+    config = nix_config()
+    system = config["system"]
+    cmd = nix_eval(
+        [
+            f"{flake_url}#clanInternals.machines.{system}.{machine_name}",
+            "--apply",
+            "machine: { inherit (machine.config.clanCore) machineDescription machineIcon machineName; }",
+            "--json",
+        ]
+    )
+    proc = run_no_stdout(cmd)
+    res = proc.stdout.strip()
+    machine = json.loads(res)
+
+    return MachineInfo(
+        machine_name=machine.get("machineName"),
+        machine_description=machine.get("machineDescription", None),
+        machine_icon=machine.get("machineIcon", None),
+    )
+
+
+def show_command(args: argparse.Namespace) -> None:
+    flake_path = Path(args.flake).resolve()
+    machine = show_machine(flake_path, args.machine, args.debug)
+    print(f"Name: {machine.machine_name}")
+    print(f"Description: {machine.machine_description or ''}")
+    print(f"Icon: {machine.machine_icon or ''}")
+
+
+def register_show_parser(parser: argparse.ArgumentParser) -> None:
+    parser.set_defaults(func=show_command)
+    machine_parser = parser.add_argument(
+        "machine", help="the name of the machine", type=machine_name_type
+    )
+    add_dynamic_completer(machine_parser, complete_machines)

pkgs/clan-cli/clan_cli/machines/update.py

@@ -3,9 +3,10 @@ import json
 import logging
 import os
 import shlex
-import subprocess
 import sys
 
+from ..cmd import run
+from ..completions import add_dynamic_completer, complete_machines
 from ..errors import ClanError
 from ..facts.generate import generate_facts
 from ..facts.upload import upload_secrets
@@ -53,11 +54,7 @@ def upload_sources(
                     path,
                 ]
             )
-            proc = subprocess.run(cmd, stdout=subprocess.PIPE, env=env, check=False)
-            if proc.returncode != 0:
-                raise ClanError(
-                    f"failed to upload sources: {shlex.join(cmd)} failed with {proc.returncode}"
-                )
+            run(cmd, env=env, error_msg="failed to upload sources")
             return path
 
     # Slow path: we need to upload all sources to the remote machine
@@ -73,16 +70,13 @@ def upload_sources(
         ]
     )
     log.info("run %s", shlex.join(cmd))
-    proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=False)
-    if proc.returncode != 0:
-        raise ClanError(
-            f"failed to upload sources: {shlex.join(cmd)} failed with {proc.returncode}"
-        )
+    proc = run(cmd, error_msg="failed to upload sources")
+
     try:
         return json.loads(proc.stdout)["path"]
     except (json.JSONDecodeError, OSError) as e:
         raise ClanError(
-            f"failed to parse output of {shlex.join(cmd)}: {e}\nGot: {proc.stdout.decode('utf-8', 'replace')}"
+            f"failed to parse output of {shlex.join(cmd)}: {e}\nGot: {proc.stdout}"
         )
 
 
@@ -98,7 +92,7 @@ def deploy_nixos(machines: MachineGroup) -> None:
         env = os.environ.copy()
         env["NIX_SSHOPTS"] = ssh_arg
 
-        generate_facts([machine])
+        generate_facts([machine], None, False)
         upload_secrets(machine)
 
         path = upload_sources(".", target)
@@ -110,11 +104,9 @@ def deploy_nixos(machines: MachineGroup) -> None:
 
         ssh_arg += " -i " + host.key if host.key else ""
 
-        extra_args = host.meta.get("extra_args", [])
         cmd = [
             "nixos-rebuild",
             "switch",
-            *extra_args,
             "--fast",
             "--option",
             "keep-going",
@@ -124,6 +116,7 @@ def deploy_nixos(machines: MachineGroup) -> None:
             "true",
             "--build-host",
             "",
+            *machine.nix_options,
             "--flake",
             f"{path}#{machine.name}",
         ]
@@ -143,7 +136,9 @@ def update(args: argparse.Namespace) -> None:
         raise ClanError("Could not find clan flake toplevel directory")
     machines = []
     if len(args.machines) == 1 and args.target_host is not None:
-        machine = Machine(name=args.machines[0], flake=args.flake)
+        machine = Machine(
+            name=args.machines[0], flake=args.flake, nix_options=args.option
+        )
         machine.target_host_address = args.target_host
         machines.append(machine)
 
@@ -153,7 +148,7 @@ def update(args: argparse.Namespace) -> None:
     else:
         if len(args.machines) == 0:
             ignored_machines = []
-            for machine in get_all_machines(args.flake):
+            for machine in get_all_machines(args.flake, args.option):
                 if machine.deployment_info.get("requireExplicitUpdate", False):
                     continue
                 try:
@@ -173,13 +168,13 @@ def update(args: argparse.Namespace) -> None:
                     print(machine, file=sys.stderr)
 
         else:
-            machines = get_selected_machines(args.flake, args.machines)
+            machines = get_selected_machines(args.flake, args.option, args.machines)
 
     deploy_nixos(MachineGroup(machines))
 
 
 def register_update_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machines",
         type=str,
         nargs="*",
@@ -187,6 +182,9 @@ def register_update_parser(parser: argparse.ArgumentParser) -> None:
         metavar="MACHINE",
         help="machine to update. If no machine is specified, all machines will be updated.",
     )
+
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.add_argument(
         "--target-host",
         type=str,

pkgs/clan-cli/clan_cli/nix.py

@@ -106,7 +106,7 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
     if os.environ.get("IN_NIX_SANDBOX"):
         return cmd
     return [
-        *nix_command(["shell", "--offline", "--inputs-from", f"{nixpkgs_flake()!s}"]),
+        *nix_command(["shell", "--inputs-from", f"{nixpkgs_flake()!s}"]),
         *packages,
         "-c",
         *cmd,

pkgs/clan-cli/clan_cli/secrets/machines.py

@@ -1,6 +1,7 @@
 import argparse
 from pathlib import Path
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..errors import ClanError
 from ..git import commit_files
 from ..machines.types import machine_name_type, validate_hostname
@@ -147,25 +148,28 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
 
     # Parser
     get_parser = subparser.add_parser("get", help="get a machine public key")
-    get_parser.add_argument(
+    get_machine_parser = get_parser.add_argument(
         "machine", help="the name of the machine", type=machine_name_type
     )
+    add_dynamic_completer(get_machine_parser, complete_machines)
     get_parser.set_defaults(func=get_command)
 
     # Parser
     remove_parser = subparser.add_parser("remove", help="remove a machine")
-    remove_parser.add_argument(
+    remove_machine_parser = remove_parser.add_argument(
         "machine", help="the name of the machine", type=machine_name_type
     )
+    add_dynamic_completer(remove_machine_parser, complete_machines)
     remove_parser.set_defaults(func=remove_command)
 
     # Parser
     add_secret_parser = subparser.add_parser(
         "add-secret", help="allow a machine to access a secret"
     )
-    add_secret_parser.add_argument(
+    machine_add_secret_parser = add_secret_parser.add_argument(
         "machine", help="the name of the machine", type=machine_name_type
     )
+    add_dynamic_completer(machine_add_secret_parser, complete_machines)
     add_secret_parser.add_argument(
         "secret", help="the name of the secret", type=secret_name_type
     )
@@ -175,9 +179,10 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
     remove_secret_parser = subparser.add_parser(
         "remove-secret", help="remove a group's access to a secret"
     )
-    remove_secret_parser.add_argument(
-        "machine", help="the name of the group", type=machine_name_type
+    machine_remove_parser = remove_secret_parser.add_argument(
+        "machine", help="the name of the machine", type=machine_name_type
     )
+    add_dynamic_completer(machine_remove_parser, complete_machines)
     remove_secret_parser.add_argument(
         "secret", help="the name of the secret", type=secret_name_type
     )

pkgs/clan-cli/clan_cli/secrets/secrets.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from typing import IO
 
 from .. import tty
+from ..completions import add_dynamic_completer, complete_secrets
 from ..errors import ClanError
 from ..git import commit_files
 from .folders import (
@@ -153,8 +154,12 @@ def remove_command(args: argparse.Namespace) -> None:
     remove_secret(Path(args.flake), args.secret)
 
 
-def add_secret_argument(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument("secret", help="the name of the secret", type=secret_name_type)
+def add_secret_argument(parser: argparse.ArgumentParser, autocomplete: bool) -> None:
+    secrets_parser = parser.add_argument(
+        "secret", help="the name of the secret", type=secret_name_type
+    )
+    if autocomplete:
+        add_dynamic_completer(secrets_parser, complete_secrets)
 
 
 def machines_folder(flake_dir: Path, group: str) -> Path:
@@ -323,11 +328,11 @@ def register_secrets_parser(subparser: argparse._SubParsersAction) -> None:
     parser_list.set_defaults(func=list_command)
 
     parser_get = subparser.add_parser("get", help="get a secret")
-    add_secret_argument(parser_get)
+    add_secret_argument(parser_get, True)
     parser_get.set_defaults(func=get_command)
 
     parser_set = subparser.add_parser("set", help="set a secret")
-    add_secret_argument(parser_set)
+    add_secret_argument(parser_set, False)
     parser_set.add_argument(
         "--group",
         type=str,
@@ -359,10 +364,10 @@ def register_secrets_parser(subparser: argparse._SubParsersAction) -> None:
     parser_set.set_defaults(func=set_command)
 
     parser_rename = subparser.add_parser("rename", help="rename a secret")
-    add_secret_argument(parser_rename)
+    add_secret_argument(parser_rename, True)
     parser_rename.add_argument("new_name", type=str, help="the new name of the secret")
     parser_rename.set_defaults(func=rename_command)
 
     parser_remove = subparser.add_parser("remove", help="remove a secret")
-    add_secret_argument(parser_remove)
+    add_secret_argument(parser_remove, True)
     parser_remove.set_defaults(func=remove_command)

pkgs/clan-cli/clan_cli/vms/run.py

@@ -69,7 +69,7 @@ def get_secrets(
     secret_facts_module = importlib.import_module(machine.secret_facts_module)
     secret_facts_store = secret_facts_module.SecretStore(machine=machine)
 
-    generate_facts([machine])
+    generate_facts([machine], None, False)
 
     secret_facts_store.upload(secrets_dir)
     return secrets_dir

pkgs/clan-cli/default.nix

@@ -29,6 +29,7 @@
   mypy,
   nixpkgs,
   clan-core-path,
+  gitMinimal,
 }:
 let
   # Dependencies that are directly used in the project
@@ -83,7 +84,7 @@ let
     cp -r ${./.} $out
     chmod -R +w $out
     rm $out/clan_cli/config/jsonschema
-    ln -s ${nixpkgs'} $out/clan_cli/nixpkgs
+    ln -sf ${nixpkgs'} $out/clan_cli/nixpkgs
     cp -r ${../../lib/jsonschema} $out/clan_cli/config/jsonschema
   '';
 
@@ -101,7 +102,7 @@ let
       outputs = _inputs: { };
     }
     EOF
-    ln -s ${nixpkgs} $out/path
+    ln -sf ${nixpkgs} $out/path
     nix flake update $out \
       --store ./. \
       --extra-experimental-features 'nix-command flakes'
@@ -113,7 +114,13 @@ python3.pkgs.buildPythonApplication {
   format = "pyproject";
 
   # Arguments for the wrapper to unset LD_LIBRARY_PATH to avoid glibc version issues
-  makeWrapperArgs = [ "--unset LD_LIBRARY_PATH" ];
+  makeWrapperArgs = [
+    "--unset LD_LIBRARY_PATH"
+    "--suffix"
+    "PATH"
+    ":"
+    "${gitMinimal}/bin/git"
+  ];
 
   # Build-time dependencies.
   nativeBuildInputs = [

pkgs/clan-cli/docs.py

@@ -22,7 +22,7 @@ class Option:
         md_li += indent_next(
             f"\n{self.description.strip()}" if self.description else ""
         )
-        md_li += indent_next(f"\n{self.epilog.strip()}" if self.epilog else "")
+        # md_li += indent_next(f"\n{self.epilog.strip()}" if self.epilog else "")
 
         return md_li
 
@@ -82,13 +82,54 @@ class Category:
             md_li += indent_all(
                 f"{self.description.strip()}\n" if self.description else "", 4
             )
-            md_li += "\n"
-            md_li += indent_all(f"{self.epilog.strip()}\n" if self.epilog else "", 4)
-            md_li += "\n"
 
         return md_li
 
 
+def epilog_to_md(text: str) -> str:
+    """
+    Convert the epilog to md
+    """
+    after_examples = False
+    md = ""
+    # md += text
+    for line in text.split("\n"):
+        if line.strip() == "Examples:":
+            after_examples = True
+            md += "### Examples"
+            md += "\n"
+        else:
+            if after_examples:
+                if line.strip().startswith("$"):
+                    md += f"`{line}`"
+                    md += "\n"
+                    md += "\n"
+                else:
+                    if contains_https_link(line):
+                        line = convert_to_markdown_link(line)
+                    md += line
+                    md += "\n"
+            else:
+                md += line
+                md += "\n"
+    return md
+
+
+import re
+
+
+def contains_https_link(line: str) -> bool:
+    pattern = r"https://\S+"
+    return re.search(pattern, line) is not None
+
+
+def convert_to_markdown_link(line: str) -> str:
+    pattern = r"(https://\S+)"
+
+    # Replacement pattern to convert it to a Markdown link
+    return re.sub(pattern, r"[\1](\1)", line)
+
+
 def indent_next(text: str, indent_size: int = 4) -> str:
     """
     Indent all lines in a string except the first line.
@@ -136,7 +177,7 @@ def get_subcommands(
             continue
 
         if isinstance(action, argparse._SubParsersAction):
-            continue  # Subparsers handled sperately
+            continue  # Subparsers handled separately
 
         option_strings = ", ".join(action.option_strings)
         if option_strings:
@@ -178,7 +219,7 @@ def get_subcommands(
                     Category(
                         title=f"{parent} {name}",
                         description=subparser.description,
-                        epilog=subparser.epilog,
+                        # epilog=subparser.epilog,
                         level=level,
                         options=_options,
                         positionals=_positionals,
@@ -222,6 +263,7 @@ def collect_commands() -> list[Category]:
                         options=_options,
                         positionals=_positionals,
                         subcommands=_subcommands,
+                        epilog=subparser.epilog,
                         level=1,
                     )
                 )
@@ -280,6 +322,7 @@ def build_command_reference() -> None:
         markdown = files.get(folder / f"{filename}.md", "")
 
         markdown += f"{'#'*(cmd.level)} {cmd.title.capitalize()}\n\n"
+
         markdown += f"{cmd.description}\n\n" if cmd.description else ""
 
         # usage: clan vms run [-h] machine
@@ -320,6 +363,8 @@ def build_command_reference() -> None:
             markdown += indent_all(commands_fmt)
             markdown += "\n"
 
+        markdown += f"{epilog_to_md(cmd.epilog)}\n\n" if cmd.epilog else ""
+
         files[folder / f"{filename}.md"] = markdown
 
     for fname, content in files.items():

pkgs/clan-cli/tests/test_backups.py

@@ -11,10 +11,10 @@ def test_backups(
 
     cli.run(
         [
-            "--flake",
-            str(test_flake_with_core.path),
             "backups",
             "list",
+            "--flake",
+            str(test_flake_with_core.path),
             "vm1",
         ]
     )

pkgs/clan-cli/tests/test_config.py

@@ -1,7 +1,4 @@
-import json
-import tempfile
 from pathlib import Path
-from typing import Any
 
 import pytest
 from cli import Cli
@@ -14,46 +11,6 @@ from clan_cli.errors import ClanError
 example_options = f"{Path(config.__file__).parent}/jsonschema/options.json"
 
 
-# use pytest.parametrize
-@pytest.mark.parametrize(
-    "args,expected",
-    [
-        (["name", "DavHau"], {"name": "DavHau"}),
-        (
-            ["kernelModules", "foo", "bar", "baz"],
-            {"kernelModules": ["foo", "bar", "baz"]},
-        ),
-        (["services.opt", "test"], {"services": {"opt": "test"}}),
-        (["userIds.DavHau", "42"], {"userIds": {"DavHau": 42}}),
-    ],
-)
-def test_set_some_option(
-    args: list[str],
-    expected: dict[str, Any],
-    test_flake: FlakeForTest,
-) -> None:
-    # create temporary file for out_file
-    with tempfile.NamedTemporaryFile() as out_file:
-        with open(out_file.name, "w") as f:
-            json.dump({}, f)
-        cli = Cli()
-        cli.run(
-            [
-                "--flake",
-                str(test_flake.path),
-                "config",
-                "--quiet",
-                "--options-file",
-                example_options,
-                "--settings-file",
-                out_file.name,
-                *args,
-            ]
-        )
-        json_out = json.loads(open(out_file.name).read())
-        assert json_out == expected
-
-
 def test_configure_machine(
     test_flake: FlakeForTest,
     temporary_home: Path,
@@ -62,25 +19,14 @@ def test_configure_machine(
 ) -> None:
     cli = Cli()
 
-    cli.run(
-        [
-            "--flake",
-            str(test_flake.path),
-            "config",
-            "-m",
-            "machine1",
-            "clan.jitsi.enable",
-            "true",
-        ]
-    )
     # clear the output buffer
     capsys.readouterr()
     # read a option value
     cli.run(
         [
+            "config",
             "--flake",
             str(test_flake.path),
-            "config",
             "-m",
             "machine1",
             "clan.jitsi.enable",
@@ -88,7 +34,7 @@ def test_configure_machine(
     )
 
     # read the output
-    assert capsys.readouterr().out == "true\n"
+    assert capsys.readouterr().out == "false\n"
 
 
 def test_walk_jsonschema_all_types() -> None:

pkgs/clan-cli/tests/test_create_flake.py

@@ -17,10 +17,51 @@ def test_create_flake(
     capsys: pytest.CaptureFixture,
     temporary_home: Path,
     cli: Cli,
+    clan_core: Path,
 ) -> None:
     flake_dir = temporary_home / "test-flake"
 
-    cli.run(["flakes", "create", str(flake_dir)])
+    url = f"{clan_core}#default"
+    cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
+    assert (flake_dir / ".clan-flake").exists()
+    monkeypatch.chdir(flake_dir)
+    cli.run(["machines", "create", "machine1"])
+    capsys.readouterr()  # flush cache
+
+    # create a hardware-configuration.nix that doesn't throw an eval error
+
+    for patch_machine in ["jon", "sara"]:
+        with open(
+            flake_dir / "machines" / f"{patch_machine}/hardware-configuration.nix", "w"
+        ) as hw_config_nix:
+            hw_config_nix.write("{}")
+
+    cli.run(["machines", "list"])
+    assert "machine1" in capsys.readouterr().out
+    flake_show = subprocess.run(
+        ["nix", "flake", "show", "--json"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    flake_outputs = json.loads(flake_show.stdout)
+    try:
+        flake_outputs["nixosConfigurations"]["machine1"]
+    except KeyError:
+        pytest.fail("nixosConfigurations.machine1 not found in flake outputs")
+
+
+@pytest.mark.impure
+def test_ui_template(
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture,
+    temporary_home: Path,
+    cli: Cli,
+    clan_core: Path,
+) -> None:
+    flake_dir = temporary_home / "test-flake"
+    url = f"{clan_core}#empty"
+    cli.run(["flakes", "create", str(flake_dir), f"--url={url}"])
     assert (flake_dir / ".clan-flake").exists()
     monkeypatch.chdir(flake_dir)
     cli.run(["machines", "create", "machine1"])
@@ -39,16 +80,3 @@ def test_create_flake(
         flake_outputs["nixosConfigurations"]["machine1"]
     except KeyError:
         pytest.fail("nixosConfigurations.machine1 not found in flake outputs")
-    # configure machine1
-    capsys.readouterr()
-    cli.run(["config", "--machine", "machine1", "services.openssh.enable", ""])
-    capsys.readouterr()
-    cli.run(
-        [
-            "config",
-            "--machine",
-            "machine1",
-            "services.openssh.enable",
-            "true",
-        ]
-    )

pkgs/clan-cli/tests/test_flakes_cli.py

@@ -15,10 +15,10 @@ def test_flakes_inspect(
     cli = Cli()
     cli.run(
         [
-            "--flake",
-            str(test_flake_with_core.path),
             "flakes",
             "inspect",
+            "--flake",
+            str(test_flake_with_core.path),
             "--machine",
             "vm1",
         ]

pkgs/clan-cli/tests/test_import_sops_cli.py

@@ -21,55 +21,55 @@ def test_import_sops(
     monkeypatch.setenv("SOPS_AGE_KEY", age_keys[1].privkey)
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "machines",
             "add",
+            "--flake",
+            str(test_flake.path),
             "machine1",
             age_keys[0].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake.path),
             "user1",
             age_keys[1].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake.path),
             "user2",
             age_keys[2].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "user1",
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "user2",
         ]
@@ -78,10 +78,10 @@ def test_import_sops(
     # To edit:
     # SOPS_AGE_KEY=AGE-SECRET-KEY-1U5ENXZQAY62NC78Y2WC0SEGRRMAEEKH79EYY5TH4GPFWJKEAY0USZ6X7YQ sops --age age14tva0txcrl0zes05x7gkx56qd6wd9q3nwecjac74xxzz4l47r44sv3fz62 ./data/secrets.yaml
     cmd = [
-        "--flake",
-        str(test_flake.path),
         "secrets",
         "import-sops",
+        "--flake",
+        str(test_flake.path),
         "--group",
         "group1",
         "--machine",
@@ -91,10 +91,10 @@ def test_import_sops(
 
     cli.run(cmd)
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake.path), "secrets", "users", "list"])
+    cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)])
     users = sorted(capsys.readouterr().out.rstrip().split())
     assert users == ["user1", "user2"]
 
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake.path), "secrets", "get", "secret-key"])
+    cli.run(["secrets", "get", "--flake", str(test_flake.path), "secret-key"])
     assert capsys.readouterr().out == "secret-value"

pkgs/clan-cli/tests/test_machines_cli.py

@@ -9,20 +9,33 @@ def test_machine_subcommands(
 ) -> None:
     cli = Cli()
     cli.run(
-        ["--flake", str(test_flake_with_core.path), "machines", "create", "machine1"]
+        ["machines", "create", "--flake", str(test_flake_with_core.path), "machine1"]
     )
 
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"])
+    cli.run(["machines", "list", "--flake", str(test_flake_with_core.path)])
 
     out = capsys.readouterr()
-    assert "machine1\nvm1\nvm2\n" == out.out
+
+    assert "machine1" in out.out
+    assert "vm1" in out.out
+    assert "vm2" in out.out
+
+    capsys.readouterr()
+    cli.run(["machines", "show", "--flake", str(test_flake_with_core.path), "machine1"])
+    out = capsys.readouterr()
+    assert "machine1" in out.out
+    assert "Description" in out.out
+    print(out)
 
     cli.run(
-        ["--flake", str(test_flake_with_core.path), "machines", "delete", "machine1"]
+        ["machines", "delete", "--flake", str(test_flake_with_core.path), "machine1"]
     )
 
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake_with_core.path), "machines", "list"])
+    cli.run(["machines", "list", "--flake", str(test_flake_with_core.path)])
     out = capsys.readouterr()
-    assert "vm1\nvm2\n" == out.out
+
+    assert "machine1" not in out.out
+    assert "vm1" in out.out
+    assert "vm2" in out.out

pkgs/clan-cli/tests/test_secrets_cli.py

@@ -27,11 +27,11 @@ def _test_identities(
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             what,
             "add",
+            "--flake",
+            str(test_flake.path),
             "foo",
             age_keys[0].pubkey,
         ]
@@ -41,11 +41,11 @@ def _test_identities(
     with pytest.raises(ClanError):  # raises "foo already exists"
         cli.run(
             [
-                "--flake",
-                str(test_flake.path),
                 "secrets",
                 what,
                 "add",
+                "--flake",
+                str(test_flake.path),
                 "foo",
                 age_keys[0].pubkey,
             ]
@@ -54,11 +54,11 @@ def _test_identities(
     # rotate the key
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             what,
             "add",
+            "--flake",
+            str(test_flake.path),
             "-f",
             "foo",
             age_keys[1].privkey,
@@ -68,11 +68,11 @@ def _test_identities(
     capsys.readouterr()  # empty the buffer
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             what,
             "get",
+            "--flake",
+            str(test_flake.path),
             "foo",
         ]
     )
@@ -80,18 +80,18 @@ def _test_identities(
     assert age_keys[1].pubkey in out.out
 
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", what, "list"])
+    cli.run(["secrets", what, "list", "--flake", str(test_flake.path)])
     out = capsys.readouterr()  # empty the buffer
     assert "foo" in out.out
 
-    cli.run(["--flake", str(test_flake.path), "secrets", what, "remove", "foo"])
+    cli.run(["secrets", what, "remove", "--flake", str(test_flake.path), "foo"])
     assert not (sops_folder / what / "foo" / "key.json").exists()
 
     with pytest.raises(ClanError):  # already removed
-        cli.run(["--flake", str(test_flake.path), "secrets", what, "remove", "foo"])
+        cli.run(["secrets", what, "remove", "--flake", str(test_flake.path), "foo"])
 
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake.path), "secrets", what, "list"])
+    cli.run(["secrets", what, "list", "--flake", str(test_flake.path)])
     out = capsys.readouterr()
     assert "foo" not in out.out
 
@@ -113,17 +113,17 @@ def test_groups(
 ) -> None:
     cli = Cli()
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "groups", "list"])
+    cli.run(["secrets", "groups", "list", "--flake", str(test_flake.path)])
     assert capsys.readouterr().out == ""
 
     with pytest.raises(ClanError):  # machine does not exist yet
         cli.run(
             [
-                "--flake",
-                str(test_flake.path),
                 "secrets",
                 "groups",
                 "add-machine",
+                "--flake",
+                str(test_flake.path),
                 "group1",
                 "machine1",
             ]
@@ -131,33 +131,33 @@ def test_groups(
     with pytest.raises(ClanError):  # user does not exist yet
         cli.run(
             [
-                "--flake",
-                str(test_flake.path),
                 "secrets",
                 "groups",
                 "add-user",
+                "--flake",
+                str(test_flake.path),
                 "groupb1",
                 "user1",
             ]
         )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "machines",
             "add",
+            "--flake",
+            str(test_flake.path),
             "machine1",
             age_keys[0].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-machine",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "machine1",
         ]
@@ -166,11 +166,11 @@ def test_groups(
     # Should this fail?
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-machine",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "machine1",
         ]
@@ -178,51 +178,51 @@ def test_groups(
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake.path),
             "user1",
             age_keys[0].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "user1",
         ]
     )
 
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "groups", "list"])
+    cli.run(["secrets", "groups", "list", "--flake", str(test_flake.path)])
     out = capsys.readouterr().out
     assert "user1" in out
     assert "machine1" in out
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "remove-user",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "user1",
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "remove-machine",
+            "--flake",
+            str(test_flake.path),
             "group1",
             "machine1",
         ]
@@ -251,90 +251,90 @@ def test_secrets(
 ) -> None:
     cli = Cli()
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "list"])
+    cli.run(["secrets", "list", "--flake", str(test_flake.path)])
     assert capsys.readouterr().out == ""
 
     monkeypatch.setenv("SOPS_NIX_SECRET", "foo")
     monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake.path / ".." / "age.key"))
-    cli.run(["--flake", str(test_flake.path), "secrets", "key", "generate"])
+    cli.run(["secrets", "key", "generate", "--flake", str(test_flake.path)])
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "key", "show"])
+    cli.run(["secrets", "key", "show", "--flake", str(test_flake.path)])
     key = capsys.readouterr().out
     assert key.startswith("age1")
     cli.run(
-        ["--flake", str(test_flake.path), "secrets", "users", "add", "testuser", key]
+        ["secrets", "users", "add", "--flake", str(test_flake.path), "testuser", key]
     )
 
     with pytest.raises(ClanError):  # does not exist yet
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "nonexisting"])
-    cli.run(["--flake", str(test_flake.path), "secrets", "set", "initialkey"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "nonexisting"])
+    cli.run(["secrets", "set", "--flake", str(test_flake.path), "initialkey"])
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake.path), "secrets", "get", "initialkey"])
+    cli.run(["secrets", "get", "--flake", str(test_flake.path), "initialkey"])
     assert capsys.readouterr().out == "foo"
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake.path), "secrets", "users", "list"])
+    cli.run(["secrets", "users", "list", "--flake", str(test_flake.path)])
     users = capsys.readouterr().out.rstrip().split("\n")
     assert len(users) == 1, f"users: {users}"
     owner = users[0]
 
     monkeypatch.setenv("EDITOR", "cat")
-    cli.run(["--flake", str(test_flake.path), "secrets", "set", "--edit", "initialkey"])
+    cli.run(["secrets", "set", "--edit", "--flake", str(test_flake.path), "initialkey"])
     monkeypatch.delenv("EDITOR")
 
-    cli.run(["--flake", str(test_flake.path), "secrets", "rename", "initialkey", "key"])
+    cli.run(["secrets", "rename", "--flake", str(test_flake.path), "initialkey", "key"])
 
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "list"])
+    cli.run(["secrets", "list", "--flake", str(test_flake.path)])
     assert capsys.readouterr().out == "key\n"
 
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "list", "nonexisting"])
+    cli.run(["secrets", "list", "--flake", str(test_flake.path), "nonexisting"])
     assert capsys.readouterr().out == ""
 
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "list", "key"])
+    cli.run(["secrets", "list", "--flake", str(test_flake.path), "key"])
     assert capsys.readouterr().out == "key\n"
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "machines",
             "add",
+            "--flake",
+            str(test_flake.path),
             "machine1",
             age_keys[1].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "machines",
             "add-secret",
+            "--flake",
+            str(test_flake.path),
             "machine1",
             "key",
         ]
     )
     capsys.readouterr()
-    cli.run(["--flake", str(test_flake.path), "secrets", "machines", "list"])
+    cli.run(["secrets", "machines", "list", "--flake", str(test_flake.path)])
     assert capsys.readouterr().out == "machine1\n"
 
     with use_key(age_keys[1].privkey, monkeypatch):
         capsys.readouterr()
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "key"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"])
 
         assert capsys.readouterr().out == "foo"
 
     # rotate machines key
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "machines",
             "add",
+            "--flake",
+            str(test_flake.path),
             "-f",
             "machine1",
             age_keys[0].privkey,
@@ -344,17 +344,17 @@ def test_secrets(
     # should also rotate the encrypted secret
     with use_key(age_keys[0].privkey, monkeypatch):
         capsys.readouterr()
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "key"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"])
 
         assert capsys.readouterr().out == "foo"
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "machines",
             "remove-secret",
+            "--flake",
+            str(test_flake.path),
             "machine1",
             "key",
         ]
@@ -362,37 +362,37 @@ def test_secrets(
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake.path),
             "user1",
             age_keys[1].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "add-secret",
+            "--flake",
+            str(test_flake.path),
             "user1",
             "key",
         ]
     )
     capsys.readouterr()
     with use_key(age_keys[1].privkey, monkeypatch):
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "key"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"])
     assert capsys.readouterr().out == "foo"
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "remove-secret",
+            "--flake",
+            str(test_flake.path),
             "user1",
             "key",
         ]
@@ -401,44 +401,44 @@ def test_secrets(
     with pytest.raises(ClanError):  # does not exist yet
         cli.run(
             [
-                "--flake",
-                str(test_flake.path),
                 "secrets",
                 "groups",
                 "add-secret",
+                "--flake",
+                str(test_flake.path),
                 "admin-group",
                 "key",
             ]
         )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake.path),
             "admin-group",
             "user1",
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake.path),
             "admin-group",
             owner,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-secret",
+            "--flake",
+            str(test_flake.path),
             "admin-group",
             "key",
         ]
@@ -447,10 +447,10 @@ def test_secrets(
     capsys.readouterr()  # empty the buffer
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "set",
+            "--flake",
+            str(test_flake.path),
             "--group",
             "admin-group",
             "key2",
@@ -459,28 +459,28 @@ def test_secrets(
 
     with use_key(age_keys[1].privkey, monkeypatch):
         capsys.readouterr()
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "key"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"])
         assert capsys.readouterr().out == "foo"
 
     # extend group will update secrets
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake.path),
             "user2",
             age_keys[2].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake.path),
             "admin-group",
             "user2",
         ]
@@ -488,16 +488,16 @@ def test_secrets(
 
     with use_key(age_keys[2].privkey, monkeypatch):  # user2
         capsys.readouterr()
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "key"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"])
         assert capsys.readouterr().out == "foo"
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "remove-user",
+            "--flake",
+            str(test_flake.path),
             "admin-group",
             "user2",
         ]
@@ -505,24 +505,24 @@ def test_secrets(
     with pytest.raises(ClanError), use_key(age_keys[2].privkey, monkeypatch):
         # user2 is not in the group anymore
         capsys.readouterr()
-        cli.run(["--flake", str(test_flake.path), "secrets", "get", "key"])
+        cli.run(["secrets", "get", "--flake", str(test_flake.path), "key"])
         print(capsys.readouterr().out)
 
     cli.run(
         [
-            "--flake",
-            str(test_flake.path),
             "secrets",
             "groups",
             "remove-secret",
+            "--flake",
+            str(test_flake.path),
             "admin-group",
             "key",
         ]
     )
 
-    cli.run(["--flake", str(test_flake.path), "secrets", "remove", "key"])
-    cli.run(["--flake", str(test_flake.path), "secrets", "remove", "key2"])
+    cli.run(["secrets", "remove", "--flake", str(test_flake.path), "key"])
+    cli.run(["secrets", "remove", "--flake", str(test_flake.path), "key2"])
 
     capsys.readouterr()  # empty the buffer
-    cli.run(["--flake", str(test_flake.path), "secrets", "list"])
+    cli.run(["secrets", "list", "--flake", str(test_flake.path)])
     assert capsys.readouterr().out == ""

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -24,27 +24,27 @@ def test_generate_secret(
     cli = Cli()
     cli.run(
         [
-            "--flake",
-            str(test_flake_with_core.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake_with_core.path),
             "user1",
             age_keys[0].pubkey,
         ]
     )
     cli.run(
         [
-            "--flake",
-            str(test_flake_with_core.path),
             "secrets",
             "groups",
             "add-user",
+            "--flake",
+            str(test_flake_with_core.path),
             "admins",
             "user1",
         ]
     )
-    cmd = ["--flake", str(test_flake_with_core.path), "facts", "generate", "vm1"]
+    cmd = ["facts", "generate", "--flake", str(test_flake_with_core.path), "vm1"]
     cli.run(cmd)
     has_secret(test_flake_with_core.path, "vm1-age.key")
     has_secret(test_flake_with_core.path, "vm1-zerotier-identity-secret")
@@ -60,7 +60,7 @@ def test_generate_secret(
     secret1_mtime = identity_secret.lstat().st_mtime_ns
 
     # test idempotency for vm1 and also generate for vm2
-    cli.run(["facts", "generate"])
+    cli.run(["facts", "generate", "--flake", str(test_flake_with_core.path)])
     assert age_key.lstat().st_mtime_ns == age_key_mtime
     assert identity_secret.lstat().st_mtime_ns == secret1_mtime
 

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -23,11 +23,11 @@ def test_secrets_upload(
     cli = Cli()
     cli.run(
         [
-            "--flake",
-            str(test_flake_with_core.path),
             "secrets",
             "users",
             "add",
+            "--flake",
+            str(test_flake_with_core.path),
             "user1",
             age_keys[0].pubkey,
         ]
@@ -35,18 +35,18 @@ def test_secrets_upload(
 
     cli.run(
         [
-            "--flake",
-            str(test_flake_with_core.path),
             "secrets",
             "machines",
             "add",
+            "--flake",
+            str(test_flake_with_core.path),
             "vm1",
             age_keys[1].pubkey,
         ]
     )
     monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
     cli.run(
-        ["--flake", str(test_flake_with_core.path), "secrets", "set", "vm1-age.key"]
+        ["secrets", "set", "--flake", str(test_flake_with_core.path), "vm1-age.key"]
     )
 
     flake = test_flake_with_core.path.joinpath("flake.nix")
@@ -55,7 +55,7 @@ def test_secrets_upload(
     new_text = flake.read_text().replace("__CLAN_TARGET_ADDRESS__", addr)
 
     flake.write_text(new_text)
-    cli.run(["--flake", str(test_flake_with_core.path), "facts", "upload", "vm1"])
+    cli.run(["facts", "upload", "--flake", str(test_flake_with_core.path), "vm1"])
 
     # the flake defines this path as the location where the sops key should be installed
     sops_key = test_flake_with_core.path.joinpath("key.txt")

pkgs/clan-cli/tests/test_vms_cli.py

@@ -86,7 +86,7 @@ def test_inspect(
     test_flake_with_core: FlakeForTest, capsys: pytest.CaptureFixture
 ) -> None:
     cli = Cli()
-    cli.run(["--flake", str(test_flake_with_core.path), "vms", "inspect", "vm1"])
+    cli.run(["vms", "inspect", "--flake", str(test_flake_with_core.path), "vm1"])
     out = capsys.readouterr()  # empty the buffer
     assert "Cores" in out.out
 

pkgs/clan-vm-manager/clan_vm_manager/app.py

@@ -47,7 +47,7 @@ class MainApplication(Adw.Application):
             None,
         )
 
-        self.window: "MainWindow" | None = None
+        self.window: MainWindow | None = None
         self.connect("activate", self.on_activate)
         self.connect("shutdown", self.on_shutdown)
 
@@ -106,7 +106,7 @@ class MainApplication(Adw.Application):
     def on_activate(self, source: "MainApplication") -> None:
         if not self.window:
             self.init_style()
-            self.window = MainWindow(config=ClanConfig(initial_view="webview"))
+            self.window = MainWindow(config=ClanConfig(initial_view="list"))
             self.window.set_application(self)
 
         self.window.show()

pkgs/clan-vm-manager/clan_vm_manager/components/gkvstore.py

@@ -29,7 +29,7 @@ class GKVStore(GObject.GObject, Gio.ListModel, Generic[K, V]):
         self.gtype = gtype
         self.key_gen = key_gen
         # From Python 3.7 onwards dictionaries are ordered by default
-        self._items: "dict[K, V]" = dict()
+        self._items: dict[K, V] = dict()
 
     ##################################
     #                                #

pkgs/clan-vm-manager/clan_vm_manager/views/webview.py

@@ -1,3 +1,4 @@
+import dataclasses
 import json
 import logging
 import sys
@@ -8,6 +9,7 @@ from threading import Lock
 from typing import Any
 
 import gi
+from clan_cli.api import API
 
 gi.require_version("WebKit", "6.0")
 
@@ -22,11 +24,34 @@ site_index: Path = (
 log = logging.getLogger(__name__)
 
 
+def dataclass_to_dict(obj: Any) -> Any:
+    """
+    Utility function to convert dataclasses to dictionaries
+    It converts all nested dataclasses, lists, tuples, and dictionaries to dictionaries
+
+    It does NOT convert member functions.
+    """
+    if dataclasses.is_dataclass(obj):
+        return {k: dataclass_to_dict(v) for k, v in dataclasses.asdict(obj).items()}
+    elif isinstance(obj, list | tuple):
+        return [dataclass_to_dict(item) for item in obj]
+    elif isinstance(obj, dict):
+        return {k: dataclass_to_dict(v) for k, v in obj.items()}
+    else:
+        return obj
+
+
 class WebView:
     def __init__(self, methods: dict[str, Callable]) -> None:
         self.method_registry: dict[str, Callable] = methods
 
         self.webview = WebKit.WebView()
+
+        settings = self.webview.get_settings()
+        # settings.
+        settings.set_property("enable-developer-extras", True)
+        self.webview.set_settings(settings)
+
         self.manager = self.webview.get_user_content_manager()
         # Can be called with: window.webkit.messageHandlers.gtk.postMessage("...")
         # Important: it seems postMessage must be given some payload, otherwise it won't trigger the event
@@ -77,12 +102,35 @@ class WebView:
         self.queue_size += 1
 
     def threaded_handler(
-        self, handler_fn: Callable[[Any], Any], data: Any, method_name: str
+        self,
+        handler_fn: Callable[
+            ...,
+            Any,
+        ],
+        data: dict[str, Any] | None,
+        method_name: str,
     ) -> None:
         with self.mutex_lock:
             log.debug("Executing... ", method_name)
-            result = handler_fn(data)
-            serialized = json.dumps(result)
+            log.debug(f"{data}")
+            if data is None:
+                result = handler_fn()
+            else:
+                reconciled_arguments = {}
+                for k, v in data.items():
+                    # Some functions expect to be called with dataclass instances
+                    # But the js api returns dictionaries.
+                    # Introspect the function and create the expected dataclass from dict dynamically
+                    # Depending on the introspected argument_type
+                    arg_type = API.get_method_argtype(method_name, k)
+                    if dataclasses.is_dataclass(arg_type):
+                        reconciled_arguments[k] = arg_type(**v)
+                    else:
+                        reconciled_arguments[k] = v
+
+                result = handler_fn(**reconciled_arguments)
+
+            serialized = json.dumps(dataclass_to_dict(result))
 
             # Use idle_add to queue the response call to js on the main GTK thread
             GLib.idle_add(self.return_data_to_js, method_name, serialized)

pkgs/clan-vm-manager/clan_vm_manager/windows/main_window.py

@@ -26,7 +26,7 @@ log = logging.getLogger(__name__)
 class MainWindow(Adw.ApplicationWindow):
     def __init__(self, config: ClanConfig) -> None:
         super().__init__()
-        self.set_title("cLAN Manager")
+        self.set_title("Clan Manager")
         self.set_default_size(980, 850)
 
         overlay = ToastOverlay.use().overlay
@@ -62,8 +62,7 @@ class MainWindow(Adw.ApplicationWindow):
         stack_view.add_named(Logs(), "logs")
 
         webview = WebView(methods=API._registry)
-
-        stack_view.add_named(webview.get_webview(), "list")
+        stack_view.add_named(webview.get_webview(), "webview")
 
         stack_view.set_visible_child_name(config.initial_view)
 

pkgs/clan-vm-manager/default.nix

@@ -27,7 +27,7 @@ let
     name = "org.clan.vm-manager";
     exec = "clan-vm-manager %u";
     icon = ./clan_vm_manager/assets/clan_white.png;
-    desktopName = "cLAN Manager";
+    desktopName = "Clan Manager";
     startupWMClass = "clan";
     mimeTypes = [ "x-scheme-handler/clan" ];
   };

pkgs/editor/clan-edit-codium.nix

@@ -0,0 +1,40 @@
+{
+  lib,
+  coreutils,
+  nil,
+  nixd,
+  nixpkgs-fmt,
+  direnv,
+  vscode-extensions,
+  vscode-with-extensions,
+  vscodium,
+  writeShellApplication,
+}:
+let
+  codium = vscode-with-extensions.override {
+    vscode = vscodium;
+    vscodeExtensions = [
+      vscode-extensions.jnoortheen.nix-ide
+      vscode-extensions.mkhl.direnv
+    ];
+  };
+in
+writeShellApplication {
+  name = "clan-edit-codium";
+  runtimeInputs = [
+    coreutils
+    nil
+    nixd
+    nixpkgs-fmt
+    direnv
+  ];
+  text = ''
+    set -eux
+    DATA_DIR="''${XDG_CACHE_HOME:-$HOME/.cache}/clan-edit-codium"
+    SETTINGS="$DATA_DIR"/User/settings.json
+    ${coreutils}/bin/mkdir -p "$DATA_DIR/User"
+    cat ${./settings.json} > "$SETTINGS"
+
+    exec ${lib.getExe codium} --user-data-dir "$DATA_DIR" "$@"
+  '';
+}

pkgs/editor/default.nix

@@ -0,0 +1,4 @@
+{ pkgs }:
+{
+  clan-edit-codium = pkgs.callPackage ./clan-edit-codium.nix;
+}

pkgs/editor/settings.json

@@ -0,0 +1,23 @@
+{
+  "security.workspace.trust.enabled": false,
+  "nix.enableLanguageServer": true,
+  "nix.serverPath": "nixd",
+  "nix.formatterPath": "nixpkgs-fmt",
+  "nix.serverSettings": {
+    "nixd": {
+      "formatting": {
+        "command": "nixpkgs-fmt"
+      },
+      "options": {
+        "nixos": {
+          "expr": "(let pkgs = import <nixpkgs> { }; in (pkgs.lib.evalModules { modules
+         =  (import <nixpkgs/nixos/modules/module-list.nix>) ++ [ ({...}: {
+         nixpkgs.hostPlatform = builtins.currentSystem;} ) ] ; })).options"
+        },
+        "home-manager": {
+          "expr": "(builtins.getFlake \"github:nix-community/home-manager\").homeConfigurations.<name>.options"
+        }
+      }
+    }
+  }
+}

pkgs/flake-module.nix

@@ -7,6 +7,7 @@
     ./installer/flake-module.nix
     ./schemas/flake-module.nix
     ./webview-ui/flake-module.nix
+    ./gui-installer/flake-module.nix
   ];
 
   perSystem =
@@ -25,9 +26,9 @@
           moonlight-sunshine-accept = pkgs.callPackage ./moonlight-sunshine-accept { };
           merge-after-ci = pkgs.callPackage ./merge-after-ci { inherit (config.packages) tea-create-pr; };
           pending-reviews = pkgs.callPackage ./pending-reviews { };
+          editor = pkgs.callPackage ./editor/clan-edit-codium.nix { };
         }
         // lib.optionalAttrs pkgs.stdenv.isLinux {
-          wayland-proxy-virtwl = pkgs.callPackage ./wayland-proxy-virtwl { };
           # halalify zerotierone
           zerotierone = pkgs.zerotierone.overrideAttrs (_old: {
             meta = _old.meta // {

pkgs/gui-installer/flake-module.nix

@@ -0,0 +1,33 @@
+{
+  perSystem =
+    { pkgs, ... }:
+    let
+      nfpmConfig = pkgs.writeText "clan-nfpm-config.yaml" (
+
+        builtins.toJSON {
+          name = "clan-gui-installer";
+          contents = [
+            {
+              src = "${./gui-installer.sh}";
+              dst = "/usr/bin/clan-vm-manager";
+            }
+          ];
+        }
+      );
+      installerFor =
+        packager:
+        pkgs.runCommand "gui-installer" { } ''
+          mkdir build
+          cd build
+          ${pkgs.nfpm}/bin/nfpm package --config ${nfpmConfig} --packager ${packager}
+          mkdir $out
+          mv * $out/
+        '';
+    in
+    {
+      packages.gui-installer-apk = installerFor "apk";
+      packages.gui-installer-archlinux = installerFor "archlinux";
+      packages.gui-installer-deb = installerFor "deb";
+      packages.gui-installer-rpm = installerFor "rpm";
+    };
+}

pkgs/gui-installer/gui-installer.sh

@@ -0,0 +1,66 @@
+#! /bin/sh
+
+# create temp dir and ensure it is always cleaned
+trap 'clean_temp_dir' EXIT
+temp_dir=$(mktemp -d)
+
+clean_temp_dir() {
+  rm -rf "$temp_dir"
+}
+
+is_installed() {
+  name=$1
+  if [ -n "$(command -v "$name")" ]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+install_nix() {
+  if is_installed curl; then
+    curl --proto '=https' --tlsv1.2 -sSf -L \
+        https://install.determinate.systems/nix \
+      > "$temp_dir"/install_nix.sh
+  elif is_installed wget; then
+    wget -qO- \
+        https://install.determinate.systems/nix \
+      > "$temp_dir"/install_nix.sh
+  else
+    echo "Either curl or wget is required to install Nix. Exiting."
+    exit 1
+  fi
+  NIX_INSTALLER_DIAGNOSTIC_ENDPOINT="" sh "$temp_dir"/install_nix.sh install
+}
+
+ask_then_install_nix() {
+  echo "Clan requires Nix to be installed. Would you like to install it now? (y/n)"
+  read -r response
+  if [ "$response" = "y" ]; then
+    install_nix
+  else
+    echo "Clan cannot run without Nix. Exiting."
+    exit 1
+  fi
+}
+
+ensure_nix_installed() {
+  if ! is_installed nix; then
+    ask_then_install_nix
+  fi
+}
+
+start_clan_gui() {
+  PATH="${PATH:+$PATH:}/nix/var/nix/profiles/default/bin" \
+    exec nix run \
+      https://git.clan.lol/clan/clan-core/archive/main.tar.gz#clan-vm-manager \
+      --no-accept-flake-config \
+      --extra-experimental-features flakes nix-command -- "$@"
+}
+
+main() {
+  ensure_nix_installed
+  start_clan_gui "$@"
+}
+
+main "$@"

pkgs/installer/flake-module.nix

@@ -98,9 +98,6 @@ let
 in
 {
   clan = {
-    clanName = "clan-core";
-    directory = self;
-
     # To build a generic installer image (without ssh pubkeys),
     # use the following command:
     # $ nix build .#iso-installer

pkgs/merge-after-ci/merge-after-ci.py

@@ -1,4 +1,5 @@
 import argparse
+import shlex
 import subprocess
 
 parser = argparse.ArgumentParser()
@@ -11,14 +12,17 @@ args = parser.parse_args()
 if not args.reviewers and not args.no_review:
     parser.error("either --reviewers or --no-review must be given")
 
-subprocess.run(
-    [
-        "tea-create-pr",
-        "origin",
-        "main",
-        "--assignees",
-        ",".join(["clan-bot", *args.reviewers]),
-        *(["--labels", "needs-review"] if not args.no_review else []),
-        *args.args,
-    ]
-)
+cmd = [
+    "tea-create-pr",
+    "origin",
+    "upstream",
+    "main",
+    "--assignees",
+    ",".join(["clan-bot", *args.reviewers]),
+    *(["--labels", "needs-review"] if not args.no_review else []),
+    *args.args,
+]
+
+print("Running:", shlex.join(cmd))
+
+subprocess.run(cmd)