Merge pull request 'docs: fix flash example' (#1745) from joerg-ci into main

Reviewed-on: clan/clan-core#1744

.envrc

@@ -1,3 +1,4 @@
+# shellcheck shell=bash
 if ! has nix_direnv_version || ! nix_direnv_version 3.0.4; then
   source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/3.0.4/direnvrc" "sha256-DzlYZ33mWF/Gs8DDeyjr8mnVmQGx7ASYqA5WlxwvBG4="
 fi
@@ -5,7 +6,7 @@ fi
 watch_file .direnv/selected-shell 
 
 if [ -e .direnv/selected-shell ]; then
-  use flake .#$(cat .direnv/selected-shell)
+  use flake ".#$(cat .direnv/selected-shell)"
 else
   use flake
 fi

.gitignore

@@ -3,6 +3,7 @@
 out.log
 .coverage.*
 **/qubeclan
+pkgs/repro-hook
 **/testdir
 democlan
 example_clan
@@ -35,4 +36,4 @@ repo
 # node
 node_modules
 dist
-.webui
+.webui

README.md

@@ -1,6 +1,6 @@
-# Clan Core Repository
+# Clan core repository
 
-Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
+Welcome to the Clan core repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
 ## Why Clan?
 
@@ -14,13 +14,13 @@ Our mission is simple: to democratize computing by providing tools that empower
 - **Robust Backup Management:** Long-term, self-hosted data preservation.
 - **Intuitive Secret Management:** Simplified encryption and password management processes.
 
-## Getting Started with Clan
+## Getting started with Clan
 
 If you're new to Clan and eager to dive in, start with our quickstart guide and explore the core functionalities that Clan offers:
 
 - **Quickstart Guide**: Check out [getting started](https://docs.clan.lol/#starting-with-a-new-clan-project)<!-- [docs/site/index.md](docs/site/index.md) --> to get up and running with Clan in no time.
 
-### Managing Secrets
+### Managing secrets
 
 In the Clan ecosystem, security is paramount. Learn how to handle secrets effectively:
 
@@ -32,14 +32,14 @@ The Clan project thrives on community contributions. We welcome everyone to cont
 
 - **Contribution Guidelines**: Make a meaningful impact by following the steps in [contributing](https://docs.clan.lol/contributing/contributing/)<!-- [contributing.md](docs/CONTRIBUTING.md) -->.
 
-## Join the Revolution
+## Join the revolution
 
 Clan is more than a tool; it's a movement towards a better digital future. By contributing to the Clan project, you're part of changing technology for the better, together.
 
-### Community and Support
+### Community and support
 
 Connect with us and the Clan community for support and discussion:
 
-- [Matrix channel](https://matrix.to/#/#clan:lassul.us) for live discussions.
+- [Matrix channel](https://matrix.to/#/#clan:clan.lol) for live discussions.
 - IRC bridges (coming soon) for real-time chat support.
 

checks/backups/flake-module.nix

@@ -27,7 +27,7 @@
           self.clanModules.localbackup
           self.clanModules.sshd
         ];
-        clan.networking.targetHost = "machine";
+        clan.core.networking.targetHost = "machine";
         networking.hostName = "machine";
         services.openssh.settings.UseDns = false;
 
@@ -68,17 +68,9 @@
             };
           };
         };
-        clanCore.facts.secretStore = "vm";
+        clan.core.facts.secretStore = "vm";
 
-        environment.systemPackages = [
-          self.packages.${pkgs.system}.clan-cli
-          (pkgs.writeShellScriptBin "pre-restore-command" ''
-            touch /var/test-service/pre-restore-command
-          '')
-          (pkgs.writeShellScriptBin "post-restore-command" ''
-            touch /var/test-service/post-restore-command
-          '')
-        ];
+        environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
         environment.etc.install-closure.source = "${closureInfo}/store-paths";
         nix.settings = {
           substituters = lib.mkForce [ ];
@@ -87,11 +79,18 @@
           flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
         };
         system.extraDependencies = dependencies;
-        clanCore.state.test-backups.folders = [ "/var/test-backups" ];
+        clan.core.state.test-backups.folders = [ "/var/test-backups" ];
 
-        clanCore.state.test-service = {
-          preRestoreCommand = "pre-restore-command";
-          postRestoreCommand = "post-restore-command";
+        clan.core.state.test-service = {
+          preBackupScript = ''
+            touch /var/test-service/pre-backup-command
+          '';
+          preRestoreScript = ''
+            touch /var/test-service/pre-restore-command
+          '';
+          postRestoreScript = ''
+            touch /var/test-service/post-restore-command
+          '';
           folders = [ "/var/test-service" ];
         };
         clan.borgbackup.destinations.test-backup.repo = "borg@machine:.";
@@ -164,13 +163,15 @@
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
+            machine.succeed("test -f /var/test-service/pre-backup-command")
 
             ## localbackup restore
-            machine.succeed("rm -f /var/test-backups/somefile /var/test-service/{pre,post}-restore-command")
+            machine.succeed("rm -rf /var/test-backups/somefile /var/test-service/ && mkdir -p /var/test-service")
             machine.succeed(f"clan backups restore --debug --flake ${self} test-backup localbackup '{localbackup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
+            machine.succeed("test -f /var/test-service/pre-backup-command")
           '';
         } { inherit pkgs self; };
       };

checks/borgbackup/default.nix

@@ -16,9 +16,9 @@
             };
           }
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
-            clanCore.state.testState.folders = [ "/etc/state" ];
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
+            clan.core.state.testState.folders = [ "/etc/state" ];
             environment.etc.state.text = "hello world";
             systemd.tmpfiles.settings."vmsecrets" = {
               "/etc/secrets/borgbackup.ssh" = {
@@ -36,7 +36,7 @@
                 };
               };
             };
-            clanCore.facts.secretStore = "vm";
+            clan.core.facts.secretStore = "vm";
 
             clan.borgbackup.destinations.test.repo = "borg@localhost:.";
           }

checks/deltachat/default.nix

@@ -10,8 +10,8 @@
           self.clanModules.deltachat
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
           }
         ];
       };

checks/flake-module.nix

@@ -23,7 +23,7 @@
                 options =
                   (pkgs.nixos {
                     imports = [ self.nixosModules.clanCore ];
-                    clanCore.clanDir = ./.;
+                    clan.core.clanDir = ./.;
                   }).options;
                 warningsAreErrors = false;
               };
@@ -44,6 +44,7 @@
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
+            postgresql = import ./postgresql nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
 

checks/installation/flake-module.nix

@@ -1,7 +1,7 @@
 { self, lib, ... }:
 {
   clan.machines.test_install_machine = {
-    clan.networking.targetHost = "test_install_machine";
+    clan.core.networking.targetHost = "test_install_machine";
     fileSystems."/".device = lib.mkDefault "/dev/vdb";
     boot.loader.grub.device = lib.mkDefault "/dev/vdb";
 
@@ -12,7 +12,7 @@
       { lib, modulesPath, ... }:
       {
         imports = [
-          self.clanModules.disk-layouts
+          "${self}/nixosModules/disk-layouts"
           (modulesPath + "/testing/test-instrumentation.nix") # we need these 2 modules always to be able to run the tests
           (modulesPath + "/profiles/qemu-guest.nix")
         ];

checks/lib/container-driver/test_driver/__init__.py

@@ -151,7 +151,7 @@ class Machine:
         """
 
         # Always run command with shell opts
-        command = f"set -euo pipefail; {command}"
+        command = f"set -eo pipefail; source /etc/profile; set -u; {command}"
 
         proc = subprocess.run(
             [

checks/matrix-synapse/default.nix

@@ -4,26 +4,61 @@
     name = "matrix-synapse";
 
     nodes.machine =
-      { self, lib, ... }:
+      {
+        config,
+        self,
+        lib,
+        ...
+      }:
       {
         imports = [
           self.clanModules.matrix-synapse
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
-            clan.matrix-synapse = {
-              enable = true;
-              domain = "clan.test";
-            };
-          }
-          {
-            # secret override
-            clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path = "${./synapse-registration_shared_secret}";
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
+
             services.nginx.virtualHosts."matrix.clan.test" = {
               enableACME = lib.mkForce false;
               forceSSL = lib.mkForce false;
             };
+            clan.matrix-synapse.domain = "clan.test";
+            clan.matrix-synapse.users.admin.admin = true;
+            clan.matrix-synapse.users.someuser = { };
+
+            clan.core.facts.secretStore = "vm";
+
+            # because we use systemd-tmpfiles to copy the secrets, we need to a seperate systemd-tmpfiles call to provison them.
+            boot.postBootCommands = "${config.systemd.package}/bin/systemd-tmpfiles --create /etc/tmpfiles.d/00-vmsecrets.conf";
+
+            systemd.tmpfiles.settings."00-vmsecrets" = {
+              # run before 00-nixos.conf
+              "/etc/secrets" = {
+                d.mode = "0700";
+                z.mode = "0700";
+              };
+              "/etc/secrets/synapse-registration_shared_secret" = {
+                f.argument = "supersecret";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/matrix-password-admin" = {
+                f.argument = "matrix-password1";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/matrix-password-someuser" = {
+                f.argument = "matrix-password2";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+            };
           }
         ];
       };
@@ -32,6 +67,12 @@
       machine.wait_for_unit("matrix-synapse")
       machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
       machine.succeed("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
+
+      machine.systemctl("restart matrix-synapse >&2") # check if user creation is idempotent
+      machine.execute("journalctl -u matrix-synapse --no-pager >&2")
+      machine.wait_for_unit("matrix-synapse")
+      machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
+      machine.succeed("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
     '';
   }
 )

checks/postgresql/default.nix

@@ -0,0 +1,72 @@
+(import ../lib/container-test.nix) ({
+  name = "postgresql";
+
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [
+        self.nixosModules.clanCore
+        self.clanModules.postgresql
+        self.clanModules.localbackup
+      ];
+      clan.postgresql.users.test = { };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+
+      systemd.services.sample-service = {
+        wantedBy = [ "multi-user.target" ];
+        script = ''
+          while true; do
+            echo "Hello, world!"
+            sleep 5
+          done
+        '';
+      };
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+      machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("sample-service")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
+      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clan.core.state.test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
+
+      # check if restore works if the database does not exist
+      machine.succeed("runuser -u postgres -- dropdb test")
+      machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+    '';
+})

checks/secrets/default.nix

@@ -10,8 +10,8 @@
       environment.etc."group-secret".source = config.sops.secrets.group-secret.path;
       sops.age.keyFile = "/etc/privkey.age";
 
-      clanCore.clanDir = "${./.}";
-      clanCore.machineName = "machine";
+      clan.core.clanDir = "${./.}";
+      clan.core.machineName = "machine";
 
       networking.hostName = "machine";
     };

checks/syncthing/default.nix

@@ -12,14 +12,14 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "introducer";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "introducer";
+            clan.core.clanDir = ./.;
             environment.etc = {
               "syncthing.pam".source = ./introducer/introducer_test_cert;
               "syncthing.key".source = ./introducer/introducer_test_key;
               "syncthing.api".source = ./introducer/introducer_test_api;
             };
-            clanCore.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
+            clan.core.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
             services.syncthing.cert = "/etc/syncthing.pam";
             services.syncthing.key = "/etc/syncthing.key";
             # Doesn't test zerotier!
@@ -53,8 +53,8 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "peer1";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "peer1";
+            clan.core.clanDir = ./.;
             clan.syncthing.introducer = lib.strings.removeSuffix "\n" (
               builtins.readFile ./introducer/introducer_device_id
             );
@@ -75,8 +75,8 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "peer2";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "peer2";
+            clan.core.clanDir = ./.;
             clan.syncthing.introducer = lib.strings.removeSuffix "\n" (
               builtins.readFile ./introducer/introducer_device_id
             );

checks/wayland-proxy-virtwl/default.nix

@@ -14,8 +14,8 @@ import ../lib/test-base.nix (
         imports = [
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
           }
         ];
         services.wayland-proxy-virtwl.enable = true;

checks/zt-tcp-relay/default.nix

@@ -10,8 +10,8 @@
           self.nixosModules.clanCore
           self.clanModules.zt-tcp-relay
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
           }
         ];
       };

clanModules/borgbackup-static/README.md

@@ -0,0 +1,11 @@
+---
+description = "Statically configure borgbackup with sane defaults."
+---
+This module implements the `borgbackup` backend and implements sane defaults
+for backup management through `borgbackup` for members of the clan.
+
+Configure target machines where the backups should be sent to through `targets`.
+
+Configure machines that should be backuped either through `includeMachines`
+which will exclusively add the included machines to be backuped, or through
+`excludeMachines`, which will add every machine except the excluded machine to the backup.

clanModules/borgbackup-static/default.nix

@@ -0,0 +1,101 @@
+{ lib, config, ... }:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+in
+lib.warn "This module is deprecated use the service via the inventory interface instead." {
+  imports = [ ../borgbackup ];
+
+  options.clan.borgbackup-static = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be backuped.
+        Mutually exclusive with includeMachines.
+        If this is not empty, every other machine except the targets in the clan will be backuped by this module.
+        If includeMachines is set, only the included machines will be backuped.
+      '';
+    };
+    includeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should be backuped.
+        Mutually exclusive with excludeMachines.
+      '';
+    };
+    targets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Machines that should act as target machines for backups.
+      '';
+    };
+  };
+
+  config.services.borgbackup.repos =
+    let
+      machines = builtins.readDir machineDir;
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      filteredMachines =
+        if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then
+          lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines
+        else
+          lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines;
+      machinesMaybeKey = lib.mapAttrsToList (
+        machine: _:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) filteredMachines;
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    lib.mkIf
+      (builtins.any (
+        target: target == config.clan.core.machineName
+      ) config.clan.borgbackup-static.targets)
+      (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { });
+
+  config.clan.borgbackup.destinations =
+    let
+      destinations = builtins.map (d: {
+        name = d;
+        value = {
+          repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}";
+        };
+      }) config.clan.borgbackup-static.targets;
+    in
+    lib.mkIf (builtins.any (
+      target: target == config.clan.core.machineName
+    ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations);
+
+  config.assertions = [
+    {
+      assertion =
+        !(
+          ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0)
+          && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0)
+        );
+      message = ''
+        The options:
+        config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}]
+        and
+        config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}]
+        are mutually exclusive.
+        Use excludeMachines to exclude certain machines and backup the other clan machines.
+        Use include machines to only backup certain machines.
+      '';
+    }
+  ];
+}

clanModules/borgbackup/README.md

@@ -1,2 +1,13 @@
-Efficient, deduplicating backup program with optional compression and secure encryption.
----
+---
+description = "Efficient, deduplicating backup program with optional compression and secure encryption."
+categories = ["backup"]
+---
+BorgBackup (short: Borg) gives you:
+
+- Space efficient storage of backups.
+- Secure, authenticated encryption.
+- Compression: lz4, zstd, zlib, lzma or none.
+- Mountable backups with FUSE.
+- Easy installation on multiple platforms: Linux, macOS, BSD, ...
+- Free software (BSD license).
+- Backed by a large and active open source community.

clanModules/borgbackup/default.nix

@@ -6,8 +6,73 @@
 }:
 let
   cfg = config.clan.borgbackup;
+  preBackupScript = ''
+    declare -A preCommandErrors
+
+    ${lib.concatMapStringsSep "\n" (
+      state:
+      lib.optionalString (state.preBackupCommand != null) ''
+        echo "Running pre-backup command for ${state.name}"
+        if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+          preCommandErrors["${state.name}"]=1
+        fi
+      ''
+    ) (lib.attrValues config.clan.core.state)}
+
+    if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+      echo "pre-backup commands failed for the following services:"
+      for state in "''${!preCommandErrors[@]}"; do
+        echo "  $state"
+      done
+      exit 1
+    fi
+  '';
 in
+# Each .nix file in the roles directory is a role
+# TODO: Helper function to set available roles within module meta.
+# roles =
+#   if builtins.pathExists ./roles then
+#     lib.pipe ./roles [
+#       builtins.readDir
+#       (lib.filterAttrs (_n: v: v == "regular"))
+#       lib.attrNames
+#       (map (fileName: lib.removeSuffix ".nix" fileName))
+#     ]
+#   else
+#     null;
+# TODO: make this an interface of every module
+# Maybe load from readme.md
+# metaInfoOption = lib.mkOption {
+#   readOnly = true;
+#   description = ''
+#     Meta is used to retrieve information about this module.
+#     - `availableRoles` is a list of roles that can be assigned via the inventory.
+#     - `category` is used to group services in the clan marketplace.
+#     - `description` is a short description of the service for the clan marketplace.
+#   '';
+#   default = {
+#     description = "Borgbackup is a backup program. Optionally, it supports compression and authenticated encryption.";
+#     availableRoles = roles;
+#     category = "backup";
+#   };
+#   type = lib.types.submodule {
+#     options = {
+#       description = lib.mkOption { type = lib.types.str; };
+#       availableRoles = lib.mkOption { type = lib.types.nullOr (lib.types.listOf lib.types.str); };
+#       category = lib.mkOption {
+#         description = "A category for the service. This is used to group services in the clan ui";
+#         type = lib.types.enum [
+#           "backup"
+#           "network"
+#         ];
+#       };
+#     };
+#   };
+# };
 {
+
+  # options.clan.borgbackup.meta = metaInfoOption;
+
   options.clan.borgbackup.destinations = lib.mkOption {
     type = lib.types.attrsOf (
       lib.types.submodule (
@@ -26,9 +91,9 @@ in
             rsh = lib.mkOption {
               type = lib.types.str;
               default = "ssh -i ${
-                config.clanCore.facts.services.borgbackup.secret."borgbackup.ssh".path
-              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
-              defaultText = "ssh -i \${config.clanCore.facts.services.borgbackup.secret.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+                config.clan.core.facts.services.borgbackup.secret."borgbackup.ssh".path
+              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes";
+              defaultText = "ssh -i \${config.clan.core.facts.services.borgbackup.secret.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
               description = "the rsh to use for the backup";
             };
           };
@@ -50,21 +115,30 @@ in
   ];
 
   config = lib.mkIf (cfg.destinations != { }) {
+    systemd.services = lib.mapAttrs' (
+      _: dest:
+      lib.nameValuePair "borgbackup-job-${dest.name}" {
+        # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
+        serviceConfig.ExecStartPre = [
+          ''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}''
+        ];
+      }
+    ) cfg.destinations;
+
     services.borgbackup.jobs = lib.mapAttrs (_: dest: {
-      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
+      paths = lib.unique (
+        lib.flatten (map (state: state.folders) (lib.attrValues config.clan.core.state))
+      );
       exclude = [ "*.pyc" ];
       repo = dest.repo;
       environment.BORG_RSH = dest.rsh;
       compression = "auto,zstd";
       startAt = "*-*-* 01:00:00";
       persistentTimer = true;
-      preHook = ''
-        set -x
-      '';
 
       encryption = {
         mode = "repokey";
-        passCommand = "cat ${config.clanCore.facts.services.borgbackup.secret."borgbackup.repokey".path}";
+        passCommand = "cat ${config.clan.core.facts.services.borgbackup.secret."borgbackup.repokey".path}";
       };
 
       prune.keep = {
@@ -75,7 +149,7 @@ in
       };
     }) cfg.destinations;
 
-    clanCore.facts.services.borgbackup = {
+    clan.core.facts.services.borgbackup = {
       public."borgbackup.ssh.pub" = { };
       secret."borgbackup.ssh" = { };
       secret."borgbackup.repokey" = { };
@@ -111,7 +185,7 @@ in
       (pkgs.writeShellScriptBin "borgbackup-restore" ''
         set -efux
         cd /
-        IFS=';' read -ra FOLDER <<< "$FOLDERS"
+        IFS=':' read -ra FOLDER <<< "$FOLDERS"
         job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
         backup_name=''${NAME#"$job_name"::}
         if ! command -v borg-job-"$job_name" &> /dev/null; then
@@ -122,7 +196,7 @@ in
       '')
     ];
 
-    clanCore.backups.providers.borgbackup = {
+    clan.core.backups.providers.borgbackup = {
       list = "borgbackup-list";
       create = "borgbackup-create";
       restore = "borgbackup-restore";

clanModules/borgbackup/roles/client.nix

@@ -0,0 +1,30 @@
+{ config, lib, ... }:
+let
+  instances = config.clan.inventory.services.borgbackup;
+  # roles = { ${role_name} :: { machines :: [string] } }
+  allServers = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if builtins.elem machineName instanceConfig.roles.client.machines then
+        instanceConfig.roles.server.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+
+  inherit (config.clan.core) machineName;
+in
+{
+  config.clan.borgbackup.destinations =
+    let
+
+      destinations = builtins.map (serverName: {
+        name = serverName;
+        value = {
+          repo = "borg@${serverName}:/var/lib/borgbackup/${machineName}";
+        };
+      }) allServers;
+    in
+    (builtins.listToAttrs destinations);
+}

clanModules/borgbackup/roles/server.nix

@@ -0,0 +1,45 @@
+{ config, lib, ... }:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+  inherit (config.clan.core) machineName;
+
+  instances = config.clan.inventory.services.borgbackup;
+
+  # roles = { ${role_name} :: { machines :: [string] } }
+
+  allClients = lib.foldlAttrs (
+    acc: _instanceName: instanceConfig:
+    acc
+    ++ (
+      if (builtins.elem machineName instanceConfig.roles.server.machines) then
+        instanceConfig.roles.client.machines
+      else
+        [ ]
+    )
+  ) [ ] instances;
+in
+{
+  config.services.borgbackup.repos =
+    let
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      machinesMaybeKey = builtins.map (
+        machine:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) allClients;
+
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    if (builtins.listToAttrs hosts) != [ ] then builtins.listToAttrs hosts else { };
+}

clanModules/deltachat/README.md

@@ -1,4 +1,5 @@
-Email-based instant messaging for Desktop.
+---
+description = "Email-based instant messaging for Desktop."
 ---
 
 !!! warning "Under construction"

clanModules/deltachat/default.nix

@@ -5,7 +5,7 @@
 
   services.maddy =
     let
-      domain = "${config.clanCore.machineName}.local";
+      domain = "${config.clan.core.machineName}.local";
     in
     {
       enable = true;

clanModules/disk-layouts/README.md

@@ -1,2 +0,0 @@
-Automatically format a disk drive on clan installation
----

clanModules/ergochat/README.md

@@ -1,2 +1,3 @@
-A modern IRC server
+---
+description = "A modern IRC server"
 ---

clanModules/ergochat/default.nix

@@ -10,5 +10,5 @@ _: {
     };
   };
 
-  clanCore.state.ergochat.folders = [ "/var/lib/ergo" ];
+  clan.core.state.ergochat.folders = [ "/var/lib/ergo" ];
 }

clanModules/flake-module.nix

@@ -1,21 +1,23 @@
 { ... }:
 {
   flake.clanModules = {
-    disk-layouts = {
-      imports = [ ./disk-layouts ];
-    };
     borgbackup = ./borgbackup;
+    borgbackup-static = ./borgbackup-static;
     deltachat = ./deltachat;
     ergochat = ./ergochat;
     localbackup = ./localbackup;
     localsend = ./localsend;
+    single-disk = ./single-disk;
     matrix-synapse = ./matrix-synapse;
     moonlight = ./moonlight;
+    packages = ./packages;
+    postgresql = ./postgresql;
     root-password = ./root-password;
     sshd = ./sshd;
     sunshine = ./sunshine;
     static-hosts = ./static-hosts;
     syncthing = ./syncthing;
+    syncthing-static-peers = ./syncthing-static-peers;
     thelounge = ./thelounge;
     trusted-nix-caches = ./trusted-nix-caches;
     user-password = ./user-password;

clanModules/localbackup/README.md

@@ -1,2 +1,3 @@
-Automatically backups current machine to local directory.
+---
+description = "Automatically backups current machine to local directory."
 ---

clanModules/localbackup/default.nix

@@ -6,7 +6,10 @@
 }:
 let
   cfg = config.clan.localbackup;
-  rsnapshotConfig = target: states: ''
+  uniqueFolders = lib.unique (
+    lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clan.core.state)
+  );
+  rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
     sync_first	1
@@ -17,12 +20,6 @@ let
     cmd_logger	${pkgs.inetutils}/bin/logger
     cmd_du	${pkgs.coreutils}/bin/du
     cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-    ${lib.optionalString (target.preBackupHook != null) ''
-      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
-        set -efu -o pipefail
-        ${target.preBackupHook}
-      ''}
-    ''}
 
     ${lib.optionalString (target.postBackupHook != null) ''
       cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
@@ -31,11 +28,9 @@ let
       ''}
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
-    ${lib.concatMapStringsSep "\n" (state: ''
-      ${lib.concatMapStringsSep "\n" (folder: ''
-        backup	${folder}	${config.networking.hostName}/
-      '') state.folders}
-    '') states}
+    ${lib.concatMapStringsSep "\n" (folder: ''
+      backup	${folder}	${config.networking.hostName}/
+    '') uniqueFolders}
   '';
 in
 {
@@ -129,14 +124,29 @@ in
               ]
             }
             ${lib.concatMapStringsSep "\n" (target: ''
-              (
-                ${mountHook target}
-                echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
-              )
-            '') (builtins.attrValues cfg.targets)}
-          '')
+              ${mountHook target}
+              echo "Creating backup '${target.name}'"
+
+              ${lib.optionalString (target.preBackupHook != null) ''
+                (
+                  ${target.preBackupHook}
+                )
+              ''}
+
+              declare -A preCommandErrors
+              ${lib.concatMapStringsSep "\n" (
+                state:
+                lib.optionalString (state.preBackupCommand != null) ''
+                  echo "Running pre-backup command for ${state.name}"
+                  if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+                    preCommandErrors["${state.name}"]=1
+                  fi
+                ''
+              ) (builtins.attrValues config.clan.core.state)}
+
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+            '') (builtins.attrValues cfg.targets)}'')
           (pkgs.writeShellScriptBin "localbackup-list" ''
             set -efu -o pipefail
             export PATH=${
@@ -167,6 +177,14 @@ in
                 pkgs.gawk
               ]
             }
+            if [[ "''${NAME:-}" == "" ]]; then
+              echo "No backup name given via NAME environment variable"
+              exit 1
+            fi
+            if [[ "''${FOLDERS:-}" == "" ]]; then
+              echo "No folders given via FOLDERS environment variable"
+              exit 1
+            fi
             name=$(awk -F'::' '{print $1}' <<< $NAME)
             backupname=''${NAME#$name::}
 
@@ -182,8 +200,9 @@ in
               exit 1
             fi
 
-            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            IFS=':' read -ra FOLDER <<< "''$FOLDERS"
             for folder in "''${FOLDER[@]}"; do
+              mkdir -p "$folder"
               rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
             done
           '')
@@ -213,7 +232,7 @@ in
           ''
         ) cfg.targets;
 
-      clanCore.backups.providers.localbackup = {
+      clan.core.backups.providers.localbackup = {
         # TODO list needs to run locally or on the remote machine
         list = "localbackup-list";
         create = "localbackup-create";

clanModules/localsend/README.md

@@ -1,2 +1,3 @@
-Securely sharing files and messages over a local network without internet connectivity.
+---
+description = "Securely sharing files and messages over a local network without internet connectivity."
 ---

clanModules/localsend/default.nix

@@ -18,7 +18,7 @@
   };
 
   config = lib.mkIf config.clan.localsend.enable {
-    clanCore.state.localsend.folders = [
+    clan.core.state.localsend.folders = [
       "/var/localsend"
       config.clan.localsend.defaultLocation
     ];

clanModules/matrix-synapse/README.md

@@ -1,2 +1,3 @@
-A federated messaging server with end-to-end encryption.
+---
+description = "A federated messaging server with end-to-end encryption."
 ---

clanModules/matrix-synapse/default.nix

@@ -6,16 +6,65 @@
 }:
 let
   cfg = config.clan.matrix-synapse;
+  nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
+  element-web =
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
+      ''
+        cp -r ${pkgs.element-web} $out
+        chmod -R u+w $out
+        jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
+          > $out/config.json < ${pkgs.element-web}/config.json
+        ln -s $out/config.json $out/config.${nginx-vhost}.json
+      '';
 in
+# FIXME: This was taken from upstream. Drop this when our patch is upstream
 {
+  options.services.matrix-synapse.package = lib.mkOption { readOnly = false; };
   options.clan.matrix-synapse = {
-    enable = lib.mkEnableOption "Enable matrix-synapse";
     domain = lib.mkOption {
       type = lib.types.str;
       description = "The domain name of the matrix server";
+      example = "example.com";
+    };
+    users = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "The name of the user";
+              };
+
+              admin = lib.mkOption {
+                type = lib.types.bool;
+                default = false;
+                description = "Whether the user should be an admin";
+              };
+            };
+          }
+        )
+      );
+      description = "A list of users. Not that only new users will be created and existing ones are not modified.";
+      example.alice = {
+        admin = true;
+      };
     };
   };
-  config = lib.mkIf cfg.enable {
+  imports = [
+    ../postgresql
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "matrix-synapse"
+      "enable"
+    ] "Importing the module will already enable the service.")
+
+    ../postgresql
+  ];
+  config = {
     services.matrix-synapse = {
       enable = true;
       settings = {
@@ -29,6 +78,7 @@ in
           "turn:turn.matrix.org?transport=udp"
           "turn:turn.matrix.org?transport=tcp"
         ];
+        registration_shared_secret_path = "/run/synapse-registration-shared-secret";
         listeners = [
           {
             port = 8008;
@@ -49,45 +99,76 @@ in
           }
         ];
       };
-      extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
-    };
-    systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
-      "+${pkgs.writeScript "copy_registration_shared_secret" ''
-        #!/bin/sh
-        cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
-      ''}"
-    ];
-
-    clanCore.facts.services."matrix-synapse" = {
-      secret."synapse-registration_shared_secret" = { };
-      generator.path = with pkgs; [
-        coreutils
-        pwgen
-      ];
-      generator.script = ''
-        echo "registration_shared_secret: $(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
-      '';
     };
 
-    services.postgresql.enable = true;
-    # we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
-    services.postgresql = {
-      ensureDatabases = [ "matrix-synapse" ];
-      ensureUsers = [
-        {
-          name = "matrix-synapse";
-          ensureDBOwnership = true;
+    systemd.tmpfiles.settings."01-matrix" = {
+      "/run/synapse-registration-shared-secret" = {
+        C.argument =
+          config.clan.core.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
+        z = {
+          mode = "0400";
+          user = "matrix-synapse";
+        };
+      };
+    };
+
+    clan.postgresql.users.matrix-synapse = { };
+    clan.postgresql.databases.matrix-synapse.create.options = {
+      TEMPLATE = "template0";
+      LC_COLLATE = "C";
+      LC_CTYPE = "C";
+      ENCODING = "UTF8";
+      OWNER = "matrix-synapse";
+    };
+    clan.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
+
+    clan.core.facts.services =
+      {
+        "matrix-synapse" = {
+          secret."synapse-registration_shared_secret" = { };
+          generator.path = with pkgs; [
+            coreutils
+            pwgen
+          ];
+          generator.script = ''
+            echo -n "$(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
+          '';
+        };
+      }
+      // lib.mapAttrs' (
+        name: user:
+        lib.nameValuePair "matrix-password-${user.name}" {
+          secret."matrix-password-${user.name}" = { };
+          generator.path = with pkgs; [ xkcdpass ];
+          generator.script = ''
+            xkcdpass -n 4 -d - > "$secrets"/${lib.escapeShellArg "matrix-password-${user.name}"}
+          '';
         }
-      ];
-      initialScript = pkgs.writeText "synapse-init.sql" ''
-        CREATE DATABASE "matrix-synapse"
-          TEMPLATE template0
-          LC_COLLATE = "C"
-          LC_CTYPE = "C";
-      '';
-    };
+      ) cfg.users;
+
+    systemd.services.matrix-synapse =
+      let
+        usersScript =
+          ''
+            while ! ${pkgs.netcat}/bin/nc -z -v ::1 8008; do
+              if ! kill -0 "$MAINPID"; then exit 1; fi
+              sleep 1;
+            done
+          ''
+          + lib.concatMapStringsSep "\n" (user: ''
+            # only create user if it doesn't exist
+            /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --exists-ok --password-file ${
+              config.clan.core.facts.services."matrix-password-${user.name}".secret."matrix-password-${user.name}".path
+            } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"}
+          '') (lib.attrValues cfg.users);
+      in
+      {
+        path = [ pkgs.curl ];
+        serviceConfig.ExecStartPost = [
+          (''+${pkgs.writeShellScript "matrix-synapse-create-users" usersScript}'')
+        ];
+      };
+
     services.nginx = {
       enable = true;
       virtualHosts = {
@@ -102,7 +183,7 @@ in
             return 200 '${
               builtins.toJSON {
                 "m.homeserver" = {
-                  "base_url" = "https://matrix.${cfg.domain}";
+                  "base_url" = "https://${nginx-vhost}";
                 };
                 "m.identity_server" = {
                   "base_url" = "https://vector.im";
@@ -111,15 +192,12 @@ in
             }';
           '';
         };
-        "matrix.${cfg.domain}" = {
+        ${nginx-vhost} = {
           forceSSL = true;
           enableACME = true;
-          locations."/_matrix" = {
-            proxyPass = "http://localhost:8008";
-          };
-          locations."/test".extraConfig = ''
-            return 200 "Hello, world!";
-          '';
+          locations."/_matrix".proxyPass = "http://localhost:8008";
+          locations."/_synapse".proxyPass = "http://localhost:8008";
+          locations."/".root = element-web;
         };
       };
     };

clanModules/moonlight/README.md

@@ -1,2 +1,3 @@
-A desktop streaming client optimized for remote gaming and synchronized movie viewing.
+---
+description = "A desktop streaming client optimized for remote gaming and synchronized movie viewing."
 ---

clanModules/moonlight/default.nix

@@ -13,10 +13,10 @@ in
   systemd.tmpfiles.rules = [
     "d '/var/lib/moonlight' 0770 'user' 'users' - -"
     "C '/var/lib/moonlight/moonlight.cert' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.moonlight.secret."moonlight.cert".path or ""
+      config.clan.core.facts.services.moonlight.secret."moonlight.cert".path or ""
     }"
     "C '/var/lib/moonlight/moonlight.key' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.moonlight.secret."moonlight.key".path or ""
+      config.clan.core.facts.services.moonlight.secret."moonlight.key".path or ""
     }"
   ];
 
@@ -45,7 +45,7 @@ in
   systemd.user.services.moonlight-join = {
     description = "Join sunshine hosts";
     script = ''${ms-accept}/bin/moonlight-sunshine-accept moonlight join --port ${builtins.toString defaultPort} --cert '${
-      config.clanCore.facts.services.moonlight.public."moonlight.cert".value or ""
+      config.clan.core.facts.services.moonlight.public."moonlight.cert".value or ""
     }' --host fd2e:25da:6035:c98f:cd99:93e0:b9b8:9ca1'';
     serviceConfig = {
       Type = "oneshot";
@@ -68,7 +68,7 @@ in
     };
   };
 
-  clanCore.facts.services.moonlight = {
+  clan.core.facts.services.moonlight = {
     secret."moonlight.key" = { };
     secret."moonlight.cert" = { };
     public."moonlight.cert" = { };

clanModules/packages/README.md

@@ -0,0 +1,4 @@
+---
+description = "Define package sets from nixpkgs and install them on one or more machines"
+categories = ["packages"]
+---

clanModules/packages/default.nix

@@ -0,0 +1,19 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.clan.packages = {
+    packages = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "The packages to install on the machine";
+    };
+  };
+  config = {
+    environment.systemPackages = map (
+      pName: lib.getAttrFromPath (lib.splitString "." pName) pkgs
+    ) config.clan.packages.packages;
+  };
+}

clanModules/packages/roles/default.nix

@@ -0,0 +1 @@
+{ }

clanModules/postgresql/README.md

@@ -0,0 +1,3 @@
+---
+description = "A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance."
+---

clanModules/postgresql/default.nix

@@ -0,0 +1,226 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  createDatatbaseState =
+    db:
+    let
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
+      '';
+      postRestoreScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
+    };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+      )
+    }
+  '';
+  cfg = config.clan.postgresql;
+
+  userClauses = lib.mapAttrsToList (
+    _: user:
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+  ) cfg.databases;
+in
+{
+  options.clan.postgresql = {
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
+    databases = lib.mkOption {
+      description = "Databases to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Database name.";
+              };
+              service = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Service name that we associate with the database.";
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Create the database if it does not exist.";
+              };
+              create.options = lib.mkOption {
+                description = "Options to pass to the CREATE DATABASE command.";
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of systemd services to stop before restoring the database.";
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      description = "Users to create";
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              description = "User name";
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
+    };
+  };
+  config = {
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
+    services.postgresql.enable = true;
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+
+    clan.core.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair db.service (createDatatbaseState db)
+    ) config.clan.postgresql.databases;
+
+    environment.systemPackages = builtins.map (
+      db:
+      let
+        folder = "/var/backup/postgres/${db.name}";
+        current = "${folder}/pg-dump";
+      in
+      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            ${
+              lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+                systemctl stop ${builtins.toString db.restore.stopOnRestore}
+                trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+              ''
+            }
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      ''
+    ) (builtins.attrValues config.clan.postgresql.databases);
+  };
+}

clanModules/root-password/README.md

@@ -1,4 +1,5 @@
-Automatically generates and configures a password for the root user.
+---
+description = "Automatically generates and configures a password for the root user."
 ---
 
 After the system was installed/deployed the following command can be used to display the root-password:

clanModules/root-password/default.nix

@@ -2,9 +2,9 @@
 {
   users.mutableUsers = false;
   users.users.root.hashedPasswordFile =
-    config.clanCore.facts.services.root-password.secret.password-hash.path;
-  sops.secrets."${config.clanCore.machineName}-password-hash".neededForUsers = true;
-  clanCore.facts.services.root-password = {
+    config.clan.core.facts.services.root-password.secret.password-hash.path;
+  sops.secrets."${config.clan.core.machineName}-password-hash".neededForUsers = true;
+  clan.core.facts.services.root-password = {
     secret.password = { };
     secret.password-hash = { };
     generator.path = with pkgs; [
@@ -13,8 +13,8 @@
       mkpasswd
     ];
     generator.script = ''
-      xkcdpass --numwords 3 --delimiter - --count 1 > $secrets/password
-      cat $secrets/password | mkpasswd -s -m sha-512 > $secrets/password-hash
+      xkcdpass --numwords 3 --delimiter - --count 1 | tr -d "\n" > $secrets/password
+      cat $secrets/password | mkpasswd -s -m sha-512 | tr -d "\n" > $secrets/password-hash
     '';
   };
 }

clanModules/single-disk/README.md

@@ -0,0 +1,42 @@
+---
+description = "Configures partitioning of the main disk"
+categories = ["disk-layout"]
+---
+# Primary Disk Layout
+
+A module for the "disk-layout" category MUST be choosen.
+
+There is exactly one slot for this type of module in the UI, if you don't fill the slot, your machine cannot boot
+
+This module is a good choice for most machines. In the future clan will offer a broader choice of disk-layouts
+
+The UI will ask for the options of this module:
+
+`device: "/dev/null"`
+
+# Usage example
+
+`inventory.json`
+```json
+"services": {
+    "single-disk": {
+        "default": {
+            "meta": {
+                "name": "single-disk"
+            },
+            "roles": {
+                "default": {
+                    "machines": ["jon"]
+                }
+            },
+            "machines": {
+                "jon": {
+                    "config": {
+                        "device": "/dev/null"
+                    }
+                }
+            }
+        }
+    }
+}
+```

clanModules/single-disk/default.nix

@@ -0,0 +1,52 @@
+{ lib, config, ... }:
+{
+  options.clan.single-disk = {
+    device = lib.mkOption {
+      type = lib.types.str;
+      description = "The primary disk device to install the system on";
+      # Question: should we set a default here?
+      # default = "/dev/null";
+    };
+  };
+  config = {
+    boot.loader.grub.efiSupport = lib.mkDefault true;
+    boot.loader.grub.efiInstallAsRemovable = lib.mkDefault true;
+    disko.devices = {
+      disk = {
+        main = {
+          type = "disk";
+          # This is set through the UI
+          device = config.clan.single-disk.device;
+
+          content = {
+            type = "gpt";
+            partitions = {
+              boot = {
+                size = "1M";
+                type = "EF02"; # for grub MBR
+                priority = 1;
+              };
+              ESP = {
+                size = "512M";
+                type = "EF00";
+                content = {
+                  type = "filesystem";
+                  format = "vfat";
+                  mountpoint = "/boot";
+                };
+              };
+              root = {
+                size = "100%";
+                content = {
+                  type = "filesystem";
+                  format = "ext4";
+                  mountpoint = "/";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

clanModules/single-disk/roles/default.nix

@@ -0,0 +1 @@
+{ }

clanModules/sshd/README.md

@@ -1,2 +1,3 @@
-Enables secure remote access to the machine over ssh
+---
+description = "Enables secure remote access to the machine over ssh"
 ---

clanModules/sshd/default.nix

@@ -5,12 +5,12 @@
 
   services.openssh.hostKeys = [
     {
-      path = config.clanCore.facts.services.openssh.secret."ssh.id_ed25519".path;
+      path = config.clan.core.facts.services.openssh.secret."ssh.id_ed25519".path;
       type = "ed25519";
     }
   ];
 
-  clanCore.facts.services.openssh = {
+  clan.core.facts.services.openssh = {
     secret."ssh.id_ed25519" = { };
     public."ssh.id_ed25519.pub" = { };
     generator.path = [

clanModules/static-hosts/README.md

@@ -1,2 +1,3 @@
-Statically configure the host names of machines based on their respective zerotier-ip.
+---
+description = "Statically configure the host names of machines based on their respective zerotier-ip."
 ---

clanModules/static-hosts/default.nix

@@ -4,7 +4,7 @@
     excludeHosts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
       default =
-        if config.clan.static-hosts.topLevelDomain != "" then [ ] else [ config.clanCore.machineName ];
+        if config.clan.static-hosts.topLevelDomain != "" then [ ] else [ config.clan.core.machineName ];
       description = "Hosts that should be excluded";
     };
     topLevelDomain = lib.mkOption {
@@ -16,13 +16,23 @@
 
   config.networking.hosts =
     let
-      clanDir = config.clanCore.clanDir;
+      clanDir = config.clan.core.clanDir;
       machineDir = clanDir + "/machines/";
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
-      machines = builtins.readDir machineDir;
+      machinesFileSet = builtins.readDir machineDir;
+      machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
+      networkIpsUnchecked = builtins.map (
+        machine:
+        let
+          fullPath = zerotierIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) machines;
+      networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
+      machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
       filteredMachines = lib.filterAttrs (
         name: _: !(lib.elem name config.clan.static-hosts.excludeHosts)
-      ) machines;
+      ) machinesWithIp;
     in
     lib.filterAttrs (_: value: value != null) (
       lib.mapAttrs' (
@@ -38,7 +48,7 @@
               [ "${machine}.${config.clan.static-hosts.topLevelDomain}" ]
           )
         else
-          null
+          { }
       ) filteredMachines
     );
 }

clanModules/sunshine/README.md

@@ -1,2 +1,3 @@
-A desktop streaming server optimized for remote gaming and synchronized movie viewing.
+---
+description = "A desktop streaming server optimized for remote gaming and synchronized movie viewing."
 ---

clanModules/sunshine/default.nix

@@ -97,10 +97,10 @@ in
   systemd.tmpfiles.rules = [
     "d '/var/lib/sunshine' 0770 'user' 'users' - -"
     "C '/var/lib/sunshine/sunshine.cert' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.sunshine.secret."sunshine.cert".path or ""
+      config.clan.core.facts.services.sunshine.secret."sunshine.cert".path or ""
     }"
     "C '/var/lib/sunshine/sunshine.key' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.sunshine.secret."sunshine.key".path or ""
+      config.clan.core.facts.services.sunshine.secret."sunshine.key".path or ""
     }"
   ];
 
@@ -117,8 +117,8 @@ in
       RestartSec = "5s";
       ReadWritePaths = [ "/var/lib/sunshine" ];
       ReadOnlyPaths = [
-        (config.clanCore.facts.services.sunshine.secret."sunshine.key".path or "")
-        (config.clanCore.facts.services.sunshine.secret."sunshine.cert".path or "")
+        (config.clan.core.facts.services.sunshine.secret."sunshine.key".path or "")
+        (config.clan.core.facts.services.sunshine.secret."sunshine.cert".path or "")
       ];
     };
     wantedBy = [ "graphical-session.target" ];
@@ -137,7 +137,7 @@ in
     startLimitIntervalSec = 500;
     script = ''
       ${ms-accept}/bin/moonlight-sunshine-accept sunshine init-state --uuid ${
-        config.clanCore.facts.services.sunshine.public.sunshine-uuid.value or null
+        config.clan.core.facts.services.sunshine.public.sunshine-uuid.value or null
       } --state-file /var/lib/sunshine/state.json
     '';
     serviceConfig = {
@@ -173,9 +173,9 @@ in
     startLimitIntervalSec = 500;
     script = ''
       ${ms-accept}/bin/moonlight-sunshine-accept sunshine listen --port ${builtins.toString listenPort} --uuid ${
-        config.clanCore.facts.services.sunshine.public.sunshine-uuid.value or null
+        config.clan.core.facts.services.sunshine.public.sunshine-uuid.value or null
       }  --state /var/lib/sunshine/state.json --cert '${
-        config.clanCore.facts.services.sunshine.public."sunshine.cert".value or null
+        config.clan.core.facts.services.sunshine.public."sunshine.cert".value or null
       }'
     '';
     serviceConfig = {
@@ -187,7 +187,7 @@ in
     wantedBy = [ "graphical-session.target" ];
   };
 
-  clanCore.facts.services.ergochat = {
+  clan.core.facts.services.ergochat = {
     secret."sunshine.key" = { };
     secret."sunshine.cert" = { };
     public."sunshine-uuid" = { };

clanModules/syncthing-static-peers/README.md

@@ -0,0 +1,3 @@
+---
+description = "Statically configure syncthing peers through clan"
+---

clanModules/syncthing-static-peers/default.nix

@@ -0,0 +1,108 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+  syncthingPublicKeyPath = machines: machineDir + machines + "/facts/syncthing.pub";
+  machinesFileSet = builtins.readDir machineDir;
+  machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
+  syncthingPublicKeysUnchecked = builtins.map (
+    machine:
+    let
+      fullPath = syncthingPublicKeyPath machine;
+    in
+    if builtins.pathExists fullPath then machine else null
+  ) machines;
+  syncthingPublicKeyMachines = lib.filter (machine: machine != null) syncthingPublicKeysUnchecked;
+  zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
+  networkIpsUnchecked = builtins.map (
+    machine:
+    let
+      fullPath = zerotierIpMachinePath machine;
+    in
+    if builtins.pathExists fullPath then machine else null
+  ) machines;
+  networkIpMachines = lib.filter (machine: machine != null) networkIpsUnchecked;
+  devices = builtins.map (machine: {
+    name = machine;
+    value = {
+      name = machine;
+      id = (lib.removeSuffix "\n" (builtins.readFile (syncthingPublicKeyPath machine)));
+      addresses =
+        [ "dynamic" ]
+        ++ (
+          if (lib.elem machine networkIpMachines) then
+            [ "tcp://[${(lib.removeSuffix "\n" (builtins.readFile (zerotierIpMachinePath machine)))}]:22000" ]
+          else
+            [ ]
+        );
+    };
+  }) syncthingPublicKeyMachines;
+in
+{
+  options.clan.syncthing-static-peers = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be added.
+      '';
+    };
+  };
+
+  config.services.syncthing.settings.devices = (builtins.listToAttrs devices);
+
+  imports = [
+    {
+      # Syncthing ports: 8384 for remote access to GUI
+      # 22000 TCP and/or UDP for sync traffic
+      # 21027/UDP for discovery
+      # source: https://docs.syncthing.net/users/firewall.html
+      networking.firewall.interfaces."zt+".allowedTCPPorts = [
+        8384
+        22000
+      ];
+      networking.firewall.allowedTCPPorts = [ 8384 ];
+      networking.firewall.interfaces."zt+".allowedUDPPorts = [
+        22000
+        21027
+      ];
+
+      # Activates inotify compatibility on syncthing
+      # use mkOverride 900 here as it otherwise would collide with the default of the
+      # upstream nixos xserver.nix
+      boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkOverride 900 524288;
+
+      services.syncthing = {
+        enable = true;
+        configDir = "/var/lib/syncthing";
+        group = "syncthing";
+
+        key = lib.mkDefault config.clan.core.facts.services.syncthing.secret."syncthing.key".path or null;
+        cert = lib.mkDefault config.clan.core.facts.services.syncthing.secret."syncthing.cert".path or null;
+      };
+
+      clan.core.facts.services.syncthing = {
+        secret."syncthing.key" = { };
+        secret."syncthing.cert" = { };
+        public."syncthing.pub" = { };
+        generator.path = [
+          pkgs.coreutils
+          pkgs.gnugrep
+          pkgs.syncthing
+        ];
+        generator.script = ''
+          syncthing generate --config "$secrets"
+          mv "$secrets"/key.pem "$secrets"/syncthing.key
+          mv "$secrets"/cert.pem "$secrets"/syncthing.cert
+          cat "$secrets"/config.xml | grep -oP '(?<=<device id=")[^"]+' | uniq > "$facts"/syncthing.pub
+        '';
+      };
+    }
+  ];
+}

clanModules/syncthing/README.md

@@ -1,4 +1,5 @@
-A secure, file synchronization app for devices over networks, offering a private alternative to cloud services.
+---
+description = "A secure, file synchronization app for devices over networks, offering a private alternative to cloud services."
 ---
 ## Usage
 

clanModules/syncthing/default.nix

@@ -7,10 +7,14 @@
 {
   options.clan.syncthing = {
     id = lib.mkOption {
+      description = ''
+        The ID of the machine.
+        It is generated automatically by default.
+      '';
       type = lib.types.nullOr lib.types.str;
       example = "BABNJY4-G2ICDLF-QQEG7DD-N3OBNGF-BCCOFK6-MV3K7QJ-2WUZHXS-7DTW4AS";
-      default = config.clanCore.facts.services.syncthing.public."syncthing.pub".value or null;
-      defaultText = "config.clanCore.facts.services.syncthing.public.\"syncthing.pub\".value";
+      default = config.clan.core.facts.services.syncthing.public."syncthing.pub".value or null;
+      defaultText = "config.clan.core.facts.services.syncthing.public.\"syncthing.pub\".value";
     };
     introducer = lib.mkOption {
       description = ''
@@ -94,7 +98,7 @@
         settings = {
           options = {
             urAccepted = -1;
-            allowedNetworks = [ config.clan.networking.zerotier.subnet ];
+            allowedNetworks = [ config.clan.core.networking.zerotier.subnet ];
           };
           devices =
             { }
@@ -119,7 +123,7 @@
           getPendingDevices = "/rest/cluster/pending/devices";
           postNewDevice = "/rest/config/devices";
           SharedFolderById = "/rest/config/folders/";
-          apiKey = config.clanCore.facts.services.syncthing.secret."syncthing.api".path or null;
+          apiKey = config.clan.core.facts.services.syncthing.secret."syncthing.api".path or null;
         in
         lib.mkIf config.clan.syncthing.autoAcceptDevices {
           description = "Syncthing auto accept devices";
@@ -161,7 +165,7 @@
 
       systemd.services.syncthing-init-api-key =
         let
-          apiKey = config.clanCore.facts.services.syncthing.secret."syncthing.api".path or null;
+          apiKey = config.clan.core.facts.services.syncthing.secret."syncthing.api".path or null;
         in
         lib.mkIf config.clan.syncthing.autoAcceptDevices {
           description = "Set the api key";
@@ -183,7 +187,7 @@
           };
         };
 
-      clanCore.facts.services.syncthing = {
+      clan.core.facts.services.syncthing = {
         secret."syncthing.key" = { };
         secret."syncthing.cert" = { };
         secret."syncthing.api" = { };

clanModules/thelounge/README.md

@@ -1,2 +1,3 @@
-Modern web IRC client
+---
+description = "Modern web IRC client"
 ---

clanModules/thelounge/default.nix

@@ -11,5 +11,5 @@ _: {
     };
   };
 
-  clanCore.state.thelounde.folders = [ "/var/lib/thelounge" ];
+  clan.core.state.thelounde.folders = [ "/var/lib/thelounge" ];
 }

clanModules/trusted-nix-caches/README.md

@@ -1,2 +1,3 @@
-This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.
+---
+description = "This module sets the `clan.lol` and `nix-community` cache up as a trusted cache."
 ----

clanModules/user-password/README.md

@@ -1,4 +1,5 @@
-Automatically generates and configures a password for the specified user account.
+---
+description = "Automatically generates and configures a password for the specified user account."
 ---
 
 If setting the option prompt to true, the user will be prompted to type in their desired password.

clanModules/user-password/default.nix

@@ -22,9 +22,9 @@
   config = {
     users.mutableUsers = false;
     users.users.${config.clan.user-password.user}.hashedPasswordFile =
-      config.clanCore.facts.services.user-password.secret.user-password-hash.path;
-    sops.secrets."${config.clanCore.machineName}-user-password-hash".neededForUsers = true;
-    clanCore.facts.services.user-password = {
+      config.clan.core.facts.services.user-password.secret.user-password-hash.path;
+    sops.secrets."${config.clan.core.machineName}-user-password-hash".neededForUsers = true;
+    clan.core.facts.services.user-password = {
       secret.user-password = { };
       secret.user-password-hash = { };
       generator.prompt = (
@@ -37,12 +37,12 @@
         mkpasswd
       ];
       generator.script = ''
-        if [[ -n $prompt_value ]]; then
-          echo $prompt_value > $secrets/user-password
+        if [[ -n ''${prompt_value-} ]]; then
+          echo $prompt_value | tr -d "\n" > $secrets/user-password
         else
-          xkcdpass --numwords 3 --delimiter - --count 1 > $secrets/user-password
+          xkcdpass --numwords 3 --delimiter - --count 1 | tr -d "\n" > $secrets/user-password
         fi
-        cat $secrets/user-password | mkpasswd -s -m sha-512 > $secrets/user-password-hash
+        cat $secrets/user-password | mkpasswd -s -m sha-512 | tr -d "\n" > $secrets/user-password-hash
       '';
     };
   };

clanModules/xfce/README.md

@@ -1,2 +1,3 @@
-A lightweight desktop manager
+---
+description = "A lightweight desktop manager"
 ---

clanModules/zerotier-static-peers/README.md

@@ -1,4 +1,5 @@
-Statically configure the `zerotier` peers of a clan network.
+---
+description = "Statically configure the `zerotier` peers of a clan network."
 ---
 Statically configure the `zerotier` peers of a clan network.
 

clanModules/zerotier-static-peers/default.nix

@@ -2,11 +2,10 @@
   lib,
   config,
   pkgs,
-  inputs,
   ...
 }:
 let
-  clanDir = config.clanCore.clanDir;
+  clanDir = config.clan.core.clanDir;
   machineDir = clanDir + "/machines/";
   machinesFileSet = builtins.readDir machineDir;
   machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
@@ -28,44 +27,61 @@ in
   options.clan.zerotier-static-peers = {
     excludeHosts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = [ config.clanCore.machineName ];
+      default = [ config.clan.core.machineName ];
       description = "Hosts that should be excluded";
     };
+    networkIps = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Extra zerotier network Ips that should be accepted";
+    };
+    networkIds = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Extra zerotier network Ids that should be accepted";
+    };
   };
 
   config.systemd.services.zerotier-static-peers-autoaccept =
     let
-      machines = builtins.readDir machineDir;
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
+      networkIpsUnchecked = builtins.map (
+        machine:
+        let
+          fullPath = zerotierIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) machines;
+      networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
+      machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
       filteredMachines = lib.filterAttrs (
         name: _: !(lib.elem name config.clan.zerotier-static-peers.excludeHosts)
-      ) machines;
+      ) machinesWithIp;
       hosts = lib.mapAttrsToList (host: _: host) (
         lib.mapAttrs' (
           machine: _:
           let
             fullPath = zerotierIpMachinePath machine;
           in
-          if builtins.pathExists fullPath then
-            lib.nameValuePair (builtins.readFile fullPath) [ machine ]
-          else
-            null
+          lib.nameValuePair (builtins.readFile fullPath) [ machine ]
         ) filteredMachines
       );
+      allHostIPs = config.clan.zerotier-static-peers.networkIps ++ hosts;
     in
-    lib.mkIf (config.clan.networking.zerotier.controller.enable) {
+    lib.mkIf (config.clan.core.networking.zerotier.controller.enable) {
       wantedBy = [ "multi-user.target" ];
       after = [ "zerotierone.service" ];
-      path = [ pkgs.zerotierone ];
+      path = [ config.clan.core.clanPkgs.zerotierone ];
       serviceConfig.ExecStart = pkgs.writeScript "static-zerotier-peers-autoaccept" ''
         #!/bin/sh
         ${lib.concatMapStringsSep "\n" (host: ''
-          ${
-            inputs.clan-core.packages.${pkgs.system}.zerotier-members
-          }/bin/zerotier-members allow --member-ip ${host}
-        '') hosts}
+          ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow --member-ip ${host}
+        '') allHostIPs}
+        ${lib.concatMapStringsSep "\n" (host: ''
+          ${config.clan.core.clanPkgs.zerotier-members}/bin/zerotier-members allow ${host}
+        '') config.clan.zerotier-static-peers.networkIds}
       '';
     };
 
-  config.clan.networking.zerotier.networkId = lib.mkDefault networkId;
+  config.clan.core.networking.zerotier.networkId = lib.mkDefault networkId;
 }

clanModules/zt-tcp-relay/README.md

@@ -1,2 +1,3 @@
-Enable ZeroTier VPN over TCP for networks where UDP is blocked.
+---
+description = "Enable ZeroTier VPN over TCP for networks where UDP is blocked."
 ---

devShell.nix

@@ -26,6 +26,7 @@
       devShells.default = pkgs.mkShell {
         packages = [
           select-shell
+          pkgs.nix-unit
           pkgs.tea
           # Better error messages than nix 2.18
           pkgs.nixVersions.latest

docs/.envrc

@@ -1,6 +1,8 @@
+# shellcheck shell=bash
 source_up
 
-watch_file $(find ./nix -name "*.nix" -printf '%p ')
+mapfile -d '' -t nix_files < <(find ./nix -name "*.nix" -print0)
+watch_file "${nix_files[@]}"
 
 # Because we depend on nixpkgs sources, uploading to builders takes a long time
 use flake .#docs --builders ''

docs/CONTRIBUTING.md

@@ -44,6 +44,14 @@ Let's get your development environment up and running:
          ```bash
          git remote add upstream gitea@git.clan.lol:clan/clan-core.git
          ```
+5. **Create an access token**:
+      - Log in to Gitea.
+      - Go to your account settings.
+      - Navigate to the Applications section.
+      - Click Generate New Token.
+      - Name your token and select all available scopes.
+      - Generate the token and copy it for later use.
+      - Your access token is now ready to use with all permissions.
 
 5. **Register Your Gitea Account Locally**:
 
@@ -54,9 +62,8 @@ Let's get your development environment up and running:
       - Fill out the prompt as follows:
         - URL of Gitea instance: `https://git.clan.lol`
         - Name of new Login [git.clan.lol]:
-        - Do you have an access token? No
-        - Username: YourUsername
-        - Password: YourPassword
+        - Do you have an access token? Yes
+        - Token: <yourtoken>
         - Set Optional settings: No
 
 
@@ -121,7 +128,7 @@ git+file:///home/lhebendanz/Projects/clan-core
 │       ├───clan-cli omitted (use '--all-systems' to show)
 │       ├───clan-cli-docs omitted (use '--all-systems' to show)
 │       ├───clan-ts-api omitted (use '--all-systems' to show)
-│       ├───clan-vm-manager omitted (use '--all-systems' to show)
+│       ├───clan-app omitted (use '--all-systems' to show)
 │       ├───default omitted (use '--all-systems' to show)
 │       ├───deploy-docs omitted (use '--all-systems' to show)
 │       ├───docs omitted (use '--all-systems' to show)

docs/mkdocs.yml

@@ -14,6 +14,7 @@ markdown_extensions:
   - attr_list
   - footnotes
   - md_in_html
+  - def_list
   - meta
   - plantuml_markdown
   - pymdownx.emoji:
@@ -38,8 +39,6 @@ exclude_docs: |
   /drafts/
 
 nav:
-  - Blog:
-      - blog/index.md
   - Getting started:
       - index.md
       - Installer: getting-started/installer.md
@@ -49,21 +48,28 @@ nav:
       - Mesh VPN: getting-started/mesh-vpn.md
       - Backup & Restore: getting-started/backups.md
       - Flake-parts: getting-started/flake-parts.md
-  - Modules:
+  - Concepts:
+      - Configuration: concepts/configuration.md
+      - Inventory: concepts/inventory.md
+  - Reference:
       - Clan Modules:
+          - reference/clanModules/borgbackup-static.md
           - reference/clanModules/borgbackup.md
           - reference/clanModules/deltachat.md
-          - reference/clanModules/disk-layouts.md
           - reference/clanModules/ergochat.md
           - reference/clanModules/localbackup.md
           - reference/clanModules/localsend.md
           - reference/clanModules/matrix-synapse.md
           - reference/clanModules/moonlight.md
+          - reference/clanModules/packages.md
+          - reference/clanModules/postgresql.md
           - reference/clanModules/root-password.md
+          - reference/clanModules/single-disk.md
           - reference/clanModules/sshd.md
-          - reference/clanModules/sunshine.md
-          - reference/clanModules/syncthing.md
           - reference/clanModules/static-hosts.md
+          - reference/clanModules/sunshine.md
+          - reference/clanModules/syncthing-static-peers.md
+          - reference/clanModules/syncthing.md
           - reference/clanModules/thelounge.md
           - reference/clanModules/trusted-nix-caches.md
           - reference/clanModules/user-password.md
@@ -71,16 +77,18 @@ nav:
           - reference/clanModules/zerotier-static-peers.md
           - reference/clanModules/zt-tcp-relay.md
       - CLI:
-          - reference/cli/index.md
           - reference/cli/backups.md
           - reference/cli/config.md
           - reference/cli/facts.md
           - reference/cli/flakes.md
           - reference/cli/flash.md
           - reference/cli/history.md
+          - reference/cli/index.md
           - reference/cli/machines.md
           - reference/cli/secrets.md
+          - reference/cli/show.md
           - reference/cli/ssh.md
+          - reference/cli/state.md
           - reference/cli/vms.md
       - Clan Core:
           - reference/clan-core/index.md
@@ -88,7 +96,11 @@ nav:
           - reference/clan-core/facts.md
           - reference/clan-core/sops.md
           - reference/clan-core/state.md
+          - reference/clan-core/deployment.md
+          - reference/clan-core/networking.md
   - Contributing: contributing/contributing.md
+  - Blog:
+      - blog/index.md
 
 docs_dir: site
 site_dir: out

docs/nix/default.nix

@@ -26,6 +26,7 @@ pkgs.stdenv.mkDerivation {
       mkdocs-material
       mkdocs-rss-plugin
       mkdocs-macros
+      filelock # FIXME: this should be already provided by mkdocs-rss-plugin
     ]);
   configurePhase = ''
     mkdir -p ./site/reference/cli

docs/nix/flake-module.nix

@@ -12,13 +12,14 @@
       # { clanCore = «derivation JSON»; clanModules = { ${name} = «derivation JSON» }; }
       jsonDocs = import ./get-module-docs.nix {
         inherit (inputs) nixpkgs;
-        inherit pkgs self;
+        inherit pkgs;
         inherit (self.nixosModules) clanCore;
         inherit (self) clanModules;
       };
 
       clanModulesFileInfo = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModules);
-      clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes);
+      # clanModulesReadmes = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesReadmes);
+      # clanModulesMeta = pkgs.writeText "info.json" (builtins.toJSON jsonDocs.clanModulesMeta);
 
       # Simply evaluated options (JSON)
       renderOptions =
@@ -29,6 +30,7 @@
             nativeBuildInputs = [
               pkgs.python3
               pkgs.mypy
+              self'.packages.clan-cli
             ];
           }
           ''
@@ -36,7 +38,7 @@
             patchShebangs --build $out
 
             ruff format --check --diff $out
-            ruff --line-length 88 $out
+            ruff check --line-length 88 $out
             mypy --strict $out
           '';
 
@@ -49,24 +51,35 @@
         sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc=";
       };
 
-      module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
-        export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
-        # A file that contains the links to all clanModule docs
-        export CLAN_MODULES=${clanModulesFileInfo}
-        export CLAN_MODULES_READMES=${clanModulesReadmes}
+      module-docs =
+        pkgs.runCommand "rendered"
+          {
+            nativeBuildInputs = [
+              pkgs.python3
+              self'.packages.clan-cli
+            ];
+          }
+          ''
+            export CLAN_CORE_PATH=${self}
+            export CLAN_CORE_DOCS=${jsonDocs.clanCore}/share/doc/nixos/options.json
+            # A file that contains the links to all clanModule docs
+            export CLAN_MODULES=${clanModulesFileInfo}
 
-        mkdir $out
+            mkdir $out
 
-        # The python script will place mkDocs files in the output directory
-        python3 ${renderOptions}
-      '';
+            # The python script will place mkDocs files in the output directory
+            python3 ${renderOptions}
+          '';
     in
     {
       devShells.docs = pkgs.callPackage ./shell.nix {
         inherit (self'.packages) docs clan-cli-docs;
-        inherit module-docs;
-        inherit asciinema-player-js;
-        inherit asciinema-player-css;
+        inherit
+          asciinema-player-js
+          asciinema-player-css
+          module-docs
+          self'
+          ;
       };
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {

docs/nix/get-module-docs.nix

@@ -3,7 +3,6 @@
   pkgs,
   clanCore,
   clanModules,
-  self,
 }:
 let
   allNixosModules = (import "${nixpkgs}/nixos/modules/module-list.nix") ++ [
@@ -13,7 +12,7 @@ let
 
   clanCoreNixosModules = [
     clanCore
-    { clanCore.clanDir = ./.; }
+    { clan.core.clanDir = ./.; }
   ] ++ allNixosModules;
 
   # TODO: optimally we would not have to evaluate all nixos modules for every page
@@ -25,27 +24,24 @@ let
   #   improves eval performance slightly (10%)
   getOptions = modules: (clanCoreNixos.extendModules { inherit modules; }).options;
 
+  getOptionsWithoutCore = modules: builtins.removeAttrs (getOptions modules) [ "core" ];
+
   evalDocs =
     options:
     pkgs.nixosOptionsDoc {
       options = options;
-      warningsAreErrors = false;
+      warningsAreErrors = true;
     };
 
   # clanModules docs
   clanModulesDocs = builtins.mapAttrs (
-    name: module: (evalDocs ((getOptions [ module ]).clan.${name} or { })).optionsJSON
-  ) clanModules;
-
-  clanModulesReadmes = builtins.mapAttrs (
-    module_name: _module: self.lib.modules.getReadme module_name
+    name: module: (evalDocs ((getOptionsWithoutCore [ module ]).clan.${name} or { })).optionsJSON
   ) clanModules;
 
   # clanCore docs
-  clanCoreDocs = (evalDocs (getOptions [ ]).clanCore).optionsJSON;
+  clanCoreDocs = (evalDocs (getOptions [ ]).clan.core).optionsJSON;
 in
 {
-  inherit clanModulesReadmes;
   clanCore = clanCoreDocs;
   clanModules = clanModulesDocs;
 }

docs/nix/scripts/renderOptions.py

@@ -28,10 +28,12 @@ import os
 from pathlib import Path
 from typing import Any
 
+from clan_cli.api.modules import Frontmatter, extract_frontmatter, get_roles
+
 # Get environment variables
-CLAN_CORE = os.getenv("CLAN_CORE")
+CLAN_CORE_PATH = os.getenv("CLAN_CORE_PATH")
+CLAN_CORE_DOCS = os.getenv("CLAN_CORE_DOCS")
 CLAN_MODULES = os.environ.get("CLAN_MODULES")
-CLAN_MODULES_READMES = os.environ.get("CLAN_MODULES_READMES")
 
 OUT = os.environ.get("out")
 
@@ -76,7 +78,9 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
 
     res = f"""
 {"#" * level} {sanitize(name)}
-{"Readonly" if read_only else ""}
+
+{"**Readonly**" if read_only else ""}
+
 {option.get("description", "No description available.")}
 
 **Type**: `{option["type"]}`
@@ -109,18 +113,19 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
 """
 
     decls = option.get("declarations", [])
-    source_path, name = replace_store_path(decls[0])
-    print(source_path, name)
-    res += f"""
+    if decls:
+        source_path, name = replace_store_path(decls[0])
+        print(source_path, name)
+        res += f"""
 :simple-git: [{name}]({source_path})
 """
-    res += "\n"
+        res += "\n"
 
     return res
 
 
 def module_header(module_name: str) -> str:
-    return f"# {module_name}\n"
+    return f"# {module_name}\n\n"
 
 
 def module_usage(module_name: str) -> str:
@@ -137,7 +142,7 @@ To use this module, import it like this:
 """
 
 
-clan_core_descr = """ClanCore delivers all the essential features for every clan. 
+clan_core_descr = """ClanCore delivers all the essential features for every clan.
 It's always included in your setup, and you can customize your clan's behavior with the configuration [options](#module-options) provided below.
 
 """
@@ -146,9 +151,9 @@ options_head = "\n## Module Options\n"
 
 
 def produce_clan_core_docs() -> None:
-    if not CLAN_CORE:
+    if not CLAN_CORE_DOCS:
         raise ValueError(
-            f"Environment variables are not set correctly: $CLAN_CORE={CLAN_CORE}"
+            f"Environment variables are not set correctly: $CLAN_CORE_DOCS={CLAN_CORE_DOCS}"
         )
 
     if not OUT:
@@ -156,14 +161,14 @@ def produce_clan_core_docs() -> None:
 
     # A mapping of output file to content
     core_outputs: dict[str, str] = {}
-    with open(CLAN_CORE) as f:
+    with open(CLAN_CORE_DOCS) as f:
         options: dict[str, dict[str, Any]] = json.load(f)
         module_name = "clan-core"
         for option_name, info in options.items():
             outfile = f"{module_name}/index.md"
 
             # Create separate files for nested options
-            if len(option_name.split(".")) <= 2:
+            if len(option_name.split(".")) <= 3:
                 # i.e. clan-core.clanDir
                 output = core_outputs.get(
                     outfile,
@@ -174,7 +179,7 @@ def produce_clan_core_docs() -> None:
                 core_outputs[outfile] = output
             else:
                 # Clan sub-options
-                [_, sub] = option_name.split(".")[0:2]
+                [_, sub] = option_name.split(".")[1:3]
                 outfile = f"{module_name}/{sub}.md"
                 # Get the content or write the header
                 output = core_outputs.get(outfile, render_option_header(sub))
@@ -188,14 +193,42 @@ def produce_clan_core_docs() -> None:
                 of.write(output)
 
 
+def render_roles(roles: list[str] | None, module_name: str) -> str:
+    if roles:
+        roles_list = "\n".join([f"    - `{r}`" for r in roles])
+        return f"""
+???+ tip "Inventory usage"
+
+    Predefined roles:
+
+{roles_list}
+
+    Usage:
+
+    ```{{.nix hl_lines="4"}}
+    buildClan {{
+      inventory.services = {{
+        {module_name}.instance_1 = {{
+            roles.{roles[0]}.machines = [ "sara_machine" ];
+            # ...
+        }};
+      }};
+    }}
+    ```
+
+"""
+    return ""
+
+
 def produce_clan_modules_docs() -> None:
     if not CLAN_MODULES:
         raise ValueError(
             f"Environment variables are not set correctly: $CLAN_MODULES={CLAN_MODULES}"
         )
-    if not CLAN_MODULES_READMES:
+
+    if not CLAN_CORE_PATH:
         raise ValueError(
-            f"Environment variables are not set correctly: $CLAN_MODULES_READMES={CLAN_MODULES_READMES}"
+            f"Environment variables are not set correctly: $CLAN_CORE_PATH={CLAN_CORE_PATH}"
         )
 
     if not OUT:
@@ -204,18 +237,36 @@ def produce_clan_modules_docs() -> None:
     with open(CLAN_MODULES) as f:
         links: dict[str, str] = json.load(f)
 
-    with open(CLAN_MODULES_READMES) as readme:
-        readme_map: dict[str, str] = json.load(readme)
+    # with open(CLAN_MODULES_READMES) as readme:
+    #     readme_map: dict[str, str] = json.load(readme)
+
+    # with open(CLAN_MODULES_META) as f:
+    #     meta_map: dict[str, Any] = json.load(f)
+    #     print(meta_map)
 
     # {'borgbackup': '/nix/store/hi17dwgy7963ddd4ijh81fv0c9sbh8sw-options.json', ... }
     for module_name, options_file in links.items():
+        readme_file = Path(CLAN_CORE_PATH) / "clanModules" / module_name / "README.md"
+        print(module_name, readme_file)
+        with open(readme_file) as f:
+            readme = f.read()
+            frontmatter: Frontmatter
+            frontmatter, readme_content = extract_frontmatter(readme, str(readme_file))
+            print(frontmatter, readme_content)
+
         with open(Path(options_file) / "share/doc/nixos/options.json") as f:
             options: dict[str, dict[str, Any]] = json.load(f)
             print(f"Rendering options for {module_name}...")
             output = module_header(module_name)
 
-            if readme_map.get(module_name, None):
-                output += f"{readme_map[module_name]}\n"
+            if frontmatter.description:
+                output += f"**{frontmatter.description}**\n\n"
+            output += f"{readme_content}\n"
+
+            # get_roles(str) -> list[str] | None
+            roles = get_roles(str(Path(CLAN_CORE_PATH) / "clanModules" / module_name))
+            if roles:
+                output += render_roles(roles, module_name)
 
             output += module_usage(module_name)
 

docs/nix/shell.nix

@@ -7,10 +7,14 @@
   asciinema-player-css,
   roboto,
   fira-code,
+  self',
   ...
 }:
 pkgs.mkShell {
-  inputsFrom = [ docs ];
+  inputsFrom = [
+    docs
+    self'.devShells.default
+  ];
   shellHook = ''
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/

docs/site/blog/posts/backups.md

@@ -0,0 +1,132 @@
+---
+title: "Dev Report: Declarative Backups and Restore"
+description: "An extension to the NixOS module system to declaratively describe how application state is backed up and restored."
+authors:
+  - Mic92
+date: 2024-06-24
+slug: backups
+---
+
+Our goal with [Clan](https://clan.lol/) is to give users control over their data.
+However, with great power comes great responsibility, and owning your data means you also need to take care of backups yourself.
+
+In our experience, setting up automatic backups is often a tedious process as it requires custom integration of the backup software and
+the services that produce the state. More important than the backup is the restore.
+Restores are often not well tested or documented, and if not working correctly, they can render the backup useless.
+
+In Clan, we want to make backup and restore a first-class citizen.
+Every service should describe what state it produces and how it can be backed up and restored.
+
+In this article, we will discuss how our backup interface in Clan works.
+The interface allows different backup software to be used interchangeably and
+allows module authors to define custom backup and restore logic for their services.
+
+
+## First Comes the State
+
+Our services are built from Clan modules. Clan modules are essentially [NixOS modules](https://wiki.nixos.org/wiki/NixOS_modules), the basic configuration components of NixOS.
+However, we have enhanced them with additional features provided by Clan and restricted certain option types to enable configuration through a [graphical interface](https://docs.clan.lol/blog/2024/05/25/jsonschema-converter/).
+
+In a simple case, this can be just a bunch of directories, such as what we define for our [ZeroTier](https://www.zerotier.com/) VPN service.
+
+```nix
+{
+  clan.core.state.zerotier.folders =  [ "/var/lib/zerotier-one" ];
+}
+```
+
+For other systems, we need more complex backup and restore logic.
+For each state, we can provide custom command hooks for backing up and restoring.
+
+In our PostgreSQL module, for example, we define `preBackupCommand` and `postRestoreCommand` to use `pg_dump` and `pg_restore` to backup and restore individual databases:
+
+```nix
+preBackupCommand = ''
+  # ...
+  runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+  # ...
+'';
+postRestoreCommand = ''
+  # ...
+  runuser -u postgres -- dropdb "${db.name}"
+  runuser -u postgres -- pg_restore -C -d postgres "${current}"
+  # ...
+'';
+```
+
+## Then the Backup
+
+Our CLI unifies the different backup providers in one [interface](https://docs.clan.lol/reference/cli/backups/).
+
+As of now, we support backups using [BorgBackup](https://www.borgbackup.org/) and
+a backup module called "localbackup" based on [rsnapshot](https://rsnapshot.org/), optimized for backup on locally attached storage media.
+
+To use different backup software, a module needs to set the options provided by our backup interface.
+The following Nix code is a toy example that uses the `tar` program to perform backup and restore to illustrate how the backup interface works:
+
+```nix
+  clan.core.backups.providers.tar = {
+    list = ''
+      echo /var/lib/system-back.tar
+    '';
+    create = let
+      uniqueFolders = lib.unique (
+        lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clan.core.state)
+      );
+    in ''
+      # FIXME: a proper implementation should also run `state.preBackupCommand` of each state
+      if [ -f /var/lib/system-back.tar ]; then
+        tar -uvpf /var/lib/system-back.tar ${builtins.toString uniqueFolders}
+      else
+        tar -cvpf /var/lib/system-back.tar ${builtins.toString uniqueFolders}
+      fi
+    '';
+    restore = ''
+      IFS=':' read -ra FOLDER <<< "''$FOLDERS"
+      echo "${FOLDER[@]}" > /run/folders-to-restore.txt
+      tar -xvpf /var/lib/system-back.tar -C / -T /run/folders-to-restore.txt
+    '';
+  };
+```
+
+For better real-world implementations, check out the implementations for [BorgBackup](https://git.clan.lol/clan/clan-core/src/branch/main/clanModules/borgbackup/default.nix)
+and [localbackup](https://git.clan.lol/clan/clan-core/src/branch/main/clanModules/localbackup/default.nix).
+
+## What It Looks Like to the End User
+
+After following the guide for configuring a [backup](https://docs.clan.lol/getting-started/backups/),
+users can use the CLI to create backups, list them, and restore them.
+
+Backups can be created through the CLI like this:
+
+```
+clan backups create web01
+```
+
+BorgBackup will also create backups itself every day by default.
+
+Completed backups can be listed like this:
+
+```
+clan backups list web01
+...
+web01::u366395@u366395.your-storagebox.de:/./borgbackup::web01-web01-2024-06-18T01:00:00
+web03::u366395@u366395.your-storagebox.de:/./borgbackup::web01-web01-2024-06-19T01:00:00
+```
+One cool feature of our backup system is that it is aware of individual services/applications.
+Let's say we want to restore the state of our [Matrix](https://matrix.org/) chat server; we can just specify it like this:
+
+```
+clan backups restore --service matrix-synapse web01 borgbackup web03::u366395@u366395.your-storagebox.de:/./borgbackup::web01-web01-2024-06-19T01:00:00
+```
+
+In this case, it will first stop the matrix-synapse systemd service, then delete the [PostgreSQL](https://www.postgresql.org/) database, restore the database from the backup, and then start the matrix-synapse service again.
+
+## Future work
+
+As of now we implemented our backup and restore for a handful of services and we expect to refine the interface as we test the interface for more applications.
+
+Currently, our backup implementation backs up filesystem state from running services.
+This can lead to inconsistencies if applications change the state while the backup is running.
+In the future, we hope to make backups more atomic by backing up a filesystem snapshot instead of normal directories.
+This, however, requires the use of modern filesystems that support these features.

docs/site/blog/posts/hello-world.md

@@ -61,7 +61,7 @@ Clan is for anyone and everyone who believes in the power of open source technol
 
 Ready to control your digital world? Clan is more than a tool—it's a movement. Secure your data, manage your systems easily, or connect with others how you like. Start with Clan for a better digital future.
 
-Connect with us on our [Matrix channel at clan.lol](https://matrix.to/#/#clan:lassul.us) or through our IRC bridges (coming soon).
+Connect with us on our [Matrix channel at clan.lol](https://matrix.to/#/#clan:clan.lol) or through our IRC bridges (coming soon).
 
 Want to see the code? Check it out [on our Gitea](https://git.clan.lol/clan/clan-core) or [on GitHub](https://github.com/clan-lol/clan-core).
 

docs/site/blog/posts/jsonschema.md

@@ -190,5 +190,5 @@ We hope these experiments inspire the community, encourage contributions and fur
 - [Comments on NixOS Discourse](https://discourse.nixos.org/t/introducing-the-nixos-to-json-schema-converter/45948)
 - [Source Code of the JSON Schema Library](https://git.clan.lol/clan/clan-core/src/branch/main/lib/jsonschema)
 - [Our Issue Tracker](https://git.clan.lol/clan/clan-core/issues)
-- [Our Matrix Channel](https://matrix.to/#/#clan:lassul.us)
+- [Our Matrix Channel](https://matrix.to/#/#clan:clan.lol)
 - [react-jsonschema-form Playground](https://rjsf-team.github.io/react-jsonschema-form/)

docs/site/blog/posts/new-docs.md

@@ -10,4 +10,4 @@ Last week, we added a new documentation hub for clan at [docs.clan.lol](https://
 We are still working on improving the installation procedures, so stay tuned.
 We now have weekly office hours where people are invited to hangout and ask questions.
 They are every Wednesday 15:30 UTC (17:30 CEST) in our [jitsi](https://jitsi.lassul.us/clan.lol).
-Otherwise drop by in our [matrix channel](https://matrix.to/#/#clan:lassul.us).
+Otherwise drop by in our [matrix channel](https://matrix.to/#/#clan:clan.lol).

docs/site/concepts/configuration.md

@@ -0,0 +1,56 @@
+# Configuration
+
+## Introduction
+
+When managing machine configuration this can be done through many possible ways.
+Ranging from writing `nix` expression in a `flake.nix` file; placing `autoincluded` files into your machine directory; or configuring everything in a simple UI (upcomming).
+
+clan currently offers the following methods to configure machines:
+
+!!! Success "Recommended for nix people"
+
+    - flake.nix (i.e. via `buildClan`)
+        - `machine` argument
+        - `inventory` argument
+
+    - machines/`machine_name`/configuration.nix (`autoincluded` if it exists)
+
+???+ Note "Used by CLI & UI"
+
+    - inventory.json
+    - machines/`machine_name`/hardware-configuration.nix (`autoincluded` if it exists)
+
+
+!!! Warning "Deprecated"
+
+    machines/`machine_name`/settings.json
+
+## BuildClan
+
+The core function that produces a clan. It returns a set of consistent configurations for all machines with ready-to-use secrets, backups and other services.
+
+### Inputs
+
+`directory`
+: The directory containing the machines subdirectory
+
+`machines`
+: Allows to include machine-specific modules i.e. machines.${name} = { ... }
+
+`meta`
+: An optional set
+
+: `{ name :: string, icon :: string, description :: string }`
+
+`inventory`
+: Service set for easily configuring distributed services, such as backups
+
+: For more details see [Inventory](./inventory.md)
+
+`specialArgs`
+: Extra arguments to pass to nixosSystem i.e. useful to make self available
+
+`pkgsForSystem`
+: A function that maps from architecture to pkgs, if specified this nixpkgs will be only imported once for each system.
+  This improves performance, but all nipxkgs.* options will be ignored.
+  `(string -> pkgs )`

docs/site/concepts/inventory.md

@@ -0,0 +1,206 @@
+# Inventory
+
+`Inventory` is an abstract service layer for consistently configuring distributed services across machine boundaries.
+
+## Meta
+
+Metadata about the clan, will be displayed upfront in the upcomming clan-app, make sure to choose a unique name.
+
+```{.nix hl_lines="3-8"}
+buildClan {
+    inventory = {
+        meta = {
+            # The following options are available
+            # name:        string # Required, name of the clan.
+            # description: null | string
+            # icon:        null | string
+        };
+    };
+}
+```
+
+## Machines
+
+Machines and a small pieve of their configuration can be added via `inventory.machines`.
+
+!!! Note
+    It doesn't matter where the machine gets introduced to buildClan - All delarations are valid, duplications are merged.
+
+    However the clan-app (UI) will create machines in the inventory, because it cannot create arbitrary nixos configs.
+
+In the following example `backup_server` is one machine - it may specify parts of its configuration in different places.
+
+```{.nix hl_lines="3-5 12-20"}
+buildClan {
+    machines = {
+        "backup_server" = {
+            # Any valid nixos config
+        };
+        "jon" = {
+            # Any valid nixos config
+        };
+    };
+    inventory = {
+        machines = {
+            "backup_server" = {
+                # Don't include any nixos config here
+                # The following fields are avilable
+                # description: null | string
+                # icon:        null | string
+                # name:        string
+                # system:      null | string
+                # tags:        [...string]
+            };
+            "jon" = {
+                # Same as above
+            };
+        };
+    };
+}
+```
+
+## Services
+
+### Available clanModules
+
+Currently the inventory interface is implemented by the following clanModules
+
+- [borgbackup](../reference/clanModules/borgbackup.md)
+- [packages](../reference/clanModules/packages.md)
+- [single-disk](../reference/clanModules/single-disk.md)
+
+See the respective module documentation for available roles.
+
+### Adding services to machines
+
+A module can be added to one or multiple machines via `Roles`. clan's `Role` interface provide sane defaults for a module this allows the module author to reduce the configuration overhead to a minimum.
+
+Each service can still be customized and configured according to the modules options.
+
+- Per instance configuration via `services.<serviceName>.<instanceName>.config`
+- Per machine configuration via `services.<serviceName>.<instanceName>.machines.<machineName>.config`
+
+### Configuration Examples
+
+!!! Example "Borgbackup Example"
+
+    To configure a service it needs to be added to the machine.
+    It is required to assign the service (`borgbackup`) an arbitrary instance name. (`instance_1`)
+
+    See also: [Multiple Service Instances](#multiple-service-instances)
+
+    ```{.nix hl_lines="14-17"}
+    buildClan {
+        inventory = {
+            machines = {
+                "backup_server" = {
+                    # Don't include any nixos config here
+                    # See inventory.Machines for available options
+                };
+                "jon" = {
+                    # Don't include any nixos config here
+                    # See inventory.Machines for available options
+                };
+            };
+            services = {
+                borgbackup.instance_1 = {
+                    roles.client.machines = [ "jon" ];
+                    roles.server.machines = [ "backup_server" ];
+                };
+            };
+        };
+    }
+    ```
+
+!!! Example "Packages Example"
+
+    This example shows how to add `pkgs.firefox` via the inventory interface.
+
+    ```{.nix hl_lines="8-11"}
+    buildClan {
+        inventory = {
+            machines = {
+                "sara" = {};
+                "jon" = {};
+            };
+            services = {
+                packages.set_1 = {
+                    roles.default.machines = [ "jon" "sara" ];
+                    # Packages is a configuration option of the "packages" clanModule
+                    config.packages = ["firefox"];
+                };
+            };
+        };
+    }
+    ```
+
+### Tags
+
+It is possible to add services to multiple machines via tags. The service instance gets added in the specified role. In this case `role = "default"`
+
+!!! Example "Tags Example"
+
+    ```{.nix hl_lines="5 8 13"}
+    buildClan {
+        inventory = {
+            machines = {
+                "sara" = {
+                    tags = ["browsing"];
+                };
+                "jon" = {
+                    tags = ["browsing"];
+                };
+            };
+            services = {
+                packages.set_1 = {
+                    roles.default.tags = [ "browsing" ];
+                    config.packages = ["firefox"];
+                };
+            };
+        };
+    }
+    ```
+
+### Multiple Service Instances
+
+!!! danger "Important"
+    Not all modules support multiple instances yet.
+
+Some modules have support for adding multiple instances of the same service in different roles or configurations.
+
+!!! Example
+
+    In this example `backup_server` has role `client` and `server` in different instances.
+
+    ```{.nix hl_lines="11 14"}
+    buildClan {
+        inventory = {
+            machines = {
+                "jon" = {};
+                "backup_server" = {};
+                "backup_backup_server" = {}
+            };
+            services = {
+                borgbackup.instance_1 = {
+                    roles.client.machines = [ "jon" ];
+                    roles.server.machines = [ "backup_server" ];
+                };
+                borgbackup.instance_1 = {
+                    roles.client.machines = [ "backup_server" ];
+                    roles.server.machines = [ "backup_backup_server" ];
+                };
+            };
+        };
+    }
+    ```
+
+### Schema specification
+
+The complete schema specification can be retrieved via:
+
+```sh
+nix build git+https://git.clan.lol/clan/clan-core#inventory-schema
+> result
+> ├── schema.cue
+> └── schema.json
+```

docs/site/getting-started/backups.md

@@ -98,7 +98,7 @@ Start by indicating where your backup data should be sent. Replace `hostname` wi
 Decide which folders you want to back up. For example, to backup your home and root directories:
 
 ```nix
-{ clanCore.state.userdata.folders = [ "/home" "/root" ]; }
+{ clan.core.state.userdata.folders = [ "/home" "/root" ]; }
 ```
 
 3. **Generate Backup Credentials:**
@@ -116,7 +116,7 @@ On the server where backups will be stored, enable the SSH daemon and set up a r
   services.borgbackup.repos.myhostname = {
     path = "/var/lib/borgbackup/myhostname";
     authorizedKeys = [
-      (builtins.readFile ./machines/myhostname/facts/borgbackup.ssh.pub)
+      (builtins.readFile  (config.clan.core.clanDir + "/machines/myhostname/facts/borgbackup.ssh.pub"))
     ];
   };
 }

docs/site/getting-started/configure.md

@@ -4,14 +4,14 @@
 
 In the `flake.nix` file:
 
-- [x] set a unique `clanName`.
+- [x] set a unique `name`.
 
 === "**buildClan**"
 
     ```nix title="clan-core.lib.buildClan"
     buildClan {
-        # Set a unique name 
-        clanName = "Lobsters";
+        # Set a unique name
+        meta.name = "Lobsters";
         # Should usually point to the directory of flake.nix
         directory = ./.;
 
@@ -30,8 +30,8 @@ In the `flake.nix` file:
 
     ```nix title="clan-core.flakeModules.default"
     clan = {
-        # Set a unique name 
-        clanName = "Lobsters";
+        # Set a unique name
+        meta.name = "Lobsters";
 
         machines = {
             jon = {
@@ -61,13 +61,13 @@ Adding or configuring a new machine requires two simple steps:
 
     ```{.shellSession hl_lines="6" .no-copy}
     NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
-    sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G 
-    ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M 
+    sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G
+    ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M
     ├─sda2      usb-ST_16GB_AA6271026J1000000509-0:0-part2      vfat     100M /boot
     └─sda3      usb-ST_16GB_AA6271026J1000000509-0:0-part3      ext4     2.9G /
-    nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G 
-    ├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M 
-    ├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G 
+    nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G
+    ├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M
+    ├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G
     └─nvme0n1p3 nvme-eui.e8238fa6bf530001001b448b4aec2929-part3 swap    16.8G
     ```
 
@@ -89,7 +89,7 @@ Adding or configuring a new machine requires two simple steps:
 
               # Change this to the correct ip-address or hostname
               # The hostname is the machine name by default
-              clan.networking.targetHost = pkgs.lib.mkDefault "root@jon"
+              clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon"
 
               # Change this to the ID-LINK of the desired disk shown by 'lsblk'
               disko.devices.disk.main = {
@@ -122,7 +122,7 @@ Adding or configuring a new machine requires two simple steps:
 
               # Change this to the correct ip-address or hostname
               # The hostname is the machine name by default
-              clan.networking.targetHost = pkgs.lib.mkDefault "root@jon"
+              clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon"
 
               # Change this to the ID-LINK of the desired disk shown by 'lsblk'
               disko.devices.disk.main = {
@@ -150,10 +150,17 @@ These steps will allow you to update your machine later.
 Generate the `hardware-configuration.nix` file for your machine by executing the following command:
 
 ```bash
-ssh root@flash-installer.local nixos-generate-config --no-filesystems --show-hardware-config > machines/jon/hardware-configuration.nix
+clan machines hw-generate [MACHINE_NAME] [HOSTNAME]
 ```
 
-This command connects to `flash-installer.local` as `root`, runs `nixos-generate-config` to detect hardware configurations (excluding filesystems), and writes them to `machines/jon/hardware-configuration.nix`.
+replace `[MACHINE_NAME]` with the name of the machine i.e. `jon` and `[HOSTNAME]` with the `ip_adress` or `hostname` of the machine within the network. i.e. `flash-installer.local`
+
+!!! Example
+    ```bash
+    clan machines hw-generate jon flash-installer.local
+    ```
+
+    This command connects to `flash-installer.local` as `root`, runs `nixos-generate-config` to detect hardware configurations (excluding filesystems), and writes them to `machines/jon/hardware-configuration.nix`.
 
 ### Step 3: Custom Disk Formatting
 

docs/site/getting-started/deploy.md

@@ -160,7 +160,7 @@ buildClan {
             # Set this for clan commands use ssh i.e. `clan machines update`
             # If you change the hostname, you need to update this line to root@<new-hostname>
             # This only works however if you have avahi running on your admin machine else use IP
-            clan.networking.targetHost = pkgs.lib.mkDefault "root@jon";
+            clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon";
         };
     };
 };
@@ -197,7 +197,7 @@ buildClan {
     # ...
     machines = {
         "jon" = {
-            clan.networking.buildHost = "root@<host_or_ip>";
+            clan.core.networking.buildHost = "root@<host_or_ip>";
         };
     };
 };

docs/site/getting-started/flake-parts.md

@@ -16,7 +16,7 @@ inputs = {
   # New flake-parts input
   flake-parts.url = "github:hercules-ci/flake-parts";
   flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
-  
+
   clan-core = {
     url = "git+https://git.clan.lol/clan/clan-core";
     inputs.nixpkgs.follows = "nixpkgs"; # Needed if your configuration uses nixpkgs unstable.
@@ -35,7 +35,7 @@ After updating your flake inputs, the next step is to import the `clan-core` fla
     inputs@{ flake-parts, ... }:
     flake-parts.lib.mkFlake { inherit inputs; } (
       {
-        # 
+        #
         imports = [
           inputs.clan-core.flakeModules.default
         ];
@@ -63,7 +63,7 @@ Below is a guide on how to structure this in your flake.nix:
       # Define your clan
       clan = {
         # Clan wide settings. (Required)
-        clanName = ""; # Ensure to choose a unique name.
+        meta.name = ""; # Ensure to choose a unique name.
 
         machines = {
           jon = {
@@ -75,7 +75,7 @@ Below is a guide on how to structure this in your flake.nix:
             nixpkgs.hostPlatform = "x86_64-linux";
 
             # Set this for clan commands use ssh i.e. `clan machines update`
-            clan.networking.targetHost = pkgs.lib.mkDefault "root@jon";
+            clan.core.networking.targetHost = pkgs.lib.mkDefault "root@jon";
 
             # remote> lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
             disko.devices.disk.main = {
@@ -83,8 +83,8 @@ Below is a guide on how to structure this in your flake.nix:
             };
 
             # There needs to be exactly one controller per clan
-            clan.networking.zerotier.controller.enable = true;
-            
+            clan.core.networking.zerotier.controller.enable = true;
+
           };
         };
       };

docs/site/getting-started/installer.md

@@ -48,13 +48,18 @@ sudo umount /dev/sdb1
     ```bash
     clan flash --flake git+https://git.clan.lol/clan/clan-core \
       --ssh-pubkey $HOME/.ssh/id_ed25519.pub \
-      --keymap en \
-      --language en \
+      --keymap us \
+      --language en_US.utf-8 \
       --disk main /dev/sd<X> \
       flash-installer
     ```
 
     The `--ssh-pubkey`, `--language` and `--keymap` are optional.
+    You can get a list of all keymaps with the following command:
+
+    ```
+    $ find $(nix-build 'https://github.com/NixOS/nixpkgs/archive/refs/heads/nixpkgs-unstable.tar.gz' --no-out-link -A kbd)/share/keymaps -type f -name '*.map.gz'
+    ```
     Replace `$HOME/.ssh/id_ed25519.pub` with a path to your SSH public key.
     If you do not have an ssh key yet, you can generate one with `ssh-keygen -t ed25519` command.
 

docs/site/getting-started/mesh-vpn.md

@@ -29,7 +29,7 @@ peers. Once addresses are allocated, the controller's continuous operation is no
 2. **Add Configuration**: Input the following configuration to the NixOS
    configuration of the controller machine:
    ```nix
-   clan.networking.zerotier.controller = {
+   clan.core.networking.zerotier.controller = {
      enable = true;
      public = true;
    };
@@ -48,7 +48,7 @@ To introduce a new machine to the VPN, adhere to the following steps:
    configuration, substituting `<CONTROLLER>` with the controller machine name:
    ```nix
    { config, ... }: {
-     clan.networking.zerotier.networkId = builtins.readFile (config.clanCore.clanDir + "/machines/<CONTROLLER>/facts/zerotier-network-id");
+     clan.core.networking.zerotier.networkId = builtins.readFile (config.clan.core.clanDir + "/machines/<CONTROLLER>/facts/zerotier-network-id");
    }
    ```
 1. **Update the New Machine**: Execute:

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717177033,
-        "narHash": "sha256-G3CZJafCO8WDy3dyA2EhpUJEmzd5gMJ2IdItAg0Hijw=",
+        "lastModified": 1720661479,
+        "narHash": "sha256-nsGgA14vVn0GGiqEfomtVgviRJCuSR3UEopfP8ixW1I=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "0274af4c92531ebfba4a5bd493251a143bc51f3c",
+        "rev": "786965e1b1ed3fd2018d78399984f461e2a44689",
         "type": "github"
       },
       "original": {
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717285511,
-        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
+        "lastModified": 1719994518,
+        "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
+        "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7",
         "type": "github"
       },
       "original": {
@@ -40,71 +40,19 @@
         "type": "github"
       }
     },
-    "nixlib": {
-      "locked": {
-        "lastModified": 1712450863,
-        "narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=",
-        "owner": "nix-community",
-        "repo": "nixpkgs.lib",
-        "rev": "3c62b6a12571c9a7f65ab037173ee153d539905f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nixpkgs.lib",
-        "type": "github"
-      }
-    },
-    "nixos-2311": {
-      "locked": {
-        "lastModified": 1717017538,
-        "narHash": "sha256-S5kltvDDfNQM3xx9XcvzKEOyN2qk8Sa+aSOLqZ+1Ujc=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "64e468fd2652105710d86cd2ae3e65a5a6d58dec",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixos-generators": {
-      "inputs": {
-        "nixlib": "nixlib",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1716210724,
-        "narHash": "sha256-iqQa3omRcHGpWb1ds75jS9ruA5R39FTmAkeR3J+ve1w=",
-        "owner": "nix-community",
-        "repo": "nixos-generators",
-        "rev": "d14b286322c7f4f897ca4b1726ce38cb68596c94",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nixos-generators",
-        "type": "github"
-      }
-    },
     "nixos-images": {
       "inputs": {
-        "nixos-2311": "nixos-2311",
+        "nixos-stable": [],
         "nixos-unstable": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1717040312,
-        "narHash": "sha256-yI/en4IxuCEClIUpIs3QTyYCCtmSPLOhwLJclfNwdeg=",
+        "lastModified": 1720659757,
+        "narHash": "sha256-ltzUuCsEfPA9CYM9BAnwObBGqDyQIs2OLkbVMeOOk00=",
         "owner": "nix-community",
         "repo": "nixos-images",
-        "rev": "47bfb55316e105390dd761e0b6e8e0be09462b67",
+        "rev": "5eddae0afbcfd4283af5d6676d08ad059ca04b70",
         "type": "github"
       },
       "original": {
@@ -115,11 +63,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1717298511,
-        "narHash": "sha256-9sXuJn/nL+9ImeYtlspTvjt83z1wIgU+9AwfNbnq+tI=",
+        "lastModified": 1720977633,
+        "narHash": "sha256-if0qaFmAe8X01NsVRK5e9Asg9mEWVkHrA9WuqM5jB70=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6634a0509e9e81e980b129435fbbec518ab246d0",
+        "rev": "0af9d835c27984b3265145f8e3cbc6c153479196",
         "type": "github"
       },
       "original": {
@@ -133,7 +81,6 @@
       "inputs": {
         "disko": "disko",
         "flake-parts": "flake-parts",
-        "nixos-generators": "nixos-generators",
         "nixos-images": "nixos-images",
         "nixpkgs": "nixpkgs",
         "sops-nix": "sops-nix",
@@ -148,11 +95,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1717297459,
-        "narHash": "sha256-cZC2f68w5UrJ1f+2NWGV9Gx0dEYmxwomWN2B0lx0QRA=",
+        "lastModified": 1720926522,
+        "narHash": "sha256-eTpnrT6yu1vp8C0B5fxHXhgKxHoYMoYTEikQx///jxY=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "ab2a43b0d21d1d37d4d5726a892f714eaeb4b075",
+        "rev": "0703ba03fd9c1665f8ab68cc3487302475164617",
         "type": "github"
       },
       "original": {
@@ -168,11 +115,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1717278143,
-        "narHash": "sha256-u10aDdYrpiGOLoxzY/mJ9llST9yO8Q7K/UlROoNxzDw=",
+        "lastModified": 1720930114,
+        "narHash": "sha256-VZK73b5hG5bSeAn97TTcnPjXUXtV7j/AtS4KN8ggCS0=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "3eb96ca1ae9edf792a8e0963cc92fddfa5a87706",
+        "rev": "b92afa1501ac73f1d745526adc4f89b527595f14",
         "type": "github"
       },
       "original": {

flake.nix

@@ -13,10 +13,10 @@
     sops-nix.url = "github:Mic92/sops-nix";
     sops-nix.inputs.nixpkgs.follows = "nixpkgs";
     sops-nix.inputs.nixpkgs-stable.follows = "";
-    nixos-generators.url = "github:nix-community/nixos-generators";
-    nixos-generators.inputs.nixpkgs.follows = "nixpkgs";
     nixos-images.url = "github:nix-community/nixos-images";
     nixos-images.inputs.nixos-unstable.follows = "nixpkgs";
+    # unused input
+    nixos-images.inputs.nixos-stable.follows = "";
     flake-parts.url = "github:hercules-ci/flake-parts";
     flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
     treefmt-nix.url = "github:numtide/treefmt-nix";
@@ -29,7 +29,7 @@
       { ... }:
       {
         clan = {
-          # meta.name = "clan-core";
+          meta.name = "clan-core";
           directory = self;
         };
         systems = [
@@ -49,6 +49,7 @@
           ./formatter.nix
           ./lib/flake-module.nix
           ./nixosModules/flake-module.nix
+          ./nixosModules/clanCore/vars/flake-module.nix
           ./pkgs/flake-module.nix
           ./templates/flake-module.nix
         ];

flakeModules/clan.nix

@@ -13,7 +13,6 @@ let
     inherit lib clan-core;
     inherit (inputs) nixpkgs;
   };
-
   cfg = config.clan;
 in
 {
@@ -91,6 +90,11 @@ in
     clanInternals = lib.mkOption {
       type = lib.types.submodule {
         options = {
+          inventory = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
+          inventoryFile = lib.mkOption { type = lib.types.unspecified; };
+
+          clanModules = lib.mkOption { type = lib.types.attrsOf lib.types.path; };
+          source = lib.mkOption { type = lib.types.path; };
           meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };

formatter.nix

@@ -1,50 +1,73 @@
-{ lib, inputs, ... }:
+{ inputs, ... }:
 {
   imports = [ inputs.treefmt-nix.flakeModule ];
   perSystem =
     { self', pkgs, ... }:
     {
-      treefmt.projectRootFile = "flake.nix";
+      treefmt.projectRootFile = ".git/config";
       treefmt.programs.shellcheck.enable = true;
 
       treefmt.programs.mypy.enable = true;
+      treefmt.programs.nixfmt.enable = true;
+      treefmt.programs.nixfmt.package = pkgs.nixfmt-rfc-style;
+      treefmt.programs.deadnix.enable = true;
+
       treefmt.programs.mypy.directories = {
         "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
-        "pkgs/clan-vm-manager".extraPythonPackages =
-          # clan-vm-manager currently only exists on linux 
-          (self'.packages.clan-vm-manager.externalTestDeps or [ ])
-          ++ self'.packages.clan-cli.testDependencies;
+        "pkgs/clan-app".extraPythonPackages =
+          # clan-app currently only exists on linux
+          (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies;
       };
+      treefmt.programs.ruff.check = true;
+      treefmt.programs.ruff.format = true;
 
-      treefmt.settings.formatter.nix = {
-        command = "sh";
-        options = [
-          "-eucx"
-          ''
-            # First deadnix
-            ${lib.getExe pkgs.deadnix} --edit "$@"
-            # Then nixpkgs-fmt
-            ${lib.getExe pkgs.nixfmt-rfc-style} "$@"
-          ''
-          "--" # this argument is ignored by bash
-        ];
-        includes = [ "*.nix" ];
-        excludes = [
-          # Was copied from nixpkgs. Keep diff minimal to simplify upstreaming.
-          "pkgs/builders/script-writers.nix"
-        ];
-      };
-      treefmt.settings.formatter.python = {
-        command = "sh";
-        options = [
-          "-eucx"
-          ''
-            ${lib.getExe pkgs.ruff} check --fix "$@"
-            ${lib.getExe pkgs.ruff} format "$@"
-          ''
-          "--" # this argument is ignored by bash
-        ];
-        includes = [ "*.py" ];
-      };
+      # FIXME: currently broken in CI
+      #treefmt.settings.formatter.vale =
+      #  let
+      #    vocab = "cLAN";
+      #    style = "Docs";
+      #    config = pkgs.writeText "vale.ini" ''
+      #      StylesPath = ${styles}
+      #      Vocab = ${vocab}
+
+      #      [*.md]
+      #      BasedOnStyles = Vale, ${style}
+      #      Vale.Terms = No
+      #    '';
+      #    styles = pkgs.symlinkJoin {
+      #      name = "vale-style";
+      #      paths = [
+      #        accept
+      #        headings
+      #      ];
+      #    };
+      #    accept = pkgs.writeTextDir "config/vocabularies/${vocab}/accept.txt" ''
+      #      Nix
+      #      NixOS
+      #      Nixpkgs
+      #      clan.lol
+      #      Clan
+      #      monorepo
+      #    '';
+      #    headings = pkgs.writeTextDir "${style}/headings.yml" ''
+      #      extends: capitalization
+      #      message: "'%s' should be in sentence case"
+      #      level: error
+      #      scope: heading
+      #      # $title, $sentence, $lower, $upper, or a pattern.
+      #      match: $sentence
+      #    '';
+      #  in
+      #  {
+      #    command = "${pkgs.vale}/bin/vale";
+      #    options = [ "--config=${config}" ];
+      #    includes = [ "*.md" ];
+      #    # TODO: too much at once, fix piecemeal
+      #    excludes = [
+      #      "docs/*"
+      #      "clanModules/*"
+      #      "pkgs/*"
+      #    ];
+      #  };
     };
 }

inventory.json

@@ -0,0 +1,77 @@
+{
+  "meta": {
+    "name": "clan-core"
+  },
+  "machines": {
+    "minimal-inventory-machine": {
+      "name": "foo",
+      "system": "x86_64-linux",
+      "description": "A nice thing",
+      "icon": "./path/to/icon.png",
+      "tags": ["1", "2", "3"]
+    }
+  },
+  "services": {
+    "packages": {
+      "editors": {
+        "meta": {
+          "name": "Some editor packages"
+        },
+        "roles": {
+          "default": {
+            "machines": ["minimal-inventory-machine"]
+          }
+        },
+        "machines": {
+          "minimal-inventory-machine": {
+            "config": {
+              "packages": ["zed-editor"]
+            }
+          }
+        },
+        "config": {
+          "packages": ["vim"]
+        }
+      },
+      "browsing": {
+        "meta": {
+          "name": "Web browsing packages"
+        },
+        "roles": {
+          "default": {
+            "machines": ["minimal-inventory-machine"]
+          }
+        },
+        "machines": {
+          "minimal-inventory-machine": {
+            "config": {
+              "packages": ["chromium"]
+            }
+          }
+        },
+        "config": {
+          "packages": ["firefox"]
+        }
+      }
+    },
+    "single-disk": {
+      "default": {
+        "meta": {
+          "name": "single-disk"
+        },
+        "roles": {
+          "default": {
+            "machines": ["minimal-inventory-machine"]
+          }
+        },
+        "machines": {
+          "minimal-inventory-machine": {
+            "config": {
+              "device": "/dev/null"
+            }
+          }
+        }
+      }
+    }
+  }
+}

lib/build-clan/default.nix

@@ -12,73 +12,132 @@
   # DEPRECATED: use meta.icon instead
   clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
   meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string
-  pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
-# This improves performance, but all nipxkgs.* options will be ignored.
+  # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
+  # This improves performance, but all nipxkgs.* options will be ignored.
+  pkgsForSystem ? (_system: null),
+  /*
+    Low level inventory configuration.
+    Overrides the services configuration.
+  */
+  inventory ? { },
 }:
 let
-  deprecationWarnings = [
-    (lib.warnIf (
-      clanName != null
-    ) "clanName is deprecated, please use meta.name instead. ${clanName}" null)
-    (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null)
-  ];
+  # Internal inventory, this is the result of merging all potential inventory sources:
+  # - Default instances configured via 'services'
+  # - The inventory overrides
+  #   - Machines that exist in inventory.machines
+  # - Machines explicitly configured via 'machines' argument
+  # - Machines that exist in the machines directory
+  # Checks on the module level:
+  # - Each service role must reference a valid machine after all machines are merged
+  mergedInventory =
+    (lib.evalModules {
+      modules = [
+        clan-core.lib.inventory.interface
+        { inherit meta; }
+        (
+          if
+            builtins.pathExists "${directory}/inventory.json"
+          # Is recursively applied. Any explicit nix will override.
+          then
+            (builtins.fromJSON (builtins.readFile "${directory}/inventory.json"))
+          else
+            { }
+        )
+        inventory
+        # Machines explicitly configured via 'machines' argument
+        {
+          # { ${name} :: meta // { name, tags } }
+          machines = lib.mapAttrs (
+            name: config:
+            (lib.attrByPath [
+              "clan"
+              "meta"
+            ] { } config)
+            // {
+              # meta.name default is the attribute name of the machine
+              name = lib.mkDefault (
+                lib.attrByPath [
+                  "clan"
+                  "meta"
+                  "name"
+                ] name config
+              );
+              tags = lib.attrByPath [
+                "clan"
+                "tags"
+              ] [ ] config;
+
+              system = lib.attrByPath [
+                "nixpkgs"
+                "hostSystem"
+              ] null config;
+            }
+          ) machines;
+        }
+
+        # Will be deprecated
+        {
+          machines = lib.mapAttrs (
+            name: _:
+            # Use mkForce to make sure users migrate to the inventory system.
+            # When the settings.json exists the evaluation will print the deprecation warning.
+            lib.mkForce {
+              inherit name;
+              system = (machineSettings name).nixpkgs.hostSystem or null;
+            }
+          ) machinesDirs;
+        }
+
+        # Deprecated interface
+        (if clanName != null then { meta.name = clanName; } else { })
+        (if clanIcon != null then { meta.icon = clanIcon; } else { })
+      ];
+    }).config;
+
+  inherit (clan-core.lib.inventory) buildInventory;
+
+  # map from machine name to service configuration
+  # { ${machineName} :: Config }
+  serviceConfigs = buildInventory mergedInventory;
 
   machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
     builtins.readDir (directory + /machines)
   );
 
-  mergedMeta =
-    let
-      metaFromFile =
-        if (builtins.pathExists "${directory}/clan/meta.json") then
-          let
-            settings = builtins.fromJSON (builtins.readFile "${directory}/clan/meta.json");
-          in
-          settings
-        else
-          { };
-      legacyMeta = lib.filterAttrs (_: v: v != null) {
-        name = clanName;
-        icon = clanIcon;
-      };
-      optionsMeta = lib.filterAttrs (_: v: v != null) meta;
-
-      warnings =
-        builtins.map (
-          name:
-          if
-            metaFromFile.${name} or null != optionsMeta.${name} or null && optionsMeta.${name} or null != null
-          then
-            lib.warn "meta.${name} is set in different places. (exlicit option meta.${name} overrides ${directory}/clan/meta.json)" null
-          else
-            null
-        ) (builtins.attrNames metaFromFile)
-        ++ [ (if (res.name or null == null) then (throw "meta.name should be set") else null) ];
-      res = metaFromFile // legacyMeta // optionsMeta;
-    in
-    # Print out warnings before returning the merged result
-    builtins.deepSeq warnings res;
-
   machineSettings =
     machineName:
+    let
+      warn = lib.warn ''
+        The use of ./machines/<machine>/settings.json is deprecated.
+        If your settings.json is empty, you can safely remove it.
+        !!! Consider using the inventory system. !!!
+
+        File: ${directory + /machines/${machineName}/settings.json}
+
+        If there are still features missing in the inventory system, please open an issue on the clan-core repository.
+      '';
+    in
     # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
     # This is useful for doing a dry-run before writing changes into the settings.json
     # Using CLAN_MACHINE_SETTINGS_FILE requires passing --impure to nix eval
     if builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE" != "" then
-      builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE"))
+      warn (builtins.fromJSON (builtins.readFile (builtins.getEnv "CLAN_MACHINE_SETTINGS_FILE")))
     else
       lib.optionalAttrs (builtins.pathExists "${directory}/machines/${machineName}/settings.json") (
-        builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))
+        warn (builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json)))
       );
 
-  # Read additional imports specified via a config option in settings.json
-  # This is not an infinite recursion, because the imports are discovered here
-  #   before calling evalModules.
-  # It is still useful to have the imports as an option, as this allows for type
-  #   checking and easy integration with the config frontend(s)
   machineImports =
     machineSettings: map (module: clan-core.clanModules.${module}) (machineSettings.clanImports or [ ]);
 
+  deprecationWarnings = [
+    (lib.warnIf (
+      clanName != null
+    ) "clanName in buildClan is deprecated, please use meta.name instead." null)
+    (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null)
+  ];
+
   # TODO: remove default system once we have a hardware-config mechanism
   nixosConfiguration =
     {
@@ -94,20 +153,30 @@ let
         in
         (machineImports settings)
         ++ [
+          {
+            # Autoinclude configuration.nix and hardware-configuration.nix
+            imports = builtins.filter (p: builtins.pathExists p) [
+              "${directory}/machines/${name}/configuration.nix"
+              "${directory}/machines/${name}/hardware-configuration.nix"
+            ];
+          }
           settings
           clan-core.nixosModules.clanCore
           extraConfig
           (machines.${name} or { })
+          # Inherit the inventory assertions ?
+          { inherit (mergedInventory) assertions; }
+          { imports = serviceConfigs.${name} or { }; }
           (
             {
               # Settings
-              clanCore.clanDir = directory;
+              clan.core.clanDir = directory;
               # Inherited from clan wide settings
-              clanCore.clanName = meta.name or clanName;
-              clanCore.clanIcon = meta.icon or clanIcon;
+              clan.core.clanName = meta.name or clanName;
+              clan.core.clanIcon = meta.icon or clanIcon;
 
               # Machine specific settings
-              clanCore.machineName = name;
+              clan.core.machineName = name;
               networking.hostName = lib.mkDefault name;
               nixpkgs.hostPlatform = lib.mkDefault system;
 
@@ -125,7 +194,7 @@ let
       } // specialArgs;
     };
 
-  allMachines = machinesDirs // machines;
+  allMachines = mergedInventory.machines or { };
 
   supportedSystems = [
     "x86_64-linux"
@@ -177,9 +246,13 @@ builtins.deepSeq deprecationWarnings {
   inherit nixosConfigurations;
 
   clanInternals = {
-    # Evaluated clan meta
-    # Merged /clan/meta.json with overrides from buildClan
-    meta = mergedMeta;
+    inherit (clan-core) clanModules;
+    source = "${clan-core}";
+
+    meta = mergedInventory.meta;
+    inventory = mergedInventory;
+
+    inventoryFile = "${directory}/inventory.json";
 
     # machine specifics
     machines = configsPerSystem;

lib/default.nix

@@ -5,7 +5,10 @@
   ...
 }:
 {
+  evalClanModules = import ./eval-clan-modules { inherit clan-core nixpkgs lib; };
+  buildClan = import ./build-clan { inherit clan-core lib nixpkgs; };
+  facts = import ./facts.nix { inherit lib; };
+  inventory = import ./inventory { inherit lib clan-core; };
   jsonschema = import ./jsonschema { inherit lib; };
   modules = import ./description.nix { inherit clan-core lib; };
-  buildClan = import ./build-clan { inherit clan-core lib nixpkgs; };
 }

lib/description.nix

@@ -1,5 +1,4 @@
-{ lib, clan-core, ... }:
-
+{ clan-core, lib }:
 rec {
   getReadme =
     modulename:
@@ -16,18 +15,20 @@ rec {
   getShortDescription =
     modulename:
     let
-      content = (getReadme modulename);
+      content = getReadme modulename;
       parts = lib.splitString "---" content;
-      description = builtins.head parts;
-      number_of_newlines = builtins.length (lib.splitString "\n" description);
+      # Partition the parts into the first part (the readme content) and the rest (the metadata)
+      parsed = builtins.partition ({ index, ... }: if index >= 2 then false else true) (
+        lib.filter ({ index, ... }: index != 0) (lib.imap0 (index: part: { inherit index part; }) parts)
+      );
+
+      # Use this if the content is needed
+      # readmeContent = lib.concatMapStrings (v: "---" + v.part) parsed.wrong;
+
+      meta = builtins.fromTOML (builtins.head parsed.right).part;
     in
-    if (builtins.length parts) > 1 then
-      if number_of_newlines > 4 then
-        throw "Short description in README.md for module ${modulename} is too long. Max 3 newlines."
-      else if number_of_newlines <= 1 then
-        throw "Missing short description in README.md for module ${modulename}."
-      else
-        description
+    if (builtins.length parts >= 3) then
+      meta.description
     else
       throw "Short description delimiter `---` not found in README.md for module ${modulename}";
 }

lib/eval-clan-modules/default.nix

@@ -0,0 +1,34 @@
+{
+  nixpkgs,
+  clan-core,
+  lib,
+}:
+let
+  inherit (import nixpkgs { system = "x86_64-linux"; }) pkgs;
+
+  inherit (clan-core) clanModules;
+
+  baseModule = {
+    imports = (import (pkgs.path + "/nixos/modules/module-list.nix")) ++ [
+      {
+        nixpkgs.hostPlatform = "x86_64-linux";
+        clan.core.clanName = "dummy";
+      }
+    ];
+  };
+
+  # This function takes a list of module names and evaluates them
+  # evalClanModules :: [ String ] -> { config, options, ... }
+  evalClanModules =
+    modulenames:
+    let
+      evaled = lib.evalModules {
+        modules = [
+          baseModule
+          clan-core.nixosModules.clanCore
+        ] ++ (map (name: clanModules.${name}) modulenames);
+      };
+    in
+    evaled;
+in
+evalClanModules

lib/facts.nix

@@ -0,0 +1,71 @@
+{ lib, ... }:
+clanDir:
+let
+
+  allMachineNames = lib.mapAttrsToList (name: _: name) (builtins.readDir clanDir);
+
+  getFactPath = machine: fact: "${clanDir}/machines/${machine}/facts/${fact}";
+
+  readFact =
+    machine: fact:
+    let
+      path = getFactPath machine fact;
+    in
+    if builtins.pathExists path then builtins.readFile path else null;
+
+  # Example:
+  #
+  # readFactFromAllMachines zerotier-ip
+  # => {
+  #   machineA = "1.2.3.4";
+  #   machineB = "5.6.7.8";
+  # };
+  readFactFromAllMachines =
+    fact:
+    let
+      machines = allMachineNames;
+      facts = lib.genAttrs machines (machine: readFact machine fact);
+      filteredFacts = lib.filterAttrs (_machine: fact: fact != null) facts;
+    in
+    filteredFacts;
+
+  # all given facts are are set and factvalues are never null.
+  #
+  # Example:
+  #
+  # readFactsFromAllMachines [ "zerotier-ip" "syncthing.pub" ]
+  # => {
+  #   machineA =
+  #     {
+  #       "zerotier-ip" = "1.2.3.4";
+  #       "synching.pub" = "1234";
+  #     };
+  #   machineB =
+  #     {
+  #       "zerotier-ip" = "5.6.7.8";
+  #       "synching.pub" = "23456719";
+  #     };
+  # };
+  readFactsFromAllMachines =
+    facts:
+    let
+      # machine -> fact -> factvalue
+      machinesFactsAttrs = lib.genAttrs allMachineNames (
+        machine: lib.genAttrs facts (fact: readFact machine fact)
+      );
+      # remove all machines which don't have all facts set
+      filteredMachineFactAttrs = lib.filterAttrs (
+        _machine: values: builtins.all (fact: values.${fact} != null) facts
+      ) machinesFactsAttrs;
+    in
+    filteredMachineFactAttrs;
+in
+{
+  inherit
+    allMachineNames
+    getFactPath
+    readFact
+    readFactFromAllMachines
+    readFactsFromAllMachines
+    ;
+}

lib/flake-module.nix

@@ -5,9 +5,12 @@
   ...
 }:
 {
-  imports = [ ./jsonschema/flake-module.nix ];
+  imports = [
+    ./jsonschema/flake-module.nix
+    ./inventory/flake-module.nix
+  ];
   flake.lib = import ./default.nix {
-    inherit lib;
+    inherit lib inputs;
     inherit (inputs) nixpkgs;
     clan-core = self;
   };

lib/inventory/.envrc

@@ -0,0 +1,6 @@
+# shellcheck shell=bash
+source_up
+
+watch_file flake-module.nix
+
+use flake .#inventory-schema --builders ''

lib/inventory/README.md

@@ -0,0 +1,90 @@
+# Inventory
+
+The inventory is our concept for distributed services. Users can configure multiple machines with minimal effort.
+
+- The inventory acts as a declarative source of truth for all machine configurations.
+- Users can easily add or remove machines to/from services.
+- Ensures that all machines and services are configured consistently, across multiple nixosConfigs.
+- Defaults and predefined roles in our modules minimizes the need for manual configuration.
+
+Open questions:
+
+- [ ] How do we set default role, description and other metadata?
+  - It must be accessible from Python.
+  - It must set the value in the module system.
+
+- [ ] Inventory might use assertions. Should each machine inherit the inventory assertions ?
+
+- [ ] Is the service config interface the same as the module config interface ?
+
+- [ ] As a user do I want to see borgbackup as the high level category?
+
+
+Architecture
+
+```
+nixosConfig < machine_module        < inventory
+---------------------------------------------
+nixos   < borgbackup            <- inventory <-> UI
+
+        creates the config      Maps from high level services to the borgbackup clan module
+        for ONE machine         Inventory is completely serializable.
+                                UI can interact with the inventory to define machines, and services
+                                Defining Users is out of scope for the first prototype.
+```
+
+## Provides a specification for the inventory
+
+It is used for design phase and as validation helper.
+
+> Cue is less verbose and easier to understand and maintain than json-schema.
+> Json-schema, if needed can be easily generated on-the fly.
+
+## Checking validity
+
+Directly check a json against the schema
+
+`cue vet inventory.json root.cue -d '#Root'`
+
+## Json schema
+
+Export the json-schema i.e. for usage in python / javascript / nix
+
+`cue export --out openapi root.cue`
+
+## Usage
+
+Comments are rendered as descriptions in the json schema.
+
+```cue
+// A name of the clan (primarily shown by the UI)
+name: string
+```
+
+Cue open sets. In the following `foo = {...}` means that the key `foo` can contain any arbitrary json object.
+
+```cue
+foo: { ... }
+```
+
+Cue dynamic keys.
+
+```cue
+[string]: {
+    attr: string
+}
+```
+
+This is the schema of
+
+```json
+{
+    "a": {
+        "attr": "foo"
+    },
+    "b": {
+        "attr": "bar"
+    }
+    // ... Indefinitely more dynamic keys of type "string"
+}
+```

lib/inventory/build-inventory/default.nix

@@ -0,0 +1,117 @@
+# Generate partial NixOS configurations for every machine in the inventory
+# This function is responsible for generating the module configuration for every machine in the inventory.
+{ lib, clan-core }:
+inventory:
+let
+  machines = machinesFromInventory inventory;
+
+  resolveTags =
+    # Inventory, { machines :: [string], tags :: [string] }
+    inventory: members: {
+      machines =
+        members.machines or [ ]
+        ++ (builtins.foldl' (
+          acc: tag:
+          let
+            # For error printing
+            availableTags = lib.foldlAttrs (
+              acc: _: v:
+              v.tags or [ ] ++ acc
+            ) [ ] inventory.machines;
+
+            tagMembers = builtins.attrNames (
+              lib.filterAttrs (_n: v: builtins.elem tag v.tags or [ ]) inventory.machines
+            );
+          in
+          if tagMembers == [ ] then
+            throw "Tag: '${tag}' not found. Available tags: ${builtins.toJSON (lib.unique availableTags)}"
+          else
+            acc ++ tagMembers
+        ) [ ] members.tags or [ ]);
+    };
+
+  /*
+    Returns a NixOS configuration for every machine in the inventory.
+
+    machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
+  */
+  machinesFromInventory =
+    inventory:
+    # For every machine in the inventory, build a NixOS configuration
+    # For each machine generate config, forEach service, if the machine is used.
+    builtins.mapAttrs (
+      machineName: machineConfig:
+      lib.foldlAttrs (
+        # [ Modules ], String, { ${instance_name} :: ServiceConfig }
+        acc: moduleName: serviceConfigs:
+        acc
+        # Collect service config
+        ++ (lib.foldlAttrs (
+          # [ Modules ], String, ServiceConfig
+          acc2: instanceName: serviceConfig:
+          let
+            resolvedRoles = builtins.mapAttrs (
+              _roleName: members: resolveTags inventory members
+            ) serviceConfig.roles;
+
+            isInService = builtins.any (members: builtins.elem machineName members.machines) (
+              builtins.attrValues resolvedRoles
+            );
+
+            # Inverse map of roles. Allows for easy lookup of roles for a given machine.
+            # { ${machine_name} :: [roles]
+            inverseRoles = lib.foldlAttrs (
+              acc: roleName:
+              { machines }:
+              acc
+              // builtins.foldl' (
+                acc2: machineName: acc2 // { ${machineName} = (acc.${machineName} or [ ]) ++ [ roleName ]; }
+              ) { } machines
+            ) { } resolvedRoles;
+
+            machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
+            globalConfig = serviceConfig.config or { };
+
+            # TODO: maybe optimize this dont lookup the role in inverse roles. Imports are not lazy
+            roleModules = builtins.map (
+              role:
+              let
+                path = "${clan-core.clanModules.${moduleName}}/roles/${role}.nix";
+              in
+              if builtins.pathExists path then
+                path
+              else
+                throw "Module doesn't have role: '${role}'. Path: ${path} not found."
+            ) inverseRoles.${machineName} or [ ];
+          in
+          if isInService then
+            acc2
+            ++ [
+              {
+                imports = [ clan-core.clanModules.${moduleName} ] ++ roleModules;
+                config.clan.${moduleName} = lib.mkMerge [
+                  globalConfig
+                  machineServiceConfig
+                ];
+              }
+              {
+                config.clan.inventory.services.${moduleName}.${instanceName} = {
+                  roles = resolvedRoles;
+                  # TODO: Add inverseRoles to the service config if needed
+                  # inherit inverseRoles;
+                };
+              }
+            ]
+          else
+            acc2
+        ) [ ] serviceConfigs)
+      ) [ ] inventory.services
+      # Append each machine config
+      ++ [
+        (lib.optionalAttrs (machineConfig.system or null != null) {
+          config.nixpkgs.hostPlatform = machineConfig.system;
+        })
+      ]
+    ) inventory.machines or { };
+in
+machines

lib/inventory/build-inventory/interface.nix

@@ -0,0 +1,140 @@
+{ config, lib, ... }:
+let
+  t = lib.types;
+
+  metaOptions = {
+    name = lib.mkOption { type = t.str; };
+    description = lib.mkOption {
+      default = null;
+      type = t.nullOr t.str;
+    };
+    icon = lib.mkOption {
+      default = null;
+      type = t.nullOr t.str;
+    };
+  };
+
+  machineRef = lib.mkOptionType {
+    name = "str";
+    description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
+    check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
+    merge = lib.mergeEqualOption;
+  };
+
+  allTags = lib.unique (
+    lib.foldlAttrs (
+      tags: _: m:
+      tags ++ m.tags or [ ]
+    ) [ ] config.machines
+  );
+
+  tagRef = lib.mkOptionType {
+    name = "str";
+    description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
+    check = v: lib.isString v && builtins.elem v allTags;
+    merge = lib.mergeEqualOption;
+  };
+
+  moduleConfig = lib.mkOption {
+    default = { };
+    type = t.attrsOf t.anything;
+  };
+in
+{
+  options = {
+    assertions = lib.mkOption {
+      type = t.listOf t.unspecified;
+      internal = true;
+      visible = false;
+      default = [ ];
+    };
+    meta = metaOptions;
+
+    machines = lib.mkOption {
+      default = { };
+      type = t.attrsOf (
+        t.submodule {
+          options = {
+            inherit (metaOptions) name description icon;
+            tags = lib.mkOption {
+              default = [ ];
+              apply = lib.unique;
+              type = t.listOf t.str;
+            };
+            system = lib.mkOption {
+              default = null;
+              type = t.nullOr t.str;
+            };
+          };
+        }
+      );
+    };
+
+    services = lib.mkOption {
+      default = { };
+      type = t.attrsOf (
+        t.attrsOf (
+          t.submodule {
+            options.meta = metaOptions;
+            options.config = moduleConfig;
+            options.machines = lib.mkOption {
+              default = { };
+              type = t.attrsOf (t.submodule { options.config = moduleConfig; });
+            };
+            options.roles = lib.mkOption {
+              default = { };
+              type = t.attrsOf (
+                t.submodule {
+                  options.machines = lib.mkOption {
+                    default = [ ];
+                    type = t.listOf machineRef;
+                  };
+                  options.tags = lib.mkOption {
+                    default = [ ];
+                    apply = lib.unique;
+                    type = t.listOf tagRef;
+                  };
+                }
+              );
+            };
+          }
+        )
+      );
+    };
+  };
+
+  # Smoke validation of the inventory
+  config.assertions =
+    let
+      # Inventory assertions
+      # - All referenced machines must exist in the top-level machines
+      serviceAssertions = lib.foldlAttrs (
+        ass1: serviceName: c:
+        ass1
+        ++ lib.foldlAttrs (
+          ass2: instanceName: instanceConfig:
+          let
+            serviceMachineNames = lib.attrNames instanceConfig.machines;
+            topLevelMachines = lib.attrNames config.machines;
+            # All machines must be defined in the top-level machines
+            assertions = builtins.map (m: {
+              assertion = builtins.elem m topLevelMachines;
+              message = "${serviceName}.${instanceName}.machines.${m}. Should be one of [ ${builtins.concatStringsSep " | " topLevelMachines} ]";
+            }) serviceMachineNames;
+          in
+          ass2 ++ assertions
+        ) [ ] c
+      ) [ ] config.services;
+
+      # Machine assertions
+      # - A machine must define their host system
+      machineAssertions = map (
+        { name, ... }:
+        {
+          assertion = true;
+          message = "Machine ${name} should define its host system in the inventory. ()";
+        }
+      ) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines));
+    in
+    machineAssertions ++ serviceAssertions;
+}