Merge pull request 'fix fake-etc build on machines without sandbox' (#1645) from fix-macos-deploy into main

If we have no sandbox enabled or on macos with sandbox enabled, /etc
contains a lot more files than we actually want.
Instead of copying some random files, we now just create those files
ourself.

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Contributing to cLAN
+# Contributing to Clan
 
 ## Live-reloading documentation
 

README.md

@@ -1,6 +1,6 @@
 # Clan Core Repository
 
-Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the cLAN project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
+Welcome to the Clan Core Repository, the heart of the [clan.lol](https://clan.lol/) project! This monorepo is the foundation of Clan, a revolutionary open-source project aimed at restoring fun, freedom, and functionality to computing. Here, you'll find all the essential packages, NixOS modules, CLI tools, and tests needed to contribute to and work with the Clan project. Clan leverages the Nix system to ensure reliability, security, and seamless management of digital environments, putting the power back into the hands of users.
 
 ## Why Clan?
 
@@ -40,6 +40,6 @@ Clan is more than a tool; it's a movement towards a better digital future. By co
 
 Connect with us and the Clan community for support and discussion:
 
-- [Matrix channel](https://matrix.to/#/#clan:lassul.us) for live discussions.
+- [Matrix channel](https://matrix.to/#/#clan:clan.lol) for live discussions.
 - IRC bridges (coming soon) for real-time chat support.
 

checks/backups/flake-module.nix

@@ -68,17 +68,9 @@
             };
           };
         };
-        clanCore.facts.secretStore = "vm";
+        clan.core.facts.secretStore = "vm";
 
-        environment.systemPackages = [
-          self.packages.${pkgs.system}.clan-cli
-          (pkgs.writeShellScriptBin "pre-restore-command" ''
-            touch /var/test-service/pre-restore-command
-          '')
-          (pkgs.writeShellScriptBin "post-restore-command" ''
-            touch /var/test-service/post-restore-command
-          '')
-        ];
+        environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
         environment.etc.install-closure.source = "${closureInfo}/store-paths";
         nix.settings = {
           substituters = lib.mkForce [ ];
@@ -87,11 +79,18 @@
           flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
         };
         system.extraDependencies = dependencies;
-        clanCore.state.test-backups.folders = [ "/var/test-backups" ];
+        clan.core.state.test-backups.folders = [ "/var/test-backups" ];
 
-        clanCore.state.test-service = {
-          preRestoreCommand = "pre-restore-command";
-          postRestoreCommand = "post-restore-command";
+        clan.core.state.test-service = {
+          preBackupScript = ''
+            touch /var/test-service/pre-backup-command
+          '';
+          preRestoreScript = ''
+            touch /var/test-service/pre-restore-command
+          '';
+          postRestoreScript = ''
+            touch /var/test-service/post-restore-command
+          '';
           folders = [ "/var/test-service" ];
         };
         clan.borgbackup.destinations.test-backup.repo = "borg@machine:.";
@@ -145,14 +144,14 @@
             machine.succeed("echo testing > /var/test-backups/somefile")
 
             # create
-            machine.succeed("clan --debug --flake ${self} backups create test-backup")
+            machine.succeed("clan backups create --debug --flake ${self} test-backup")
             machine.wait_until_succeeds("! systemctl is-active borgbackup-job-test-backup >&2")
             machine.succeed("test -f /run/mount-external-disk")
             machine.succeed("test -f /run/unmount-external-disk")
 
             # list
             backup_id = json.loads(machine.succeed("borg-job-test-backup list --json"))["archives"][0]["archive"]
-            out = machine.succeed("clan --debug --flake ${self} backups list test-backup").strip()
+            out = machine.succeed("clan backups list --debug --flake ${self} test-backup").strip()
             print(out)
             assert backup_id in out, f"backup {backup_id} not found in {out}"
             localbackup_id = "hdd::/mnt/external-disk/snapshot.0"
@@ -160,17 +159,19 @@
 
             ## borgbackup restore
             machine.succeed("rm -f /var/test-backups/somefile")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
+            machine.succeed(f"clan backups restore --debug --flake ${self} test-backup borgbackup 'test-backup::borg@machine:.::{backup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
+            machine.succeed("test -f /var/test-service/pre-backup-command")
 
             ## localbackup restore
-            machine.succeed("rm -f /var/test-backups/somefile /var/test-service/{pre,post}-restore-command")
-            machine.succeed(f"clan --debug --flake ${self} backups restore test-backup localbackup '{localbackup_id}' >&2")
+            machine.succeed("rm -rf /var/test-backups/somefile /var/test-service/ && mkdir -p /var/test-service")
+            machine.succeed(f"clan backups restore --debug --flake ${self} test-backup localbackup '{localbackup_id}' >&2")
             assert machine.succeed("cat /var/test-backups/somefile").strip() == "testing", "restore failed"
             machine.succeed("test -f /var/test-service/pre-restore-command")
             machine.succeed("test -f /var/test-service/post-restore-command")
+            machine.succeed("test -f /var/test-service/pre-backup-command")
           '';
         } { inherit pkgs self; };
       };

checks/borgbackup/default.nix

@@ -16,9 +16,9 @@
             };
           }
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
-            clanCore.state.testState.folders = [ "/etc/state" ];
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
+            clan.core.state.testState.folders = [ "/etc/state" ];
             environment.etc.state.text = "hello world";
             systemd.tmpfiles.settings."vmsecrets" = {
               "/etc/secrets/borgbackup.ssh" = {
@@ -36,7 +36,7 @@
                 };
               };
             };
-            clanCore.facts.secretStore = "vm";
+            clan.core.facts.secretStore = "vm";
 
             clan.borgbackup.destinations.test.repo = "borg@localhost:.";
           }

checks/deltachat/default.nix

@@ -10,8 +10,8 @@
           self.clanModules.deltachat
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
           }
         ];
       };

checks/flake-module.nix

@@ -23,7 +23,7 @@
                 options =
                   (pkgs.nixos {
                     imports = [ self.nixosModules.clanCore ];
-                    clanCore.clanDir = ./.;
+                    clan.core.clanDir = ./.;
                   }).options;
                 warningsAreErrors = false;
               };
@@ -44,6 +44,7 @@
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
+            postgresql = import ./postgresql nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
 

checks/flash/flake-module.nix

@@ -1,33 +1,50 @@
-{ ... }:
+{ self, ... }:
 {
   perSystem =
-    { ... }:
     {
-      # checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
-      #   flash = (import ../lib/test-base.nix) {
-      #     name = "flash";
-      #     nodes.target = {
-      #       virtualisation.emptyDiskImages = [ 4096 ];
-      #       virtualisation.memorySize = 3000;
-      #       environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
-      #       environment.etc."install-closure".source = "${closureInfo}/store-paths"; 
+      nodes,
+      pkgs,
+      lib,
+      ...
+    }:
+    let
+      dependencies = [
+        pkgs.disko
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.toplevel
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.diskoScript
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.build.diskoScript.drvPath
+        self.clanInternals.machines.${pkgs.hostPlatform.system}.test_install_machine.config.system.clan.deployment.file
+      ] ++ builtins.map (i: i.outPath) (builtins.attrValues self.inputs);
+      closureInfo = pkgs.closureInfo { rootPaths = dependencies; };
+    in
+    {
+      # Currently disabled...
+      checks = pkgs.lib.mkIf (pkgs.stdenv.isLinux) {
+        flash = (import ../lib/test-base.nix) {
+          name = "flash";
+          nodes.target = {
+            virtualisation.emptyDiskImages = [ 4096 ];
+            virtualisation.memorySize = 3000;
+            environment.systemPackages = [ self.packages.${pkgs.system}.clan-cli ];
+            environment.etc."install-closure".source = "${closureInfo}/store-paths";
 
-      #       nix.settings = {
-      #         substituters = lib.mkForce [ ];
-      #         hashed-mirrors = null;
-      #         connect-timeout = lib.mkForce 3;
-      #         flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
-      #         experimental-features = [
-      #           "nix-command"
-      #           "flakes"
-      #         ];
-      #       };
-      #     };
-      #     testScript = ''
-      #       start_all()
-      #       machine.succeed("clan --debug --flake ${../..} flash --yes --disk main /dev/vdb test_install_machine")
-      #     '';
-      #   } { inherit pkgs self; };
-      # };
+            nix.settings = {
+              substituters = lib.mkForce [ ];
+              hashed-mirrors = null;
+              connect-timeout = lib.mkForce 3;
+              flake-registry = pkgs.writeText "flake-registry" ''{"flakes":[],"version":2}'';
+              experimental-features = [
+                "nix-command"
+                "flakes"
+              ];
+            };
+          };
+          testScript = ''
+            start_all()
+
+            machine.succeed("clan flash --debug --flake ${../..} --yes --disk main /dev/vdb test_install_machine")
+          '';
+        } { inherit pkgs self; };
+      };
     };
 }

checks/impure/flake-module.nix

@@ -7,6 +7,8 @@
         #!${pkgs.bash}/bin/bash
         set -euo pipefail
 
+        unset CLAN_DIR
+
         export PATH="${
           lib.makeBinPath [
             pkgs.gitMinimal

checks/installation/flake-module.nix

@@ -2,8 +2,8 @@
 {
   clan.machines.test_install_machine = {
     clan.networking.targetHost = "test_install_machine";
-    fileSystems."/".device = lib.mkDefault "/dev/null";
-    boot.loader.grub.device = lib.mkDefault "/dev/null";
+    fileSystems."/".device = lib.mkDefault "/dev/vdb";
+    boot.loader.grub.device = lib.mkDefault "/dev/vdb";
 
     imports = [ self.nixosModules.test_install_machine ];
   };
@@ -98,7 +98,7 @@
             client.succeed("${pkgs.coreutils}/bin/install -Dm 600 ${../lib/ssh/privkey} /root/.ssh/id_ed25519")
             client.wait_until_succeeds("ssh -o StrictHostKeyChecking=accept-new -v root@target hostname")
 
-            client.succeed("clan --debug --flake ${../..} machines install --yes test_install_machine root@target >&2")
+            client.succeed("clan machines install --debug --flake ${../..} --yes test_install_machine root@target >&2")
             try:
               target.shutdown()
             except BrokenPipeError:

checks/lib/container-driver/test_driver/__init__.py

@@ -151,7 +151,7 @@ class Machine:
         """
 
         # Always run command with shell opts
-        command = f"set -euo pipefail; {command}"
+        command = f"set -eo pipefail; source /etc/profile; set -u; {command}"
 
         proc = subprocess.run(
             [

checks/lib/container-test.nix

@@ -10,6 +10,7 @@ in
     hostPkgs = pkgs;
     # speed-up evaluation
     defaults = {
+      nix.package = pkgs.nixVersions.latest;
       documentation.enable = lib.mkDefault false;
       boot.isContainer = true;
 

checks/lib/test-base.nix

@@ -10,6 +10,7 @@ in
   defaults = {
     documentation.enable = lib.mkDefault false;
     nix.settings.min-free = 0;
+    nix.package = pkgs.nixVersions.latest;
   };
 
   # to accept external dependencies such as disko

checks/matrix-synapse/default.nix

@@ -4,26 +4,61 @@
     name = "matrix-synapse";
 
     nodes.machine =
-      { self, lib, ... }:
+      {
+        config,
+        self,
+        lib,
+        ...
+      }:
       {
         imports = [
           self.clanModules.matrix-synapse
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
-            clan.matrix-synapse = {
-              enable = true;
-              domain = "clan.test";
-            };
-          }
-          {
-            # secret override
-            clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path = "${./synapse-registration_shared_secret}";
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
+
             services.nginx.virtualHosts."matrix.clan.test" = {
               enableACME = lib.mkForce false;
               forceSSL = lib.mkForce false;
             };
+            clan.matrix-synapse.domain = "clan.test";
+            clan.matrix-synapse.users.admin.admin = true;
+            clan.matrix-synapse.users.someuser = { };
+
+            clan.core.facts.secretStore = "vm";
+
+            # because we use systemd-tmpfiles to copy the secrets, we need to a seperate systemd-tmpfiles call to provison them.
+            boot.postBootCommands = "${config.systemd.package}/bin/systemd-tmpfiles --create /etc/tmpfiles.d/00-vmsecrets.conf";
+
+            systemd.tmpfiles.settings."00-vmsecrets" = {
+              # run before 00-nixos.conf
+              "/etc/secrets" = {
+                d.mode = "0700";
+                z.mode = "0700";
+              };
+              "/etc/secrets/synapse-registration_shared_secret" = {
+                f.argument = "supersecret";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/matrix-password-admin" = {
+                f.argument = "matrix-password1";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+              "/etc/secrets/matrix-password-someuser" = {
+                f.argument = "matrix-password2";
+                z = {
+                  mode = "0400";
+                  user = "root";
+                };
+              };
+            };
           }
         ];
       };
@@ -32,6 +67,12 @@
       machine.wait_for_unit("matrix-synapse")
       machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
       machine.succeed("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
+
+      machine.systemctl("restart matrix-synapse >&2") # check if user creation is idempotent
+      machine.execute("journalctl -u matrix-synapse --no-pager >&2")
+      machine.wait_for_unit("matrix-synapse")
+      machine.succeed("${pkgs.netcat}/bin/nc -z -v ::1 8008")
+      machine.succeed("${pkgs.curl}/bin/curl -Ssf -L http://localhost/_matrix/static/ -H 'Host: matrix.clan.test'")
     '';
   }
 )

checks/postgresql/default.nix

@@ -0,0 +1,72 @@
+(import ../lib/container-test.nix) ({
+  name = "postgresql";
+
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [
+        self.nixosModules.clanCore
+        self.clanModules.postgresql
+        self.clanModules.localbackup
+      ];
+      clan.postgresql.users.test = { };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+
+      systemd.services.sample-service = {
+        wantedBy = [ "multi-user.target" ];
+        script = ''
+          while true; do
+            echo "Hello, world!"
+            sleep 5
+          done
+        '';
+      };
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+      machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("sample-service")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
+      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clan.core.state.test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
+
+      # check if restore works if the database does not exist
+      machine.succeed("runuser -u postgres -- dropdb test")
+      machine.succeed("${nodes.machine.clan.core.state.test.postRestoreCommand}")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+    '';
+})

checks/secrets/default.nix

@@ -10,8 +10,8 @@
       environment.etc."group-secret".source = config.sops.secrets.group-secret.path;
       sops.age.keyFile = "/etc/privkey.age";
 
-      clanCore.clanDir = "${./.}";
-      clanCore.machineName = "machine";
+      clan.core.clanDir = "${./.}";
+      clan.core.machineName = "machine";
 
       networking.hostName = "machine";
     };

checks/syncthing/default.nix

@@ -12,14 +12,14 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "introducer";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "introducer";
+            clan.core.clanDir = ./.;
             environment.etc = {
               "syncthing.pam".source = ./introducer/introducer_test_cert;
               "syncthing.key".source = ./introducer/introducer_test_key;
               "syncthing.api".source = ./introducer/introducer_test_api;
             };
-            clanCore.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
+            clan.core.facts.services.syncthing.secret."syncthing.api".path = "/etc/syncthing.api";
             services.syncthing.cert = "/etc/syncthing.pam";
             services.syncthing.key = "/etc/syncthing.key";
             # Doesn't test zerotier!
@@ -53,8 +53,8 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "peer1";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "peer1";
+            clan.core.clanDir = ./.;
             clan.syncthing.introducer = lib.strings.removeSuffix "\n" (
               builtins.readFile ./introducer/introducer_device_id
             );
@@ -75,8 +75,8 @@
           self.clanModules.syncthing
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "peer2";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "peer2";
+            clan.core.clanDir = ./.;
             clan.syncthing.introducer = lib.strings.removeSuffix "\n" (
               builtins.readFile ./introducer/introducer_device_id
             );

checks/wayland-proxy-virtwl/default.nix

@@ -14,8 +14,8 @@ import ../lib/test-base.nix (
         imports = [
           self.nixosModules.clanCore
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
           }
         ];
         services.wayland-proxy-virtwl.enable = true;

checks/zt-tcp-relay/default.nix

@@ -10,8 +10,8 @@
           self.nixosModules.clanCore
           self.clanModules.zt-tcp-relay
           {
-            clanCore.machineName = "machine";
-            clanCore.clanDir = ./.;
+            clan.core.machineName = "machine";
+            clan.core.clanDir = ./.;
           }
         ];
       };

clanModules/borgbackup-static/README.md

@@ -0,0 +1,10 @@
+Statically configure borgbackup with sane defaults.
+---
+This module implements the `borgbackup` backend and implements sane defaults 
+for backup management through `borgbackup` for members of the clan.
+
+Configure target machines where the backups should be sent to through `targets`.
+
+Configure machines that should be backuped either through `includeMachines`
+which will exclusively add the included machines to be backuped, or through 
+`excludeMachines`, which will add every machine except the excluded machine to the backup.

clanModules/borgbackup-static/default.nix

@@ -0,0 +1,101 @@
+{ lib, config, ... }:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+in
+{
+  imports = [ ../borgbackup ];
+
+  options.clan.borgbackup-static = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be backuped.
+        Mutually exclusive with includeMachines.
+        If this is not empty, every other machine except the targets in the clan will be backuped by this module.
+        If includeMachines is set, only the included machines will be backuped.
+      '';
+    };
+    includeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should be backuped.
+        Mutually exclusive with excludeMachines.
+      '';
+    };
+    targets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Machines that should act as target machines for backups.
+      '';
+    };
+  };
+
+  config.services.borgbackup.repos =
+    let
+      machines = builtins.readDir machineDir;
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      filteredMachines =
+        if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then
+          lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines
+        else
+          lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines;
+      machinesMaybeKey = lib.mapAttrsToList (
+        machine: _:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) filteredMachines;
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    lib.mkIf
+      (builtins.any (
+        target: target == config.clan.core.machineName
+      ) config.clan.borgbackup-static.targets)
+      (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { });
+
+  config.clan.borgbackup.destinations =
+    let
+      destinations = builtins.map (d: {
+        name = d;
+        value = {
+          repo = "borg@${d}:/var/lib/borgbackup/${config.clan.core.machineName}";
+        };
+      }) config.clan.borgbackup-static.targets;
+    in
+    lib.mkIf (builtins.any (
+      target: target == config.clan.core.machineName
+    ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations);
+
+  config.assertions = [
+    {
+      assertion =
+        !(
+          ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0)
+          && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0)
+        );
+      message = ''
+        The options:
+        config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}]
+        and
+        config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}]
+        are mutually exclusive.
+        Use excludeMachines to exclude certain machines and backup the other clan machines.
+        Use include machines to only backup certain machines.
+      '';
+    }
+  ];
+}

clanModules/borgbackup/default.nix

@@ -6,6 +6,27 @@
 }:
 let
   cfg = config.clan.borgbackup;
+  preBackupScript = ''
+    declare -A preCommandErrors
+
+    ${lib.concatMapStringsSep "\n" (
+      state:
+      lib.optionalString (state.preBackupCommand != null) ''
+        echo "Running pre-backup command for ${state.name}"
+        if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+          preCommandErrors["${state.name}"]=1
+        fi
+      ''
+    ) (lib.attrValues config.clan.core.state)}
+
+    if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+      echo "pre-backup commands failed for the following services:"
+      for state in "''${!preCommandErrors[@]}"; do
+        echo "  $state"
+      done
+      exit 1
+    fi
+  '';
 in
 {
   options.clan.borgbackup.destinations = lib.mkOption {
@@ -26,9 +47,9 @@ in
             rsh = lib.mkOption {
               type = lib.types.str;
               default = "ssh -i ${
-                config.clanCore.facts.services.borgbackup.secret."borgbackup.ssh".path
-              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
-              defaultText = "ssh -i \${config.clanCore.facts.services.borgbackup.secret.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+                config.clan.core.facts.services.borgbackup.secret."borgbackup.ssh".path
+              } -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o IdentitiesOnly=Yes";
+              defaultText = "ssh -i \${config.clan.core.facts.services.borgbackup.secret.\"borgbackup.ssh\".path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
               description = "the rsh to use for the backup";
             };
           };
@@ -50,21 +71,30 @@ in
   ];
 
   config = lib.mkIf (cfg.destinations != { }) {
+    systemd.services = lib.mapAttrs' (
+      _: dest:
+      lib.nameValuePair "borgbackup-job-${dest.name}" {
+        # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
+        serviceConfig.ExecStartPre = [
+          (''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'')
+        ];
+      }
+    ) cfg.destinations;
+
     services.borgbackup.jobs = lib.mapAttrs (_: dest: {
-      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
+      paths = lib.unique (
+        lib.flatten (map (state: state.folders) (lib.attrValues config.clan.core.state))
+      );
       exclude = [ "*.pyc" ];
       repo = dest.repo;
       environment.BORG_RSH = dest.rsh;
       compression = "auto,zstd";
       startAt = "*-*-* 01:00:00";
       persistentTimer = true;
-      preHook = ''
-        set -x
-      '';
 
       encryption = {
         mode = "repokey";
-        passCommand = "cat ${config.clanCore.facts.services.borgbackup.secret."borgbackup.repokey".path}";
+        passCommand = "cat ${config.clan.core.facts.services.borgbackup.secret."borgbackup.repokey".path}";
       };
 
       prune.keep = {
@@ -75,7 +105,7 @@ in
       };
     }) cfg.destinations;
 
-    clanCore.facts.services.borgbackup = {
+    clan.core.facts.services.borgbackup = {
       public."borgbackup.ssh.pub" = { };
       secret."borgbackup.ssh" = { };
       secret."borgbackup.repokey" = { };
@@ -111,7 +141,7 @@ in
       (pkgs.writeShellScriptBin "borgbackup-restore" ''
         set -efux
         cd /
-        IFS=';' read -ra FOLDER <<< "$FOLDERS"
+        IFS=':' read -ra FOLDER <<< "$FOLDERS"
         job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
         backup_name=''${NAME#"$job_name"::}
         if ! command -v borg-job-"$job_name" &> /dev/null; then
@@ -122,7 +152,7 @@ in
       '')
     ];
 
-    clanCore.backups.providers.borgbackup = {
+    clan.core.backups.providers.borgbackup = {
       list = "borgbackup-list";
       create = "borgbackup-create";
       restore = "borgbackup-restore";

clanModules/deltachat/default.nix

@@ -5,7 +5,7 @@
 
   services.maddy =
     let
-      domain = "${config.clanCore.machineName}.local";
+      domain = "${config.clan.core.machineName}.local";
     in
     {
       enable = true;

clanModules/ergochat/default.nix

@@ -10,5 +10,5 @@ _: {
     };
   };
 
-  clanCore.state.ergochat.folders = [ "/var/lib/ergo" ];
+  clan.core.state.ergochat.folders = [ "/var/lib/ergo" ];
 }

clanModules/flake-module.nix

@@ -5,18 +5,22 @@
       imports = [ ./disk-layouts ];
     };
     borgbackup = ./borgbackup;
+    borgbackup-static = ./borgbackup-static;
     deltachat = ./deltachat;
     ergochat = ./ergochat;
     localbackup = ./localbackup;
     localsend = ./localsend;
     matrix-synapse = ./matrix-synapse;
     moonlight = ./moonlight;
+    postgresql = ./postgresql;
     root-password = ./root-password;
     sshd = ./sshd;
     sunshine = ./sunshine;
     static-hosts = ./static-hosts;
     syncthing = ./syncthing;
+    syncthing-static-peers = ./syncthing-static-peers;
     thelounge = ./thelounge;
+    trusted-nix-caches = ./trusted-nix-caches;
     user-password = ./user-password;
     xfce = ./xfce;
     zerotier-static-peers = ./zerotier-static-peers;

clanModules/localbackup/default.nix

@@ -6,7 +6,10 @@
 }:
 let
   cfg = config.clan.localbackup;
-  rsnapshotConfig = target: states: ''
+  uniqueFolders = lib.unique (
+    lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clan.core.state)
+  );
+  rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
     sync_first	1
@@ -17,12 +20,6 @@ let
     cmd_logger	${pkgs.inetutils}/bin/logger
     cmd_du	${pkgs.coreutils}/bin/du
     cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-    ${lib.optionalString (target.preBackupHook != null) ''
-      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
-        set -efu -o pipefail
-        ${target.preBackupHook}
-      ''}
-    ''}
 
     ${lib.optionalString (target.postBackupHook != null) ''
       cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
@@ -31,11 +28,9 @@ let
       ''}
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
-    ${lib.concatMapStringsSep "\n" (state: ''
-      ${lib.concatMapStringsSep "\n" (folder: ''
-        backup	${folder}	${config.networking.hostName}/
-      '') state.folders}
-    '') states}
+    ${lib.concatMapStringsSep "\n" (folder: ''
+      backup	${folder}	${config.networking.hostName}/
+    '') uniqueFolders}
   '';
 in
 {
@@ -129,14 +124,29 @@ in
               ]
             }
             ${lib.concatMapStringsSep "\n" (target: ''
-              (
-                ${mountHook target}
-                echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
-              )
-            '') (builtins.attrValues cfg.targets)}
-          '')
+              ${mountHook target}
+              echo "Creating backup '${target.name}'"
+
+              ${lib.optionalString (target.preBackupHook != null) ''
+                (
+                  ${target.preBackupHook}
+                )
+              ''}
+
+              declare -A preCommandErrors
+              ${lib.concatMapStringsSep "\n" (
+                state:
+                lib.optionalString (state.preBackupCommand != null) ''
+                  echo "Running pre-backup command for ${state.name}"
+                  if ! /run/current-system/sw/bin/${state.preBackupCommand}; then
+                    preCommandErrors["${state.name}"]=1
+                  fi
+                ''
+              ) (builtins.attrValues config.clan.core.state)}
+
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+            '') (builtins.attrValues cfg.targets)}'')
           (pkgs.writeShellScriptBin "localbackup-list" ''
             set -efu -o pipefail
             export PATH=${
@@ -167,6 +177,14 @@ in
                 pkgs.gawk
               ]
             }
+            if [[ "''${NAME:-}" == "" ]]; then
+              echo "No backup name given via NAME environment variable"
+              exit 1
+            fi
+            if [[ "''${FOLDERS:-}" == "" ]]; then
+              echo "No folders given via FOLDERS environment variable"
+              exit 1
+            fi
             name=$(awk -F'::' '{print $1}' <<< $NAME)
             backupname=''${NAME#$name::}
 
@@ -182,8 +200,9 @@ in
               exit 1
             fi
 
-            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            IFS=':' read -ra FOLDER <<< "''$FOLDERS"
             for folder in "''${FOLDER[@]}"; do
+              mkdir -p "$folder"
               rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
             done
           '')
@@ -213,7 +232,7 @@ in
           ''
         ) cfg.targets;
 
-      clanCore.backups.providers.localbackup = {
+      clan.core.backups.providers.localbackup = {
         # TODO list needs to run locally or on the remote machine
         list = "localbackup-list";
         create = "localbackup-create";

clanModules/localsend/default.nix

@@ -9,7 +9,7 @@
   # - cli frontend: https://github.com/localsend/localsend/issues/11
   # - ipv6 support: https://github.com/localsend/localsend/issues/549
   options.clan.localsend = {
-    enable = lib.mkEnableOption (lib.mdDoc "enable the localsend module");
+    enable = lib.mkEnableOption "enable the localsend module";
     defaultLocation = lib.mkOption {
       type = lib.types.str;
       description = "The default download location";
@@ -18,7 +18,7 @@
   };
 
   config = lib.mkIf config.clan.localsend.enable {
-    clanCore.state.localsend.folders = [
+    clan.core.state.localsend.folders = [
       "/var/localsend"
       config.clan.localsend.defaultLocation
     ];

clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch

@@ -0,0 +1,100 @@
+From bc199a27f23b0fcf175b116f7cf606c0d22b422a Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
+Date: Tue, 11 Jun 2024 11:40:47 +0200
+Subject: [PATCH 1/2] register_new_matrix_user: add password-file flag
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+getpass in python expects stdin to be a tty, hence we cannot just pipe
+into register_new_matrix_user. --password-file instead works better and
+it would also allow the use of stdin if /dev/stdin is passed.
+
+Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
+Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
+---
+ changelog.d/17294.feature                    |  2 ++
+ debian/register_new_matrix_user.ronn         |  9 +++++++--
+ synapse/_scripts/register_new_matrix_user.py | 20 +++++++++++++++-----
+ 3 files changed, 24 insertions(+), 7 deletions(-)
+ create mode 100644 changelog.d/17294.feature
+
+diff --git a/changelog.d/17294.feature b/changelog.d/17294.feature
+new file mode 100644
+index 000000000..33aac7b0b
+--- /dev/null
++++ b/changelog.d/17294.feature
+@@ -0,0 +1,2 @@
++`register_new_matrix_user` now supports a --password-file flag, which
++is useful for scripting.
+diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn
+index 0410b1f4c..d99e9215a 100644
+--- a/debian/register_new_matrix_user.ronn
++++ b/debian/register_new_matrix_user.ronn
+@@ -31,8 +31,13 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
+     Local part of the new user. Will prompt if omitted.
+ 
+   * `-p`, `--password`:
+-    New password for user. Will prompt if omitted. Supplying the password
+-    on the command line is not recommended. Use the STDIN instead.
++    New password for user. Will prompt if this option and `--password-file` are omitted.
++    Supplying the password on the command line is not recommended.
++    Use `--password-file` if possible.
++
++  * `--password-file`:
++    File containing the new password for user. If set, overrides `--password`.
++    This is a more secure alternative to specifying the password on the command line.
+ 
+   * `-a`, `--admin`:
+     Register new user as an admin. Will prompt if omitted.
+diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
+index 77a7129ee..972b35e2d 100644
+--- a/synapse/_scripts/register_new_matrix_user.py
++++ b/synapse/_scripts/register_new_matrix_user.py
+@@ -173,11 +173,18 @@ def main() -> None:
+         default=None,
+         help="Local part of the new user. Will prompt if omitted.",
+     )
+-    parser.add_argument(
++    password_group = parser.add_mutually_exclusive_group()
++    password_group.add_argument(
+         "-p",
+         "--password",
+         default=None,
+-        help="New password for user. Will prompt if omitted.",
++        help="New password for user. Will prompt for a password if "
++        "this flag and `--password-file` are both omitted.",
++    )
++    password_group.add_argument(
++        "--password-file",
++        default=None,
++        help="File containing the new password for user. If set, will override `--password`.",
+     )
+     parser.add_argument(
+         "-t",
+@@ -247,6 +254,11 @@ def main() -> None:
+             print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr)
+             sys.exit(1)
+ 
++    if args.password_file:
++        password = _read_file(args.password_file, "password-file").strip()
++    else:
++        password = args.password
++
+     if args.server_url:
+         server_url = args.server_url
+     elif config is not None:
+@@ -269,9 +281,7 @@ def main() -> None:
+     if args.admin or args.no_admin:
+         admin = args.admin
+ 
+-    register_new_user(
+-        args.user, args.password, server_url, secret, admin, args.user_type
+-    )
++    register_new_user(args.user, password, server_url, secret, admin, args.user_type)
+ 
+ 
+ def _read_file(file_path: Any, config_path: str) -> str:
+-- 
+2.44.1
+

clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch

@@ -0,0 +1,94 @@
+From 1789416df425d22693b0055a6688d8686e0ee4a1 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
+Date: Thu, 13 Jun 2024 14:38:19 +0200
+Subject: [PATCH 2/2] register-new-matrix-user: add a flag to ignore already
+ existing users
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This allows to register users in a more declarative and stateless way.
+
+Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
+---
+ synapse/_scripts/register_new_matrix_user.py | 22 ++++++++++++++++++--
+ 1 file changed, 20 insertions(+), 2 deletions(-)
+
+diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
+index 972b35e2d..233e7267d 100644
+--- a/synapse/_scripts/register_new_matrix_user.py
++++ b/synapse/_scripts/register_new_matrix_user.py
+@@ -52,6 +52,7 @@ def request_registration(
+     user_type: Optional[str] = None,
+     _print: Callable[[str], None] = print,
+     exit: Callable[[int], None] = sys.exit,
++    exists_ok: bool = False,
+ ) -> None:
+     url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),)
+ 
+@@ -97,6 +98,10 @@ def request_registration(
+     r = requests.post(url, json=data)
+ 
+     if r.status_code != 200:
++        response = r.json()
++        if exists_ok and response["errcode"] == "M_USER_IN_USE":
++            _print("User already exists. Skipping.")
++            return
+         _print("ERROR! Received %d %s" % (r.status_code, r.reason))
+         if 400 <= r.status_code < 500:
+             try:
+@@ -115,6 +120,7 @@ def register_new_user(
+     shared_secret: str,
+     admin: Optional[bool],
+     user_type: Optional[str],
++    exists_ok: bool = False,
+ ) -> None:
+     if not user:
+         try:
+@@ -154,7 +160,13 @@ def register_new_user(
+             admin = False
+ 
+     request_registration(
+-        user, password, server_location, shared_secret, bool(admin), user_type
++        user,
++        password,
++        server_location,
++        shared_secret,
++        bool(admin),
++        user_type,
++        exists_ok=exists_ok,
+     )
+ 
+ 
+@@ -173,6 +185,11 @@ def main() -> None:
+         default=None,
+         help="Local part of the new user. Will prompt if omitted.",
+     )
++    parser.add_argument(
++        "--exists-ok",
++        action="store_true",
++        help="Do not fail if user already exists.",
++    )
+     password_group = parser.add_mutually_exclusive_group()
+     password_group.add_argument(
+         "-p",
+@@ -192,6 +209,7 @@ def main() -> None:
+         default=None,
+         help="User type as specified in synapse.api.constants.UserTypes",
+     )
++
+     admin_group = parser.add_mutually_exclusive_group()
+     admin_group.add_argument(
+         "-a",
+@@ -281,7 +299,7 @@ def main() -> None:
+     if args.admin or args.no_admin:
+         admin = args.admin
+ 
+-    register_new_user(args.user, password, server_url, secret, admin, args.user_type)
++    register_new_user(args.user, password, server_url, secret, admin, args.user_type, exists_ok=args.exists_ok)
+ 
+ 
+ def _read_file(file_path: Any, config_path: str) -> str:
+-- 
+2.44.1
+

clanModules/matrix-synapse/default.nix

@@ -6,17 +6,93 @@
 }:
 let
   cfg = config.clan.matrix-synapse;
+  nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
+  element-web =
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
+      ''
+        cp -r ${pkgs.element-web} $out
+        chmod -R u+w $out
+        jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
+          > $out/config.json < ${pkgs.element-web}/config.json
+        ln -s $out/config.json $out/config.${nginx-vhost}.json
+      '';
+
+  # FIXME: This was taken from upstream. Drop this when our patch is upstream
+  synapseCfg = config.services.matrix-synapse;
+  wantedExtras =
+    synapseCfg.extras
+    ++ lib.optional (synapseCfg.settings ? oidc_providers) "oidc"
+    ++ lib.optional (synapseCfg.settings ? jwt_config) "jwt"
+    ++ lib.optional (synapseCfg.settings ? saml2_config) "saml2"
+    ++ lib.optional (synapseCfg.settings ? redis) "redis"
+    ++ lib.optional (synapseCfg.settings ? sentry) "sentry"
+    ++ lib.optional (synapseCfg.settings ? user_directory) "user-search"
+    ++ lib.optional (synapseCfg.settings.url_preview_enabled) "url-preview"
+    ++ lib.optional (synapseCfg.settings.database.name == "psycopg2") "postgres";
 in
 {
+  options.services.matrix-synapse.package = lib.mkOption { readOnly = false; };
   options.clan.matrix-synapse = {
-    enable = lib.mkEnableOption "Enable matrix-synapse";
     domain = lib.mkOption {
       type = lib.types.str;
       description = "The domain name of the matrix server";
+      example = "example.com";
+    };
+    users = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "The name of the user";
+              };
+
+              admin = lib.mkOption {
+                type = lib.types.bool;
+                default = false;
+                description = "Whether the user should be an admin";
+              };
+            };
+          }
+        )
+      );
+      description = "A list of users. Not that only new users will be created and existing ones are not modified.";
+      example.alice = {
+        admin = true;
+      };
     };
   };
-  config = lib.mkIf cfg.enable {
+  imports = [
+    ../postgresql
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "matrix-synapse"
+      "enable"
+    ] "Importing the module will already enable the service.")
+
+    ../postgresql
+  ];
+  config = {
     services.matrix-synapse = {
+      package = lib.mkForce (
+        pkgs.matrix-synapse.override {
+          matrix-synapse-unwrapped = pkgs.matrix-synapse.unwrapped.overrideAttrs (_old: {
+            doInstallCheck = false; # too slow, nixpkgs maintainer already run this.
+            patches = [
+              # see: https://github.com/element-hq/synapse/pull/17304
+              ./0001-register_new_matrix_user-add-password-file-flag.patch
+              ./0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch
+            ];
+          });
+          extras = wantedExtras;
+          plugins = synapseCfg.plugins;
+        }
+      );
+
       enable = true;
       settings = {
         server_name = cfg.domain;
@@ -29,6 +105,7 @@ in
           "turn:turn.matrix.org?transport=udp"
           "turn:turn.matrix.org?transport=tcp"
         ];
+        registration_shared_secret_path = "/run/synapse-registration-shared-secret";
         listeners = [
           {
             port = 8008;
@@ -49,45 +126,76 @@ in
           }
         ];
       };
-      extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
-    };
-    systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
-      "+${pkgs.writeScript "copy_registration_shared_secret" ''
-        #!/bin/sh
-        cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
-      ''}"
-    ];
-
-    clanCore.facts.services."matrix-synapse" = {
-      secret."synapse-registration_shared_secret" = { };
-      generator.path = with pkgs; [
-        coreutils
-        pwgen
-      ];
-      generator.script = ''
-        echo "registration_shared_secret: $(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
-      '';
     };
 
-    services.postgresql.enable = true;
-    # we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
-    services.postgresql = {
-      ensureDatabases = [ "matrix-synapse" ];
-      ensureUsers = [
-        {
-          name = "matrix-synapse";
-          ensureDBOwnership = true;
+    systemd.tmpfiles.settings."01-matrix" = {
+      "/run/synapse-registration-shared-secret" = {
+        C.argument =
+          config.clan.core.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
+        z = {
+          mode = "0400";
+          user = "matrix-synapse";
+        };
+      };
+    };
+
+    clan.postgresql.users.matrix-synapse = { };
+    clan.postgresql.databases.matrix-synapse.create.options = {
+      TEMPLATE = "template0";
+      LC_COLLATE = "C";
+      LC_CTYPE = "C";
+      ENCODING = "UTF8";
+      OWNER = "matrix-synapse";
+    };
+    clan.postgresql.databases.matrix-synapse.restore.stopOnRestore = [ "matrix-synapse" ];
+
+    clan.core.facts.services =
+      {
+        "matrix-synapse" = {
+          secret."synapse-registration_shared_secret" = { };
+          generator.path = with pkgs; [
+            coreutils
+            pwgen
+          ];
+          generator.script = ''
+            echo -n "$(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
+          '';
+        };
+      }
+      // lib.mapAttrs' (
+        name: user:
+        lib.nameValuePair "matrix-password-${user.name}" {
+          secret."matrix-password-${user.name}" = { };
+          generator.path = with pkgs; [ xkcdpass ];
+          generator.script = ''
+            xkcdpass -n 4 -d - > "$secrets"/${lib.escapeShellArg "matrix-password-${user.name}"}
+          '';
         }
-      ];
-      initialScript = pkgs.writeText "synapse-init.sql" ''
-        CREATE DATABASE "matrix-synapse"
-          TEMPLATE template0
-          LC_COLLATE = "C"
-          LC_CTYPE = "C";
-      '';
-    };
+      ) cfg.users;
+
+    systemd.services.matrix-synapse =
+      let
+        usersScript =
+          ''
+            while ! ${pkgs.netcat}/bin/nc -z -v ::1 8008; do
+              if ! kill -0 "$MAINPID"; then exit 1; fi
+              sleep 1;
+            done
+          ''
+          + lib.concatMapStringsSep "\n" (user: ''
+            # only create user if it doesn't exist
+            /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --exists-ok --password-file ${
+              config.clan.core.facts.services."matrix-password-${user.name}".secret."matrix-password-${user.name}".path
+            } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"}
+          '') (lib.attrValues cfg.users);
+      in
+      {
+        path = [ pkgs.curl ];
+        serviceConfig.ExecStartPost = [
+          (''+${pkgs.writeShellScript "matrix-synapse-create-users" usersScript}'')
+        ];
+      };
+
     services.nginx = {
       enable = true;
       virtualHosts = {
@@ -102,7 +210,7 @@ in
             return 200 '${
               builtins.toJSON {
                 "m.homeserver" = {
-                  "base_url" = "https://matrix.${cfg.domain}";
+                  "base_url" = "https://${nginx-vhost}";
                 };
                 "m.identity_server" = {
                   "base_url" = "https://vector.im";
@@ -111,15 +219,12 @@ in
             }';
           '';
         };
-        "matrix.${cfg.domain}" = {
+        ${nginx-vhost} = {
           forceSSL = true;
           enableACME = true;
-          locations."/_matrix" = {
-            proxyPass = "http://localhost:8008";
-          };
-          locations."/test".extraConfig = ''
-            return 200 "Hello, world!";
-          '';
+          locations."/_matrix".proxyPass = "http://localhost:8008";
+          locations."/_synapse".proxyPass = "http://localhost:8008";
+          locations."/".root = element-web;
         };
       };
     };

clanModules/moonlight/default.nix

@@ -13,10 +13,10 @@ in
   systemd.tmpfiles.rules = [
     "d '/var/lib/moonlight' 0770 'user' 'users' - -"
     "C '/var/lib/moonlight/moonlight.cert' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.moonlight.secret."moonlight.cert".path or ""
+      config.clan.core.facts.services.moonlight.secret."moonlight.cert".path or ""
     }"
     "C '/var/lib/moonlight/moonlight.key' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.moonlight.secret."moonlight.key".path or ""
+      config.clan.core.facts.services.moonlight.secret."moonlight.key".path or ""
     }"
   ];
 
@@ -45,7 +45,7 @@ in
   systemd.user.services.moonlight-join = {
     description = "Join sunshine hosts";
     script = ''${ms-accept}/bin/moonlight-sunshine-accept moonlight join --port ${builtins.toString defaultPort} --cert '${
-      config.clanCore.facts.services.moonlight.public."moonlight.cert".value or ""
+      config.clan.core.facts.services.moonlight.public."moonlight.cert".value or ""
     }' --host fd2e:25da:6035:c98f:cd99:93e0:b9b8:9ca1'';
     serviceConfig = {
       Type = "oneshot";
@@ -68,7 +68,7 @@ in
     };
   };
 
-  clanCore.facts.services.moonlight = {
+  clan.core.facts.services.moonlight = {
     secret."moonlight.key" = { };
     secret."moonlight.cert" = { };
     public."moonlight.cert" = { };

clanModules/postgresql/README.md

@@ -0,0 +1,2 @@
+A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
+---

clanModules/postgresql/default.nix

@@ -0,0 +1,222 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  createDatatbaseState =
+    db:
+    let
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
+      '';
+      postRestoreScript = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
+    };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+      )
+    }
+  '';
+  cfg = config.clan.postgresql;
+
+  userClauses = lib.mapAttrsToList (
+    _: user:
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+  ) cfg.databases;
+in
+{
+  options.clan.postgresql = {
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
+    databases = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Database name.";
+              };
+              service = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+                description = "Service name that we associate with the database.";
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Create the database if it does not exist.";
+              };
+              create.options = lib.mkOption {
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of systemd services to stop before restoring the database.";
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
+    };
+  };
+  config = {
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
+    services.postgresql.enable = true;
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+
+    clan.core.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair db.service (createDatatbaseState db)
+    ) config.clan.postgresql.databases;
+
+    environment.systemPackages = builtins.map (
+      db:
+      let
+        folder = "/var/backup/postgres/${db.name}";
+        current = "${folder}/pg-dump";
+      in
+      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+            pkgs.gnugrep
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            ${
+              lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+                systemctl stop ${builtins.toString db.restore.stopOnRestore}
+                trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+              ''
+            }
+
+            mkdir -p "${folder}"
+            if runuser -u postgres -- psql -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${db.name}'" | grep -q 1; then
+              runuser -u postgres -- dropdb "${db.name}"
+            fi
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      ''
+    ) (builtins.attrValues config.clan.postgresql.databases);
+  };
+}

clanModules/root-password/default.nix

@@ -2,9 +2,9 @@
 {
   users.mutableUsers = false;
   users.users.root.hashedPasswordFile =
-    config.clanCore.facts.services.root-password.secret.password-hash.path;
-  sops.secrets."${config.clanCore.machineName}-password-hash".neededForUsers = true;
-  clanCore.facts.services.root-password = {
+    config.clan.core.facts.services.root-password.secret.password-hash.path;
+  sops.secrets."${config.clan.core.machineName}-password-hash".neededForUsers = true;
+  clan.core.facts.services.root-password = {
     secret.password = { };
     secret.password-hash = { };
     generator.path = with pkgs; [

clanModules/sshd/default.nix

@@ -5,12 +5,12 @@
 
   services.openssh.hostKeys = [
     {
-      path = config.clanCore.facts.services.openssh.secret."ssh.id_ed25519".path;
+      path = config.clan.core.facts.services.openssh.secret."ssh.id_ed25519".path;
       type = "ed25519";
     }
   ];
 
-  clanCore.facts.services.openssh = {
+  clan.core.facts.services.openssh = {
     secret."ssh.id_ed25519" = { };
     public."ssh.id_ed25519.pub" = { };
     generator.path = [

clanModules/static-hosts/default.nix

@@ -3,20 +3,36 @@
   options.clan.static-hosts = {
     excludeHosts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = [ config.clanCore.machineName ];
+      default =
+        if config.clan.static-hosts.topLevelDomain != "" then [ ] else [ config.clan.core.machineName ];
       description = "Hosts that should be excluded";
     };
+    topLevelDomain = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      description = "Top level domain to reach hosts";
+    };
   };
 
   config.networking.hosts =
     let
-      clanDir = config.clanCore.clanDir;
+      clanDir = config.clan.core.clanDir;
       machineDir = clanDir + "/machines/";
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
-      machines = builtins.readDir machineDir;
+      machinesFileSet = builtins.readDir machineDir;
+      machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
+      networkIpsUnchecked = builtins.map (
+        machine:
+        let
+          fullPath = zerotierIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) machines;
+      networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
+      machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
       filteredMachines = lib.filterAttrs (
         name: _: !(lib.elem name config.clan.static-hosts.excludeHosts)
-      ) machines;
+      ) machinesWithIp;
     in
     lib.filterAttrs (_: value: value != null) (
       lib.mapAttrs' (
@@ -24,7 +40,15 @@
         let
           path = zerotierIpMachinePath machine;
         in
-        if builtins.pathExists path then lib.nameValuePair (builtins.readFile path) [ machine ] else null
+        if builtins.pathExists path then
+          lib.nameValuePair (builtins.readFile path) (
+            if (config.clan.static-hosts.topLevelDomain == "") then
+              [ machine ]
+            else
+              [ "${machine}.${config.clan.static-hosts.topLevelDomain}" ]
+          )
+        else
+          { }
       ) filteredMachines
     );
 }

clanModules/sunshine/default.nix

@@ -97,10 +97,10 @@ in
   systemd.tmpfiles.rules = [
     "d '/var/lib/sunshine' 0770 'user' 'users' - -"
     "C '/var/lib/sunshine/sunshine.cert' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.sunshine.secret."sunshine.cert".path or ""
+      config.clan.core.facts.services.sunshine.secret."sunshine.cert".path or ""
     }"
     "C '/var/lib/sunshine/sunshine.key' 0644 'user' 'users' - ${
-      config.clanCore.facts.services.sunshine.secret."sunshine.key".path or ""
+      config.clan.core.facts.services.sunshine.secret."sunshine.key".path or ""
     }"
   ];
 
@@ -117,8 +117,8 @@ in
       RestartSec = "5s";
       ReadWritePaths = [ "/var/lib/sunshine" ];
       ReadOnlyPaths = [
-        (config.clanCore.facts.services.sunshine.secret."sunshine.key".path or "")
-        (config.clanCore.facts.services.sunshine.secret."sunshine.cert".path or "")
+        (config.clan.core.facts.services.sunshine.secret."sunshine.key".path or "")
+        (config.clan.core.facts.services.sunshine.secret."sunshine.cert".path or "")
       ];
     };
     wantedBy = [ "graphical-session.target" ];
@@ -137,7 +137,7 @@ in
     startLimitIntervalSec = 500;
     script = ''
       ${ms-accept}/bin/moonlight-sunshine-accept sunshine init-state --uuid ${
-        config.clanCore.facts.services.sunshine.public.sunshine-uuid.value or null
+        config.clan.core.facts.services.sunshine.public.sunshine-uuid.value or null
       } --state-file /var/lib/sunshine/state.json
     '';
     serviceConfig = {
@@ -173,9 +173,9 @@ in
     startLimitIntervalSec = 500;
     script = ''
       ${ms-accept}/bin/moonlight-sunshine-accept sunshine listen --port ${builtins.toString listenPort} --uuid ${
-        config.clanCore.facts.services.sunshine.public.sunshine-uuid.value or null
+        config.clan.core.facts.services.sunshine.public.sunshine-uuid.value or null
       }  --state /var/lib/sunshine/state.json --cert '${
-        config.clanCore.facts.services.sunshine.public."sunshine.cert".value or null
+        config.clan.core.facts.services.sunshine.public."sunshine.cert".value or null
       }'
     '';
     serviceConfig = {
@@ -187,7 +187,7 @@ in
     wantedBy = [ "graphical-session.target" ];
   };
 
-  clanCore.facts.services.ergochat = {
+  clan.core.facts.services.ergochat = {
     secret."sunshine.key" = { };
     secret."sunshine.cert" = { };
     public."sunshine-uuid" = { };

clanModules/syncthing-static-peers/README.md

@@ -0,0 +1,2 @@
+Statically configure syncthing peers through clan
+---

clanModules/syncthing-static-peers/default.nix

@@ -0,0 +1,108 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+  clanDir = config.clan.core.clanDir;
+  machineDir = clanDir + "/machines/";
+  syncthingPublicKeyPath = machines: machineDir + machines + "/facts/syncthing.pub";
+  machinesFileSet = builtins.readDir machineDir;
+  machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
+  syncthingPublicKeysUnchecked = builtins.map (
+    machine:
+    let
+      fullPath = syncthingPublicKeyPath machine;
+    in
+    if builtins.pathExists fullPath then machine else null
+  ) machines;
+  syncthingPublicKeyMachines = lib.filter (machine: machine != null) syncthingPublicKeysUnchecked;
+  zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
+  networkIpsUnchecked = builtins.map (
+    machine:
+    let
+      fullPath = zerotierIpMachinePath machine;
+    in
+    if builtins.pathExists fullPath then machine else null
+  ) machines;
+  networkIpMachines = lib.filter (machine: machine != null) networkIpsUnchecked;
+  devices = builtins.map (machine: {
+    name = machine;
+    value = {
+      name = machine;
+      id = (lib.removeSuffix "\n" (builtins.readFile (syncthingPublicKeyPath machine)));
+      addresses =
+        [ "dynamic" ]
+        ++ (
+          if (lib.elem machine networkIpMachines) then
+            [ "tcp://[${(lib.removeSuffix "\n" (builtins.readFile (zerotierIpMachinePath machine)))}]:22000" ]
+          else
+            [ ]
+        );
+    };
+  }) syncthingPublicKeyMachines;
+in
+{
+  options.clan.syncthing-static-peers = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clan.core.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be added.
+      '';
+    };
+  };
+
+  config.services.syncthing.settings.devices = (builtins.listToAttrs devices);
+
+  imports = [
+    {
+      # Syncthing ports: 8384 for remote access to GUI
+      # 22000 TCP and/or UDP for sync traffic
+      # 21027/UDP for discovery
+      # source: https://docs.syncthing.net/users/firewall.html
+      networking.firewall.interfaces."zt+".allowedTCPPorts = [
+        8384
+        22000
+      ];
+      networking.firewall.allowedTCPPorts = [ 8384 ];
+      networking.firewall.interfaces."zt+".allowedUDPPorts = [
+        22000
+        21027
+      ];
+
+      # Activates inotify compatibility on syncthing
+      # use mkOverride 900 here as it otherwise would collide with the default of the
+      # upstream nixos xserver.nix
+      boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkOverride 900 524288;
+
+      services.syncthing = {
+        enable = true;
+        configDir = "/var/lib/syncthing";
+        group = "syncthing";
+
+        key = lib.mkDefault config.clan.core.facts.services.syncthing.secret."syncthing.key".path or null;
+        cert = lib.mkDefault config.clan.core.facts.services.syncthing.secret."syncthing.cert".path or null;
+      };
+
+      clan.core.facts.services.syncthing = {
+        secret."syncthing.key" = { };
+        secret."syncthing.cert" = { };
+        public."syncthing.pub" = { };
+        generator.path = [
+          pkgs.coreutils
+          pkgs.gnugrep
+          pkgs.syncthing
+        ];
+        generator.script = ''
+          syncthing generate --config "$secrets"
+          mv "$secrets"/key.pem "$secrets"/syncthing.key
+          mv "$secrets"/cert.pem "$secrets"/syncthing.cert
+          cat "$secrets"/config.xml | grep -oP '(?<=<device id=")[^"]+' | uniq > "$facts"/syncthing.pub
+        '';
+      };
+    }
+  ];
+}

clanModules/syncthing/default.nix

@@ -9,8 +9,8 @@
     id = lib.mkOption {
       type = lib.types.nullOr lib.types.str;
       example = "BABNJY4-G2ICDLF-QQEG7DD-N3OBNGF-BCCOFK6-MV3K7QJ-2WUZHXS-7DTW4AS";
-      default = config.clanCore.facts.services.syncthing.public."syncthing.pub".value or null;
-      defaultText = "config.clanCore.facts.services.syncthing.public.\"syncthing.pub\".value";
+      default = config.clan.core.facts.services.syncthing.public."syncthing.pub".value or null;
+      defaultText = "config.clan.core.facts.services.syncthing.public.\"syncthing.pub\".value";
     };
     introducer = lib.mkOption {
       description = ''
@@ -119,7 +119,7 @@
           getPendingDevices = "/rest/cluster/pending/devices";
           postNewDevice = "/rest/config/devices";
           SharedFolderById = "/rest/config/folders/";
-          apiKey = config.clanCore.facts.services.syncthing.secret."syncthing.api".path or null;
+          apiKey = config.clan.core.facts.services.syncthing.secret."syncthing.api".path or null;
         in
         lib.mkIf config.clan.syncthing.autoAcceptDevices {
           description = "Syncthing auto accept devices";
@@ -161,7 +161,7 @@
 
       systemd.services.syncthing-init-api-key =
         let
-          apiKey = config.clanCore.facts.services.syncthing.secret."syncthing.api".path or null;
+          apiKey = config.clan.core.facts.services.syncthing.secret."syncthing.api".path or null;
         in
         lib.mkIf config.clan.syncthing.autoAcceptDevices {
           description = "Set the api key";
@@ -183,7 +183,7 @@
           };
         };
 
-      clanCore.facts.services.syncthing = {
+      clan.core.facts.services.syncthing = {
         secret."syncthing.key" = { };
         secret."syncthing.cert" = { };
         secret."syncthing.api" = { };

clanModules/thelounge/default.nix

@@ -11,5 +11,5 @@ _: {
     };
   };
 
-  clanCore.state.thelounde.folders = [ "/var/lib/thelounge" ];
+  clan.core.state.thelounde.folders = [ "/var/lib/thelounge" ];
 }

clanModules/trusted-nix-caches/README.md

@@ -0,0 +1,2 @@
+This module sets the `clan.lol` and `nix-community` cache up as a trusted cache.
+----

clanModules/trusted-nix-caches/default.nix

@@ -0,0 +1,10 @@
+{
+  nix.settings.trusted-substituters = [
+    "https://cache.clan.lol"
+    "https://nix-community.cachix.org"
+  ];
+  nix.settings.trusted-public-keys = [
+    "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="
+    "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28="
+  ];
+}

clanModules/user-password/default.nix

@@ -22,9 +22,9 @@
   config = {
     users.mutableUsers = false;
     users.users.${config.clan.user-password.user}.hashedPasswordFile =
-      config.clanCore.facts.services.user-password.secret.user-password-hash.path;
-    sops.secrets."${config.clanCore.machineName}-user-password-hash".neededForUsers = true;
-    clanCore.facts.services.user-password = {
+      config.clan.core.facts.services.user-password.secret.user-password-hash.path;
+    sops.secrets."${config.clan.core.machineName}-user-password-hash".neededForUsers = true;
+    clan.core.facts.services.user-password = {
       secret.user-password = { };
       secret.user-password-hash = { };
       generator.prompt = (

clanModules/zerotier-static-peers/default.nix

@@ -6,7 +6,7 @@
   ...
 }:
 let
-  clanDir = config.clanCore.clanDir;
+  clanDir = config.clan.core.clanDir;
   machineDir = clanDir + "/machines/";
   machinesFileSet = builtins.readDir machineDir;
   machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
@@ -20,7 +20,7 @@ let
     if builtins.pathExists fullPath then builtins.readFile fullPath else null
   ) machines;
   networkIds = lib.filter (machine: machine != null) networkIdsUnchecked;
-  networkId = builtins.elemAt networkIds 0;
+  networkId = if builtins.length networkIds == 0 then null else builtins.elemAt networkIds 0;
 in
 #TODO:trace on multiple found network-ids
 #TODO:trace on no single found networkId
@@ -28,28 +28,33 @@ in
   options.clan.zerotier-static-peers = {
     excludeHosts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = [ config.clanCore.machineName ];
+      default = [ config.clan.core.machineName ];
       description = "Hosts that should be excluded";
     };
   };
 
   config.systemd.services.zerotier-static-peers-autoaccept =
     let
-      machines = builtins.readDir machineDir;
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
-      filteredMachines = lib.filterAttrs (
-        name: _: !(lib.elem name config.clan.static-hosts.excludeHosts)
+      networkIpsUnchecked = builtins.map (
+        machine:
+        let
+          fullPath = zerotierIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
       ) machines;
+      networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
+      machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
+      filteredMachines = lib.filterAttrs (
+        name: _: !(lib.elem name config.clan.zerotier-static-peers.excludeHosts)
+      ) machinesWithIp;
       hosts = lib.mapAttrsToList (host: _: host) (
         lib.mapAttrs' (
           machine: _:
           let
             fullPath = zerotierIpMachinePath machine;
           in
-          if builtins.pathExists fullPath then
-            lib.nameValuePair (builtins.readFile fullPath) [ machine ]
-          else
-            null
+          lib.nameValuePair (builtins.readFile fullPath) [ machine ]
         ) filteredMachines
       );
     in

devShell.nix

@@ -1,3 +1,4 @@
+{ ... }:
 {
   perSystem =
     {
@@ -26,7 +27,8 @@
         packages = [
           select-shell
           pkgs.tea
-          pkgs.nix
+          # Better error messages than nix 2.18
+          pkgs.nixVersions.latest
           self'.packages.tea-create-pr
           self'.packages.merge-after-ci
           self'.packages.pending-reviews
@@ -34,9 +36,6 @@
           config.treefmt.build.wrapper
         ];
         shellHook = ''
-          # no longer used
-          rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
-
           echo -e "${ansiEscapes.green}switch to another dev-shell using: select-shell${ansiEscapes.reset}"
         '';
       };

docs/.gitignore

@@ -1 +1,3 @@
-/site/reference
+/site/reference
+/site/static/Roboto-Regular.ttf
+/site/static/FiraCode-VF.ttf

docs/CONTRIBUTING.md

@@ -15,92 +15,124 @@ Let's get your development environment up and running:
 
 1. **Install Nix Package Manager**:
 
-   - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
-     ```bash
-     curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
-     ```
+      - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
+        ```bash
+        curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
+        ```
 
 2. **Install direnv**:
 
-   - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command:
-     ```bash
-     curl -sfL https://direnv.net/install.sh | bash
-     ```
+      - To automatically setup a devshell on entering the directory
+        ```bash
+        nix profile install nixpkgs#nix-direnv-flakes
+        ```
 
 3. **Add direnv to your shell**:
 
-   - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
-     You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
+      - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
+        You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
 
-   ```bash
-   echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
-   ```
-
-4. **Clone the Repository and Navigate**:
-
-   - Clone this repository and navigate to it.
-
-5. **Allow .envrc**:
-
-   - When you enter the directory, you'll receive an error message like this:
-     ```bash
-     direnv: error .envrc is blocked. Run `direnv allow` to approve its content
-     ```
-   - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
-
-# Setting Up Your Git Workflow
-
-Let's set up your Git workflow to collaborate effectively:
-
-1. **Register Your Gitea Account Locally**:
-
-   - Execute the following command to add your Gitea account locally:
-     ```bash
-     tea login add
-     ```
-   - Fill out the prompt as follows:
-     - URL of Gitea instance: `https://git.clan.lol`
-     - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171`
-     - Do you have an access token? No
-     - Username: YourUsername
-     - Password: YourPassword
-     - Set Optional settings: No
-
-2. **Git Workflow**:
-
-   1. Add your changes to Git using `git add <file1> <file2>`.
-   2. Run `nix fmt` to lint your files.
-   3. Commit your changes with a descriptive message: `git commit -a -m "My descriptive commit message"`.
-   4. Make sure your branch has the latest changes from upstream by executing:
       ```bash
-      git fetch && git rebase origin/main --autostash
+      echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
       ```
-   5. Use `git status` to check for merge conflicts.
-   6. If conflicts exist, resolve them. Here's a tutorial for resolving conflicts in [VSCode](https://code.visualstudio.com/docs/sourcecontrol/overview#_merge-conflicts).
-   7. After resolving conflicts, execute `git merge --continue` and repeat step 5 until there are no conflicts.
 
-3. **Create a Pull Request**:
+4. **Create a Gitea Account**:
+      - Register an account on https://git.clan.lol
+      - Fork the [clan-core](https://git.clan.lol/clan/clan-core) repository
+      - Clone the repository and navigate to it
+      - Add a new remote called upstream:
+         ```bash
+         git remote add upstream gitea@git.clan.lol:clan/clan-core.git
+         ```
 
-   - To automatically open a pull request that gets merged if all tests pass, execute:
-     ```bash
-     merge-after-ci
-     ```
+5. **Register Your Gitea Account Locally**:
 
-4. **Review Your Pull Request**:
+      - Execute the following command to add your Gitea account locally:
+        ```bash
+        tea login add
+        ```
+      - Fill out the prompt as follows:
+        - URL of Gitea instance: `https://git.clan.lol`
+        - Name of new Login [git.clan.lol]:
+        - Do you have an access token? No
+        - Username: YourUsername
+        - Password: YourPassword
+        - Set Optional settings: No
 
-   - Visit https://git.clan.lol and go to the project page. Check under "Pull Requests" for any issues with your pull request.
 
-5. **Push Your Changes**:
-   - If there are issues, fix them and redo step 2. Afterward, execute:
-     ```bash
-     git push origin HEAD:YourUsername-main
-     ```
-   - This will directly push to your open pull request.
+6. **Allow .envrc**:
+
+      - When you enter the directory, you'll receive an error message like this:
+        ```bash
+        direnv: error .envrc is blocked. Run `direnv allow` to approve its content
+        ```
+      - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
+
+7. **(Optional) Install Git Hooks**:
+      - To syntax check your code you can run:
+         ```bash
+         nix fmt
+         ```
+      - To make this automatic install the git hooks
+         ```bash
+         ./scripts/pre-commit
+         ```
+
+8. **Open a Pull Request**:
+      - To automatically open up a pull request you can use our tool called:
+      ```
+      merge-after-ci --reviewers Mic92 Lassulus Qubasa
+      ```
 
 # Debugging
 
 Here are some methods for debugging and testing the clan-cli:
 
+## See all possible packages and tests
+
+To quickly show all possible packages and tests execute:
+
+```bash
+nix flake show --system no-eval
+```
+
+Under `checks` you will find all tests that are executed in our CI. Under `packages` you find all our projects.
+
+```
+git+file:///home/lhebendanz/Projects/clan-core
+├───apps
+│   └───x86_64-linux
+│       ├───install-vm: app
+│       └───install-vm-nogui: app
+├───checks
+│   └───x86_64-linux
+│       ├───borgbackup omitted (use '--all-systems' to show)
+│       ├───check-for-breakpoints omitted (use '--all-systems' to show)
+│       ├───clan-dep-age omitted (use '--all-systems' to show)
+│       ├───clan-dep-bash omitted (use '--all-systems' to show)
+│       ├───clan-dep-e2fsprogs omitted (use '--all-systems' to show)
+│       ├───clan-dep-fakeroot omitted (use '--all-systems' to show)
+│       ├───clan-dep-git omitted (use '--all-systems' to show)
+│       ├───clan-dep-nix omitted (use '--all-systems' to show)
+│       ├───clan-dep-openssh omitted (use '--all-systems' to show)
+│       ├───"clan-dep-python3.11-mypy" omitted (use '--all-systems' to show)
+├───packages
+│   └───x86_64-linux
+│       ├───clan-cli omitted (use '--all-systems' to show)
+│       ├───clan-cli-docs omitted (use '--all-systems' to show)
+│       ├───clan-ts-api omitted (use '--all-systems' to show)
+│       ├───clan-app omitted (use '--all-systems' to show)
+│       ├───default omitted (use '--all-systems' to show)
+│       ├───deploy-docs omitted (use '--all-systems' to show)
+│       ├───docs omitted (use '--all-systems' to show)
+│       ├───editor omitted (use '--all-systems' to show)
+└───templates
+    ├───default: template: Initialize a new clan flake
+    └───new-clan: template: Initialize a new clan flake
+```
+
+You can execute every test separately by following the tree path `nix build .#checks.x86_64-linux.clan-pytest` for example.
+
 ## Test Locally in Devshell with Breakpoints
 
 To test the cli locally in a development environment and set breakpoints for debugging, follow these steps:
@@ -150,12 +182,14 @@ If you need to inspect the Nix sandbox while running tests, follow these steps:
 2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example:
 
    ```bash
-   cntr exec -w your_sandbox_name
    psgrep -a -x your_python_process_name
+   cntr attach <container id, container name or process id>
    ```
 
+Or you can also use the [nix breakpoint hook](https://nixos.org/manual/nixpkgs/stable/#breakpointhook)
+
 
 # Standards
 
-Every new module name should be in kebab-case.
-Every fact definition, where possible should be in kebab-case.
+- Every new module name should be in kebab-case.
+- Every fact definition, where possible should be in kebab-case.

docs/_internal/use-cases/01-self-host.md

@@ -20,11 +20,11 @@ There are several reasons for choosing to self-host. These can include:
 
 Alice wants to self-host a mumble server for her family.
 
-- She visits to the cLAN website, and follows the instructions on how to install cLAN-OS on her server.
-- Alice logs into a terminal on her server via SSH (alternatively uses cLAN GUI app)
-- Using the cLAN CLI or GUI tool, alice creates a new private network for her family (VPN)
-- Alice now browses a list of curated cLAN modules and finds a module for mumble.
-- She adds this module to her network using the cLAN tool.
+- She visits to the Clan website, and follows the instructions on how to install Clan-OS on her server.
+- Alice logs into a terminal on her server via SSH (alternatively uses Clan GUI app)
+- Using the Clan CLI or GUI tool, alice creates a new private network for her family (VPN)
+- Alice now browses a list of curated Clan modules and finds a module for mumble.
+- She adds this module to her network using the Clan tool.
 - After that, she uses the clan tool to invite her family members to her network
 - Other family members join the private network via the invitation.
 - By accepting the invitation, other members automatically install all required software to interact with the network on their machine.
@@ -33,7 +33,7 @@ Alice wants to self-host a mumble server for her family.
 
 Alice wants to add a photos app to her private network
 
-- She uses the clan CLI or GUI tool to manage her existing private cLAN family network
+- She uses the clan CLI or GUI tool to manage her existing private Clan family network
 - She discovers a module for photoprism, and adds it to her server using the tool
 - Other members who are already part of her network, will receive a notification that an update is required to their environment
 - After accepting, all new software and services to interact with the new photoprism service will be installed automatically.

docs/_internal/use-cases/02-join-network.md

@@ -1,4 +1,4 @@
-# Joining a cLAN network
+# Joining a Clan network
 
 ## General Description
 
@@ -8,13 +8,13 @@ Joining a self-hosted infrastructure involves connecting to a network, server, o
 
 ### Story 1: Joining a private network
 
-Alice' son Bob has never heard of cLAN, but receives an invitation URL from Alice who already set up private cLAN network for her family.
+Alice' son Bob has never heard of Clan, but receives an invitation URL from Alice who already set up private Clan network for her family.
 
-Bob opens the invitation link and lands on the cLAN website. He quickly learns about what cLAN is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
+Bob opens the invitation link and lands on the Clan website. He quickly learns about what Clan is and can see that the invitation is for a private network of his family that hosts a number of services, like a private voice chat and a photo sharing platform.
 
-Bob decides to join the network and follows the instructions to install the cLAN tool on his computer.
+Bob decides to join the network and follows the instructions to install the Clan tool on his computer.
 
-Feeding the invitation link to the cLAN tool, bob registers his machine with the network.
+Feeding the invitation link to the Clan tool, bob registers his machine with the network.
 
 All programs required to interact with the network will be installed and configured automatically and securely.
 
@@ -22,7 +22,7 @@ Optionally, bob can customize the configuration of these programs through a simp
 
 ### Story 2: Receiving breaking changes
 
-The cLAN family network which Bob is part of received an update.
+The Clan family network which Bob is part of received an update.
 
 The existing photo sharing service has been removed and replaced with another alternative service. The new photo sharing service requires a different client app to view and upload photos.
 
@@ -30,7 +30,7 @@ Bob accepts the update. Now his environment will be updated. The old client soft
 
 Because Bob has customized the previous photo viewing app, he is notified that this customization is no longer valid, as the software has been removed (deprecation message).l
 
-Optionally, Bob can now customize the new photo viewing software through his cLAN configuration app or via a config file.
+Optionally, Bob can now customize the new photo viewing software through his Clan configuration app or via a config file.
 
 ## Challenges
 

docs/_internal/use-cases/03-maintaining-clan-modules.md

@@ -1,10 +1,10 @@
-# cLAN module maintaining
+# Clan module maintaining
 
 ## General Description
 
-cLAN modules are pieces of software that can be used by admins to build a private or public infrastructure.
+Clan modules are pieces of software that can be used by admins to build a private or public infrastructure.
 
-cLAN modules should have the following properties:
+Clan modules should have the following properties:
 
 1. Documented: It should be clear what the module does and how to use it.
 1. Self contained: A module should be usable as is. If it requires any other software or settings, those should be delivered with the module itself.

docs/main.py

@@ -16,15 +16,26 @@ def define_env(env: Any) -> None:
     @env.macro
     def asciinema(name: str) -> str:
         return f"""<div id="{name}">
-            <script src="{asciinema_dir}/asciinema-player.min.js"></script>
             <script>
-                AsciinemaPlayer.create('{video_dir + name}',
-                document.getElementById("{name}"), {{
-                    loop: true,
-                    autoPlay: true,
-                    controls: false,
-                    speed: 1.5,
-                    theme: "solarized-light"
-                }});
+                // Function to load the script and then create the Asciinema player
+                function loadAsciinemaPlayer() {{
+                    var script = document.createElement('script');
+                    script.src = "{asciinema_dir}/asciinema-player.min.js";
+                    script.onload = function() {{
+                        AsciinemaPlayer.create('{video_dir + name}', document.getElementById("{name}"), {{
+                            loop: true,
+                            autoPlay: true,
+                            controls: false,
+                            speed: 1.5,
+                            theme: "solarized-light"
+                    }});
+                    }};
+                    document.head.appendChild(script);
+                }}
+
+                // Load the Asciinema player script
+                loadAsciinemaPlayer();
             </script>
+
+            <link rel="stylesheet" type="text/css" href="{asciinema_dir}/asciinema-player.css" />
         </div>"""

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Clan Docs
+site_name: Clan Documentation
 site_url: https://docs.clan.lol
 repo_url: https://git.clan.lol/clan/clan-core/
 repo_name: clan-core
@@ -28,6 +28,7 @@ markdown_extensions:
   - pymdownx.highlight:
       use_pygments: true
       anchor_linenums: true
+  - pymdownx.keys
   - toc:
       title: On this page
 
@@ -38,7 +39,7 @@ exclude_docs: |
 
 nav:
   - Blog:
-    - blog/index.md
+      - blog/index.md
   - Getting started:
       - index.md
       - Installer: getting-started/installer.md
@@ -50,6 +51,7 @@ nav:
       - Flake-parts: getting-started/flake-parts.md
   - Modules:
       - Clan Modules:
+          - reference/clanModules/borgbackup-static.md
           - reference/clanModules/borgbackup.md
           - reference/clanModules/deltachat.md
           - reference/clanModules/disk-layouts.md
@@ -58,12 +60,15 @@ nav:
           - reference/clanModules/localsend.md
           - reference/clanModules/matrix-synapse.md
           - reference/clanModules/moonlight.md
+          - reference/clanModules/postgresql.md
           - reference/clanModules/root-password.md
           - reference/clanModules/sshd.md
-          - reference/clanModules/sunshine.md
-          - reference/clanModules/syncthing.md
           - reference/clanModules/static-hosts.md
+          - reference/clanModules/sunshine.md
+          - reference/clanModules/syncthing-static-peers.md
+          - reference/clanModules/syncthing.md
           - reference/clanModules/thelounge.md
+          - reference/clanModules/trusted-nix-caches.md
           - reference/clanModules/user-password.md
           - reference/clanModules/xfce.md
           - reference/clanModules/zerotier-static-peers.md
@@ -78,6 +83,7 @@ nav:
           - reference/cli/history.md
           - reference/cli/machines.md
           - reference/cli/secrets.md
+          - reference/cli/show.md
           - reference/cli/ssh.md
           - reference/cli/vms.md
       - Clan Core:
@@ -92,8 +98,9 @@ docs_dir: site
 site_dir: out
 
 theme:
-  logo: static/clan-white.png
-  favicon: static/clan-dark.png
+  font: false
+  logo: https://clan.lol/static/logo/clan-white.png
+  favicon: https://clan.lol/static/dark-favicon/128x128.png
   name: material
   features:
     - navigation.instant
@@ -102,7 +109,8 @@ theme:
     - content.code.copy
     - content.tabs.link
   icon:
-    repo: fontawesome/brands/git
+    repo: fontawesome/brands/git-alt
+  custom_dir: overrides
 
   palette:
     # Palette toggle for light mode
@@ -124,8 +132,7 @@ theme:
         name: Switch to light mode
 
 extra_css:
-  - static/asciinema-player/custom-theme.css
-  - static/asciinema-player/asciinema-player.css
+  - static/extra.css
 
 extra:
   social:
@@ -138,7 +145,6 @@ extra:
     - icon: fontawesome/solid/rss
       link: /feed_rss_created.xml
 
-
 plugins:
   - search
   - blog

docs/nix/default.nix

@@ -2,6 +2,10 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
+  roboto,
+  fira-code,
   ...
 }:
 let
@@ -27,6 +31,14 @@ pkgs.stdenv.mkDerivation {
     mkdir -p ./site/reference/cli
     cp -af ${module-docs}/* ./site/reference/
     cp -af ${clan-cli-docs}/* ./site/reference/cli/
+
+    mkdir -p ./site/static/asciinema-player
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
+
+    # Link to fonts
+    ln -snf ${roboto}/share/fonts/truetype/Roboto-Regular.ttf ./site/static/
+    ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
   '';
 
   buildPhase = ''

docs/nix/flake-module.nix

@@ -40,8 +40,17 @@
             mypy --strict $out
           '';
 
+      asciinema-player-js = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.min.js";
+        sha256 = "sha256-Ymco/+FinDr5YOrV72ehclpp4amrczjo5EU3jfr/zxs=";
+      };
+      asciinema-player-css = pkgs.fetchurl {
+        url = "https://github.com/asciinema/asciinema-player/releases/download/v3.7.0/asciinema-player.css";
+        sha256 = "sha256-GZMeZFFGvP5GMqqh516mjJKfQaiJ6bL38bSYOXkaohc=";
+      };
+
       module-docs = pkgs.runCommand "rendered" { nativeBuildInputs = [ pkgs.python3 ]; } ''
-        export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json 
+        export CLAN_CORE=${jsonDocs.clanCore}/share/doc/nixos/options.json
         # A file that contains the links to all clanModule docs
         export CLAN_MODULES=${clanModulesFileInfo}
         export CLAN_MODULES_READMES=${clanModulesReadmes}
@@ -56,12 +65,16 @@
       devShells.docs = pkgs.callPackage ./shell.nix {
         inherit (self'.packages) docs clan-cli-docs;
         inherit module-docs;
+        inherit asciinema-player-js;
+        inherit asciinema-player-css;
       };
       packages = {
         docs = pkgs.python3.pkgs.callPackage ./default.nix {
           inherit (self'.packages) clan-cli-docs;
           inherit (inputs) nixpkgs;
           inherit module-docs;
+          inherit asciinema-player-js;
+          inherit asciinema-player-css;
         };
         deploy-docs = pkgs.callPackage ./deploy-docs.nix { inherit (config.packages) docs; };
         inherit module-docs;

docs/nix/get-module-docs.nix

@@ -13,7 +13,7 @@ let
 
   clanCoreNixosModules = [
     clanCore
-    { clanCore.clanDir = ./.; }
+    { clan.core.clanDir = ./.; }
   ] ++ allNixosModules;
 
   # TODO: optimally we would not have to evaluate all nixos modules for every page
@@ -25,6 +25,8 @@ let
   #   improves eval performance slightly (10%)
   getOptions = modules: (clanCoreNixos.extendModules { inherit modules; }).options;
 
+  getOptionsWithoutCore = modules: builtins.removeAttrs (getOptions modules) [ "core" ];
+
   evalDocs =
     options:
     pkgs.nixosOptionsDoc {
@@ -34,7 +36,7 @@ let
 
   # clanModules docs
   clanModulesDocs = builtins.mapAttrs (
-    name: module: (evalDocs ((getOptions [ module ]).clan.${name} or { })).optionsJSON
+    name: module: (evalDocs ((getOptionsWithoutCore [ module ]).clan.${name} or { })).optionsJSON
   ) clanModules;
 
   clanModulesReadmes = builtins.mapAttrs (
@@ -42,7 +44,7 @@ let
   ) clanModules;
 
   # clanCore docs
-  clanCoreDocs = (evalDocs (getOptions [ ]).clanCore).optionsJSON;
+  clanCoreDocs = (evalDocs (getOptions [ ]).clan.core).optionsJSON;
 in
 {
   inherit clanModulesReadmes;

docs/nix/scripts/renderOptions.py

@@ -40,13 +40,14 @@ def sanitize(text: str) -> str:
     return text.replace(">", "\\>")
 
 
-def replace_store_path(text: str) -> Path:
+def replace_store_path(text: str) -> tuple[str, str]:
     res = text
     if text.startswith("/nix/store/"):
         res = "https://git.clan.lol/clan/clan-core/src/branch/main/" + str(
             Path(*Path(text).parts[4:])
         )
-    return Path(res)
+    name = Path(res).name
+    return (res, name)
 
 
 def render_option_header(name: str) -> str:
@@ -108,9 +109,10 @@ def render_option(name: str, option: dict[str, Any], level: int = 3) -> str:
 """
 
     decls = option.get("declarations", [])
-    source_path = replace_store_path(decls[0])
+    source_path, name = replace_store_path(decls[0])
+    print(source_path, name)
     res += f"""
-:simple-git: [{source_path.name}]({source_path})
+:simple-git: [{name}]({source_path})
 """
     res += "\n"
 
@@ -135,7 +137,7 @@ To use this module, import it like this:
 """
 
 
-clan_core_descr = """ClanCore delivers all the essential features for every clan. 
+clan_core_descr = """ClanCore delivers all the essential features for every clan.
 It's always included in your setup, and you can customize your clan's behavior with the configuration [options](#module-options) provided below.
 
 """
@@ -160,8 +162,8 @@ def produce_clan_core_docs() -> None:
         for option_name, info in options.items():
             outfile = f"{module_name}/index.md"
 
-            # Create seperate files for nested options
-            if len(option_name.split(".")) <= 2:
+            # Create separate files for nested options
+            if len(option_name.split(".")) <= 3:
                 # i.e. clan-core.clanDir
                 output = core_outputs.get(
                     outfile,
@@ -172,7 +174,7 @@ def produce_clan_core_docs() -> None:
                 core_outputs[outfile] = output
             else:
                 # Clan sub-options
-                [_, sub] = option_name.split(".")[0:2]
+                [_, sub] = option_name.split(".")[1:3]
                 outfile = f"{module_name}/{sub}.md"
                 # Get the content or write the header
                 output = core_outputs.get(outfile, render_option_header(sub))

docs/nix/shell.nix

@@ -3,6 +3,10 @@
   pkgs,
   module-docs,
   clan-cli-docs,
+  asciinema-player-js,
+  asciinema-player-css,
+  roboto,
+  fira-code,
   ...
 }:
 pkgs.mkShell {
@@ -14,5 +18,14 @@ pkgs.mkShell {
     chmod +w ./site/reference/*
 
     echo "Generated API documentation in './site/reference/' "
+
+    mkdir -p ./site/static/asciinema-player
+
+    ln -snf ${asciinema-player-js} ./site/static/asciinema-player/asciinema-player.min.js
+    ln -snf ${asciinema-player-css} ./site/static/asciinema-player/asciinema-player.css
+
+    # Link to fonts
+    ln -snf ${roboto}/share/fonts/truetype/Roboto-Regular.ttf ./site/static/
+    ln -snf ${fira-code}/share/fonts/truetype/FiraCode-VF.ttf ./site/static/
   '';
 }

docs/overrides/main.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block extrahead %}
+<meta property="og:title" content="Clan - Documentation, Blog & Getting Started Guide" />
+<meta property="og:description" content="Documentation for Clan. The peer-to-peer machine deployment framework." />
+<meta property="og:image" content="https://clan.lol/static/dark-favicon/128x128.png" />
+<meta property="og:url" content="https://docs.clan.lol" />
+<meta property="og:type" content="website" />
+<meta property="og:site_name" content="Clan" />
+<meta property="og:locale" content="en_US" />
+
+{% endblock %}

docs/site/blog/.authors.yml

@@ -1,21 +1,26 @@
 authors:
+  DavHau:
+    name: "DavHau"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/davhau.jpg"
+    url: "https://DavHau.com"
   Lassulus:
     name: "Lassulus"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/621759?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/lassulus.jpg"
     url: "https://http://lassul.us/"
   Mic92:
     name: "Mic92"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/96200?v=4"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/mic92.jpg"
     url: "https://thalheim.io"
   W:
     name: "W"
     description: "Founder of Clan"
-    avatar: "/static/w_profile.webp"
+    avatar: "https://clan.lol/static/profiles/w_profile.webp"
     url: ""
   Qubasa:
     name: "Qubasa"
-    description: "Contributor to Clan"
-    avatar: "https://avatars.githubusercontent.com/u/22085373?v=4"
-    url: "https://github.com/Qubasa"
+    description: "Core Developer"
+    avatar: "https://clan.lol/static/profiles/qubasa.png"
+    url: "https://github.com/Qubasa"

docs/site/blog/posts/hello-world.md

@@ -61,7 +61,7 @@ Clan is for anyone and everyone who believes in the power of open source technol
 
 Ready to control your digital world? Clan is more than a tool—it's a movement. Secure your data, manage your systems easily, or connect with others how you like. Start with Clan for a better digital future.
 
-Connect with us on our [Matrix channel at clan.lol](https://matrix.to/#/#clan:lassul.us) or through our IRC bridges (coming soon).
+Connect with us on our [Matrix channel at clan.lol](https://matrix.to/#/#clan:clan.lol) or through our IRC bridges (coming soon).
 
 Want to see the code? Check it out [on our Gitea](https://git.clan.lol/clan/clan-core) or [on GitHub](https://github.com/clan-lol/clan-core).
 

docs/site/blog/posts/jsonschema.md

@@ -0,0 +1,194 @@
+---
+title: "Dev Report: Introducing the NixOS to JSON Schema Converter"
+description: "Discover our new library designed to extract JSON schema interfaces from NixOS modules, streamlining frontend development"
+authors:
+  - DavHau
+date: 2024-05-25
+slug: jsonschema-converter
+---
+
+## Overview
+
+We’ve developed a new library designed to extract interfaces from NixOS modules and convert them into JSON schemas, paving the way for effortless GUI generation. This blog post outlines the motivations behind this development, demonstrates the capabilities of the library, and guides you through leveraging it to create GUIs seamlessly.
+
+## Motivation
+
+In recent months, our team has been exploring various graphical user interfaces (GUIs) to streamline NixOS machine configuration. While our opinionated Clan modules simplify NixOS configurations, there's a need to configure these modules from diverse frontends, such as:
+
+- Command-line interfaces (CLIs)
+- Web-based UIs
+- Desktop applications
+- Mobile applications
+- Large Language Models (LLMs)
+
+Given this need, a universal format like JSON is a natural choice. It is already possible as of now, to import json based NixOS configurations, as illustrated below:
+
+`configuration.json`:
+```json
+{ "networking": { "hostName": "my-machine" } }
+```
+
+This configuration can be then imported inside a classic NixOS config:
+```nix
+{config, lib, pkgs, ...}: {
+  imports = [
+    (lib.importJSON ./configuration.json)
+  ];
+}
+```
+
+This straightforward approach allows us to build a frontend that generates JSON, enabling the configuration of NixOS machines. But, two critical questions arise:
+
+1. How does the frontend learn about existing configuration options?
+2. How can it verify user input without running Nix?
+
+Introducing [JSON schema](https://json-schema.org/), a widely supported standard that defines interfaces in JSON and validates input against them.
+
+Example schema for `networking.hostName`:
+```json
+{
+  "type": "object",
+  "properties": {
+    "networking": {
+      "type": "object",
+      "properties": {
+        "hostName": {
+          "type": "string",
+          "pattern": "^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$"
+        }
+      }
+    }
+  }
+}
+```
+
+## Client-Side Input Validation
+
+Validating input against JSON schemas is both efficient and well-supported across numerous programming languages. Using JSON schema validators, you can accurately check configurations like our `configuration.json`.
+
+Validation example:
+
+```shell
+$ nix-shell -p check-jsonschema
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[SUCCESS]===(./configuration.json)===
+```
+
+In case of invalid input, schema validators provide explicit error messages:
+
+```shell
+$ echo '{ "networking": { "hostName": "my/machine" } }' > configuration.json
+$ jsonschema -o pretty ./schema.json -i ./configuration.json
+===[ValidationError]===(./configuration.json)===
+
+'my/machine' does not match '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$'
+
+Failed validating 'pattern' in schema['properties']['networking']['properties']['hostName']:
+    {'pattern': '^$|^[a-z0-9]([a-z0-9_-]{0,61}[a-z0-9])?$',
+     'type': 'string'}
+
+On instance['networking']['hostName']:
+    'my/machine'
+```
+
+## Automatic GUI Generation
+
+Certain libraries facilitate straightforward GUI generation from JSON schemas. For instance, the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) auto-generates a form for any given schema.
+
+## NixOS Module to JSON Schema Converter
+
+To enable the development of responsive frontends, our library allows the extraction of interfaces from NixOS modules to JSON schemas. Open-sourced for community collaboration, this library supports building sophisticated user interfaces for NixOS.
+
+Here’s a preview of our library's functions exposed through the [clan-core](https://git.clan.lol/clan/clan-core) flake:
+
+- `lib.jsonschema.parseModule` - Generates a schema for a NixOS module.
+- `lib.jsonschema.parseOption` - Generates a schema for a single NixOS option.
+- `lib.jsonschema.parseOptions` - Generates a schema from an attrset of NixOS options.
+
+Example:
+`module.nix`:
+```nix
+{lib, config, pkgs, ...}: {
+  # a simple service with two options
+  options.services.example-web-service = {
+    enable = lib.mkEnableOption "Example web service";
+    port = lib.mkOption {
+      type = lib.types.int;
+      description = "Port used to serve the content";
+    };
+  };
+}
+```
+
+Converted, using the `parseModule` function:
+```shell
+$ cd clan-core
+$ nix eval --json --impure --expr \
+  '(import ./lib/jsonschema {}).parseModule ./module.nix' | jq | head
+{
+  "properties": {
+    "services": {
+      "properties": {
+        "example-web-service": {
+          "properties": {
+            "enable": {
+              "default": false,
+              "description": "Whether to enable Example web service.",
+              "examples": [
+...
+```
+
+This utility can also generate interfaces for existing NixOS modules or options.
+
+## GUI for NGINX in Under a Minute
+
+Creating a prototype GUI for the NGINX module using our library and [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/) can be done quickly:
+
+1. Export all NGINX options into a JSON schema using a Nix expression:
+
+```nix
+# export.nix
+let
+  pkgs = import <nixpkgs> {};
+  clan-core = builtins.getFlake "git+https://git.clan.lol/clan/clan-core";
+  options = (pkgs.nixos {}).options.services.nginx;
+in
+  clan-core.lib.jsonschema.parseOption options
+```
+
+2. Write the schema into a file:
+```shell
+$ nix eval --json -f ./export.nix | jq > nginx.json
+```
+
+3. Open the [react-jsonschema-form playground](https://rjsf-team.github.io/react-jsonschema-form/), select `Blank` and paste the `nginx.json` contents.
+
+This provides a quick look at a potential GUI (screenshot is cropped).
+
+![Image title](https://clan.lol/static/blog-post-jsonschema/nginx-gui.jpg)
+
+## Limitations
+
+### Laziness
+
+JSON schema mandates the declaration of all required fields upfront, which might be configured implicitly or remain unused. For instance, `services.nginx.virtualHosts.<name>.sslCertificate` must be specified even if SSL isn’t enabled.
+
+### Limited Types
+
+Certain NixOS module types, like `types.functionTo` and `types.package`, do not map straightforwardly to JSON. For full compatibility, adjustments to NixOS modules might be necessary, such as substituting `listOf package` with `listOf str`.
+
+### Parsing NixOS Modules
+
+Currently, our converter relies on the `options` attribute of evaluated NixOS modules, extracting information from the `type.name` attribute, which is suboptimal. Enhanced introspection capabilities within the NixOS module system would be beneficial.
+
+## Future Prospects
+
+We hope these experiments inspire the community, encourage contributions and further development in this space. Share your ideas and contributions through our issue tracker or matrix channel!
+
+## Links
+
+- [Comments on NixOS Discourse](https://discourse.nixos.org/t/introducing-the-nixos-to-json-schema-converter/45948)
+- [Source Code of the JSON Schema Library](https://git.clan.lol/clan/clan-core/src/branch/main/lib/jsonschema)
+- [Our Issue Tracker](https://git.clan.lol/clan/clan-core/issues)
+- [Our Matrix Channel](https://matrix.to/#/#clan:clan.lol)
+- [react-jsonschema-form Playground](https://rjsf-team.github.io/react-jsonschema-form/)

docs/site/blog/posts/new-docs.md

@@ -10,4 +10,4 @@ Last week, we added a new documentation hub for clan at [docs.clan.lol](https://
 We are still working on improving the installation procedures, so stay tuned.
 We now have weekly office hours where people are invited to hangout and ask questions.
 They are every Wednesday 15:30 UTC (17:30 CEST) in our [jitsi](https://jitsi.lassul.us/clan.lol).
-Otherwise drop by in our [matrix channel](https://matrix.to/#/#clan:lassul.us).
+Otherwise drop by in our [matrix channel](https://matrix.to/#/#clan:clan.lol).

docs/site/drafts/deployment-framework.md

@@ -0,0 +1,63 @@
+---
+title: "Git Based Machine Deployment with Clan-Core"
+description: ""
+authors:
+  - Qubasa
+date: 2024-05-25
+---
+## Revolutionizing Server Management
+
+In the world of server management, countless tools claim to offer seamless deployment of multiple machines. Yet, many fall short, leaving server admins and self-hosting enthusiasts grappling with complexity. Enter the Clan-Core Framework—a groundbreaking all in one solution designed to transform decentralized self-hosting into an effortless and scalable endeavor.
+
+### The Power of Clan-Core
+
+Imagine having the power to manage your servers with unparalleled ease, scaling your IT infrastructure like never before. Clan-Core empowers you to do just that. At its core, Clan-Core leverages a single Git repository to define everything about your machines. This central repository utilizes Nix or JSON files to specify configurations, including disk formatting, ensuring a streamlined and unified approach.
+
+### Simplified Deployment Process
+
+With Clan-Core, the cumbersome task of bootstrapping a specific ISO is a thing of the past. All you need is SSH access to your Linux server. Clan-Core allows you to overwrite any existing Linux distribution live over SSH, eliminating time-consuming setup processes. This capability means you can deploy updates or new configurations swiftly and efficiently, maximizing uptime and minimizing hassle.
+
+### Secure and Efficient Secret Management
+
+Security is paramount in server management, and Clan-Core takes it seriously. Passwords and other sensitive information are encrypted within the Git repository, automatically decrypted during deployment. This not only ensures the safety of your secrets but also simplifies their management. Clan-Core supports sharing secrets with other admins, fostering collaboration and maintaining reproducibillity and security without sacrificing convenience.
+
+### Services as Apps
+
+Setting up a service can be quite difficult. Many server adjustments need to be made, from setting up a database to adjusting webserver configurations and generating the correct private keys. However, Clan-Core aims to make setting up a service as easy as installing an application. Through Clan-Core's Module system, everything down to secrets can be automatically set up. This transforms the often daunting task of service setup into a smooth, automated process, making it accessible to all.
+
+### Decentralized Mesh VPN
+
+Building on these features is a self-configuring decentralized mesh VPN that interconnects all your machines into a private darknet. This ensures that sensitive services, which might have too much attack surface to be hosted on the public internet, can still be made available privately without the need to worry about potential system compromise. By creating a secure, private network, Clan-Core offers an additional layer of protection for your most critical services.
+
+### Decentralized Domain Name System
+
+Current DNS implementations are distributed but not truly decentralized. For Clan-Core, we implemented our own truly decentralized DNS module. This module uses simple flooding and caching algorithms to discover available domains inside the darknet. This approach ensures that your internal domain name system is robust, reliable, and independent of external control, enhancing the resilience and security of your infrastructure.
+
+
+### A New Era of Decentralized Self-Hosting
+
+Clan-Core is more than just a tool; it's a paradigm shift in server management. By consolidating machine definitions, secrets and network configuration, into a single, secure repository, it transforms how you manage and scale your infrastructure. Whether you're a seasoned server admin or a self-hosting enthusiast, Clan-Core offers a powerful, user-friendly solution to take your capabilities to the next level.
+
+
+### Key Features of Clan-Core:
+
+- **Unified Git Repository**: All machine configurations and secrets stored in a single repository.
+- **Live Overwrites**: Deploy configurations over existing Linux distributions via SSH.
+- **Automated Service Setup**: Easily set up services with Clan-Core's Module system.
+- **Decentralized Mesh VPN**: Securely interconnect all machines into a private darknet.
+- **Decentralized DNS**: Robust, independent DNS using flooding and caching algorithms.
+- **Automated Secret Management**: Encrypted secrets that are automatically decrypted during deployment.
+- **Collaboration Support**: Share secrets securely with other admins.
+
+
+## Clan-Cores Future
+
+Our vision for Clan-Core extends far beyond being just another deployment tool. Clan-Core is a framework we've developed to achieve something much greater. We want to put the "personal" back into "personal computing." Our goal is for everyday users to fully customize their phones or laptops and create truly private spaces for friends and family.
+
+Our first major step is to develop a Graphical User Interface (GUI) that makes configuring all this possible. Initial tests have shown that AI can be leveraged as an alternative to traditional GUIs. This paves the way for a future where people can simply talk to their computers, and they will configure themselves according to the users' wishes. 
+
+By adopting Clan, you're not just embracing a tool—you're joining a movement towards a more efficient, secure, and scalable approach to server management. Join us and revolutionize your IT infrastructure today.
+
+
+
+

docs/site/getting-started/backups.md

@@ -98,7 +98,7 @@ Start by indicating where your backup data should be sent. Replace `hostname` wi
 Decide which folders you want to back up. For example, to backup your home and root directories:
 
 ```nix
-{ clanCore.state.userdata.folders = [ "/home" "/root" ]; }
+{ clan.core.state.userdata.folders = [ "/home" "/root" ]; }
 ```
 
 3. **Generate Backup Credentials:**
@@ -116,7 +116,7 @@ On the server where backups will be stored, enable the SSH daemon and set up a r
   services.borgbackup.repos.myhostname = {
     path = "/var/lib/borgbackup/myhostname";
     authorizedKeys = [
-      (builtins.readFile ./machines/myhostname/facts/borgbackup.ssh.pub)
+      (builtins.readFile  (config.clan.core.clanDir + "/machines/myhostname/facts/borgbackup.ssh.pub"))
     ];
   };
 }

docs/site/getting-started/configure.md

@@ -10,7 +10,7 @@ In the `flake.nix` file:
 
     ```nix title="clan-core.lib.buildClan"
     buildClan {
-        # Set a unique name 
+        # Set a unique name
         clanName = "Lobsters";
         # Should usually point to the directory of flake.nix
         directory = ./.;
@@ -30,7 +30,7 @@ In the `flake.nix` file:
 
     ```nix title="clan-core.flakeModules.default"
     clan = {
-        # Set a unique name 
+        # Set a unique name
         clanName = "Lobsters";
 
         machines = {
@@ -59,15 +59,15 @@ Adding or configuring a new machine requires two simple steps:
 
     Which should show something like:
 
-    ```bash hl_lines="6"
+    ```{.shellSession hl_lines="6" .no-copy}
     NAME        ID-LINK                                         FSTYPE   SIZE MOUNTPOINT
-    sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G 
-    ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M 
+    sda         usb-ST_16GB_AA6271026J1000000509-0:0                    14.9G
+    ├─sda1      usb-ST_16GB_AA6271026J1000000509-0:0-part1                 1M
     ├─sda2      usb-ST_16GB_AA6271026J1000000509-0:0-part2      vfat     100M /boot
     └─sda3      usb-ST_16GB_AA6271026J1000000509-0:0-part3      ext4     2.9G /
-    nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G 
-    ├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M 
-    ├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G 
+    nvme0n1     nvme-eui.e8238fa6bf530001001b448b4aec2929              476.9G
+    ├─nvme0n1p1 nvme-eui.e8238fa6bf530001001b448b4aec2929-part1 vfat     512M
+    ├─nvme0n1p2 nvme-eui.e8238fa6bf530001001b448b4aec2929-part2 ext4   459.6G
     └─nvme0n1p3 nvme-eui.e8238fa6bf530001001b448b4aec2929-part3 swap    16.8G
     ```
 
@@ -150,10 +150,17 @@ These steps will allow you to update your machine later.
 Generate the `hardware-configuration.nix` file for your machine by executing the following command:
 
 ```bash
-ssh root@flash-installer.local nixos-generate-config --no-filesystems --show-hardware-config > machines/jon/hardware-configuration.nix
+clan machines hw-generate [MACHINE_NAME] [HOSTNAME]
 ```
 
-This command connects to `flash-installer.local` as `root`, runs `nixos-generate-config` to detect hardware configurations (excluding filesystems), and writes them to `machines/jon/hardware-configuration.nix`.
+replace `[MACHINE_NAME]` with the name of the machine i.e. `jon` and `[HOSTNAME]` with the `ip_adress` or `hostname` of the machine within the network. i.e. `flash-installer.local`
+
+!!! Example
+    ```bash
+    clan machines hw-generate jon flash-installer.local
+    ```
+
+    This command connects to `flash-installer.local` as `root`, runs `nixos-generate-config` to detect hardware configurations (excluding filesystems), and writes them to `machines/jon/hardware-configuration.nix`.
 
 ### Step 3: Custom Disk Formatting
 

docs/site/getting-started/deploy.md

@@ -52,7 +52,7 @@ This process involves preparing a suitable hardware and disk partitioning config
 
     This is an example of the booted installer.
 
-    ```{ .bash .annotate .no-copy }
+    ```{ .bash .annotate .no-copy .nohighlight}
     ┌─────────────────────────────────────────────────────────────────────────────────────┐
     │ ┌───────────────────────────┐                                                       │
     │ │███████████████████████████│ # This is the QR Code (1)                             │
@@ -151,7 +151,7 @@ Clan CLI enables you to remotely update your machines over SSH. This requires se
 ### Setting the Target Host
 
 Replace `root@jon` with the actual hostname or IP address of your target machine:
-```nix hl_lines="9"
+```{.nix hl_lines="9" .no-copy}
 buildClan {
     # ...
     machines = {
@@ -192,7 +192,7 @@ it is also possible to specify a build host instead.
 During an update, the cli will ssh into the build host and run `nixos-rebuild` from there.
 
 
-```nix hl_lines="5"
+```{.nix hl_lines="5" .no-copy}
 buildClan {
     # ...
     machines = {
@@ -208,7 +208,7 @@ buildClan {
 To exclude machines from being updated when running `clan machines update` without any machines specified,
 one can set the `clan.deployment.requireExplicitUpdate` option to true:
 
-```nix hl_lines="5"
+```{.nix hl_lines="5" .no-copy}
 buildClan {
     # ...
     machines = {

docs/site/getting-started/flake-parts.md

@@ -94,9 +94,4 @@ Below is a guide on how to structure this in your flake.nix:
 For detailed information about configuring `flake-parts` and the available options within Clan,
 refer to the Clan module documentation located [here](https://git.clan.lol/clan/clan-core/src/branch/main/flakeModules/clan.nix).
 
-## Whats next?
-
-- [Configure Machines](configure.md): Customize machine configuration
-- [Deploying](deploy.md): Deploying a Machine configuration
-
 ---

docs/site/getting-started/installer.md

@@ -22,7 +22,7 @@ Follow our step-by-step guide to create and transfer this image onto a bootable
     lsblk
     ```
 
-    ```shellSession hl_lines="2"
+    ```{.shellSession hl_lines="2" .no-copy}
     NAME                                          MAJ:MIN RM   SIZE RO TYPE  MOUNTPOINTS
     sdb                                             8:0    1 117,2G  0 disk
     └─sdb1                                          8:1    1 117,2G  0 part  /run/media/qubasa/INTENSO
@@ -42,13 +42,22 @@ sudo umount /dev/sdb1
 === "**Linux OS**"
     ### Step 2. Flash Custom Installer
 
-    Using clan flash enables the inclusion of ssh public keys and disables ssh password authentication.
-    It also includes the language and keymap currently used into the installer image.
+    Using clan flash enables the inclusion of ssh public keys.
+    It also allows to set language and keymap currently in the installer image.
 
     ```bash
-    clan --flake git+https://git.clan.lol/clan/clan-core flash flash-installer --disk main /dev/sd<X>
+    clan flash --flake git+https://git.clan.lol/clan/clan-core \
+      --ssh-pubkey $HOME/.ssh/id_ed25519.pub \
+      --keymap en \
+      --language en \
+      --disk main /dev/sd<X> \
+      flash-installer
     ```
 
+    The `--ssh-pubkey`, `--language` and `--keymap` are optional.
+    Replace `$HOME/.ssh/id_ed25519.pub` with a path to your SSH public key.
+    If you do not have an ssh key yet, you can generate one with `ssh-keygen -t ed25519` command.
+
     !!! Danger "Specifying the wrong device can lead to unrecoverable data loss."
 
         The `clan flash` utility will erase the disk. Make sure to specify the correct device
@@ -123,7 +132,7 @@ This will enter `iwd`
 
 Now run the following command to connect to your Wifi:
 
-```shellSession
+```{.shellSession .no-copy}
 # Identify your network device.
 device list
 
@@ -148,10 +157,10 @@ Connected network     FRITZ!Box (Your router device)
 IPv4 address          192.168.188.50 (Your new local ip)
 ```
 
-Press `ctrl-d` to exit `IWD`.
+Press ++ctrl+d++ to exit `IWD`.
 
 !!! Important
-    Press `ctrl-d` **again** to update the displayed QR code and connection information.
+    Press ++ctrl+d++ **again** to update the displayed QR code and connection information.
 
 You're all set up
 
@@ -161,4 +170,4 @@ You're all set up
 
 - [Configure Machines](configure.md): Customize machine configuration
 
----
+---

docs/site/getting-started/mesh-vpn.md

@@ -9,7 +9,7 @@ include a new machine into the VPN.
 
 By default all machines within one clan are connected via a chosen network technology.
 
-```
+```{.no-copy}
 Clan 
     Node A
     <-> (zerotier / mycelium / ...)
@@ -36,7 +36,7 @@ peers. Once addresses are allocated, the controller's continuous operation is no
    ```
 3. **Update the Controller Machine**: Execute the following:
    ```bash
-   $ clan machines update <CONTROLLER>
+   clan machines update <CONTROLLER>
    ```
    Your machine is now operational as the VPN controller.
 
@@ -48,7 +48,7 @@ To introduce a new machine to the VPN, adhere to the following steps:
    configuration, substituting `<CONTROLLER>` with the controller machine name:
    ```nix
    { config, ... }: {
-     clan.networking.zerotier.networkId = builtins.readFile (config.clanCore.clanDir + "/machines/<CONTROLLER>/facts/zerotier-network-id");
+     clan.networking.zerotier.networkId = builtins.readFile (config.clan.core.clanDir + "/machines/<CONTROLLER>/facts/zerotier-network-id");
    }
    ```
 1. **Update the New Machine**: Execute:

docs/site/getting-started/secrets.md

@@ -4,10 +4,15 @@ Clan enables encryption of secrets (such as passwords & keys) ensuring security
 
 Clan utilizes the [sops](https://github.com/getsops/sops) format and integrates with [sops-nix](https://github.com/Mic92/sops-nix) on NixOS machines.
 
+This guide will walk you through:
 
-### Create Your Master Keypair
+- **Creating a Keypair for Your User**: Learn how to generate a keypair for $USER to securely control all secrets.
+- **Creating Your First Secret**: Step-by-step instructions on creating your initial secret.
+- **Assigning Machine Access to the Secret**: Understand how to grant a machine access to the newly created secret.
 
-To get started, you'll need to create **Your master keypair**.
+## Create Your Admin Keypair
+
+To get started, you'll need to create **Your admin keypair**.
 
 !!! info
     Don't worry — if you've already made one before, this step won't change or overwrite it.
@@ -27,7 +32,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 
 !!! warning
     Make sure to keep a safe backup of the private key you've just created.
-    If it's lost, you won't be able to get to your secrets anymore because they all need the master key to be unlocked.
+    If it's lost, you won't be able to get to your secrets anymore because they all need the admin key to be unlocked.
 
 !!! note
     It's safe to add any secrets created by the clan CLI and placed in your repository to version control systems like `git`.
@@ -35,7 +40,7 @@ Also add your age public key to the repository with 'clan secrets users add YOUR
 ### Add Your Public Key
 
 ```bash
-clan secrets users add <your_username> <your_public_key>
+clan secrets users add $USER <your_public_key>
 ```
 
 It's best to choose the same username as on your Setup/Admin Machine that you use to control the deployment with.
@@ -101,17 +106,20 @@ In your nixos configuration you can get a path to secrets like this `config.sops
 
 ### Assigning Access
 
-By default, secrets are encrypted for your key. To specify which users and machines can access a secret:
+When using `clan secrets set <secret>` without arguments, secrets are encrypted for the key of the user named like your current $USER.
 
-```bash
-clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
-```
-You can also just add machines/users to existing secrets:
+To add machines/users to an existing secret use:
 
 ```bash
  clan secrets machines add-secret <machine_name> <secret_name>
 ```
 
+Alternatively specify users and machines while creating a secret:
+
+```bash
+clan secrets set --machine <machine1> --machine <machine2> --user <user1> --user <user2> <secret_name>
+```
+
 ## Advanced
 
 In this section we go into more advanced secret management topics.
@@ -183,11 +191,9 @@ Since our clan secret module will auto-import secrets that are encrypted for a p
 you can now remove `sops.secrets.<secrets> = { };` unless you need to specify more options for the secret like owner/group of the secret file.
 
 
-
 ## Indepth Explanation
 
 
-
 The secrets system conceptually knows two different entities:
 
 - **Machine**: consumes secrets

docs/site/static/asciinema-player/asciinema-player.css

@@ -0,0 +1 @@
+/nix/store/8y5h98wk5p94mv1wyb2c4gkrr7bswd19-asciinema-player.css

docs/site/static/asciinema-player/asciinema-player.min.js

@@ -0,0 +1 @@
+/nix/store/w0i3f9qzn9n6jmfnfgiw5wnab2f9ssdw-asciinema-player.min.js

docs/site/static/asciinema-player/custom-theme.css

@@ -1,45 +0,0 @@
-.asciinema-player-theme-alabaster-auto {
-        --term-color-foreground: #000000; /* Black for foreground text */
-        --term-color-background: #f7f7f7; /* Very light gray for background */
-
-        --term-color-0: #000000; /* Black */
-        --term-color-1: #aa3731; /* Red */
-        --term-color-2: #448c37; /* Green */
-        --term-color-3: #cb9000; /* Yellow */
-        --term-color-4: #325cc0; /* Blue */
-        --term-color-5: #7a3e9d; /* Magenta */
-        --term-color-6: #0083b2; /* Cyan */
-        --term-color-7: #bbbbbb; /* White */
-        --term-color-8: #777777; /* Bright black (gray) */
-        --term-color-9: #f05050; /* Bright red */
-        --term-color-10: #60cb00; /* Bright green */
-        --term-color-11: #ffbc5d; /* Bright yellow */
-        --term-color-12: #007acc; /* Bright blue */
-        --term-color-13: #e64ce6; /* Bright magenta */
-        --term-color-14: #00aacb; /* Bright cyan */
-        --term-color-15: #ffffff; /* Bright white */
-    }
-
-  @media (prefers-color-scheme: dark) {
-    .asciinema-player-theme-solarized-auto {
-      --term-color-foreground: #839496;
-      --term-color-background: #002b36;
-
-      --term-color-0: #073642;
-      --term-color-1: #dc322f;
-      --term-color-2: #859900;
-      --term-color-3: #b58900;
-      --term-color-4: #268bd2;
-      --term-color-5: #d33682;
-      --term-color-6: #2aa198;
-      --term-color-7: #eee8d5;
-      --term-color-8: #002b36;
-      --term-color-9: #cb4b16;
-      --term-color-10: #586e75;
-      --term-color-11: #657b83;
-      --term-color-12: #839496;
-      --term-color-13: #6c71c4;
-      --term-color-14: #93a1a1;
-      --term-color-15: #fdf6e3;
-    }
-  }

docs/site/static/extra.css

@@ -0,0 +1,13 @@
+@font-face {
+    font-family: "Roboto";
+    src: url(./Roboto-Regular.ttf) format('truetype');
+}
+@font-face {
+    font-family: "Fira Code";
+    src: url(./FiraCode-VF.ttf) format('truetype');
+}
+
+:root {
+    --md-text-font: "Roboto"; 
+    --md-code-font: "Fira Code";
+}

flake.lock

@@ -7,15 +7,15 @@
         ]
       },
       "locked": {
-        "lastModified": 1714400597,
-        "narHash": "sha256-AA1TCyEl4O6+6F5man/V5VH9Zl9HPBpK91tSkZ16i2E=",
-        "owner": "Qubasa",
+        "lastModified": 1717915259,
+        "narHash": "sha256-VsGPboaleIlPELHY5cNTrXK4jHVmgUra8uC6h7KVC5c=",
+        "owner": "nix-community",
         "repo": "disko",
-        "rev": "58785136b8c37aeb2f67081387b48f663b166331",
+        "rev": "1bbdb06f14e2621290b250e631cf3d8948e4d19b",
         "type": "github"
       },
       "original": {
-        "owner": "Qubasa",
+        "owner": "nix-community",
         "repo": "disko",
         "type": "github"
       }
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715865404,
-        "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=",
+        "lastModified": 1717285511,
+        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9",
+        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
         "type": "github"
       },
       "original": {
@@ -55,22 +55,6 @@
         "type": "github"
       }
     },
-    "nixos-2311": {
-      "locked": {
-        "lastModified": 1715818734,
-        "narHash": "sha256-WvAJWCwPj/6quKcsgsvQYyZRxV8ho/yUzj0HZQ34DVU=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "95742536dc6debb5a8b8b78b27001c38f369f1e7",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "nixos-generators": {
       "inputs": {
         "nixlib": "nixlib",
@@ -79,11 +63,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716123454,
-        "narHash": "sha256-U2o4UPM/UsEyIX2p11+YEQgR9HY3PmjZ2mRl/x5e4xo=",
+        "lastModified": 1716210724,
+        "narHash": "sha256-iqQa3omRcHGpWb1ds75jS9ruA5R39FTmAkeR3J+ve1w=",
         "owner": "nix-community",
         "repo": "nixos-generators",
-        "rev": "a63e0c83dd83fe28cc571b97129e13373436bd82",
+        "rev": "d14b286322c7f4f897ca4b1726ce38cb68596c94",
         "type": "github"
       },
       "original": {
@@ -94,17 +78,17 @@
     },
     "nixos-images": {
       "inputs": {
-        "nixos-2311": "nixos-2311",
+        "nixos-stable": [],
         "nixos-unstable": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1716132123,
-        "narHash": "sha256-rATSWbPaKQfZGaemu0tHL2xfCzVIVwpuTjk+KSBC+k4=",
+        "lastModified": 1717770332,
+        "narHash": "sha256-NQmFHj0hTCUgnMAsaNTu6sNTRyo0rFQEe+/lVgV5yxU=",
         "owner": "nix-community",
         "repo": "nixos-images",
-        "rev": "8c9cab8c44434c12dafc465fbf61a710c5bceb08",
+        "rev": "72771bd35f4e19e32d6f652528483b5e07fc317b",
         "type": "github"
       },
       "original": {
@@ -115,11 +99,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1716127062,
-        "narHash": "sha256-2rk8FqB/iQV2d0vQLs684/Tj5PUHaS1sFwG7fng5vXE=",
+        "lastModified": 1717926692,
+        "narHash": "sha256-THcv8qDqobZefHHluPjx/8n+MtVVb8ag/oJbKMqKNRo=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "8a2555763c48e2410054de3f52f7310ce3241ec5",
+        "rev": "7d916e720af6b2ca355e4d0cfb8e4f742c172239",
         "type": "github"
       },
       "original": {
@@ -148,11 +132,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1716087663,
-        "narHash": "sha256-zuSAGlx8Qk0OILGCC2GUyZ58/SJ5R3GZdeUNQ6IS0fQ=",
+        "lastModified": 1717902109,
+        "narHash": "sha256-OQTjaEZcByyVmHwJlKp/8SE9ikC4w+mFd3X0jJs6wiA=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "0bf1808e70ce80046b0cff821c019df2b19aabf5",
+        "rev": "f0922ad001829b400f0160ba85b47d252fa3d925",
         "type": "github"
       },
       "original": {
@@ -168,11 +152,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715940852,
-        "narHash": "sha256-wJqHMg/K6X3JGAE9YLM0LsuKrKb4XiBeVaoeMNlReZg=",
+        "lastModified": 1717850719,
+        "narHash": "sha256-npYqVg+Wk4oxnWrnVG7416fpfrlRhp/lQ6wQ4DHI8YE=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "2fba33a182602b9d49f0b2440513e5ee091d838b",
+        "rev": "4fc1c45a5f50169f9f29f6a98a438fb910b834ed",
         "type": "github"
       },
       "original": {

flake.nix

@@ -8,7 +8,7 @@
 
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
-    disko.url = "github:Qubasa/disko";
+    disko.url = "github:nix-community/disko";
     disko.inputs.nixpkgs.follows = "nixpkgs";
     sops-nix.url = "github:Mic92/sops-nix";
     sops-nix.inputs.nixpkgs.follows = "nixpkgs";
@@ -17,6 +17,8 @@
     nixos-generators.inputs.nixpkgs.follows = "nixpkgs";
     nixos-images.url = "github:nix-community/nixos-images";
     nixos-images.inputs.nixos-unstable.follows = "nixpkgs";
+    # unused input
+    nixos-images.inputs.nixos-stable.follows = "";
     flake-parts.url = "github:hercules-ci/flake-parts";
     flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
     treefmt-nix.url = "github:numtide/treefmt-nix";
@@ -24,10 +26,14 @@
   };
 
   outputs =
-    inputs@{ flake-parts, ... }:
+    inputs@{ flake-parts, self, ... }:
     flake-parts.lib.mkFlake { inherit inputs; } (
       { ... }:
       {
+        clan = {
+          # meta.name = "clan-core";
+          directory = self;
+        };
         systems = [
           "x86_64-linux"
           "aarch64-linux"
@@ -47,6 +53,8 @@
           ./nixosModules/flake-module.nix
           ./pkgs/flake-module.nix
           ./templates/flake-module.nix
+
+          ./inventory/flake-module.nix
         ];
       }
     );

flakeModules/clan.nix

@@ -17,6 +17,33 @@ let
   cfg = config.clan;
 in
 {
+  imports = [
+    # TODO: figure out how to print the deprecation warning
+    # "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix"
+    (lib.mkRenamedOptionModule
+      [
+        "clan"
+        "clanName"
+      ]
+      [
+        "clan"
+        "meta"
+        "name"
+      ]
+    )
+    (lib.mkRenamedOptionModule
+      [
+        "clan"
+        "clanIcon"
+      ]
+      [
+        "clan"
+        "meta"
+        "icon"
+      ]
+    )
+  ];
+
   options.clan = {
     directory = mkOption {
       type = types.path;
@@ -33,15 +60,27 @@ in
       default = { };
       description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
     };
-    clanName = mkOption {
-      type = types.str;
-      description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
-    };
-    clanIcon = mkOption {
-      type = types.nullOr types.path;
-      default = null;
-      description = "A path to an icon to be used for the clan, should be the same for all machines";
+
+    # Checks are performed in 'buildClan'
+    # Not everyone uses flake-parts
+    meta = {
+      name = lib.mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
+      };
+      icon = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = "A path to an icon to be used for the clan in the GUI";
+      };
+      description = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "A short description of the clan";
+      };
     };
+
     pkgsForSystem = mkOption {
       type = types.functionTo types.raw;
       default = _system: null;
@@ -52,6 +91,7 @@ in
     clanInternals = lib.mkOption {
       type = lib.types.submodule {
         options = {
+          meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
           machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
@@ -65,9 +105,8 @@ in
         directory
         specialArgs
         machines
-        clanName
-        clanIcon
         pkgsForSystem
+        meta
         ;
     };
   };

formatter.nix

@@ -10,10 +10,9 @@
       treefmt.programs.mypy.enable = true;
       treefmt.programs.mypy.directories = {
         "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
-        "pkgs/clan-vm-manager".extraPythonPackages =
-          # clan-vm-manager currently only exists on linux 
-          (self'.packages.clan-vm-manager.externalTestDeps or [ ])
-          ++ self'.packages.clan-cli.testDependencies;
+        "pkgs/clan-app".extraPythonPackages =
+          # clan-app currently only exists on linux
+          (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies;
       };
 
       treefmt.settings.formatter.nix = {

inventory/.envrc

@@ -0,0 +1,5 @@
+source_up
+
+watch_file flake-module.nix
+
+use flake .#inventory-schema --builders ''

inventory/README.md

@@ -0,0 +1,57 @@
+# Inventory
+
+This part provides a specification for the inventory.
+
+It is used for design phase and as validation helper.
+
+> Cue is less verbose and easier to understand and maintain than json-schema.
+> Json-schema, if needed can be easily generated on-the fly.
+
+## Checking validity
+
+Directly check a json against the schema
+
+`cue vet inventory.json root.cue -d '#Root'`
+
+## Json schema
+
+Export the json-schema i.e. for usage in python / javascript / nix
+
+`cue export --out openapi root.cue`
+
+## Usage
+
+Comments are rendered as descriptions in the json schema.
+
+```cue
+// A name of the clan (primarily shown by the UI)
+name: string
+```
+
+Cue open sets. In the following `foo = {...}` means that the key `foo` can contain any arbitrary json object.
+
+```cue
+foo: { ... }
+```
+
+Cue dynamic keys.
+
+```cue
+[string]: {
+    attr: string
+}
+```
+
+This is the schema of
+
+```json
+{
+    "a": {
+        "attr": "foo"
+    },
+    "b": {
+        "attr": "bar"
+    }
+    // ... Indefinitely more dynamic keys of type "string"
+}
+```

inventory/example_flake.nix

@@ -0,0 +1,137 @@
+{
+  description = "<Put your description here>";
+
+  inputs.clan-core.url = "https://git.clan.lol/clan/clan-core/archive/main.tar.gz";
+
+  outputs =
+    { clan-core, ... }:
+    let
+      pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
+      system = "x86_64-linux";
+    in
+    # Usage see: https://docs.clan.lol
+    #  nice_flake_interface -> buildInventory() -> Inventory -> buildClanFromInventory() -> nixosConfigurations
+    # buildClanFromInventory = inventory: evalModules {
+    #     extraAttrs = { inherit inventory; };
+    #     # (attrNames inventory.machines)
+    # };
+    # clan =
+    #   clan-core.lib.buildClanFromInventory [
+    #     # Inventory 0 (loads the json file managed by the Python API)
+    #     (builtins.fromJSON (builtins.readFile ./inventory.json))
+    #     # ->
+    #     # {
+    #     #   services."backups_1".autoIncludeMachines = true;
+    #     #   services."backups_1".module = "borgbackup";
+    #     #   ... etc.
+    #     # }
+    #   ]
+    #   ++ (buildInventory {
+    #     clanName = "nice_flake_interface";
+    #     description = "A nice flake interface";
+    #     icon = "assets/icon.png";
+    #     machines = {
+    #       jon = {
+    #         # Just regular nixos/clan configuration ?
+    #         # config = {
+    #         #   imports = [
+    #         #     ./modules/shared.nix
+    #         #     ./machines/jon/configuration.nix
+    #         #   ];
+    #         #   nixpkgs.hostPlatform = system;
+    #         #   # Set this for clan commands use ssh i.e. `clan machines update`
+    #         #   # If you change the hostname, you need to update this line to root@<new-hostname>
+    #         #   # This only works however if you have avahi running on your admin machine else use IP
+    #         #   clan.networking.targetHost = pkgs.lib.mkDefault "root@jon";
+    #         #   # ssh root@flash-installer.local lsblk --output NAME,ID-LINK,FSTYPE,SIZE,MOUNTPOINT
+    #         #   disko.devices.disk.main = {
+    #         #     device = "/dev/disk/by-id/__CHANGE_ME__";
+    #         #   };
+    #         #   # IMPORTANT! Add your SSH key here
+    #         #   # e.g. > cat ~/.ssh/id_ed25519.pub
+    #         #   users.users.root.openssh.authorizedKeys.keys = throw ''
+    #         #     Don't forget to add your SSH key here!
+    #         #     users.users.root.openssh.authorizedKeys.keys = [ "<YOUR SSH_KEY>" ]
+    #         #   '';
+    #         #   # Zerotier needs one controller to accept new nodes. Once accepted
+    #         #   # the controller can be offline and routing still works.
+    #         #   clan.networking.zerotier.controller.enable = true;
+    #         # };
+    #       };
+    #     };
+    #   })
+    #   ++ [
+    #     # Low level inventory overrides (comes at the end)
+    #     {
+    #       services."backups_2".autoIncludeMachines = true;
+    #       services."backups_2".module = "borgbackup";
+    #     }
+    #   ];
+    # # buildClan :: [ Partial<Inventory> ] -> Inventory
+    # # foldl' (acc: v: lib.recursiveUpdate acc v) {} []
+    # inventory = [
+    #   # import json
+    #   {...}
+    #   # power user flake
+    #   {...}
+    # ]
+    # # With Module system
+    # # Pros: Easy to understand,
+    # # Cons: Verbose, hard to maintain
+    # # buildClan :: { modules = [ { config = Partial<Inventory>; options :: InventoryOptions; } } ]; } -> Inventory
+    # eval = lib.evalModules {
+    #   modules = [
+    #     {
+    #       # Inventory Schema
+    #       # Python validation
+    #       options = {...}
+    #     }
+    #     {
+    #       config = map lib.mkDefault
+    #       (builtins.fromJSON (builtins.readFile ./inventory.json))
+    #     }
+    #     {
+    #       #  User provided
+    #       config = {...}
+    #     }
+    #     # Later overrides.
+    #     {
+    #       lib.mkForce ...
+    #     }
+    #   ];
+    # }
+    # nixosConfigurations = lib.evalModules inventory;
+    # eval.config.inventory
+    # #
+    # eval.config.machines.jon#nixosConfig
+    # eval.config.machines.sara#nixosConfig
+    #
+    # {inventory, config, ...}:{
+    #   hostname = config.machines.sara # Invalid
+    #   hostname = inventory.machines.sara.hostname # Valid
+    # }
+    /*
+      # Type
+
+      buildInventory :: {
+        clanName :: string
+        machines :: {
+          ${name} :: {
+            config :: {
+              # NixOS configuration
+            };
+          };
+        };
+        # ... More mapped inventory options
+        # i.e. shared config for all machines
+      } -> Inventory
+    */
+    {
+      # all machines managed by Clan
+      inherit (clan) nixosConfigurations clanInternals;
+      # add the Clan cli tool to the dev shell
+      devShells.${system}.default = pkgs.mkShell {
+        packages = [ clan-core.packages.${system}.clan-cli ];
+      };
+    };
+}

inventory/flake-module.nix

@@ -0,0 +1,46 @@
+{ ... }:
+{
+  perSystem =
+    { pkgs, config, ... }:
+    {
+      packages.inventory-schema = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema";
+        src = ./src;
+
+        buildInputs = [ pkgs.cue ];
+
+        installPhase = ''
+          mkdir -p $out
+        '';
+      };
+      devShells.inventory-schema = pkgs.mkShell { inputsFrom = [ config.packages.inventory-schema ]; };
+
+      checks.inventory-schema-checks = pkgs.stdenv.mkDerivation {
+        name = "inventory-schema-checks";
+        src = ./src;
+        buildInputs = [ pkgs.cue ];
+        buildPhase = ''
+          echo "Running inventory tests..."
+
+          echo "Export cue as json-schema..."
+          cue export --out openapi root.cue
+
+          echo "Validate test/*.json against inventory-schema..."
+
+          test_dir="test"
+          for file in "$test_dir"/*; do
+            # Check if the item is a file
+            if [ -f "$file" ]; then
+              # Print the filename
+              echo "Running test on: $file"
+
+              # Run the cue vet command
+              cue vet "$file" root.cue -d "#Root"
+            fi
+          done
+
+          touch $out
+        '';
+      };
+    };
+}

inventory/src/cue.mod/module.cue

@@ -0,0 +1,2 @@
+module: "clan.lol/inventory"
+language: version: "v0.8.2"

inventory/src/machines/machines.cue

@@ -0,0 +1,8 @@
+package machines
+
+
+#machine: machines: [string]: {
+    name: string,
+    description?: string,
+    icon?: string
+}

inventory/src/root.cue

@@ -0,0 +1,24 @@
+package inventory
+
+import (
+	"clan.lol/inventory/services"
+	"clan.lol/inventory/machines"
+)
+
+@jsonschema(schema="http://json-schema.org/schema#")
+#Root: {
+	meta: {
+		// A name of the clan (primarily shown by the UI)
+		name: string
+		// A description of the clan
+		description?: string
+		// The icon path
+		icon?: string
+	}
+
+	// A map of services
+	services.#service
+
+	// A map of machines
+	machines.#machine
+}

inventory/src/services/services.cue

@@ -0,0 +1,32 @@
+package services
+
+#ServiceRole: "server" | "client" | "both"
+
+#service: services: [string]: {
+    // Required meta fields
+    meta: {
+        name: string,
+        icon?: string
+        description?: string,
+    },
+    // Required module specifies the behavior of the service.
+    module: string,
+
+    // We moved the machine sepcific config to "machines".
+    // It may be moved back depending on what makes more sense in the future.
+    machineConfig: {
+        [string]: {
+            roles?: [ ...#ServiceRole ],
+            config?: {
+                ...
+            }
+        }
+    },
+
+    // Configuration for the service
+    config: {
+        // Schema depends on the module.
+        // It declares the interface how the service can be configured.
+        ...
+    }
+}

inventory/src/tests/borgbackup.json

@@ -0,0 +1,38 @@
+{
+  "machines": {
+    "camina_machine": {
+      "name": "camina"
+    },
+    "vyr_machine": {
+      "name": "vyr"
+    },
+    "vi_machine": {
+      "name": "vi"
+    }
+  },
+  "meta": {
+    "name": "kenjis clan"
+  },
+  "services": {
+    "backup": {
+      "meta": {
+        "name": "My backup"
+      },
+      "module": "borbackup-static",
+      "machineConfig": {
+        "vyr": {
+          "roles": ["server"]
+        },
+        "vi": {
+          "roles": ["client"]
+        },
+        "camina_machine": {
+          "roles": ["client"]
+        }
+      },
+      "config": {
+        "folders": ["/home", "/root", "/var", "/etc"]
+      }
+    }
+  }
+}

inventory/src/tests/syncthing.json

@@ -0,0 +1,45 @@
+{
+  "machines": {
+    "camina_machine": {
+      "name": "camina"
+    },
+    "vyr": {
+      "name": "vyr"
+    },
+    "vi": {
+      "name": "vi"
+    }
+  },
+  "meta": {
+    "name": "kenjis clan"
+  },
+  "services": {
+    "sync_files": {
+      "meta": {
+        "name": "My sync"
+      },
+      "module": "syncthing-static-peers",
+      "machineConfig": {
+        "vyr": {},
+        "vi": {},
+        "camina_machine": {}
+      },
+      "config": {
+        "folders": {
+          "test": {
+            "path": "~/data/docs",
+            "devices": ["camina", "vyr", "vi"]
+          },
+          "videos": {
+            "path": "~/data/videos",
+            "devices": ["camina", "vyr", "ezra"]
+          },
+          "playlist": {
+            "path": "~/data/playlist",
+            "devices": ["camina", "vyr", "ezra"]
+          }
+        }
+      }
+    }
+  }
+}

inventory/src/tests/zerotier.json

@@ -0,0 +1,36 @@
+{
+  "machines": {
+    "camina_machine": {
+      "name": "camina"
+    },
+    "vyr_machine": {
+      "name": "vyr"
+    },
+    "vi_machine": {
+      "name": "vi"
+    }
+  },
+  "meta": {
+    "name": "kenjis clan"
+  },
+  "services": {
+    "backup": {
+      "meta": {
+        "name": "My backup"
+      },
+      "module": "borbackup-static",
+      "machineConfig": {
+        "vyr_machine": {
+          "roles": ["server"]
+        },
+        "vi_machine": {
+          "roles": ["peer"]
+        },
+        "camina_machine": {
+          "roles": ["peer"]
+        }
+      },
+      "config": {}
+    }
+  }
+}

lib/build-clan/default.nix

@@ -7,16 +7,58 @@
   directory, # The directory containing the machines subdirectory
   specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available
   machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... }
-  clanName, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  # DEPRECATED: use meta.name instead
+  clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  # DEPRECATED: use meta.icon instead
   clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
+  meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string
   pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
 # This improves performance, but all nipxkgs.* options will be ignored.
 }:
 let
+  deprecationWarnings = [
+    (lib.warnIf (
+      clanName != null
+    ) "clanName is deprecated, please use meta.name instead. ${clanName}" null)
+    (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null)
+  ];
+
   machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
     builtins.readDir (directory + /machines)
   );
 
+  mergedMeta =
+    let
+      metaFromFile =
+        if (builtins.pathExists "${directory}/clan/meta.json") then
+          let
+            settings = builtins.fromJSON (builtins.readFile "${directory}/clan/meta.json");
+          in
+          settings
+        else
+          { };
+      legacyMeta = lib.filterAttrs (_: v: v != null) {
+        name = clanName;
+        icon = clanIcon;
+      };
+      optionsMeta = lib.filterAttrs (_: v: v != null) meta;
+
+      warnings =
+        builtins.map (
+          name:
+          if
+            metaFromFile.${name} or null != optionsMeta.${name} or null && optionsMeta.${name} or null != null
+          then
+            lib.warn "meta.${name} is set in different places. (exlicit option meta.${name} overrides ${directory}/clan/meta.json)" null
+          else
+            null
+        ) (builtins.attrNames metaFromFile)
+        ++ [ (if (res.name or null == null) then (throw "meta.name should be set") else null) ];
+      res = metaFromFile // legacyMeta // optionsMeta;
+    in
+    # Print out warnings before returning the merged result
+    builtins.deepSeq warnings res;
+
   machineSettings =
     machineName:
     # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
@@ -58,11 +100,15 @@ let
           (machines.${name} or { })
           (
             {
+              # Settings
+              clan.core.clanDir = directory;
+              # Inherited from clan wide settings
+              clan.core.clanName = meta.name or clanName;
+              clan.core.clanIcon = meta.icon or clanIcon;
+
+              # Machine specific settings
+              clan.core.machineName = name;
               networking.hostName = lib.mkDefault name;
-              clanCore.clanName = clanName;
-              clanCore.clanIcon = clanIcon;
-              clanCore.clanDir = directory;
-              clanCore.machineName = name;
               nixpkgs.hostPlatform = lib.mkDefault system;
 
               # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
@@ -127,10 +173,15 @@ let
     ) supportedSystems
   );
 in
-{
+builtins.deepSeq deprecationWarnings {
   inherit nixosConfigurations;
 
   clanInternals = {
+    # Evaluated clan meta
+    # Merged /clan/meta.json with overrides from buildClan
+    meta = mergedMeta;
+
+    # machine specifics
     machines = configsPerSystem;
     machinesFunc = configsFuncPerSystem;
     all-machines-json = lib.mapAttrs (

lib/jsonschema/default.nix

@@ -83,20 +83,20 @@ rec {
     in
 
     # either type
-    # TODO: if all nested optiosn are excluded, the parent sould be excluded too
+    # TODO: if all nested options are excluded, the parent should be excluded too
     if
-      option.type.name or null == "either"
+      option.type.name or null == "either" || option.type.name or null == "coercedTo"
     # return jsonschema property definition for either
     then
       let
         optionsList' = [
           {
-            type = option.type.nestedTypes.left;
+            type = option.type.nestedTypes.left or option.type.nestedTypes.coercedType;
             _type = "option";
             loc = option.loc;
           }
           {
-            type = option.type.nestedTypes.right;
+            type = option.type.nestedTypes.right or option.type.nestedTypes.finalType;
             _type = "option";
             loc = option.loc;
           }
@@ -157,12 +157,21 @@ rec {
 
     # TODO: Add support for intMatching in jsonschema
     # parse port type aka. "unsignedInt16"
-    else if option.type.name == "unsignedInt16" then
+    else if
+      option.type.name == "unsignedInt16"
+      || option.type.name == "unsignedInt"
+      || option.type.name == "pkcs11"
+      || option.type.name == "intBetween"
+    then
       default // example // description // { type = "integer"; }
 
     # parse string
+    # TODO: parse more precise string types
     else if
       option.type.name == "str"
+      || option.type.name == "singleLineStr"
+      || option.type.name == "passwdEntry str"
+      || option.type.name == "passwdEntry path"
     # return jsonschema property definition for string
     then
       default // example // description // { type = "string"; }

nixosModules/clanCore/backups.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 {
   imports = [ ./state.nix ];
-  options.clanCore.backups = {
+  options.clan.core.backups = {
     providers = lib.mkOption {
       type = lib.types.attrsOf (
         lib.types.submodule (

nixosModules/clanCore/facts/compat.nix

@@ -2,37 +2,43 @@
 {
   imports = [
     (lib.mkRemovedOptionModule [
-      "clanCore"
+      "clan"
+      "core"
       "secretsPrefix"
     ] "secretsPrefix was only used by the sops module and the code is now integrated in there")
     (lib.mkRenamedOptionModule
       [
-        "clanCore"
+        "clan"
+        "core"
         "secretStore"
       ]
       [
-        "clanCore"
+        "clan"
+        "core"
         "facts"
         "secretStore"
       ]
     )
     (lib.mkRemovedOptionModule [
-      "clanCore"
+      "clan"
+      "core"
       "secretsDirectory"
-    ] "clancore.secretsDirectory was removed. Use clanCore.facts.secretPathFunction instead")
+    ] "clan.core.secretsDirectory was removed. Use clan.core.facts.secretPathFunction instead")
     (lib.mkRenamedOptionModule
       [
-        "clanCore"
+        "clan"
+        "core"
         "secretsUploadDirectory"
       ]
       [
-        "clanCore"
+        "clan"
+        "core"
         "facts"
         "secretUploadDirectory"
       ]
     )
   ];
-  options.clanCore.secrets = lib.mkOption {
+  options.clan.core.secrets = lib.mkOption {
     visible = false;
     default = { };
     type = lib.types.attrsOf (
@@ -97,14 +103,14 @@
                       description = ''
                         path to a secret which is generated by the generator
                       '';
-                      default = config.clanCore.facts.secretPathFunction secret;
-                      defaultText = lib.literalExpression "config.clanCore.facts.secretPathFunction secret";
+                      default = config.clan.core.facts.secretPathFunction secret;
+                      defaultText = lib.literalExpression "config.clan.core.facts.secretPathFunction secret";
                     };
                   }
-                  // lib.optionalAttrs (config.clanCore.facts.secretStore == "sops") {
+                  // lib.optionalAttrs (config.clan.core.facts.secretStore == "sops") {
                     groups = lib.mkOption {
                       type = lib.types.listOf lib.types.str;
-                      default = config.clanCore.sops.defaultGroups;
+                      default = config.clan.core.sops.defaultGroups;
                       description = ''
                         Groups to decrypt the secret for. By default we always use the user's key.
                       '';
@@ -134,12 +140,12 @@
                       path to a fact which is generated by the generator
                     '';
                     default =
-                      config.clanCore.clanDir
-                      + "/machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}";
-                    defaultText = lib.literalExpression "\${config.clanCore.clanDir}/machines/\${config.clanCore.machineName}/facts/\${fact.config._module.args.name}";
+                      config.clan.core.clanDir
+                      + "/machines/${config.clan.core.machineName}/facts/${fact.config._module.args.name}";
+                    defaultText = lib.literalExpression "\${config.clan.core.clanDir}/machines/\${config.clan.core.machineName}/facts/\${fact.config._module.args.name}";
                   };
                   value = lib.mkOption {
-                    defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
+                    defaultText = lib.literalExpression "\${config.clan.core.clanDir}/\${fact.config.path}";
                     type = lib.types.nullOr lib.types.str;
                     default =
                       if builtins.pathExists fact.config.path then lib.strings.fileContents fact.config.path else null;
@@ -152,16 +158,16 @@
       })
     );
   };
-  config = lib.mkIf (config.clanCore.secrets != { }) {
-    clanCore.facts.services = lib.mapAttrs' (
+  config = lib.mkIf (config.clan.core.secrets != { }) {
+    clan.core.facts.services = lib.mapAttrs' (
       name: service:
-      lib.warn "clanCore.secrets.${name} is deprecated, use clanCore.facts.services.${name} instead" (
+      lib.warn "clan.core.secrets.${name} is deprecated, use clan.core.facts.services.${name} instead" (
         lib.nameValuePair name ({
           secret = service.secrets;
           public = service.facts;
           generator = service.generator;
         })
       )
-    ) config.clanCore.secrets;
+    ) config.clan.core.secrets;
   };
 }

nixosModules/clanCore/facts/default.nix

@@ -5,7 +5,7 @@
   ...
 }:
 {
-  options.clanCore.facts = {
+  options.clan.core.facts = {
     secretStore = lib.mkOption {
       type = lib.types.enum [
         "sops"
@@ -115,6 +115,7 @@
                       type = lib.types.str;
                       readOnly = true;
                       internal = true;
+                      defaultText = "read only script";
                       default = ''
                         set -eu -o pipefail
 
@@ -122,13 +123,23 @@
 
                         # prepare sandbox user
                         mkdir -p /etc
-                        cp ${
-                          pkgs.runCommand "fake-etc" { } ''
-                            export PATH="${pkgs.coreutils}/bin"
-                            mkdir -p $out
-                            cp /etc/* $out/
-                          ''
-                        }/* /etc/
+
+                        cat > /etc/group <<EOF
+                        root:x:0:
+                        nixbld:!:$(id -g):
+                        nogroup:x:65534:
+                        EOF
+
+                        cat > /etc/passwd <<EOF
+                        root:x:0:0:Nix build user:/build:/noshell
+                        nixbld:x:$(id -u):$(id -g):Nix build user:/build:/noshell
+                        nobody:x:65534:65534:Nobody:/:/noshell
+                        EOF
+
+                        cat > /etc/hosts <<EOF
+                        127.0.0.1 localhost
+                        ::1 localhost
+                        EOF
 
                         ${config.script}
                       '';
@@ -155,13 +166,13 @@
                         description = ''
                           path to a secret which is generated by the generator
                         '';
-                        default = config.clanCore.facts.secretPathFunction secret;
+                        default = config.clan.core.facts.secretPathFunction secret;
                       };
                     }
-                    // lib.optionalAttrs (config.clanCore.facts.secretModule == "clan_cli.facts.secret_modules.sops") {
+                    // lib.optionalAttrs (config.clan.core.facts.secretModule == "clan_cli.facts.secret_modules.sops") {
                       groups = lib.mkOption {
                         type = lib.types.listOf lib.types.str;
-                        default = config.clanCore.sops.defaultGroups;
+                        default = config.clan.core.sops.defaultGroups;
                         description = ''
                           Groups to decrypt the secret for. By default we always use the user's key.
                         '';
@@ -190,12 +201,12 @@
                       description = ''
                         path to a fact which is generated by the generator
                       '';
-                      defaultText = lib.literalExpression "\${config.clanCore.clanDir}/machines/\${config.clanCore.machineName}/facts/\${fact.config.name}";
+                      defaultText = lib.literalExpression "\${config.clan.core.clanDir}/machines/\${config.clan.core.machineName}/facts/\${fact.config.name}";
                       default =
-                        config.clanCore.clanDir + "/machines/${config.clanCore.machineName}/facts/${fact.config.name}";
+                        config.clan.core.clanDir + "/machines/${config.clan.core.machineName}/facts/${fact.config.name}";
                     };
                     value = lib.mkOption {
-                      defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
+                      defaultText = lib.literalExpression "\${config.clan.core.clanDir}/\${fact.config.path}";
                       type = lib.types.nullOr lib.types.str;
                       default =
                         if builtins.pathExists fact.config.path then lib.strings.fileContents fact.config.path else null;
@@ -218,5 +229,15 @@
 
     ./public/in_repo.nix
     ./public/vm.nix
+
+    # (lib.mkRenamedOptionModule
+    #   [
+    #     "clanCore"
+    #   ]
+    #   [
+    #     "clan"
+    #     "core"
+    #   ]
+    # )
   ];
 }