Merge branch 'main' into add-vale

checks/flake-module.nix

@@ -44,6 +44,7 @@
             zt-tcp-relay = import ./zt-tcp-relay nixosTestArgs;
             borgbackup = import ./borgbackup nixosTestArgs;
             syncthing = import ./syncthing nixosTestArgs;
+            postgresql = import ./postgresql nixosTestArgs;
             wayland-proxy-virtwl = import ./wayland-proxy-virtwl nixosTestArgs;
           };
 

checks/lib/container-test.nix

@@ -10,6 +10,7 @@ in
     hostPkgs = pkgs;
     # speed-up evaluation
     defaults = {
+      nix.package = pkgs.nixVersions.latest;
       documentation.enable = lib.mkDefault false;
       boot.isContainer = true;
 

checks/lib/test-base.nix

@@ -10,6 +10,7 @@ in
   defaults = {
     documentation.enable = lib.mkDefault false;
     nix.settings.min-free = 0;
+    nix.package = pkgs.nixVersions.latest;
   };
 
   # to accept external dependencies such as disko

checks/matrix-synapse/default.nix

@@ -13,7 +13,6 @@
             clanCore.machineName = "machine";
             clanCore.clanDir = ./.;
             clan.matrix-synapse = {
-              enable = true;
               domain = "clan.test";
             };
           }

checks/postgresql/default.nix

@@ -0,0 +1,67 @@
+(import ../lib/container-test.nix) ({
+  name = "postgresql";
+
+  nodes.machine =
+    { self, config, ... }:
+    {
+      imports = [
+        self.nixosModules.clanCore
+        self.clanModules.postgresql
+        self.clanModules.localbackup
+      ];
+      clan.postgresql.users.test = { };
+      clan.postgresql.databases.test.create.options.OWNER = "test";
+      clan.postgresql.databases.test.restore.stopOnRestore = [ "sample-service" ];
+      clan.localbackup.targets.hdd.directory = "/mnt/external-disk";
+
+      systemd.services.sample-service = {
+        wantedBy = [ "multi-user.target" ];
+        script = ''
+          while true; do
+            echo "Hello, world!"
+            sleep 5
+          done
+        '';
+      };
+
+      environment.systemPackages = [ config.services.postgresql.package ];
+    };
+  testScript =
+    { nodes, ... }:
+    ''
+      start_all()
+      machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("sample-service")
+      # Create a test table
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -c 'CREATE TABLE test (id serial PRIMARY KEY);' test")
+
+      machine.succeed("/run/current-system/sw/bin/localbackup-create >&2")
+      timestamp_before = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+
+      machine.succeed("test -e /mnt/external-disk/snapshot.0/machine/var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'INSERT INTO test DEFAULT VALUES;'")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'DROP TABLE test;'")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("rm -rf /var/backup/postgres")
+
+      machine.succeed("NAME=/mnt/external-disk/snapshot.0 FOLDERS=/var/backup/postgres/test /run/current-system/sw/bin/localbackup-restore >&2")
+      machine.succeed("test -e /var/backup/postgres/test/pg-dump || { echo 'pg-dump not found'; exit 1; }")
+
+      machine.succeed("""
+      set -x
+      ${nodes.machine.clanCore.state.postgresql-test.postRestoreCommand}
+      """)
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -l >&2")
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c '\dt' >&2")
+
+      timestamp_after = int(machine.succeed("systemctl show --property=ExecMainStartTimestampMonotonic sample-service | cut -d= -f2").strip())
+      assert timestamp_before < timestamp_after, f"{timestamp_before} >= {timestamp_after}: expected sample-service to be restarted after restore"
+
+      # Check that the table is still there
+      machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql -d test -c 'SELECT * FROM test;'")
+      output = machine.succeed("runuser -u postgres -- /run/current-system/sw/bin/psql --csv -c \"SELECT datdba::regrole FROM pg_database WHERE datname = 'test'\"")
+      owner = output.split("\n")[1]
+      assert owner == "test", f"Expected database owner to be 'test', got '{owner}'"
+    '';
+})

clanModules/borgbackup-static/README.md

@@ -0,0 +1,10 @@
+Statically configure borgbackup with sane defaults.
+---
+This module implements the `borgbackup` backend and implements sane defaults 
+for backup management through `borgbackup` for members of the clan.
+
+Configure target machines where the backups should be sent to through `targets`.
+
+Configure machines that should be backuped either through `includeMachines`
+which will exclusively add the included machines to be backuped, or through 
+`excludeMachines`, which will add every machine except the excluded machine to the backup.

clanModules/borgbackup-static/default.nix

@@ -0,0 +1,99 @@
+{ lib, config, ... }:
+let
+  clanDir = config.clanCore.clanDir;
+  machineDir = clanDir + "/machines/";
+in
+{
+  imports = [ ../borgbackup ];
+
+  options.clan.borgbackup-static = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clanCore.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be backuped.
+        Mutually exclusive with includeMachines.
+        If this is not empty, every other machine except the targets in the clan will be backuped by this module.
+        If includeMachines is set, only the included machines will be backuped.
+      '';
+    };
+    includeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clanCore.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should be backuped.
+        Mutually exclusive with excludeMachines.
+      '';
+    };
+    targets = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Machines that should act as target machines for backups.
+      '';
+    };
+  };
+
+  config.services.borgbackup.repos =
+    let
+      machines = builtins.readDir machineDir;
+      borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
+      filteredMachines =
+        if ((builtins.length config.clan.borgbackup-static.includeMachines) != 0) then
+          lib.filterAttrs (name: _: (lib.elem name config.clan.borgbackup-static.includeMachines)) machines
+        else
+          lib.filterAttrs (name: _: !(lib.elem name config.clan.borgbackup-static.excludeMachines)) machines;
+      machinesMaybeKey = lib.mapAttrsToList (
+        machine: _:
+        let
+          fullPath = borgbackupIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) filteredMachines;
+      machinesWithKey = lib.filter (x: x != null) machinesMaybeKey;
+      hosts = builtins.map (machine: {
+        name = machine;
+        value = {
+          path = "/var/lib/borgbackup/${machine}";
+          authorizedKeys = [ (builtins.readFile (borgbackupIpMachinePath machine)) ];
+        };
+      }) machinesWithKey;
+    in
+    lib.mkIf
+      (builtins.any (target: target == config.clanCore.machineName) config.clan.borgbackup-static.targets)
+      (if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { });
+
+  config.clan.borgbackup.destinations =
+    let
+      destinations = builtins.map (d: {
+        name = d;
+        value = {
+          repo = "borg@${d}:/var/lib/borgbackup/${config.clanCore.machineName}";
+        };
+      }) config.clan.borgbackup-static.targets;
+    in
+    lib.mkIf (builtins.any (
+      target: target == config.clanCore.machineName
+    ) config.clan.borgbackup-static.includeMachines) (builtins.listToAttrs destinations);
+
+  config.assertions = [
+    {
+      assertion =
+        !(
+          ((builtins.length config.clan.borgbackup-static.excludeMachines) != 0)
+          && ((builtins.length config.clan.borgbackup-static.includeMachines) != 0)
+        );
+      message = ''
+        The options:
+        config.clan.borgbackup-static.excludeMachines = [${builtins.toString config.clan.borgbackup-static.excludeMachines}]
+        and
+        config.clan.borgbackup-static.includeMachines = [${builtins.toString config.clan.borgbackup-static.includeMachines}]
+        are mutually exclusive.
+        Use excludeMachines to exclude certain machines and backup the other clan machines.
+        Use include machines to only backup certain machines.
+      '';
+    }
+  ];
+}

clanModules/borgbackup/default.nix

@@ -6,6 +6,27 @@
 }:
 let
   cfg = config.clan.borgbackup;
+  preBackupScript = ''
+    declare -A preCommandErrors
+
+    ${lib.concatMapStringsSep "\n" (
+      state:
+      lib.optionalString (state.preBackupCommand != null) ''
+        echo "Running pre-backup command for ${state.name}"
+        if ! ( ${state.preBackupCommand} ) then
+          preCommandErrors["${state.name}"]=1
+        fi
+      ''
+    ) (lib.attrValues config.clanCore.state)}
+
+    if [[ ''${#preCommandErrors[@]} -gt 0 ]]; then
+      echo "PreBackupCommand failed for the following services:"
+      for state in "''${!preCommandErrors[@]}"; do
+        echo "  $state"
+      done
+      exit 1
+    fi
+  '';
 in
 {
   options.clan.borgbackup.destinations = lib.mkOption {
@@ -50,17 +71,26 @@ in
   ];
 
   config = lib.mkIf (cfg.destinations != { }) {
+    systemd.services = lib.mapAttrs' (
+      _: dest:
+      lib.nameValuePair "borgbackup-job-${dest.name}" {
+        # since borgbackup mounts the system read-only, we need to run in a ExecStartPre script, so we can generate additional files.
+        serviceConfig.ExecStartPre = [
+          (''+${pkgs.writeShellScript "borgbackup-job-${dest.name}-pre-backup-commands" preBackupScript}'')
+        ];
+      }
+    ) cfg.destinations;
+
     services.borgbackup.jobs = lib.mapAttrs (_: dest: {
-      paths = lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state));
+      paths = lib.unique (
+        lib.flatten (map (state: state.folders) (lib.attrValues config.clanCore.state))
+      );
       exclude = [ "*.pyc" ];
       repo = dest.repo;
       environment.BORG_RSH = dest.rsh;
       compression = "auto,zstd";
       startAt = "*-*-* 01:00:00";
       persistentTimer = true;
-      preHook = ''
-        set -x
-      '';
 
       encryption = {
         mode = "repokey";
@@ -111,7 +141,7 @@ in
       (pkgs.writeShellScriptBin "borgbackup-restore" ''
         set -efux
         cd /
-        IFS=';' read -ra FOLDER <<< "$FOLDERS"
+        IFS=':' read -ra FOLDER <<< "$FOLDERS"
         job_name=$(echo "$NAME" | ${pkgs.gawk}/bin/awk -F'::' '{print $1}')
         backup_name=''${NAME#"$job_name"::}
         if ! command -v borg-job-"$job_name" &> /dev/null; then

clanModules/flake-module.nix

@@ -5,17 +5,20 @@
       imports = [ ./disk-layouts ];
     };
     borgbackup = ./borgbackup;
+    borgbackup-static = ./borgbackup-static;
     deltachat = ./deltachat;
     ergochat = ./ergochat;
     localbackup = ./localbackup;
     localsend = ./localsend;
     matrix-synapse = ./matrix-synapse;
     moonlight = ./moonlight;
+    postgresql = ./postgresql;
     root-password = ./root-password;
     sshd = ./sshd;
     sunshine = ./sunshine;
     static-hosts = ./static-hosts;
     syncthing = ./syncthing;
+    syncthing-static-peers = ./syncthing-static-peers;
     thelounge = ./thelounge;
     trusted-nix-caches = ./trusted-nix-caches;
     user-password = ./user-password;

clanModules/localbackup/default.nix

@@ -6,7 +6,10 @@
 }:
 let
   cfg = config.clan.localbackup;
-  rsnapshotConfig = target: states: ''
+  uniqueFolders = lib.unique (
+    lib.flatten (lib.mapAttrsToList (_name: state: state.folders) config.clanCore.state)
+  );
+  rsnapshotConfig = target: ''
     config_version	1.2
     snapshot_root	${target.directory}
     sync_first	1
@@ -17,12 +20,6 @@ let
     cmd_logger	${pkgs.inetutils}/bin/logger
     cmd_du	${pkgs.coreutils}/bin/du
     cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
-    ${lib.optionalString (target.preBackupHook != null) ''
-      cmd_preexec	${pkgs.writeShellScript "preexec.sh" ''
-        set -efu -o pipefail
-        ${target.preBackupHook}
-      ''}
-    ''}
 
     ${lib.optionalString (target.postBackupHook != null) ''
       cmd_postexec	${pkgs.writeShellScript "postexec.sh" ''
@@ -31,11 +28,9 @@ let
       ''}
     ''}
     retain	snapshot	${builtins.toString config.clan.localbackup.snapshots}
-    ${lib.concatMapStringsSep "\n" (state: ''
-      ${lib.concatMapStringsSep "\n" (folder: ''
-        backup	${folder}	${config.networking.hostName}/
-      '') state.folders}
-    '') states}
+    ${lib.concatMapStringsSep "\n" (folder: ''
+      backup	${folder}	${config.networking.hostName}/
+    '') uniqueFolders}
   '';
 in
 {
@@ -129,14 +124,30 @@ in
               ]
             }
             ${lib.concatMapStringsSep "\n" (target: ''
-              (
-                ${mountHook target}
-                echo "Creating backup '${target.name}'"
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" sync
-                rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target (lib.attrValues config.clanCore.state))}" snapshot
-              )
-            '') (builtins.attrValues cfg.targets)}
-          '')
+              ${mountHook target}
+              set -x
+              echo "Creating backup '${target.name}'"
+
+              ${lib.optionalString (target.preBackupHook != null) ''
+                (
+                  ${target.preBackupHook}
+                )
+              ''}
+
+              declare -A preCommandErrors
+              ${lib.concatMapStringsSep "\n" (
+                state:
+                lib.optionalString (state.preBackupCommand != null) ''
+                  echo "Running pre-backup command for ${state.name}"
+                  if ! ( ${state.preBackupCommand} ) then
+                    preCommandErrors["${state.name}"]=1
+                  fi
+                ''
+              ) (builtins.attrValues config.clanCore.state)}
+
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" sync
+              rsnapshot -c "${pkgs.writeText "rsnapshot.conf" (rsnapshotConfig target)}" snapshot
+            '') (builtins.attrValues cfg.targets)}'')
           (pkgs.writeShellScriptBin "localbackup-list" ''
             set -efu -o pipefail
             export PATH=${
@@ -167,6 +178,14 @@ in
                 pkgs.gawk
               ]
             }
+            if [[ "''${NAME:-}" == "" ]]; then
+              echo "No backup name given via NAME environment variable"
+              exit 1
+            fi
+            if [[ "''${FOLDERS:-}" == "" ]]; then
+              echo "No folders given via FOLDERS environment variable"
+              exit 1
+            fi
             name=$(awk -F'::' '{print $1}' <<< $NAME)
             backupname=''${NAME#$name::}
 
@@ -182,8 +201,9 @@ in
               exit 1
             fi
 
-            IFS=';' read -ra FOLDER <<< "$FOLDERS"
+            IFS=':' read -ra FOLDER <<< "''$FOLDERS"
             for folder in "''${FOLDER[@]}"; do
+              mkdir -p "$folder"
               rsync -a "$backupname/${config.networking.hostName}$folder/" "$folder"
             done
           '')

clanModules/matrix-synapse/default.nix

@@ -6,16 +6,35 @@
 }:
 let
   cfg = config.clan.matrix-synapse;
+  nginx-vhost = "matrix.${config.clan.matrix-synapse.domain}";
+  element-web =
+    pkgs.runCommand "element-web-with-config" { nativeBuildInputs = [ pkgs.buildPackages.jq ]; }
+      ''
+        cp -r ${pkgs.element-web} $out
+        chmod -R u+w $out
+        jq '."default_server_config"."m.homeserver" = { "base_url": "https://${nginx-vhost}:443", "server_name": "${config.clan.matrix-synapse.domain}" }' \
+          > $out/config.json < ${pkgs.element-web}/config.json
+        ln -s $out/config.json $out/config.${nginx-vhost}.json
+      '';
 in
 {
   options.clan.matrix-synapse = {
-    enable = lib.mkEnableOption "Enable matrix-synapse";
     domain = lib.mkOption {
       type = lib.types.str;
       description = "The domain name of the matrix server";
+      example = "example.com";
     };
   };
-  config = lib.mkIf cfg.enable {
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "clan"
+      "matrix-synapse"
+      "enable"
+    ] "Importing the module will already enable the service.")
+
+    ../postgresql
+  ];
+  config = {
     services.matrix-synapse = {
       enable = true;
       settings = {
@@ -49,16 +68,27 @@ in
           }
         ];
       };
-      extraConfigFiles = [ "/var/lib/matrix-synapse/registration_shared_secret.yaml" ];
+      extraConfigFiles = [ "/run/synapse-registration-shared-secret.yaml" ];
+    };
+    systemd.tmpfiles.settings."synapse" = {
+      "/run/synapse-registration-shared-secret.yaml" = {
+        C.argument =
+          config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
+        z = {
+          mode = "0400";
+          user = "matrix-synapse";
+        };
+      };
+    };
+
+    clan.postgresql.users.matrix-synapse = { };
+    clan.postgresql.databases.matrix-synapse.create.options = {
+      TEMPLATE = "template0";
+      LC_COLLATE = "C";
+      LC_CTYPE = "C";
+      ENCODING = "UTF8";
+      OWNER = "matrix-synapse";
     };
-    systemd.services.matrix-synapse.serviceConfig.ExecStartPre = [
-      "+${pkgs.writeScript "copy_registration_shared_secret" ''
-        #!/bin/sh
-        cp ${config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path} /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chown matrix-synapse:matrix-synapse /var/lib/matrix-synapse/registration_shared_secret.yaml
-        chmod 600 /var/lib/matrix-synapse/registration_shared_secret.yaml
-      ''}"
-    ];
 
     clanCore.facts.services."matrix-synapse" = {
       secret."synapse-registration_shared_secret" = { };
@@ -71,23 +101,6 @@ in
       '';
     };
 
-    services.postgresql.enable = true;
-    # we need to use both ensusureDatabases and initialScript, because the former runs everytime but with the wrong collation
-    services.postgresql = {
-      ensureDatabases = [ "matrix-synapse" ];
-      ensureUsers = [
-        {
-          name = "matrix-synapse";
-          ensureDBOwnership = true;
-        }
-      ];
-      initialScript = pkgs.writeText "synapse-init.sql" ''
-        CREATE DATABASE "matrix-synapse"
-          TEMPLATE template0
-          LC_COLLATE = "C"
-          LC_CTYPE = "C";
-      '';
-    };
     services.nginx = {
       enable = true;
       virtualHosts = {
@@ -102,7 +115,7 @@ in
             return 200 '${
               builtins.toJSON {
                 "m.homeserver" = {
-                  "base_url" = "https://matrix.${cfg.domain}";
+                  "base_url" = "https://${nginx-vhost}";
                 };
                 "m.identity_server" = {
                   "base_url" = "https://vector.im";
@@ -111,15 +124,12 @@ in
             }';
           '';
         };
-        "matrix.${cfg.domain}" = {
+        ${nginx-vhost} = {
           forceSSL = true;
           enableACME = true;
-          locations."/_matrix" = {
-            proxyPass = "http://localhost:8008";
-          };
-          locations."/test".extraConfig = ''
-            return 200 "Hello, world!";
-          '';
+          locations."/_matrix".proxyPass = "http://localhost:8008";
+          locations."/_synapse".proxyPass = "http://localhost:8008";
+          locations."/".root = element-web;
         };
       };
     };

clanModules/postgresql/README.md

@@ -0,0 +1,2 @@
+A free and open-source relational database management system (RDBMS) emphasizing extensibility and SQL compliance.
+---

clanModules/postgresql/default.nix

@@ -0,0 +1,166 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  createDatatbaseState =
+    db:
+    let
+      folder = "/var/backup/postgres/${db.name}";
+      current = "${folder}/pg-dump";
+      compression = lib.optionalString (lib.versionAtLeast config.services.postgresql.package.version "16") "--compress=zstd";
+    in
+    {
+      folders = [ folder ];
+      preBackupCommand = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+
+        mkdir -p "${folder}"
+        runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
+        mv "${current}.tmp" ${current}
+      '';
+      postRestoreCommand = ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
+            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
+
+            mkdir -p "${folder}"
+            runuser -u postgres -- dropdb "${db.name}"
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      '';
+    };
+
+  createDatabase = db: ''
+    CREATE DATABASE "${db.name}" ${
+      lib.concatStringsSep " " (
+        lib.mapAttrsToList (name: value: "${name} = '${value}'") db.create.options
+      )
+    }
+  '';
+  cfg = config.clan.postgresql;
+
+  userClauses = lib.mapAttrsToList (
+    _: user:
+    ''$PSQL -tAc "SELECT 1 FROM pg_roles WHERE rolname='${user.name}'" | grep -q 1 || $PSQL -tAc 'CREATE USER "${user.name}"' ''
+  ) cfg.users;
+  databaseClauses = lib.mapAttrsToList (
+    name: db:
+    lib.optionalString db.create.enable ''$PSQL -d postgres -c "SELECT 1 FROM pg_database WHERE datname = '${name}'" | grep -q 1 || $PSQL -d postgres -c ${lib.escapeShellArg (createDatabase db)} ''
+  ) cfg.databases;
+in
+{
+  options.clan.postgresql = {
+    # we are reimplemeting ensureDatabase and ensureUser options here to allow to create databases with options
+    databases = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+              };
+              # set to false, in case the upstream module uses ensureDatabase option
+              create.enable = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = "Create the database if it does not exist.";
+              };
+              create.options = lib.mkOption {
+                type = lib.types.lazyAttrsOf lib.types.str;
+                default = { };
+                example = {
+                  TEMPLATE = "template0";
+                  LC_COLLATE = "C";
+                  LC_CTYPE = "C";
+                  ENCODING = "UTF8";
+                  OWNER = "foo";
+                };
+              };
+              restore.stopOnRestore = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                description = "List of services to stop before restoring the database.";
+              };
+            };
+          }
+        )
+      );
+    };
+    users = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options.name = lib.mkOption {
+              type = lib.types.str;
+              default = name;
+            };
+          }
+        )
+      );
+    };
+  };
+  config = {
+    services.postgresql.settings = {
+      wal_level = "replica";
+      max_wal_senders = 3;
+    };
+
+    services.postgresql.enable = true;
+    # We are duplicating a bit the upstream module but allow to create databases with options
+    systemd.services.postgresql.postStart = ''
+      PSQL="psql --port=${builtins.toString config.services.postgresql.settings.port}"
+
+      while ! $PSQL -d postgres -c "" 2> /dev/null; do
+        if ! kill -0 "$MAINPID"; then exit 1; fi
+        sleep 0.1
+      done
+      ${lib.concatStringsSep "\n" userClauses}
+      ${lib.concatStringsSep "\n" databaseClauses}
+    '';
+
+    clanCore.state = lib.mapAttrs' (
+      _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
+    ) config.clan.postgresql.databases;
+  };
+}

clanModules/static-hosts/default.nix

@@ -3,7 +3,8 @@
   options.clan.static-hosts = {
     excludeHosts = lib.mkOption {
       type = lib.types.listOf lib.types.str;
-      default = [ config.clanCore.machineName ];
+      default =
+        if config.clan.static-hosts.topLevelDomain != "" then [ ] else [ config.clanCore.machineName ];
       description = "Hosts that should be excluded";
     };
     topLevelDomain = lib.mkOption {
@@ -18,10 +19,20 @@
       clanDir = config.clanCore.clanDir;
       machineDir = clanDir + "/machines/";
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
-      machines = builtins.readDir machineDir;
+      machinesFileSet = builtins.readDir machineDir;
+      machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
+      networkIpsUnchecked = builtins.map (
+        machine:
+        let
+          fullPath = zerotierIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
+      ) machines;
+      networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
+      machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
       filteredMachines = lib.filterAttrs (
         name: _: !(lib.elem name config.clan.static-hosts.excludeHosts)
-      ) machines;
+      ) machinesWithIp;
     in
     lib.filterAttrs (_: value: value != null) (
       lib.mapAttrs' (
@@ -37,7 +48,7 @@
               [ "${machine}.${config.clan.static-hosts.topLevelDomain}" ]
           )
         else
-          null
+          { }
       ) filteredMachines
     );
 }

clanModules/syncthing-static-peers/README.md

@@ -0,0 +1,2 @@
+Statically configure syncthing peers through clan
+---

clanModules/syncthing-static-peers/default.nix

@@ -0,0 +1,108 @@
+{
+  lib,
+  config,
+  pkgs,
+  ...
+}:
+let
+  clanDir = config.clanCore.clanDir;
+  machineDir = clanDir + "/machines/";
+  syncthingPublicKeyPath = machines: machineDir + machines + "/facts/syncthing.pub";
+  machinesFileSet = builtins.readDir machineDir;
+  machines = lib.mapAttrsToList (name: _: name) machinesFileSet;
+  syncthingPublicKeysUnchecked = builtins.map (
+    machine:
+    let
+      fullPath = syncthingPublicKeyPath machine;
+    in
+    if builtins.pathExists fullPath then machine else null
+  ) machines;
+  syncthingPublicKeyMachines = lib.filter (machine: machine != null) syncthingPublicKeysUnchecked;
+  zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
+  networkIpsUnchecked = builtins.map (
+    machine:
+    let
+      fullPath = zerotierIpMachinePath machine;
+    in
+    if builtins.pathExists fullPath then machine else null
+  ) machines;
+  networkIpMachines = lib.filter (machine: machine != null) networkIpsUnchecked;
+  devices = builtins.map (machine: {
+    name = machine;
+    value = {
+      name = machine;
+      id = (lib.removeSuffix "\n" (builtins.readFile (syncthingPublicKeyPath machine)));
+      addresses =
+        [ "dynamic" ]
+        ++ (
+          if (lib.elem machine networkIpMachines) then
+            [ "tcp://[${(lib.removeSuffix "\n" (builtins.readFile (zerotierIpMachinePath machine)))}]:22000" ]
+          else
+            [ ]
+        );
+    };
+  }) syncthingPublicKeyMachines;
+in
+{
+  options.clan.syncthing-static-peers = {
+    excludeMachines = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [ config.clanCore.machineName ];
+      default = [ ];
+      description = ''
+        Machines that should not be added.
+      '';
+    };
+  };
+
+  config.services.syncthing.settings.devices = (builtins.listToAttrs devices);
+
+  imports = [
+    {
+      # Syncthing ports: 8384 for remote access to GUI
+      # 22000 TCP and/or UDP for sync traffic
+      # 21027/UDP for discovery
+      # source: https://docs.syncthing.net/users/firewall.html
+      networking.firewall.interfaces."zt+".allowedTCPPorts = [
+        8384
+        22000
+      ];
+      networking.firewall.allowedTCPPorts = [ 8384 ];
+      networking.firewall.interfaces."zt+".allowedUDPPorts = [
+        22000
+        21027
+      ];
+
+      # Activates inotify compatibility on syncthing
+      # use mkOverride 900 here as it otherwise would collide with the default of the
+      # upstream nixos xserver.nix
+      boot.kernel.sysctl."fs.inotify.max_user_watches" = lib.mkOverride 900 524288;
+
+      services.syncthing = {
+        enable = true;
+        configDir = "/var/lib/syncthing";
+        group = "syncthing";
+
+        key = lib.mkDefault config.clanCore.facts.services.syncthing.secret."syncthing.key".path or null;
+        cert = lib.mkDefault config.clanCore.facts.services.syncthing.secret."syncthing.cert".path or null;
+      };
+
+      clanCore.facts.services.syncthing = {
+        secret."syncthing.key" = { };
+        secret."syncthing.cert" = { };
+        public."syncthing.pub" = { };
+        generator.path = [
+          pkgs.coreutils
+          pkgs.gnugrep
+          pkgs.syncthing
+        ];
+        generator.script = ''
+          syncthing generate --config "$secrets"
+          mv "$secrets"/key.pem "$secrets"/syncthing.key
+          mv "$secrets"/cert.pem "$secrets"/syncthing.cert
+          cat "$secrets"/config.xml | grep -oP '(?<=<device id=")[^"]+' | uniq > "$facts"/syncthing.pub
+        '';
+      };
+    }
+  ];
+}

clanModules/zerotier-static-peers/default.nix

@@ -20,7 +20,7 @@ let
     if builtins.pathExists fullPath then builtins.readFile fullPath else null
   ) machines;
   networkIds = lib.filter (machine: machine != null) networkIdsUnchecked;
-  networkId = builtins.elemAt networkIds 0;
+  networkId = if builtins.length networkIds == 0 then null else builtins.elemAt networkIds 0;
 in
 #TODO:trace on multiple found network-ids
 #TODO:trace on no single found networkId
@@ -35,21 +35,26 @@ in
 
   config.systemd.services.zerotier-static-peers-autoaccept =
     let
-      machines = builtins.readDir machineDir;
       zerotierIpMachinePath = machines: machineDir + machines + "/facts/zerotier-ip";
-      filteredMachines = lib.filterAttrs (
-        name: _: !(lib.elem name config.clan.static-hosts.excludeHosts)
+      networkIpsUnchecked = builtins.map (
+        machine:
+        let
+          fullPath = zerotierIpMachinePath machine;
+        in
+        if builtins.pathExists fullPath then machine else null
       ) machines;
+      networkIps = lib.filter (machine: machine != null) networkIpsUnchecked;
+      machinesWithIp = lib.filterAttrs (name: _: (lib.elem name networkIps)) machinesFileSet;
+      filteredMachines = lib.filterAttrs (
+        name: _: !(lib.elem name config.clan.zerotier-static-peers.excludeHosts)
+      ) machinesWithIp;
       hosts = lib.mapAttrsToList (host: _: host) (
         lib.mapAttrs' (
           machine: _:
           let
             fullPath = zerotierIpMachinePath machine;
           in
-          if builtins.pathExists fullPath then
-            lib.nameValuePair (builtins.readFile fullPath) [ machine ]
-          else
-            null
+          lib.nameValuePair (builtins.readFile fullPath) [ machine ]
         ) filteredMachines
       );
     in

devShell.nix

@@ -27,7 +27,8 @@
         packages = [
           select-shell
           pkgs.tea
-          pkgs.nix
+          # Better error messages than nix 2.18
+          pkgs.nixVersions.latest
           self'.packages.tea-create-pr
           self'.packages.merge-after-ci
           self'.packages.pending-reviews

docs/CONTRIBUTING.md

@@ -15,56 +15,74 @@ Let's get your development environment up and running:
 
 1. **Install Nix Package Manager**:
 
-   - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
-     ```bash
-     curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
-     ```
+      - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
+        ```bash
+        curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
+        ```
 
 2. **Install direnv**:
 
-   - To automatically setup a devshell on entering the directory
-     ```bash
-     nix profile install nixpkgs#nix-direnv-flakes
-     ```
+      - To automatically setup a devshell on entering the directory
+        ```bash
+        nix profile install nixpkgs#nix-direnv-flakes
+        ```
 
 3. **Add direnv to your shell**:
 
-   - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
-     You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
+      - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
+        You can do this by executing following command. The example below will setup direnv for `zsh` and `bash`
 
-   ```bash
-   echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
-   ```
+      ```bash
+      echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
+      ```
 
 4. **Create a Gitea Account**:
-   - Register an account on https://git.clan.lol
-   - Fork the [clan-core](https://git.clan.lol/clan/clan-core) repository
-   - Clone the repository and navigate to it
-   - Add a new remote called upstream:
-      ```bash
-      git remote add upstream gitea@git.clan.lol:clan/clan-core.git
+      - Register an account on https://git.clan.lol
+      - Fork the [clan-core](https://git.clan.lol/clan/clan-core) repository
+      - Clone the repository and navigate to it
+      - Add a new remote called upstream:
+         ```bash
+         git remote add upstream gitea@git.clan.lol:clan/clan-core.git
+         ```
+
+5. **Register Your Gitea Account Locally**:
+
+      - Execute the following command to add your Gitea account locally:
+        ```bash
+        tea login add
+        ```
+      - Fill out the prompt as follows:
+        - URL of Gitea instance: `https://git.clan.lol`
+        - Name of new Login [git.clan.lol]:
+        - Do you have an access token? No
+        - Username: YourUsername
+        - Password: YourPassword
+        - Set Optional settings: No
+
+
+6. **Allow .envrc**:
+
+      - When you enter the directory, you'll receive an error message like this:
+        ```bash
+        direnv: error .envrc is blocked. Run `direnv allow` to approve its content
+        ```
+      - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
+
+7. **(Optional) Install Git Hooks**:
+      - To syntax check your code you can run:
+         ```bash
+         nix fmt
+         ```
+      - To make this automatic install the git hooks
+         ```bash
+         ./scripts/pre-commit
+         ```
+
+8. **Open a Pull Request**:
+      - To automatically open up a pull request you can use our tool called:
       ```
-
-5. **Allow .envrc**:
-
-   - When you enter the directory, you'll receive an error message like this:
-     ```bash
-     direnv: error .envrc is blocked. Run `direnv allow` to approve its content
-     ```
-   - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
-
-6. **(Optional) Install Git Hooks**:
-   - To syntax check your code you can run:
-      ```bash
-      nix fmt
+      merge-after-ci --reviewers Mic92 Lassulus Qubasa
       ```
-   - To make this automatic install the git hooks
-      ```bash
-      ./scripts/pre-commit
-      ```
-
-7. **Open a Pull Request**:
-   - Go to the webinterface and open up a pull request
 
 # Debugging
 
@@ -103,7 +121,7 @@ git+file:///home/lhebendanz/Projects/clan-core
 │       ├───clan-cli omitted (use '--all-systems' to show)
 │       ├───clan-cli-docs omitted (use '--all-systems' to show)
 │       ├───clan-ts-api omitted (use '--all-systems' to show)
-│       ├───clan-vm-manager omitted (use '--all-systems' to show)
+│       ├───clan-app omitted (use '--all-systems' to show)
 │       ├───default omitted (use '--all-systems' to show)
 │       ├───deploy-docs omitted (use '--all-systems' to show)
 │       ├───docs omitted (use '--all-systems' to show)

docs/main.py

@@ -16,15 +16,26 @@ def define_env(env: Any) -> None:
     @env.macro
     def asciinema(name: str) -> str:
         return f"""<div id="{name}">
-            <script src="{asciinema_dir}/asciinema-player.min.js"></script>
             <script>
-                AsciinemaPlayer.create('{video_dir + name}',
-                document.getElementById("{name}"), {{
-                    loop: true,
-                    autoPlay: true,
-                    controls: false,
-                    speed: 1.5,
-                    theme: "solarized-light"
-                }});
+                // Function to load the script and then create the Asciinema player
+                function loadAsciinemaPlayer() {{
+                    var script = document.createElement('script');
+                    script.src = "{asciinema_dir}/asciinema-player.min.js";
+                    script.onload = function() {{
+                        AsciinemaPlayer.create('{video_dir + name}', document.getElementById("{name}"), {{
+                            loop: true,
+                            autoPlay: true,
+                            controls: false,
+                            speed: 1.5,
+                            theme: "solarized-light"
+                    }});
+                    }};
+                    document.head.appendChild(script);
+                }}
+
+                // Load the Asciinema player script
+                loadAsciinemaPlayer();
             </script>
+
+            <link rel="stylesheet" type="text/css" href="{asciinema_dir}/asciinema-player.css" />
         </div>"""

docs/mkdocs.yml

@@ -1,4 +1,4 @@
-site_name: Clan Docs
+site_name: Clan Documentation
 site_url: https://docs.clan.lol
 repo_url: https://git.clan.lol/clan/clan-core/
 repo_name: clan-core
@@ -51,6 +51,7 @@ nav:
       - Flake-parts: getting-started/flake-parts.md
   - Modules:
       - Clan Modules:
+          - reference/clanModules/borgbackup-static.md
           - reference/clanModules/borgbackup.md
           - reference/clanModules/deltachat.md
           - reference/clanModules/disk-layouts.md
@@ -59,11 +60,13 @@ nav:
           - reference/clanModules/localsend.md
           - reference/clanModules/matrix-synapse.md
           - reference/clanModules/moonlight.md
+          - reference/clanModules/postgresql.md
           - reference/clanModules/root-password.md
           - reference/clanModules/sshd.md
-          - reference/clanModules/sunshine.md
-          - reference/clanModules/syncthing.md
           - reference/clanModules/static-hosts.md
+          - reference/clanModules/sunshine.md
+          - reference/clanModules/syncthing-static-peers.md
+          - reference/clanModules/syncthing.md
           - reference/clanModules/thelounge.md
           - reference/clanModules/trusted-nix-caches.md
           - reference/clanModules/user-password.md
@@ -80,6 +83,7 @@ nav:
           - reference/cli/history.md
           - reference/cli/machines.md
           - reference/cli/secrets.md
+          - reference/cli/show.md
           - reference/cli/ssh.md
           - reference/cli/vms.md
       - Clan Core:
@@ -96,7 +100,7 @@ site_dir: out
 theme:
   font: false
   logo: https://clan.lol/static/logo/clan-white.png
-  favicon: https://clan.lol/static/logo/clan-dark.png
+  favicon: https://clan.lol/static/dark-favicon/128x128.png
   name: material
   features:
     - navigation.instant
@@ -105,7 +109,8 @@ theme:
     - content.code.copy
     - content.tabs.link
   icon:
-    repo: fontawesome/brands/git
+    repo: fontawesome/brands/git-alt
+  custom_dir: overrides
 
   palette:
     # Palette toggle for light mode
@@ -128,8 +133,6 @@ theme:
 
 extra_css:
   - static/extra.css
-  - static/asciinema-player/custom-theme.css
-  - static/asciinema-player/asciinema-player.css
 
 extra:
   social:

docs/overrides/main.html

@@ -0,0 +1,12 @@
+{% extends "base.html" %}
+
+{% block extrahead %}
+<meta property="og:title" content="Clan - Documentation, Blog & Getting Started Guide" />
+<meta property="og:description" content="Documentation for Clan. The peer-to-peer machine deployment framework." />
+<meta property="og:image" content="https://clan.lol/static/dark-favicon/128x128.png" />
+<meta property="og:url" content="https://docs.clan.lol" />
+<meta property="og:type" content="website" />
+<meta property="og:site_name" content="Clan" />
+<meta property="og:locale" content="en_US" />
+
+{% endblock %}

docs/site/getting-started/backups.md

@@ -116,7 +116,7 @@ On the server where backups will be stored, enable the SSH daemon and set up a r
   services.borgbackup.repos.myhostname = {
     path = "/var/lib/borgbackup/myhostname";
     authorizedKeys = [
-      (builtins.readFile ./machines/myhostname/facts/borgbackup.ssh.pub)
+      (builtins.readFile  (config.clanCore.clanDir + "/machines/myhostname/facts/borgbackup.ssh.pub"))
     ];
   };
 }

docs/site/getting-started/installer.md

@@ -54,9 +54,9 @@ sudo umount /dev/sdb1
       flash-installer
     ```
 
-   The `--ssh-pubkey`, `--language` and `--keymap` are optional.
-   Replace `$HOME/.ssh/id_ed25519.pub` with a path to your SSH public key.
-   If you do not have an ssh key yet, you can generate one with `ssh-keygen -t ed25519` command.
+    The `--ssh-pubkey`, `--language` and `--keymap` are optional.
+    Replace `$HOME/.ssh/id_ed25519.pub` with a path to your SSH public key.
+    If you do not have an ssh key yet, you can generate one with `ssh-keygen -t ed25519` command.
 
     !!! Danger "Specifying the wrong device can lead to unrecoverable data loss."
 

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1716773194,
-        "narHash": "sha256-rskkGmWlvYFb+CXedBiL8eWEuED0Es0XR4CkJ11RQKY=",
+        "lastModified": 1717915259,
+        "narHash": "sha256-VsGPboaleIlPELHY5cNTrXK4jHVmgUra8uC6h7KVC5c=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "10986091e47fb1180620b78438512b294b7e8f67",
+        "rev": "1bbdb06f14e2621290b250e631cf3d8948e4d19b",
         "type": "github"
       },
       "original": {
@@ -27,11 +27,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715865404,
-        "narHash": "sha256-/GJvTdTpuDjNn84j82cU6bXztE0MSkdnTWClUCRub78=",
+        "lastModified": 1717285511,
+        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8dc45382d5206bd292f9c2768b8058a8fd8311d9",
+        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
         "type": "github"
       },
       "original": {
@@ -55,22 +55,6 @@
         "type": "github"
       }
     },
-    "nixos-2311": {
-      "locked": {
-        "lastModified": 1716767563,
-        "narHash": "sha256-xaSLDTqKIU55HsCkDnzFKmPiJO2z1xAAvrhUlwlmT2M=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "0c007b36981bdbd69ccf0c7df30a174e63660667",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "release-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "nixos-generators": {
       "inputs": {
         "nixlib": "nixlib",
@@ -94,17 +78,17 @@
     },
     "nixos-images": {
       "inputs": {
-        "nixos-2311": "nixos-2311",
+        "nixos-stable": "nixos-stable",
         "nixos-unstable": [
           "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1716786664,
-        "narHash": "sha256-iszhOLhxnv+TX/XM2gAX4LhTCoMzLuG51ObZq/eyDx8=",
+        "lastModified": 1717770332,
+        "narHash": "sha256-NQmFHj0hTCUgnMAsaNTu6sNTRyo0rFQEe+/lVgV5yxU=",
         "owner": "nix-community",
         "repo": "nixos-images",
-        "rev": "2478833ef8cc6de3d9e331f53b6f3682e425f207",
+        "rev": "72771bd35f4e19e32d6f652528483b5e07fc317b",
         "type": "github"
       },
       "original": {
@@ -113,13 +97,29 @@
         "type": "github"
       }
     },
-    "nixpkgs": {
+    "nixos-stable": {
       "locked": {
-        "lastModified": 1716924492,
-        "narHash": "sha256-9/Ro5/MfI+PNMF8jzh7+gXDPUHeOzL1e/iw3p4z6Ttc=",
+        "lastModified": 1717555607,
+        "narHash": "sha256-WZ1s48OODmRJ3DHC+I/DtM3tDRuRJlNqMvxvAPTD7ec=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "4ae13643e7f2cd4bc6555fce074865d9d14e7c24",
+        "rev": "0b8e7a1ae5a94da2e1ee3f3030a32020f6254105",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1717926692,
+        "narHash": "sha256-THcv8qDqobZefHHluPjx/8n+MtVVb8ag/oJbKMqKNRo=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "7d916e720af6b2ca355e4d0cfb8e4f742c172239",
         "type": "github"
       },
       "original": {
@@ -148,11 +148,11 @@
         "nixpkgs-stable": []
       },
       "locked": {
-        "lastModified": 1716692524,
-        "narHash": "sha256-sALodaA7Zkp/JD6ehgwc0UCBrSBfB4cX66uFGTsqeFU=",
+        "lastModified": 1717902109,
+        "narHash": "sha256-OQTjaEZcByyVmHwJlKp/8SE9ikC4w+mFd3X0jJs6wiA=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "962797a8d7f15ed7033031731d0bb77244839960",
+        "rev": "f0922ad001829b400f0160ba85b47d252fa3d925",
         "type": "github"
       },
       "original": {
@@ -168,11 +168,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1715940852,
-        "narHash": "sha256-wJqHMg/K6X3JGAE9YLM0LsuKrKb4XiBeVaoeMNlReZg=",
+        "lastModified": 1717850719,
+        "narHash": "sha256-npYqVg+Wk4oxnWrnVG7416fpfrlRhp/lQ6wQ4DHI8YE=",
         "owner": "numtide",
         "repo": "treefmt-nix",
-        "rev": "2fba33a182602b9d49f0b2440513e5ee091d838b",
+        "rev": "4fc1c45a5f50169f9f29f6a98a438fb910b834ed",
         "type": "github"
       },
       "original": {

flake.nix

@@ -24,10 +24,14 @@
   };
 
   outputs =
-    inputs@{ flake-parts, ... }:
+    inputs@{ flake-parts, self, ... }:
     flake-parts.lib.mkFlake { inherit inputs; } (
       { ... }:
       {
+        clan = {
+          # meta.name = "clan-core";
+          directory = self;
+        };
         systems = [
           "x86_64-linux"
           "aarch64-linux"

flakeModules/clan.nix

@@ -17,6 +17,33 @@ let
   cfg = config.clan;
 in
 {
+  imports = [
+    # TODO: figure out how to print the deprecation warning
+    # "${inputs.nixpkgs}/nixos/modules/misc/assertions.nix"
+    (lib.mkRenamedOptionModule
+      [
+        "clan"
+        "clanName"
+      ]
+      [
+        "clan"
+        "meta"
+        "name"
+      ]
+    )
+    (lib.mkRenamedOptionModule
+      [
+        "clan"
+        "clanIcon"
+      ]
+      [
+        "clan"
+        "meta"
+        "icon"
+      ]
+    )
+  ];
+
   options.clan = {
     directory = mkOption {
       type = types.path;
@@ -33,15 +60,27 @@ in
       default = { };
       description = "Allows to include machine-specific modules i.e. machines.\${name} = { ... }";
     };
-    clanName = mkOption {
-      type = types.str;
-      description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
-    };
-    clanIcon = mkOption {
-      type = types.nullOr types.path;
-      default = null;
-      description = "A path to an icon to be used for the clan, should be the same for all machines";
+
+    # Checks are performed in 'buildClan'
+    # Not everyone uses flake-parts
+    meta = {
+      name = lib.mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.";
+      };
+      icon = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = "A path to an icon to be used for the clan in the GUI";
+      };
+      description = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "A short description of the clan";
+      };
     };
+
     pkgsForSystem = mkOption {
       type = types.functionTo types.raw;
       default = _system: null;
@@ -52,6 +91,7 @@ in
     clanInternals = lib.mkOption {
       type = lib.types.submodule {
         options = {
+          meta = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           all-machines-json = lib.mkOption { type = lib.types.attrsOf lib.types.unspecified; };
           machines = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
           machinesFunc = lib.mkOption { type = lib.types.attrsOf (lib.types.attrsOf lib.types.unspecified); };
@@ -65,9 +105,8 @@ in
         directory
         specialArgs
         machines
-        clanName
-        clanIcon
         pkgsForSystem
+        meta
         ;
     };
   };

formatter.nix

@@ -10,10 +10,9 @@
       treefmt.programs.mypy.enable = true;
       treefmt.programs.mypy.directories = {
         "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
-        "pkgs/clan-vm-manager".extraPythonPackages =
-          # clan-vm-manager currently only exists on linux 
-          (self'.packages.clan-vm-manager.externalTestDeps or [ ])
-          ++ self'.packages.clan-cli.testDependencies;
+        "pkgs/clan-app".extraPythonPackages =
+          # clan-app currently only exists on linux
+          (self'.packages.clan-app.externalTestDeps or [ ]) ++ self'.packages.clan-cli.testDependencies;
       };
 
       treefmt.settings.formatter.nix = {

lib/build-clan/default.nix

@@ -7,16 +7,58 @@
   directory, # The directory containing the machines subdirectory
   specialArgs ? { }, # Extra arguments to pass to nixosSystem i.e. useful to make self available
   machines ? { }, # allows to include machine-specific modules i.e. machines.${name} = { ... }
-  clanName, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  # DEPRECATED: use meta.name instead
+  clanName ? null, # Needs to be (globally) unique, as this determines the folder name where the flake gets downloaded to.
+  # DEPRECATED: use meta.icon instead
   clanIcon ? null, # A path to an icon to be used for the clan, should be the same for all machines
+  meta ? { }, # A set containing clan meta: name :: string, icon :: string, description :: string
   pkgsForSystem ? (_system: null), # A map from arch to pkgs, if specified this nixpkgs will be only imported once for each system.
 # This improves performance, but all nipxkgs.* options will be ignored.
 }:
 let
+  deprecationWarnings = [
+    (lib.warnIf (
+      clanName != null
+    ) "clanName is deprecated, please use meta.name instead. ${clanName}" null)
+    (lib.warnIf (clanIcon != null) "clanIcon is deprecated, please use meta.icon instead" null)
+  ];
+
   machinesDirs = lib.optionalAttrs (builtins.pathExists "${directory}/machines") (
     builtins.readDir (directory + /machines)
   );
 
+  mergedMeta =
+    let
+      metaFromFile =
+        if (builtins.pathExists "${directory}/clan/meta.json") then
+          let
+            settings = builtins.fromJSON (builtins.readFile "${directory}/clan/meta.json");
+          in
+          settings
+        else
+          { };
+      legacyMeta = lib.filterAttrs (_: v: v != null) {
+        name = clanName;
+        icon = clanIcon;
+      };
+      optionsMeta = lib.filterAttrs (_: v: v != null) meta;
+
+      warnings =
+        builtins.map (
+          name:
+          if
+            metaFromFile.${name} or null != optionsMeta.${name} or null && optionsMeta.${name} or null != null
+          then
+            lib.warn "meta.${name} is set in different places. (exlicit option meta.${name} overrides ${directory}/clan/meta.json)" null
+          else
+            null
+        ) (builtins.attrNames metaFromFile)
+        ++ [ (if (res.name or null == null) then (throw "meta.name should be set") else null) ];
+      res = metaFromFile // legacyMeta // optionsMeta;
+    in
+    # Print out warnings before returning the merged result
+    builtins.deepSeq warnings res;
+
   machineSettings =
     machineName:
     # CLAN_MACHINE_SETTINGS_FILE allows to override the settings file temporarily
@@ -58,11 +100,15 @@ let
           (machines.${name} or { })
           (
             {
-              networking.hostName = lib.mkDefault name;
-              clanCore.clanName = clanName;
-              clanCore.clanIcon = clanIcon;
+              # Settings
               clanCore.clanDir = directory;
+              # Inherited from clan wide settings
+              clanCore.clanName = meta.name or clanName;
+              clanCore.clanIcon = meta.icon or clanIcon;
+
+              # Machine specific settings
               clanCore.machineName = name;
+              networking.hostName = lib.mkDefault name;
               nixpkgs.hostPlatform = lib.mkDefault system;
 
               # speeds up nix commands by using the nixpkgs from the host system (especially useful in VMs)
@@ -127,10 +173,15 @@ let
     ) supportedSystems
   );
 in
-{
+builtins.deepSeq deprecationWarnings {
   inherit nixosConfigurations;
 
   clanInternals = {
+    # Evaluated clan meta
+    # Merged /clan/meta.json with overrides from buildClan
+    meta = mergedMeta;
+
+    # machine specifics
     machines = configsPerSystem;
     machinesFunc = configsFuncPerSystem;
     all-machines-json = lib.mapAttrs (

nixosModules/clanCore/state.nix

@@ -8,15 +8,43 @@
     default = { };
     type = lib.types.attrsOf (
       lib.types.submodule (
-        { ... }:
+        { name, ... }:
         {
           options = {
+            name = lib.mkOption {
+              type = lib.types.str;
+              default = name;
+              description = ''
+                Name of the state
+              '';
+            };
             folders = lib.mkOption {
               type = lib.types.listOf lib.types.str;
               description = ''
                 Folder where state resides in
               '';
             };
+            preBackupCommand = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              description = ''
+                script to run before backing up the state dir
+                This is for example useful for services that require an export of their state
+                e.g. a database dump
+              '';
+            };
+
+            # TODO: implement this
+            #stopOnRestore = lib.mkOption {
+            #  type = lib.types.listOf lib.types.str;
+            #  default = [];
+            #  description = ''
+            #    List of services to stop before restoring the state dir from a backup
+
+            #    Utilize this to stop services which currently access these folders or or other services affected by the restore
+            #  '';
+            #};
+
             preRestoreCommand = lib.mkOption {
               type = lib.types.nullOr lib.types.str;
               default = null;
@@ -26,6 +54,7 @@
                 Utilize this to stop services which currently access these folders
               '';
             };
+
             postRestoreCommand = lib.mkOption {
               type = lib.types.nullOr lib.types.str;
               default = null;

pkgs/clan-app/.envrc

@@ -3,4 +3,4 @@ source_up
 watch_file flake-module.nix shell.nix default.nix
 
 # Because we depend on nixpkgs sources, uploading to builders takes a long time
-use flake .#clan-vm-manager --builders ''
+use flake .#clan-app --builders ''

pkgs/clan-app/README.md

@@ -1,4 +1,4 @@
-# Clan VM Manager
+# clan app
 
 Provides users with the simple functionality to manage their locally registered clans.
 
@@ -9,19 +9,19 @@ Provides users with the simple functionality to manage their locally registered
 Run this application
 
 ```bash
-./bin/clan-vm-manager
+./bin/clan-app
 ```
 
 Join the default machine of a clan
 
 ```bash
-./bin/clan-vm-manager [clan-uri]
+./bin/clan-app [clan-uri]
 ```
 
 Join a specific machine of a clan
 
 ```bash
-./bin/clan-vm-manager [clan-uri]#[machine]
+./bin/clan-app [clan-uri]#[machine]
 ```
 
 For more available commands see the developer section below.
@@ -35,7 +35,7 @@ For more available commands see the developer section below.
 gsettings set org.gtk.Settings.Debug enable-inspector-keybinding true
 
 # Start the application with the debugger attached
-GTK_DEBUG=interactive ./bin/clan-vm-manager --debug
+GTK_DEBUG=interactive ./bin/clan-app --debug
 ```
 
 Appending `--debug` flag enables debug logging printed into the console.
@@ -45,7 +45,7 @@ Appending `--debug` flag enables debug logging printed into the console.
 To activate profiling you can run
 
 ```bash
-PERF=1 ./bin/clan-vm-manager
+PERF=1 ./bin/clan-app
 ```
 
 ### Library Components
@@ -75,20 +75,20 @@ gtk4-icon-browser
 
 ### Links
 
-Here are some important documentation links related to the Clan VM Manager:
+Here are some important documentation links related to the Clan App:
 
-- [Adw PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Adw-1): This link provides the PyGObject reference documentation for the Adw library, which is used in the Clan VM Manager. It contains detailed information about the Adw widgets and their usage.
+- [Adw PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Adw-1): This link provides the PyGObject reference documentation for the Adw library, which is used in the Clan App. It contains detailed information about the Adw widgets and their usage.
 
-- [GTK4 PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Gtk-4.0): This link provides the PyGObject reference documentation for GTK4, the toolkit used for building the user interface of the Clan VM Manager. It includes information about GTK4 widgets, signals, and other features.
+- [GTK4 PyGobject Reference](http://lazka.github.io/pgi-docs/index.html#Gtk-4.0): This link provides the PyGObject reference documentation for GTK4, the toolkit used for building the user interface of the clan app. It includes information about GTK4 widgets, signals, and other features.
 
-- [Adw Widget Gallery](https://gnome.pages.gitlab.gnome.org/libadwaita/doc/main/widget-gallery.html): This link showcases a widget gallery for Adw, allowing you to see the available widgets and their visual appearance. It can be helpful for designing the user interface of the Clan VM Manager.
+- [Adw Widget Gallery](https://gnome.pages.gitlab.gnome.org/libadwaita/doc/main/widget-gallery.html): This link showcases a widget gallery for Adw, allowing you to see the available widgets and their visual appearance. It can be helpful for designing the user interface of the clan app.
 
-- [Python + GTK3 Tutorial](https://python-gtk-3-tutorial.readthedocs.io/en/latest/textview.html): Although the Clan VM Manager uses GTK4, this tutorial for GTK3 can still be useful as it covers the basics of building GTK-based applications with Python. It includes examples and explanations for various GTK widgets, including text views.
+- [Python + GTK3 Tutorial](https://python-gtk-3-tutorial.readthedocs.io/en/latest/textview.html): Although the clan app uses GTK4, this tutorial for GTK3 can still be useful as it covers the basics of building GTK-based applications with Python. It includes examples and explanations for various GTK widgets, including text views.
 
 - [GNOME Human Interface Guidelines](https://developer.gnome.org/hig/): This link provides the GNOME Human Interface Guidelines, which offer design and usability recommendations for creating GNOME applications. It covers topics such as layout, navigation, and interaction patterns.
 
 ## Error handling
 
-> Error dialogs should be avoided where possible, since they are disruptive. 
-> 
+> Error dialogs should be avoided where possible, since they are disruptive.
+>
 > For simple non-critical errors, toasts can be a good alternative.

pkgs/clan-app/bin/clan-app

@@ -7,7 +7,7 @@ module_path = Path(__file__).parent.parent.absolute()
 sys.path.insert(0, str(module_path))
 sys.path.insert(0, str(module_path.parent / "clan_cli"))
 
-from clan_vm_manager import main  # NOQA
+from clan_app import main  # NOQA
 
 if __name__ == "__main__":
     main()

pkgs/clan-app/clan_app/__init__.py

@@ -3,7 +3,7 @@ import sys
 
 from clan_cli.profiler import profile
 
-from clan_vm_manager.app import MainApplication
+from clan_app.app import MainApplication
 
 log = logging.getLogger(__name__)
 

pkgs/clan-app/clan_app/app.py

@@ -4,8 +4,8 @@ from typing import Any, ClassVar
 
 import gi
 
-from clan_vm_manager import assets
-from clan_vm_manager.singletons.toast import InfoToast, ToastOverlay
+from clan_app import assets
+from clan_app.singletons.toast import InfoToast, ToastOverlay
 
 gi.require_version("Gtk", "4.0")
 gi.require_version("Adw", "1")
@@ -13,8 +13,8 @@ gi.require_version("Adw", "1")
 from clan_cli.custom_logger import setup_logging
 from gi.repository import Adw, Gdk, Gio, Gtk
 
-from clan_vm_manager.components.interfaces import ClanConfig
-from clan_vm_manager.singletons.use_join import GLib, GObject
+from clan_app.components.interfaces import ClanConfig
+from clan_app.singletons.use_join import GLib, GObject
 
 from .windows.main_window import MainWindow
 
@@ -106,7 +106,7 @@ class MainApplication(Adw.Application):
     def on_activate(self, source: "MainApplication") -> None:
         if not self.window:
             self.init_style()
-            self.window = MainWindow(config=ClanConfig(initial_view="list"))
+            self.window = MainWindow(config=ClanConfig(initial_view="webview"))
             self.window.set_application(self)
 
         self.window.show()

pkgs/clan-app/clan_app/components/list_splash.py

@@ -4,7 +4,7 @@ from typing import TypeVar
 
 import gi
 
-from clan_vm_manager import assets
+from clan_app import assets
 
 gi.require_version("Adw", "1")
 from gi.repository import Adw, GdkPixbuf, Gio, GObject, Gtk

pkgs/clan-app/clan_app/components/trayicon.py

@@ -562,7 +562,7 @@ class StatusNotifierImplementation(BaseImplementation):
             )
             self.tray_icon.register()
 
-            from clan_vm_manager.assets import loc
+            from clan_app.assets import loc
 
             icon_path = str(loc / "clan_white_notext.png")
             self.set_icon(icon_path)

pkgs/clan-app/clan_app/components/vmobj.py

@@ -17,8 +17,8 @@ from clan_cli.clan_uri import ClanURI
 from clan_cli.history.add import HistoryEntry
 from clan_cli.machines.machines import Machine
 
-from clan_vm_manager.components.executor import MPProcess, spawn
-from clan_vm_manager.singletons.toast import (
+from clan_app.components.executor import MPProcess, spawn
+from clan_app.singletons.toast import (
     InfoToast,
     SuccessToast,
     ToastOverlay,

pkgs/clan-app/clan_app/singletons/toast.py

@@ -9,8 +9,8 @@ gi.require_version("Adw", "1")
 
 from gi.repository import Adw
 
-from clan_vm_manager.singletons.use_views import ViewStack
-from clan_vm_manager.views.logs import Logs
+from clan_app.singletons.use_views import ViewStack
+from clan_app.views.logs import Logs
 
 log = logging.getLogger(__name__)
 

pkgs/clan-app/clan_app/singletons/use_join.py

@@ -7,8 +7,8 @@ import gi
 from clan_cli.clan_uri import ClanURI
 from clan_cli.history.add import HistoryEntry, add_history
 
-from clan_vm_manager.components.gkvstore import GKVStore
-from clan_vm_manager.singletons.use_vms import ClanStore
+from clan_app.components.gkvstore import GKVStore
+from clan_app.singletons.use_vms import ClanStore
 
 gi.require_version("Gtk", "4.0")
 gi.require_version("Adw", "1")

pkgs/clan-app/clan_app/singletons/use_vms.py

@@ -7,11 +7,11 @@ import gi
 from clan_cli.clan_uri import ClanURI
 from clan_cli.history.add import HistoryEntry
 
-from clan_vm_manager import assets
-from clan_vm_manager.components.gkvstore import GKVStore
-from clan_vm_manager.components.vmobj import VMObject
-from clan_vm_manager.singletons.use_views import ViewStack
-from clan_vm_manager.views.logs import Logs
+from clan_app import assets
+from clan_app.components.gkvstore import GKVStore
+from clan_app.components.vmobj import VMObject
+from clan_app.singletons.use_views import ViewStack
+from clan_app.views.logs import Logs
 
 gi.require_version("GObject", "2.0")
 gi.require_version("Gtk", "4.0")

pkgs/clan-app/clan_app/views/list.py

@@ -7,20 +7,20 @@ from typing import Any, TypeVar
 import gi
 from clan_cli.clan_uri import ClanURI
 
-from clan_vm_manager.components.gkvstore import GKVStore
-from clan_vm_manager.components.interfaces import ClanConfig
-from clan_vm_manager.components.list_splash import EmptySplash
-from clan_vm_manager.components.vmobj import VMObject
-from clan_vm_manager.singletons.toast import (
+from clan_app.components.gkvstore import GKVStore
+from clan_app.components.interfaces import ClanConfig
+from clan_app.components.list_splash import EmptySplash
+from clan_app.components.vmobj import VMObject
+from clan_app.singletons.toast import (
     LogToast,
     SuccessToast,
     ToastOverlay,
     WarningToast,
 )
-from clan_vm_manager.singletons.use_join import JoinList, JoinValue
-from clan_vm_manager.singletons.use_views import ViewStack
-from clan_vm_manager.singletons.use_vms import ClanStore, VMStore
-from clan_vm_manager.views.logs import Logs
+from clan_app.singletons.use_join import JoinList, JoinValue
+from clan_app.singletons.use_views import ViewStack
+from clan_app.singletons.use_vms import ClanStore, VMStore
+from clan_app.views.logs import Logs
 
 gi.require_version("Adw", "1")
 from gi.repository import Adw, Gdk, Gio, GLib, GObject, Gtk

pkgs/clan-app/clan_app/views/logs.py

@@ -5,7 +5,7 @@ import gi
 gi.require_version("Adw", "1")
 from gi.repository import Adw, Gio, Gtk
 
-from clan_vm_manager.singletons.use_views import ViewStack
+from clan_app.singletons.use_views import ViewStack
 
 log = logging.getLogger(__name__)
 

pkgs/clan-app/clan_app/views/webview.py

@@ -10,15 +10,14 @@ from typing import Any
 
 import gi
 from clan_cli.api import API
+from clan_cli.api.directory import FileRequest
 
 gi.require_version("WebKit", "6.0")
 
-from gi.repository import GLib, WebKit
+from gi.repository import Gio, GLib, Gtk, WebKit
 
 site_index: Path = (
-    Path(sys.argv[0]).absolute()
-    / Path("../..")
-    / Path("clan_vm_manager/.webui/index.html")
+    Path(sys.argv[0]).absolute() / Path("../..") / Path("clan_app/.webui/index.html")
 ).resolve()
 
 log = logging.getLogger(__name__)
@@ -41,11 +40,95 @@ def dataclass_to_dict(obj: Any) -> Any:
         return obj
 
 
+# Implement the abstract open_file function
+def open_file(file_request: FileRequest) -> str | None:
+    # Function to handle the response and stop the loop
+    selected_path = None
+
+    def on_file_select(
+        file_dialog: Gtk.FileDialog, task: Gio.Task, main_loop: GLib.MainLoop
+    ) -> None:
+        try:
+            gfile = file_dialog.open_finish(task)
+            if gfile:
+                nonlocal selected_path
+                selected_path = gfile.get_path()
+        except Exception as e:
+            print(f"Error getting selected file or directory: {e}")
+        finally:
+            main_loop.quit()
+
+    def on_folder_select(
+        file_dialog: Gtk.FileDialog, task: Gio.Task, main_loop: GLib.MainLoop
+    ) -> None:
+        try:
+            gfile = file_dialog.select_folder_finish(task)
+            if gfile:
+                nonlocal selected_path
+                selected_path = gfile.get_path()
+        except Exception as e:
+            print(f"Error getting selected directory: {e}")
+        finally:
+            main_loop.quit()
+
+    dialog = Gtk.FileDialog()
+
+    if file_request.title:
+        dialog.set_title(file_request.title)
+
+    if file_request.filters:
+        filters = Gio.ListStore.new(Gtk.FileFilter)
+        file_filters = Gtk.FileFilter()
+
+        if file_request.filters.title:
+            file_filters.set_name(file_request.filters.title)
+
+        # Create and configure a filter for image files
+        if file_request.filters.mime_types:
+            for mime in file_request.filters.mime_types:
+                file_filters.add_mime_type(mime)
+                filters.append(file_filters)
+
+        if file_request.filters.patterns:
+            for pattern in file_request.filters.patterns:
+                file_filters.add_pattern(pattern)
+
+        if file_request.filters.suffixes:
+            for suffix in file_request.filters.suffixes:
+                file_filters.add_suffix(suffix)
+
+        filters.append(file_filters)
+        dialog.set_filters(filters)
+
+    main_loop = GLib.MainLoop()
+
+    # if select_folder
+    if file_request.mode == "select_folder":
+        dialog.select_folder(
+            callback=lambda dialog, task: on_folder_select(dialog, task, main_loop)
+        )
+    elif file_request.mode == "open_file":
+        dialog.open(
+            callback=lambda dialog, task: on_file_select(dialog, task, main_loop)
+        )
+
+    # Wait for the user to select a file or directory
+    main_loop.run()  # type: ignore
+
+    return selected_path
+
+
 class WebView:
     def __init__(self, methods: dict[str, Callable]) -> None:
         self.method_registry: dict[str, Callable] = methods
 
         self.webview = WebKit.WebView()
+
+        settings = self.webview.get_settings()
+        # settings.
+        settings.set_property("enable-developer-extras", True)
+        self.webview.set_settings(settings)
+
         self.manager = self.webview.get_user_content_manager()
         # Can be called with: window.webkit.messageHandlers.gtk.postMessage("...")
         # Important: it seems postMessage must be given some payload, otherwise it won't trigger the event

pkgs/clan-app/clan_app/windows/main_window.py

@@ -5,20 +5,20 @@ import gi
 from clan_cli.api import API
 from clan_cli.history.list import list_history
 
-from clan_vm_manager.components.interfaces import ClanConfig
-from clan_vm_manager.singletons.toast import ToastOverlay
-from clan_vm_manager.singletons.use_views import ViewStack
-from clan_vm_manager.singletons.use_vms import ClanStore
-from clan_vm_manager.views.details import Details
-from clan_vm_manager.views.list import ClanList
-from clan_vm_manager.views.logs import Logs
-from clan_vm_manager.views.webview import WebView
+from clan_app.components.interfaces import ClanConfig
+from clan_app.singletons.toast import ToastOverlay
+from clan_app.singletons.use_views import ViewStack
+from clan_app.singletons.use_vms import ClanStore
+from clan_app.views.details import Details
+from clan_app.views.list import ClanList
+from clan_app.views.logs import Logs
+from clan_app.views.webview import WebView, open_file
 
 gi.require_version("Adw", "1")
 
-from gi.repository import Adw, Gio, GLib, Gtk
+from gi.repository import Adw, Gio, GLib
 
-from clan_vm_manager.components.trayicon import TrayIcon
+from clan_app.components.trayicon import TrayIcon
 
 log = logging.getLogger(__name__)
 
@@ -29,6 +29,7 @@ class MainWindow(Adw.ApplicationWindow):
         self.set_title("Clan Manager")
         self.set_default_size(980, 850)
 
+        # Overlay for GTK side exclusive toasts
         overlay = ToastOverlay.use().overlay
         view = Adw.ToolbarView()
         overlay.set_child(view)
@@ -45,28 +46,20 @@ class MainWindow(Adw.ApplicationWindow):
         # Initialize all ClanStore
         threading.Thread(target=self._populate_vms).start()
 
-        # Initialize all views
         stack_view = ViewStack.use().view
-
-        clamp = Adw.Clamp()
-        clamp.set_child(stack_view)
-        clamp.set_maximum_size(1000)
-
-        scroll = Gtk.ScrolledWindow()
-        scroll.set_propagate_natural_height(True)
-        scroll.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
-        scroll.set_child(clamp)
-
         stack_view.add_named(ClanList(config), "list")
         stack_view.add_named(Details(), "details")
         stack_view.add_named(Logs(), "logs")
 
-        webview = WebView(methods=API._registry)
-        stack_view.add_named(webview.get_webview(), "webview")
+        # Override platform specific functions
+        API.register(open_file)
 
+        webview = WebView(methods=API._registry)
+
+        stack_view.add_named(webview.get_webview(), "webview")
         stack_view.set_visible_child_name(config.initial_view)
 
-        view.set_content(scroll)
+        view.set_content(stack_view)
 
         self.connect("destroy", self.on_destroy)
 

pkgs/clan-app/default.nix

@@ -24,10 +24,10 @@
 let
   source = ./.;
   desktop-file = makeDesktopItem {
-    name = "org.clan.vm-manager";
-    exec = "clan-vm-manager %u";
-    icon = ./clan_vm_manager/assets/clan_white.png;
-    desktopName = "Clan Manager";
+    name = "org.clan.app";
+    exec = "clan-app %u";
+    icon = ./clan_app/assets/clan_white.png;
+    desktopName = "Clan App";
     startupWMClass = "clan";
     mimeTypes = [ "x-scheme-handler/clan" ];
   };
@@ -69,7 +69,7 @@ let
   pythonWithTestDeps = python3.withPackages (_ps: testDependencies);
 in
 python3.pkgs.buildPythonApplication rec {
-  name = "clan-vm-manager";
+  name = "clan-app";
   src = source;
   format = "pyproject";
 
@@ -99,8 +99,8 @@ python3.pkgs.buildPythonApplication rec {
   # also re-expose dependencies so we test them in CI
   passthru = {
     tests = {
-      clan-vm-manager-pytest =
-        runCommand "clan-vm-manager-pytest" { inherit buildInputs propagatedBuildInputs nativeBuildInputs; }
+      clan-app-pytest =
+        runCommand "clan-app-pytest" { inherit buildInputs propagatedBuildInputs nativeBuildInputs; }
           ''
             cp -r ${source} ./src
             chmod +w -R ./src
@@ -112,8 +112,8 @@ python3.pkgs.buildPythonApplication rec {
             mkdir -p .home/.local/share/fonts
             export HOME=.home
 
-            fc-cache --verbose 
-            # > fc-cache succeded 
+            fc-cache --verbose
+            # > fc-cache succeded
 
             echo "Loaded the following fonts ..."
             fc-list
@@ -124,7 +124,7 @@ python3.pkgs.buildPythonApplication rec {
             touch $out
           '';
 
-      clan-vm-manager-no-breakpoints = runCommand "clan-vm-manager-no-breakpoints" { } ''
+      clan-app-no-breakpoints = runCommand "clan-app-no-breakpoints" { } ''
         if grep --include \*.py -Rq "breakpoint()" ${source}; then
           echo "breakpoint() found in ${source}:"
           grep --include \*.py -Rn "breakpoint()" ${source}
@@ -142,10 +142,10 @@ python3.pkgs.buildPythonApplication rec {
   passthru.runtimeDependencies = runtimeDependencies;
   passthru.testDependencies = testDependencies;
 
-  # TODO: place webui in lib/python3.11/site-packages/clan_vm_manager
+  # TODO: place webui in lib/python3.11/site-packages/clan_app
   postInstall = ''
-    mkdir -p $out/clan_vm_manager/.webui
-    cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/clan_vm_manager/.webui
+    mkdir -p $out/clan_app/.webui
+    cp -r ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* $out/clan_app/.webui
   '';
 
   # Don't leak python packages into a devshell.
@@ -160,13 +160,13 @@ python3.pkgs.buildPythonApplication rec {
     mkdir -p .home/.local/share/fonts
     export HOME=.home
 
-    fc-cache --verbose 
-    # > fc-cache succeded 
+    fc-cache --verbose
+    # > fc-cache succeded
 
     echo "Loaded the following fonts ..."
     fc-list
 
-    PYTHONPATH= $out/bin/clan-vm-manager --help
+    PYTHONPATH= $out/bin/clan-app --help
   '';
   desktopItems = [ desktop-file ];
 }

pkgs/clan-app/demo.sh

@@ -41,7 +41,7 @@ democlan="$1"
 
 check_git_tag "$democlan" "v2.2"
 
-check_git_tag "." "v2.3"
+check_git_tag "." "demo-v2.3"
 
 rm -rf ~/.config/clan
 

pkgs/clan-app/flake-module.nix

@@ -0,0 +1,24 @@
+{ ... }:
+{
+  perSystem =
+    {
+      config,
+      pkgs,
+      lib,
+      system,
+      ...
+    }:
+    if lib.elem system lib.platforms.darwin then
+      { }
+    else
+      {
+        devShells.clan-app = pkgs.callPackage ./shell.nix {
+          inherit (config.packages) clan-app webview-ui;
+        };
+        packages.clan-app = pkgs.python3.pkgs.callPackage ./default.nix {
+          inherit (config.packages) clan-cli webview-ui;
+        };
+
+        checks = config.packages.clan-app.tests;
+      };
+}

pkgs/clan-app/install-desktop.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-CLAN=$(nix build .#clan-vm-manager --print-out-paths)
+CLAN=$(nix build .#clan-app --print-out-paths)
 
 if ! command -v xdg-mime &> /dev/null; then
   echo "Warning: 'xdg-mime' is not available. The desktop file cannot be installed."
@@ -8,14 +8,14 @@ fi
 
 # install desktop file
 set -eou pipefail
-DESKTOP_FILE_NAME=org.clan.vm-manager.desktop
+DESKTOP_FILE_NAME=org.clan.app.desktop
 DESKTOP_DST=~/.local/share/applications/"$DESKTOP_FILE_NAME"
 DESKTOP_SRC="$CLAN/share/applications/$DESKTOP_FILE_NAME"
-UI_BIN="$CLAN/bin/clan-vm-manager"
+UI_BIN="$CLAN/bin/clan-app"
 
 cp -f "$DESKTOP_SRC" "$DESKTOP_DST"
 sleep 2
-sed -i "s|Exec=.*clan-vm-manager|Exec=$UI_BIN|" "$DESKTOP_DST"
+sed -i "s|Exec=.*clan-app|Exec=$UI_BIN|" "$DESKTOP_DST"
 xdg-mime default "$DESKTOP_FILE_NAME"  x-scheme-handler/clan
 echo "==== Validating desktop file installation   ===="
 set -x

pkgs/clan-app/pyproject.toml

@@ -4,10 +4,10 @@ build-backend = "setuptools.build_meta"
 
 
 [project]
-name = "clan-vm-manager"
-description = "clan vm manager"
+name = "clan-app"
+description = "clan app"
 dynamic = ["version"]
-scripts = { clan-vm-manager = "clan_vm_manager:main" }
+scripts = { clan-app = "clan_app:main" }
 
 [project.urls]
 Homepage = "https://clan.lol/"
@@ -18,7 +18,7 @@ Repository = "https://git.clan.lol/clan/clan-core"
 exclude = ["result"]
 
 [tool.setuptools.package-data]
-clan_vm_manager = ["**/assets/*"]
+clan_app = ["**/assets/*"]
 
 [tool.pytest.ini_options]
 testpaths = "tests"
@@ -43,5 +43,5 @@ ignore_missing_imports = true
 [tool.ruff]
 target-version = "py311"
 line-length = 88
-lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
+lint.select = ["E", "F", "I", "U", "N", "RUF", "ANN", "A"]
 lint.ignore = ["E501", "E402", "N802", "ANN101", "ANN401", "A003"]

pkgs/clan-app/shell.nix

@@ -1,7 +1,7 @@
 {
   lib,
   stdenv,
-  clan-vm-manager,
+  clan-app,
   mkShell,
   ruff,
   desktop-file-utils,
@@ -15,7 +15,7 @@
 
 let
   devshellTestDeps =
-    clan-vm-manager.externalTestDeps
+    clan-app.externalTestDeps
     ++ (with python3.pkgs; [
       rope
       mypy
@@ -26,7 +26,7 @@ let
     ]);
 in
 mkShell {
-  inherit (clan-vm-manager) nativeBuildInputs;
+  inherit (clan-app) nativeBuildInputs;
   buildInputs =
     [
       ruff
@@ -45,18 +45,18 @@ mkShell {
 
   shellHook = ''
     export GIT_ROOT=$(git rev-parse --show-toplevel)
-    export PKG_ROOT=$GIT_ROOT/pkgs/clan-vm-manager
+    export PKG_ROOT=$GIT_ROOT/pkgs/clan-app
 
-    # Add clan-vm-manager command to PATH
+    # Add clan-app command to PATH
     export PATH="$PKG_ROOT/bin":"$PATH"
 
     # Add clan-cli to the python path so that we can import it without building it in nix first
     export PYTHONPATH="$GIT_ROOT/pkgs/clan-cli":"$PYTHONPATH"
 
     # Add the webview-ui to the .webui directory
-    rm -rf ./clan_vm_manager/.webui/*
-    mkdir -p ./clan_vm_manager/.webui
-    cp -a ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* ./clan_vm_manager/.webui
-    chmod -R +w ./clan_vm_manager/.webui
+    rm -rf ./clan_app/.webui/*
+    mkdir -p ./clan_app/.webui
+    cp -a ${webview-ui}/lib/node_modules/@clan/webview-ui/dist/* ./clan_app/.webui
+    chmod -R +w ./clan_app/.webui
   '';
 }

pkgs/clan-app/tests/conftest.py

@@ -7,9 +7,7 @@ from clan_cli.custom_logger import setup_logging
 from clan_cli.nix import nix_shell
 
 sys.path.append(str(Path(__file__).parent / "helpers"))
-sys.path.append(
-    str(Path(__file__).parent.parent)
-)  # Also add clan vm manager to PYTHONPATH
+sys.path.append(str(Path(__file__).parent.parent))  # Also add clan app to PYTHONPATH
 
 pytest_plugins = [
     "temporary_dir",

pkgs/clan-app/tests/helpers/cli.py

@@ -3,7 +3,7 @@ import shlex
 
 from clan_cli.custom_logger import get_caller
 
-from clan_vm_manager import main
+from clan_app import main
 
 log = logging.getLogger(__name__)
 

pkgs/clan-app/tests/test_cli.py

@@ -5,4 +5,4 @@ from cli import Cli
 def test_help(capfd: pytest.CaptureFixture) -> None:
     cli = Cli()
     with pytest.raises(SystemExit):
-        cli.run(["clan-vm-manager", "--help"])
+        cli.run(["clan-app", "--help"])

pkgs/clan-app/tests/wayland.py

@@ -21,7 +21,7 @@ GtkProc = NewType("GtkProc", Popen)
 
 @pytest.fixture(scope="function")
 def app() -> Generator[GtkProc, None, None]:
-    rapp = Popen([sys.executable, "-m", "clan_vm_manager"], text=True)
+    rapp = Popen([sys.executable, "-m", "clan_app"], text=True)
     yield GtkProc(rapp)
     # Cleanup: Terminate your application
     rapp.terminate()

pkgs/clan-cli/clan_cli/__init__.py

@@ -6,6 +6,12 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any
 
+# These imports are unused, but necessary for @API.register to run once.
+from clan_cli.api import directory
+from clan_cli.flakes import show
+
+__all__ = ["directory"]
+
 from . import (
     backups,
     config,
@@ -107,8 +113,32 @@ For more detailed information, visit: https://docs.clan.lol
         ),
         formatter_class=argparse.RawTextHelpFormatter,
     )
+
     subparsers = parser.add_subparsers()
 
+    # Commands directly under the root i.e. "clan show"
+    show_parser = subparsers.add_parser(
+        "show",
+        help="Show meta about the clan if present.",
+        description="Show meta about the clan if present.",
+        epilog=(
+            """
+This command prints the metadata of a clan.
+
+Examples:
+
+  $ clan show --flake [PATH]
+  Name: My Empty Clan
+  Description: some nice description
+  Icon: A path to the png
+
+Note: The meta results from clan/meta.json and manual flake arguments. It may not be present for clans not created via the clan-app.
+
+"""
+        ),
+    )
+    show_parser.set_defaults(func=show.show_command)
+
     parser_backups = subparsers.add_parser(
         "backups",
         help="manage backups of clan machines",
@@ -152,6 +182,7 @@ For more detailed information, visit: https://docs.clan.lol/getting-started
         ),
         formatter_class=argparse.RawTextHelpFormatter,
     )
+
     flakes.register_parser(parser_flake)
 
     parser_config = subparsers.add_parser(
@@ -312,7 +343,7 @@ def main() -> None:
     if len(sys.argv) == 1:
         parser.print_help()
 
-    if args.debug:
+    if getattr(args, "debug", False):
         setup_logging(logging.DEBUG, root_log_name=__name__.split(".")[0])
         log.debug("Debug log activated")
         if flatpak.is_flatpak():

pkgs/clan-cli/clan_cli/api/__init__.py

@@ -1,10 +1,12 @@
 from collections.abc import Callable
 from dataclasses import dataclass
-from typing import Any, Generic, Literal, TypeVar
+from functools import wraps
+from typing import Annotated, Any, Generic, Literal, TypeVar, get_type_hints
+
+from clan_cli.errors import ClanError
 
 T = TypeVar("T")
 
-
 ResponseDataType = TypeVar("ResponseDataType")
 
 
@@ -16,22 +18,55 @@ class ApiError:
 
 
 @dataclass
-class ApiResponse(Generic[ResponseDataType]):
-    status: Literal["success", "error"]
-    errors: list[ApiError] | None
-    data: ResponseDataType | None
+class SuccessDataClass(Generic[ResponseDataType]):
+    status: Annotated[Literal["success"], "The status of the response."]
+    data: ResponseDataType
+
+
+@dataclass
+class ErrorDataClass:
+    status: Literal["error"]
+    errors: list[ApiError]
+
+
+ApiResponse = SuccessDataClass[ResponseDataType] | ErrorDataClass
 
 
 class _MethodRegistry:
     def __init__(self) -> None:
+        self._orig: dict[str, Callable[[Any], Any]] = {}
         self._registry: dict[str, Callable[[Any], Any]] = {}
 
     def register(self, fn: Callable[..., T]) -> Callable[..., T]:
-        self._registry[fn.__name__] = fn
+        self._orig[fn.__name__] = fn
+
+        @wraps(fn)
+        def wrapper(*args: Any, **kwargs: Any) -> ApiResponse[T]:
+            try:
+                data: T = fn(*args, **kwargs)
+                return SuccessDataClass(status="success", data=data)
+            except ClanError as e:
+                return ErrorDataClass(
+                    status="error",
+                    errors=[
+                        ApiError(
+                            message=e.msg,
+                            description=e.description,
+                            location=[fn.__name__, e.location],
+                        )
+                    ],
+                )
+
+        # @wraps preserves all metadata of fn
+        # we need to update the annotation, because our wrapper changes the return type
+        # This overrides the new return type annotation with the generic typeVar filled in
+        orig_return_type = get_type_hints(fn).get("return")
+        wrapper.__annotations__["return"] = ApiResponse[orig_return_type]  # type: ignore
+
+        self._registry[fn.__name__] = wrapper
         return fn
 
     def to_json_schema(self) -> dict[str, Any]:
-        # Import only when needed
         from typing import get_type_hints
 
         from clan_cli.api.util import type_to_dict

pkgs/clan-cli/clan_cli/api/directory.py

@@ -0,0 +1,83 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+from clan_cli.errors import ClanError
+
+from . import API
+
+
+@dataclass
+class FileFilter:
+    title: str | None
+    mime_types: list[str] | None
+    patterns: list[str] | None
+    suffixes: list[str] | None
+
+
+@dataclass
+class FileRequest:
+    # Mode of the os dialog window
+    mode: Literal["open_file", "select_folder"]
+    # Title of the os dialog window
+    title: str | None = None
+    # Pre-applied filters for the file dialog
+    filters: FileFilter | None = None
+
+
+@API.register
+def open_file(file_request: FileRequest) -> str | None:
+    """
+    Abstract api method to open a file dialog window.
+    It must return the name of the selected file or None if no file was selected.
+    """
+    raise NotImplementedError("Each specific platform should implement this function.")
+
+
+@dataclass
+class File:
+    path: str
+    file_type: Literal["file", "directory", "symlink"]
+
+
+@dataclass
+class Directory:
+    path: str
+    files: list[File] = field(default_factory=list)
+
+
+@API.register
+def get_directory(current_path: str) -> Directory:
+    curr_dir = Path(current_path)
+    directory = Directory(path=str(curr_dir))
+
+    if not curr_dir.is_dir():
+        raise ClanError()
+
+    with os.scandir(curr_dir.resolve()) as it:
+        for entry in it:
+            if entry.is_symlink():
+                directory.files.append(
+                    File(
+                        path=str(curr_dir / Path(entry.name)),
+                        file_type="symlink",
+                    )
+                )
+            elif entry.is_file():
+                directory.files.append(
+                    File(
+                        path=str(curr_dir / Path(entry.name)),
+                        file_type="file",
+                    )
+                )
+
+            elif entry.is_dir():
+                directory.files.append(
+                    File(
+                        path=str(curr_dir / Path(entry.name)),
+                        file_type="directory",
+                    )
+                )
+
+    return directory

pkgs/clan-cli/clan_cli/api/util.py

@@ -1,20 +1,92 @@
+import copy
 import dataclasses
 import pathlib
 from types import NoneType, UnionType
-from typing import Any, Union
+from typing import (
+    Annotated,
+    Any,
+    Literal,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
 
 
-def type_to_dict(t: Any, scope: str = "") -> dict:
+class JSchemaTypeError(Exception):
+    pass
+
+
+# Inspect the fields of the parameterized type
+def inspect_dataclass_fields(t: type) -> dict[TypeVar, type]:
+    """
+    Returns a map of type variables to actual types for a parameterized type.
+    """
+    origin = get_origin(t)
+    type_args = get_args(t)
+    if origin is None:
+        return {}
+
+    type_params = origin.__parameters__
+    # Create a map from type parameters to actual type arguments
+    type_map = dict(zip(type_params, type_args))
+
+    return type_map
+
+
+def apply_annotations(schema: dict[str, Any], annotations: list[Any]) -> dict[str, Any]:
+    """
+    Add metadata from typing.annotations to the json Schema.
+    The annotations can be a dict, a tuple, or a string and is directly applied to the schema as shown below.
+    No further validation is done, the caller is responsible for following json-schema.
+
+    Examples
+
+    ```python
+    # String annotation
+    Annotated[int, "This is an int"] -> {"type": "integer", "description": "This is an int"}
+
+    # Dict annotation
+    Annotated[int, {"minimum": 0, "maximum": 10}] -> {"type": "integer", "minimum": 0, "maximum": 10}
+
+    # Tuple annotation
+    Annotated[int, ("minimum", 0)] -> {"type": "integer", "minimum": 0}
+    ```
+    """
+    for annotation in annotations:
+        if isinstance(annotation, dict):
+            # Assuming annotation is a dict that can directly apply to the schema
+            schema.update(annotation)
+        elif isinstance(annotation, tuple) and len(annotation) == 2:
+            # Assuming a tuple where first element is a keyword (like 'minLength') and the second is the value
+            schema[annotation[0]] = annotation[1]
+        elif isinstance(annotation, str):
+            # String annotations can be used for description
+            schema.update({"description": f"{annotation}"})
+    return schema
+
+
+def type_to_dict(t: Any, scope: str = "", type_map: dict[TypeVar, type] = {}) -> dict:
     if t is None:
         return {"type": "null"}
 
     if dataclasses.is_dataclass(t):
         fields = dataclasses.fields(t)
         properties = {
-            f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}")
+            f.name: type_to_dict(f.type, f"{scope} {t.__name__}.{f.name}", type_map)
             for f in fields
         }
-        required = [pn for pn, pv in properties.items() if "null" not in pv["type"]]
+
+        required = []
+        for pn, pv in properties.items():
+            if pv.get("type") is not None:
+                if "null" not in pv["type"]:
+                    required.append(pn)
+
+            elif pv.get("oneOf") is not None:
+                if "null" not in [i["type"] for i in pv.get("oneOf", [])]:
+                    required.append(pn)
+
         return {
             "type": "object",
             "properties": properties,
@@ -22,24 +94,54 @@ def type_to_dict(t: Any, scope: str = "") -> dict:
             # Dataclasses can only have the specified properties
             "additionalProperties": False,
         }
+
     elif type(t) is UnionType:
         return {
-            "type": [type_to_dict(arg, scope)["type"] for arg in t.__args__],
+            "oneOf": [type_to_dict(arg, scope, type_map) for arg in t.__args__],
         }
 
+    if isinstance(t, TypeVar):
+        # if t is a TypeVar, look up the type in the type_map
+        # And return the resolved type instead of the TypeVar
+        resolved = type_map.get(t)
+        if not resolved:
+            raise JSchemaTypeError(
+                f"{scope} - TypeVar {t} not found in type_map, map: {type_map}"
+            )
+        return type_to_dict(type_map.get(t), scope, type_map)
+
     elif hasattr(t, "__origin__"):  # Check if it's a generic type
-        origin = getattr(t, "__origin__", None)
+        origin = get_origin(t)
+        args = get_args(t)
 
         if origin is None:
             # Non-generic user-defined or built-in type
             # TODO: handle custom types
-            raise BaseException("Unhandled Type: ", origin)
+            raise JSchemaTypeError("Unhandled Type: ", origin)
+
+        elif origin is Literal:
+            # Handle Literal values for enums in JSON Schema
+            return {
+                "type": "string",
+                "enum": list(args),  # assumes all args are strings
+            }
+
+        elif origin is Annotated:
+            base_type, *metadata = get_args(t)
+            schema = type_to_dict(base_type, scope)  # Generate schema for the base type
+            return apply_annotations(schema, metadata)
 
         elif origin is Union:
-            return {"type": [type_to_dict(arg, scope)["type"] for arg in t.__args__]}
+            union_types = [type_to_dict(arg, scope, type_map) for arg in t.__args__]
+            return {
+                "oneOf": union_types,
+            }
 
-        elif issubclass(origin, list):
-            return {"type": "array", "items": type_to_dict(t.__args__[0], scope)}
+        elif origin in {list, set, frozenset}:
+            return {
+                "type": "array",
+                "items": type_to_dict(t.__args__[0], scope, type_map),
+            }
 
         elif issubclass(origin, dict):
             value_type = t.__args__[1]
@@ -48,10 +150,19 @@ def type_to_dict(t: Any, scope: str = "") -> dict:
             else:
                 return {
                     "type": "object",
-                    "additionalProperties": type_to_dict(value_type, scope),
+                    "additionalProperties": type_to_dict(value_type, scope, type_map),
                 }
+        # Generic dataclass with type parameters
+        elif dataclasses.is_dataclass(origin):
+            # This behavior should mimic the scoping of typeVars in dataclasses
+            # Once type_to_dict() encounters a TypeVar, it will look up the type in the type_map
+            # When type_to_dict() returns the map goes out of scope.
+            # This behaves like a stack, where the type_map is pushed and popped as we traverse the dataclass fields
+            new_map = copy.deepcopy(type_map)
+            new_map.update(inspect_dataclass_fields(t))
+            return type_to_dict(origin, scope, new_map)
 
-        raise BaseException(f"Error api type not yet supported {t!s}")
+        raise JSchemaTypeError(f"Error api type not yet supported {t!s}")
 
     elif isinstance(t, type):
         if t is str:
@@ -65,20 +176,23 @@ def type_to_dict(t: Any, scope: str = "") -> dict:
         if t is object:
             return {"type": "object"}
         if t is Any:
-            raise BaseException(
+            raise JSchemaTypeError(
                 f"Usage of the Any type is not supported for API functions. In: {scope}"
             )
-
         if t is pathlib.Path:
             return {
                 # TODO: maybe give it a pattern for URI
                 "type": "string",
             }
+        if t is dict:
+            raise JSchemaTypeError(
+                "Error: generic dict type not supported. Use dict[str. Any] instead"
+            )
 
         # Optional[T] gets internally transformed Union[T,NoneType]
         if t is NoneType:
             return {"type": "null"}
 
-        raise BaseException(f"Error primitive type not supported {t!s}")
+        raise JSchemaTypeError(f"Error primitive type not supported {t!s}")
     else:
-        raise BaseException(f"Error type not supported {t!s}")
+        raise JSchemaTypeError(f"Error type not supported {t!s}")

pkgs/clan-cli/clan_cli/backups/create.py

@@ -2,6 +2,11 @@ import argparse
 import json
 import logging
 
+from ..completions import (
+    add_dynamic_completer,
+    complete_backup_providers_for_machine,
+    complete_machines,
+)
 from ..errors import ClanError
 from ..machines.machines import Machine
 
@@ -40,8 +45,13 @@ def create_command(args: argparse.Namespace) -> None:
 
 
 def register_create_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine", type=str, help="machine in the flake to create backups of"
     )
-    parser.add_argument("--provider", type=str, help="backup provider to use")
+    add_dynamic_completer(machines_parser, complete_machines)
+
+    provider_action = parser.add_argument(
+        "--provider", type=str, help="backup provider to use"
+    )
+    add_dynamic_completer(provider_action, complete_backup_providers_for_machine)
     parser.set_defaults(func=create_command)

pkgs/clan-cli/clan_cli/backups/list.py

@@ -3,6 +3,11 @@ import json
 import subprocess
 from dataclasses import dataclass
 
+from ..completions import (
+    add_dynamic_completer,
+    complete_backup_providers_for_machine,
+    complete_machines,
+)
 from ..errors import ClanError
 from ..machines.machines import Machine
 
@@ -57,8 +62,12 @@ def list_command(args: argparse.Namespace) -> None:
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine", type=str, help="machine in the flake to show backups of"
     )
-    parser.add_argument("--provider", type=str, help="backup provider to filter by")
+    add_dynamic_completer(machines_parser, complete_machines)
+    provider_action = parser.add_argument(
+        "--provider", type=str, help="backup provider to filter by"
+    )
+    add_dynamic_completer(provider_action, complete_backup_providers_for_machine)
     parser.set_defaults(func=list_command)

pkgs/clan-cli/clan_cli/backups/restore.py

@@ -2,6 +2,11 @@ import argparse
 import json
 import subprocess
 
+from ..completions import (
+    add_dynamic_completer,
+    complete_backup_providers_for_machine,
+    complete_machines,
+)
 from ..errors import ClanError
 from ..machines.machines import Machine
 
@@ -9,10 +14,16 @@ from ..machines.machines import Machine
 def restore_service(machine: Machine, name: str, provider: str, service: str) -> None:
     backup_metadata = json.loads(machine.eval_nix("config.clanCore.backups"))
     backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
+
+    if service not in backup_folders:
+        msg = f"Service {service} not found in configuration. Available services are: {', '.join(backup_folders.keys())}"
+        raise ClanError(msg)
+
     folders = backup_folders[service]["folders"]
     env = {}
     env["NAME"] = name
-    env["FOLDERS"] = ":".join(folders)
+    # FIXME: If we have too many folder this might overflow the stack.
+    env["FOLDERS"] = ":".join(set(folders))
 
     if pre_restore := backup_folders[service]["preRestoreCommand"]:
         proc = machine.target_host.run(
@@ -53,12 +64,23 @@ def restore_backup(
     name: str,
     service: str | None = None,
 ) -> None:
+    errors = []
     if service is None:
         backup_folders = json.loads(machine.eval_nix("config.clanCore.state"))
         for _service in backup_folders:
-            restore_service(machine, name, provider, _service)
+            try:
+                restore_service(machine, name, provider, _service)
+            except ClanError as e:
+                errors.append(f"{_service}: {e}")
     else:
-        restore_service(machine, name, provider, service)
+        try:
+            restore_service(machine, name, provider, service)
+        except ClanError as e:
+            errors.append(f"{service}: {e}")
+    if errors:
+        raise ClanError(
+            "Restore failed for the following services:\n" + "\n".join(errors)
+        )
 
 
 def restore_command(args: argparse.Namespace) -> None:
@@ -74,10 +96,14 @@ def restore_command(args: argparse.Namespace) -> None:
 
 
 def register_restore_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machine_action = parser.add_argument(
         "machine", type=str, help="machine in the flake to create backups of"
     )
-    parser.add_argument("provider", type=str, help="backup provider to use")
+    add_dynamic_completer(machine_action, complete_machines)
+    provider_action = parser.add_argument(
+        "provider", type=str, help="backup provider to use"
+    )
+    add_dynamic_completer(provider_action, complete_backup_providers_for_machine)
     parser.add_argument("name", type=str, help="Name of the backup to restore")
     parser.add_argument("--service", type=str, help="name of the service to restore")
     parser.set_defaults(func=restore_command)

pkgs/clan-cli/clan_cli/cmd.py

@@ -56,7 +56,7 @@ def handle_output(process: subprocess.Popen, log: Log) -> tuple[str, str]:
             sys.stderr.buffer.write(ret)
             sys.stderr.flush()
         stderr_buf += ret
-    return stdout_buf.decode("utf-8"), stderr_buf.decode("utf-8")
+    return stdout_buf.decode("utf-8", "replace"), stderr_buf.decode("utf-8", "replace")
 
 
 class TimeTable:
@@ -101,13 +101,19 @@ TIME_TABLE = TimeTable()
 def run(
     cmd: list[str],
     *,
+    input: bytes | None = None,  # noqa: A002
     env: dict[str, str] | None = None,
     cwd: Path = Path.cwd(),
     log: Log = Log.STDERR,
     check: bool = True,
     error_msg: str | None = None,
 ) -> CmdOut:
-    glog.debug(f"$: {shlex.join(cmd)} \nCaller: {get_caller()}")
+    if input:
+        glog.debug(
+            f"""$: echo "{input.decode('utf-8', 'replace')}" | {shlex.join(cmd)} \nCaller: {get_caller()}"""
+        )
+    else:
+        glog.debug(f"$: {shlex.join(cmd)} \nCaller: {get_caller()}")
     tstart = datetime.now()
 
     # Start the subprocess
@@ -120,7 +126,10 @@ def run(
     )
     stdout_buf, stderr_buf = handle_output(process, log)
 
-    rc = process.wait()
+    if input:
+        process.communicate(input)
+    else:
+        process.wait()
     tend = datetime.now()
 
     global TIME_TABLE
@@ -136,7 +145,7 @@ def run(
         msg=error_msg,
     )
 
-    if check and rc != 0:
+    if check and process.returncode != 0:
         raise ClanCmdError(cmd_out)
 
     return cmd_out

pkgs/clan-cli/clan_cli/completions.py

@@ -0,0 +1,242 @@
+import argparse
+import json
+import subprocess
+import threading
+from collections.abc import Callable, Iterable
+from types import ModuleType
+from typing import Any
+
+from .cmd import run
+from .nix import nix_eval
+
+"""
+This module provides dynamic completions.
+The completions should feel fast.
+We target a maximum of 1second on our average machine.
+"""
+
+
+argcomplete: ModuleType | None = None
+try:
+    import argcomplete  # type: ignore[no-redef]
+except ImportError:
+    pass
+
+
+# The default completion timeout for commands
+COMPLETION_TIMEOUT: int = 3
+
+
+def clan_dir(flake: str | None) -> str | None:
+    from .dirs import get_clan_flake_toplevel_or_env
+
+    path_result = get_clan_flake_toplevel_or_env()
+    return str(path_result) if path_result is not None else None
+
+
+def complete_machines(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for machine names configured in the clan.
+    """
+    machines: list[str] = []
+
+    def run_cmd() -> None:
+        try:
+            if (clan_dir_result := clan_dir(None)) is not None:
+                flake = clan_dir_result
+            else:
+                flake = "."
+            services_result = json.loads(
+                run(
+                    nix_eval(
+                        flags=[
+                            f"{flake}#nixosConfigurations",
+                            "--apply",
+                            "builtins.attrNames",
+                        ],
+                    ),
+                ).stdout.strip()
+            )
+
+            machines.extend(services_result)
+        except subprocess.CalledProcessError:
+            pass
+
+    thread = threading.Thread(target=run_cmd)
+    thread.start()
+    thread.join(timeout=COMPLETION_TIMEOUT)
+
+    if thread.is_alive():
+        return iter([])
+
+    machines_dict = {name: "machine" for name in machines}
+    return machines_dict
+
+
+def complete_services_for_machine(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for machine facts generation services.
+    """
+    services: list[str] = []
+    # TODO: consolidate, if multiple machines are used
+    machines: list[str] = parsed_args.machines
+
+    def run_cmd() -> None:
+        try:
+            if (clan_dir_result := clan_dir(None)) is not None:
+                flake = clan_dir_result
+            else:
+                flake = "."
+            services_result = json.loads(
+                run(
+                    nix_eval(
+                        flags=[
+                            f"{flake}#nixosConfigurations.{machines[0]}.config.clanCore.facts.services",
+                            "--apply",
+                            "builtins.attrNames",
+                        ],
+                    ),
+                ).stdout.strip()
+            )
+
+            services.extend(services_result)
+        except subprocess.CalledProcessError:
+            pass
+
+    thread = threading.Thread(target=run_cmd)
+    thread.start()
+    thread.join(timeout=COMPLETION_TIMEOUT)
+
+    if thread.is_alive():
+        return iter([])
+
+    services_dict = {name: "service" for name in services}
+    return services_dict
+
+
+def complete_backup_providers_for_machine(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for machine backup providers.
+    """
+    providers: list[str] = []
+    machine: str = parsed_args.machine
+
+    def run_cmd() -> None:
+        try:
+            if (clan_dir_result := clan_dir(None)) is not None:
+                flake = clan_dir_result
+            else:
+                flake = "."
+            providers_result = json.loads(
+                run(
+                    nix_eval(
+                        flags=[
+                            f"{flake}#nixosConfigurations.{machine}.config.clanCore.backups.providers",
+                            "--apply",
+                            "builtins.attrNames",
+                        ],
+                    ),
+                ).stdout.strip()
+            )
+
+            providers.extend(providers_result)
+        except subprocess.CalledProcessError:
+            pass
+
+    thread = threading.Thread(target=run_cmd)
+    thread.start()
+    thread.join(timeout=COMPLETION_TIMEOUT)
+
+    if thread.is_alive():
+        return iter([])
+
+    providers_dict = {name: "provider" for name in providers}
+    return providers_dict
+
+
+def complete_secrets(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for clan secrets
+    """
+    from pathlib import Path
+
+    from .secrets.secrets import ListSecretsOptions, list_secrets
+
+    if (clan_dir_result := clan_dir(None)) is not None:
+        flake = clan_dir_result
+    else:
+        flake = "."
+
+    options = ListSecretsOptions(
+        flake=Path(flake),
+        pattern=None,
+    )
+
+    secrets = list_secrets(options.flake, options.pattern)
+
+    secrets_dict = {name: "secret" for name in secrets}
+    return secrets_dict
+
+
+def complete_users(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for clan users
+    """
+    from pathlib import Path
+
+    from .secrets.users import list_users
+
+    if (clan_dir_result := clan_dir(None)) is not None:
+        flake = clan_dir_result
+    else:
+        flake = "."
+
+    users = list_users(Path(flake))
+
+    users_dict = {name: "user" for name in users}
+    return users_dict
+
+
+def complete_groups(
+    prefix: str, parsed_args: argparse.Namespace, **kwargs: Any
+) -> Iterable[str]:
+    """
+    Provides completion functionality for clan groups
+    """
+    from pathlib import Path
+
+    from .secrets.groups import list_groups
+
+    if (clan_dir_result := clan_dir(None)) is not None:
+        flake = clan_dir_result
+    else:
+        flake = "."
+
+    groups_list = list_groups(Path(flake))
+    groups = [group.name for group in groups_list]
+
+    groups_dict = {name: "group" for name in groups}
+    return groups_dict
+
+
+def add_dynamic_completer(
+    action: argparse.Action,
+    completer: Callable[..., Iterable[str]],
+) -> None:
+    """
+    Add a completion function to an argparse action, this will only be added,
+    if the argcomplete module is loaded.
+    """
+    if argcomplete:
+        # mypy doesn't check this correctly, so we ignore it
+        action.completer = completer  # type: ignore[attr-defined]

pkgs/clan-cli/clan_cli/config/__init__.py

@@ -9,6 +9,7 @@ from pathlib import Path
 from typing import Any, get_origin
 
 from clan_cli.cmd import run
+from clan_cli.completions import add_dynamic_completer, complete_machines
 from clan_cli.dirs import machine_settings_file
 from clan_cli.errors import ClanError
 from clan_cli.git import commit_file
@@ -317,13 +318,14 @@ def register_parser(
 
     # inject callback function to process the input later
     parser.set_defaults(func=get_option)
-    parser.add_argument(
+    set_machine_action = parser.add_argument(
         "--machine",
         "-m",
         help="Machine to configure",
         type=str,
         default="default",
     )
+    add_dynamic_completer(set_machine_action, complete_machines)
 
     parser.add_argument(
         "--show-trace",

pkgs/clan-cli/clan_cli/errors.py

@@ -1,4 +1,5 @@
 import shutil
+from dataclasses import dataclass
 from math import floor
 from pathlib import Path
 
@@ -15,25 +16,17 @@ def text_heading(heading: str) -> str:
     return f"{'=' * filler} {heading} {'=' * filler}"
 
 
+@dataclass
 class CmdOut:
-    def __init__(
-        self,
-        stdout: str,
-        stderr: str,
-        cwd: Path,
-        command: str,
-        returncode: int,
-        msg: str | None,
-    ) -> None:
-        super().__init__()
-        self.stdout = stdout
-        self.stderr = stderr
-        self.cwd = cwd
-        self.command = command
-        self.returncode = returncode
-        self.msg = msg
+    stdout: str
+    stderr: str
+    cwd: Path
+    command: str
+    returncode: int
+    msg: str | None
 
-        self.error_str = f"""
+    def __str__(self) -> str:
+        error_str = f"""
 {text_heading(heading="Command")}
 {self.command}
 {text_heading(heading="Stderr")}
@@ -45,15 +38,30 @@ Message: {self.msg}
 Working Directory: '{self.cwd}'
 Return Code: {self.returncode}
         """
-
-    def __str__(self) -> str:
-        return self.error_str
+        return error_str
 
 
 class ClanError(Exception):
     """Base class for exceptions in this module."""
 
-    pass
+    description: str | None
+    location: str
+
+    def __init__(
+        self,
+        msg: str | None = None,
+        *,
+        description: str | None = None,
+        location: str | None = None,
+    ) -> None:
+        self.description = description
+        self.location = location or "Unknown location"
+        self.msg = msg or ""
+        if self.description:
+            exception_msg = f"{self.location}: {self.msg} - {self.description}"
+        else:
+            exception_msg = f"{self.location}: {self.msg}"
+        super().__init__(exception_msg)
 
 
 class ClanHttpError(ClanError):

pkgs/clan-cli/clan_cli/facts/check.py

@@ -2,6 +2,7 @@ import argparse
 import importlib
 import logging
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 
 log = logging.getLogger(__name__)
@@ -54,10 +55,12 @@ def check_command(args: argparse.Namespace) -> None:
 
 
 def register_check_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         help="The machine to check secrets for",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.add_argument(
         "--service",
         help="the service to check",

pkgs/clan-cli/clan_cli/facts/generate.py

@@ -9,6 +9,11 @@ from tempfile import TemporaryDirectory
 
 from clan_cli.cmd import run
 
+from ..completions import (
+    add_dynamic_completer,
+    complete_machines,
+    complete_services_for_machine,
+)
 from ..errors import ClanError
 from ..git import commit_files
 from ..machines.inventory import get_all_machines, get_selected_machines
@@ -27,6 +32,7 @@ def read_multiline_input(prompt: str = "Finish with Ctrl-D") -> str:
     """
     print(prompt, flush=True)
     proc = subprocess.run(["cat"], stdout=subprocess.PIPE, text=True)
+    log.info("Input received. Processing...")
     return proc.stdout
 
 
@@ -216,19 +222,23 @@ def generate_command(args: argparse.Namespace) -> None:
 
 
 def register_generate_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machines",
         type=str,
         help="machine to generate facts for. if empty, generate facts for all machines",
         nargs="*",
         default=[],
     )
-    parser.add_argument(
+    add_dynamic_completer(machines_parser, complete_machines)
+
+    service_parser = parser.add_argument(
         "--service",
         type=str,
         help="service to generate facts for, if empty, generate facts for every service",
         default=None,
     )
+    add_dynamic_completer(service_parser, complete_services_for_machine)
+
     parser.add_argument(
         "--regenerate",
         type=bool,

pkgs/clan-cli/clan_cli/facts/list.py

@@ -3,6 +3,7 @@ import importlib
 import json
 import logging
 
+from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 
 log = logging.getLogger(__name__)
@@ -37,8 +38,10 @@ def get_command(args: argparse.Namespace) -> None:
 
 
 def register_list_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         help="The machine to print facts for",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.set_defaults(func=get_command)

pkgs/clan-cli/clan_cli/facts/upload.py

@@ -5,6 +5,7 @@ from pathlib import Path
 from tempfile import TemporaryDirectory
 
 from ..cmd import Log, run
+from ..completions import add_dynamic_completer, complete_machines
 from ..machines.machines import Machine
 from ..nix import nix_shell
 
@@ -32,6 +33,8 @@ def upload_secrets(machine: Machine) -> None:
                     " ".join(["ssh"] + ssh_cmd[2:]),
                     "-az",
                     "--delete",
+                    "--chown=root:root",
+                    "--chmod=D700,F600",
                     f"{tempdir!s}/",
                     f"{host.user}@{host.host}:{machine.secrets_upload_directory}/",
                 ],
@@ -46,8 +49,10 @@ def upload_command(args: argparse.Namespace) -> None:
 
 
 def register_upload_parser(parser: argparse.ArgumentParser) -> None:
-    parser.add_argument(
+    machines_parser = parser.add_argument(
         "machine",
         help="The machine to upload secrets to",
     )
+    add_dynamic_completer(machines_parser, complete_machines)
+
     parser.set_defaults(func=upload_command)