Merge pull request 'matrix-synapse: user creation fixes' (#1620) from matrix into main

checks/lib/container-driver/test_driver/__init__.py

@@ -151,7 +151,7 @@ class Machine:
         """
 
         # Always run command with shell opts
-        command = f"set -euo pipefail; {command}"
+        command = f"set -eo pipefail; source /etc/profile; set -u; {command}"
 
         proc = subprocess.run(
             [

checks/matrix-synapse/default.nix

@@ -38,7 +38,7 @@
                 z.mode = "0700";
               };
               "/etc/secrets/synapse-registration_shared_secret" = {
-                f.argument = "registration_shared_secret: supersecret";
+                f.argument = "supersecret";
                 z = {
                   mode = "0400";
                   user = "root";

clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch

@@ -1,43 +1,57 @@
-From df45634a92944dcab4edb02fb5e478911c58fdd6 Mon Sep 17 00:00:00 2001
+From bc199a27f23b0fcf175b116f7cf606c0d22b422a Mon Sep 17 00:00:00 2001
 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
 Date: Tue, 11 Jun 2024 11:40:47 +0200
-Subject: [PATCH] register_new_matrix_user: add password-file flag
+Subject: [PATCH 1/2] register_new_matrix_user: add password-file flag
 MIME-Version: 1.0
 Content-Type: text/plain; charset=UTF-8
 Content-Transfer-Encoding: 8bit
 
-getpass in python exist on stdin to be a tty, hence we cannot just pipe
+getpass in python expects stdin to be a tty, hence we cannot just pipe
 into register_new_matrix_user. --password-file instead works better and
 it would also allow the use of stdin if /dev/stdin is passed.
 
+Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>
 Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
 ---
- debian/register_new_matrix_user.ronn         |  6 +++++-
- synapse/_scripts/register_new_matrix_user.py | 16 ++++++++++++++--
- 2 files changed, 19 insertions(+), 3 deletions(-)
+ changelog.d/17294.feature                    |  2 ++
+ debian/register_new_matrix_user.ronn         |  9 +++++++--
+ synapse/_scripts/register_new_matrix_user.py | 20 +++++++++++++++-----
+ 3 files changed, 24 insertions(+), 7 deletions(-)
+ create mode 100644 changelog.d/17294.feature
 
+diff --git a/changelog.d/17294.feature b/changelog.d/17294.feature
+new file mode 100644
+index 000000000..33aac7b0b
+--- /dev/null
++++ b/changelog.d/17294.feature
+@@ -0,0 +1,2 @@
++`register_new_matrix_user` now supports a --password-file flag, which
++is useful for scripting.
 diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn
-index 0410b1f4c..e39bef448 100644
+index 0410b1f4c..d99e9215a 100644
 --- a/debian/register_new_matrix_user.ronn
 +++ b/debian/register_new_matrix_user.ronn
-@@ -32,7 +32,11 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
+@@ -31,8 +31,13 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
+     Local part of the new user. Will prompt if omitted.
  
    * `-p`, `--password`:
-     New password for user. Will prompt if omitted. Supplying the password
+-    New password for user. Will prompt if omitted. Supplying the password
 -    on the command line is not recommended. Use the STDIN instead.
-+    on the command line is not recommended. Use password-file if possible.
++    New password for user. Will prompt if this option and `--password-file` are omitted.
++    Supplying the password on the command line is not recommended.
++    Use `--password-file` if possible.
 +
 +  * `--password-file`:
-+    File containing the new password for user. Will prompt if omitted.
++    File containing the new password for user. If set, overrides `--password`.
 +    This is a more secure alternative to specifying the password on the command line.
  
    * `-a`, `--admin`:
      Register new user as an admin. Will prompt if omitted.
 diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
-index 77a7129ee..f067e6832 100644
+index 77a7129ee..972b35e2d 100644
 --- a/synapse/_scripts/register_new_matrix_user.py
 +++ b/synapse/_scripts/register_new_matrix_user.py
-@@ -173,12 +173,18 @@ def main() -> None:
+@@ -173,11 +173,18 @@ def main() -> None:
          default=None,
          help="Local part of the new user. Will prompt if omitted.",
      )
@@ -47,21 +61,21 @@ index 77a7129ee..f067e6832 100644
          "-p",
          "--password",
          default=None,
-         help="New password for user. Will prompt if omitted.",
-     )
+-        help="New password for user. Will prompt if omitted.",
++        help="New password for user. Will prompt for a password if "
++        "this flag and `--password-file` are both omitted.",
++    )
 +    password_group.add_argument(
 +        "--password-file",
 +        default=None,
-+        help="File containing the new password for user. Will prompt if omitted.",
-+    )
++        help="File containing the new password for user. If set, will override `--password`.",
+     )
      parser.add_argument(
          "-t",
-         "--user_type",
-@@ -247,6 +253,12 @@ def main() -> None:
+@@ -247,6 +254,11 @@ def main() -> None:
              print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr)
              sys.exit(1)
  
-+    password = ""
 +    if args.password_file:
 +        password = _read_file(args.password_file, "password-file").strip()
 +    else:
@@ -70,15 +84,17 @@ index 77a7129ee..f067e6832 100644
      if args.server_url:
          server_url = args.server_url
      elif config is not None:
-@@ -270,7 +282,7 @@ def main() -> None:
+@@ -269,9 +281,7 @@ def main() -> None:
+     if args.admin or args.no_admin:
          admin = args.admin
  
-     register_new_user(
+-    register_new_user(
 -        args.user, args.password, server_url, secret, admin, args.user_type
-+        args.user, password, server_url, secret, admin, args.user_type
-     )
+-    )
++    register_new_user(args.user, password, server_url, secret, admin, args.user_type)
  
  
+ def _read_file(file_path: Any, config_path: str) -> str:
 -- 
 2.44.1
 

clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch

@@ -0,0 +1,94 @@
+From 1789416df425d22693b0055a6688d8686e0ee4a1 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
+Date: Thu, 13 Jun 2024 14:38:19 +0200
+Subject: [PATCH 2/2] register-new-matrix-user: add a flag to ignore already
+ existing users
+MIME-Version: 1.0
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: 8bit
+
+This allows to register users in a more declarative and stateless way.
+
+Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
+---
+ synapse/_scripts/register_new_matrix_user.py | 22 ++++++++++++++++++--
+ 1 file changed, 20 insertions(+), 2 deletions(-)
+
+diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py
+index 972b35e2d..233e7267d 100644
+--- a/synapse/_scripts/register_new_matrix_user.py
++++ b/synapse/_scripts/register_new_matrix_user.py
+@@ -52,6 +52,7 @@ def request_registration(
+     user_type: Optional[str] = None,
+     _print: Callable[[str], None] = print,
+     exit: Callable[[int], None] = sys.exit,
++    exists_ok: bool = False,
+ ) -> None:
+     url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),)
+ 
+@@ -97,6 +98,10 @@ def request_registration(
+     r = requests.post(url, json=data)
+ 
+     if r.status_code != 200:
++        response = r.json()
++        if exists_ok and response["errcode"] == "M_USER_IN_USE":
++            _print("User already exists. Skipping.")
++            return
+         _print("ERROR! Received %d %s" % (r.status_code, r.reason))
+         if 400 <= r.status_code < 500:
+             try:
+@@ -115,6 +120,7 @@ def register_new_user(
+     shared_secret: str,
+     admin: Optional[bool],
+     user_type: Optional[str],
++    exists_ok: bool = False,
+ ) -> None:
+     if not user:
+         try:
+@@ -154,7 +160,13 @@ def register_new_user(
+             admin = False
+ 
+     request_registration(
+-        user, password, server_location, shared_secret, bool(admin), user_type
++        user,
++        password,
++        server_location,
++        shared_secret,
++        bool(admin),
++        user_type,
++        exists_ok=exists_ok,
+     )
+ 
+ 
+@@ -173,6 +185,11 @@ def main() -> None:
+         default=None,
+         help="Local part of the new user. Will prompt if omitted.",
+     )
++    parser.add_argument(
++        "--exists-ok",
++        action="store_true",
++        help="Do not fail if user already exists.",
++    )
+     password_group = parser.add_mutually_exclusive_group()
+     password_group.add_argument(
+         "-p",
+@@ -192,6 +209,7 @@ def main() -> None:
+         default=None,
+         help="User type as specified in synapse.api.constants.UserTypes",
+     )
++
+     admin_group = parser.add_mutually_exclusive_group()
+     admin_group.add_argument(
+         "-a",
+@@ -281,7 +299,7 @@ def main() -> None:
+     if args.admin or args.no_admin:
+         admin = args.admin
+ 
+-    register_new_user(args.user, password, server_url, secret, admin, args.user_type)
++    register_new_user(args.user, password, server_url, secret, admin, args.user_type, exists_ok=args.exists_ok)
+ 
+ 
+ def _read_file(file_path: Any, config_path: str) -> str:
+-- 
+2.44.1
+

clanModules/matrix-synapse/default.nix

@@ -82,8 +82,11 @@ in
         pkgs.matrix-synapse.override {
           matrix-synapse-unwrapped = pkgs.matrix-synapse.unwrapped.overrideAttrs (_old: {
             doInstallCheck = false; # too slow, nixpkgs maintainer already run this.
-            # see: https://github.com/element-hq/synapse/pull/17294
-            patches = [ ./0001-register_new_matrix_user-add-password-file-flag.patch ];
+            patches = [
+              # see: https://github.com/element-hq/synapse/pull/17304
+              ./0001-register_new_matrix_user-add-password-file-flag.patch
+              ./0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch
+            ];
           });
           extras = wantedExtras;
           plugins = synapseCfg.plugins;
@@ -102,6 +105,7 @@ in
           "turn:turn.matrix.org?transport=udp"
           "turn:turn.matrix.org?transport=tcp"
         ];
+        registration_shared_secret_path = "/run/synapse-registration-shared-secret";
         listeners = [
           {
             port = 8008;
@@ -122,11 +126,10 @@ in
           }
         ];
       };
-      extraConfigFiles = [ "/run/synapse-registration-shared-secret.yaml" ];
     };
 
     systemd.tmpfiles.settings."01-matrix" = {
-      "/run/synapse-registration-shared-secret.yaml" = {
+      "/run/synapse-registration-shared-secret" = {
         C.argument =
           config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path;
         z = {
@@ -154,7 +157,7 @@ in
             pwgen
           ];
           generator.script = ''
-            echo "registration_shared_secret: $(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
+            echo -n "$(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret
           '';
         };
       }
@@ -177,21 +180,12 @@ in
               if ! kill -0 "$MAINPID"; then exit 1; fi
               sleep 1;
             done
-
-            headers=$(mktemp)
-            trap 'rm -f "$headers"' EXIT
-
-            cat > "$headers" <<EOF
-            Authorization: Bearer $(cat /run/synapse-registration-shared-secret.yaml| sed -n 's/registration_shared_secret: //p')
-            EOF
           ''
           + lib.concatMapStringsSep "\n" (user: ''
             # only create user if it doesn't exist
-            if ! curl --header "$headers" "http://localhost:8008/_synapse/admin/v1/whois/${user.name}@${cfg.domain}" >&2; then
-              /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --password-file ${
-                config.clanCore.facts.services."matrix-password-${user.name}".secret."matrix-password-${user.name}".path
-              } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"}
-            fi
+            /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --exists-ok --password-file ${
+              config.clanCore.facts.services."matrix-password-${user.name}".secret."matrix-password-${user.name}".path
+            } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"}
           '') (lib.attrValues cfg.users);
       in
       {

clanModules/postgresql/default.nix

@@ -32,38 +32,7 @@ let
         runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp"
         mv "${current}.tmp" ${current}
       '';
-      postRestoreCommand = ''
-        export PATH=${
-          lib.makeBinPath [
-            config.services.postgresql.package
-            config.systemd.package
-            pkgs.coreutils
-            pkgs.util-linux
-            pkgs.zstd
-          ]
-        }
-        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
-          sleep 1
-        done
-        echo "Waiting for postgres to be ready..."
-        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
-          if ! systemctl is-active postgresql; then exit 1; fi
-          sleep 0.1
-        done
-
-        if [[ -e "${current}" ]]; then
-          (
-            systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore}
-            trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT
-
-            mkdir -p "${folder}"
-            runuser -u postgres -- dropdb "${db.name}"
-            runuser -u postgres -- pg_restore -C -d postgres "${current}"
-          )
-        else
-          echo No database backup found, skipping restore
-        fi
-      '';
+      postRestoreCommand = "postgres-db-restore-command-${db.name}";
     };
 
   createDatabase = db: ''
@@ -162,5 +131,49 @@ in
     clanCore.state = lib.mapAttrs' (
       _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db)
     ) config.clan.postgresql.databases;
+
+    environment.systemPackages = builtins.map (
+      db:
+      let
+        folder = "/var/backup/postgres/${db.name}";
+        current = "${folder}/pg-dump";
+      in
+      pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" ''
+        export PATH=${
+          lib.makeBinPath [
+            config.services.postgresql.package
+            config.systemd.package
+            pkgs.coreutils
+            pkgs.util-linux
+            pkgs.zstd
+          ]
+        }
+        while [[ "$(systemctl is-active postgresql)" == activating ]]; do
+          sleep 1
+        done
+        echo "Waiting for postgres to be ready..."
+        while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do
+          if ! systemctl is-active postgresql; then exit 1; fi
+          sleep 0.1
+        done
+
+        if [[ -e "${current}" ]]; then
+          (
+            ${
+              lib.optionalString (db.restore.stopOnRestore != [ ]) ''
+                systemctl stop ${builtins.toString db.restore.stopOnRestore}
+                trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT
+              ''
+            }
+
+            mkdir -p "${folder}"
+            runuser -u postgres -- dropdb "${db.name}"
+            runuser -u postgres -- pg_restore -C -d postgres "${current}"
+          )
+        else
+          echo No database backup found, skipping restore
+        fi
+      ''
+    ) (builtins.attrValues config.clan.postgresql.databases);
   };
 }