diff --git a/checks/lib/container-driver/test_driver/__init__.py b/checks/lib/container-driver/test_driver/__init__.py index be9673a3..7b6df2ef 100644 --- a/checks/lib/container-driver/test_driver/__init__.py +++ b/checks/lib/container-driver/test_driver/__init__.py @@ -151,7 +151,7 @@ class Machine: """ # Always run command with shell opts - command = f"set -euo pipefail; {command}" + command = f"set -eo pipefail; source /etc/profile; set -u; {command}" proc = subprocess.run( [ diff --git a/checks/matrix-synapse/default.nix b/checks/matrix-synapse/default.nix index 10395d26..3f2d72d7 100644 --- a/checks/matrix-synapse/default.nix +++ b/checks/matrix-synapse/default.nix @@ -38,7 +38,7 @@ z.mode = "0700"; }; "/etc/secrets/synapse-registration_shared_secret" = { - f.argument = "registration_shared_secret: supersecret"; + f.argument = "supersecret"; z = { mode = "0400"; user = "root"; diff --git a/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch b/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch index 39e9ea2c..fb9e2eb2 100644 --- a/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch +++ b/clanModules/matrix-synapse/0001-register_new_matrix_user-add-password-file-flag.patch @@ -1,43 +1,57 @@ -From df45634a92944dcab4edb02fb5e478911c58fdd6 Mon Sep 17 00:00:00 2001 +From bc199a27f23b0fcf175b116f7cf606c0d22b422a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 11 Jun 2024 11:40:47 +0200 -Subject: [PATCH] register_new_matrix_user: add password-file flag +Subject: [PATCH 1/2] register_new_matrix_user: add password-file flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -getpass in python exist on stdin to be a tty, hence we cannot just pipe +getpass in python expects stdin to be a tty, hence we cannot just pipe into register_new_matrix_user. --password-file instead works better and it would also allow the use of stdin if /dev/stdin is passed. +Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Signed-off-by: Jörg Thalheim --- - debian/register_new_matrix_user.ronn | 6 +++++- - synapse/_scripts/register_new_matrix_user.py | 16 ++++++++++++++-- - 2 files changed, 19 insertions(+), 3 deletions(-) + changelog.d/17294.feature | 2 ++ + debian/register_new_matrix_user.ronn | 9 +++++++-- + synapse/_scripts/register_new_matrix_user.py | 20 +++++++++++++++----- + 3 files changed, 24 insertions(+), 7 deletions(-) + create mode 100644 changelog.d/17294.feature +diff --git a/changelog.d/17294.feature b/changelog.d/17294.feature +new file mode 100644 +index 000000000..33aac7b0b +--- /dev/null ++++ b/changelog.d/17294.feature +@@ -0,0 +1,2 @@ ++`register_new_matrix_user` now supports a --password-file flag, which ++is useful for scripting. diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn -index 0410b1f4c..e39bef448 100644 +index 0410b1f4c..d99e9215a 100644 --- a/debian/register_new_matrix_user.ronn +++ b/debian/register_new_matrix_user.ronn -@@ -32,7 +32,11 @@ A sample YAML file accepted by `register_new_matrix_user` is described below: +@@ -31,8 +31,13 @@ A sample YAML file accepted by `register_new_matrix_user` is described below: + Local part of the new user. Will prompt if omitted. * `-p`, `--password`: - New password for user. Will prompt if omitted. Supplying the password +- New password for user. Will prompt if omitted. Supplying the password - on the command line is not recommended. Use the STDIN instead. -+ on the command line is not recommended. Use password-file if possible. ++ New password for user. Will prompt if this option and `--password-file` are omitted. ++ Supplying the password on the command line is not recommended. ++ Use `--password-file` if possible. + + * `--password-file`: -+ File containing the new password for user. Will prompt if omitted. ++ File containing the new password for user. If set, overrides `--password`. + This is a more secure alternative to specifying the password on the command line. * `-a`, `--admin`: Register new user as an admin. Will prompt if omitted. diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py -index 77a7129ee..f067e6832 100644 +index 77a7129ee..972b35e2d 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py -@@ -173,12 +173,18 @@ def main() -> None: +@@ -173,11 +173,18 @@ def main() -> None: default=None, help="Local part of the new user. Will prompt if omitted.", ) @@ -47,21 +61,21 @@ index 77a7129ee..f067e6832 100644 "-p", "--password", default=None, - help="New password for user. Will prompt if omitted.", - ) +- help="New password for user. Will prompt if omitted.", ++ help="New password for user. Will prompt for a password if " ++ "this flag and `--password-file` are both omitted.", ++ ) + password_group.add_argument( + "--password-file", + default=None, -+ help="File containing the new password for user. Will prompt if omitted.", -+ ) ++ help="File containing the new password for user. If set, will override `--password`.", + ) parser.add_argument( "-t", - "--user_type", -@@ -247,6 +253,12 @@ def main() -> None: +@@ -247,6 +254,11 @@ def main() -> None: print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr) sys.exit(1) -+ password = "" + if args.password_file: + password = _read_file(args.password_file, "password-file").strip() + else: @@ -70,15 +84,17 @@ index 77a7129ee..f067e6832 100644 if args.server_url: server_url = args.server_url elif config is not None: -@@ -270,7 +282,7 @@ def main() -> None: +@@ -269,9 +281,7 @@ def main() -> None: + if args.admin or args.no_admin: admin = args.admin - register_new_user( +- register_new_user( - args.user, args.password, server_url, secret, admin, args.user_type -+ args.user, password, server_url, secret, admin, args.user_type - ) +- ) ++ register_new_user(args.user, password, server_url, secret, admin, args.user_type) + def _read_file(file_path: Any, config_path: str) -> str: -- 2.44.1 diff --git a/clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch b/clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch new file mode 100644 index 00000000..df02ad52 --- /dev/null +++ b/clanModules/matrix-synapse/0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch @@ -0,0 +1,94 @@ +From 1789416df425d22693b0055a6688d8686e0ee4a1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= +Date: Thu, 13 Jun 2024 14:38:19 +0200 +Subject: [PATCH 2/2] register-new-matrix-user: add a flag to ignore already + existing users +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This allows to register users in a more declarative and stateless way. + +Signed-off-by: Jörg Thalheim +--- + synapse/_scripts/register_new_matrix_user.py | 22 ++++++++++++++++++-- + 1 file changed, 20 insertions(+), 2 deletions(-) + +diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py +index 972b35e2d..233e7267d 100644 +--- a/synapse/_scripts/register_new_matrix_user.py ++++ b/synapse/_scripts/register_new_matrix_user.py +@@ -52,6 +52,7 @@ def request_registration( + user_type: Optional[str] = None, + _print: Callable[[str], None] = print, + exit: Callable[[int], None] = sys.exit, ++ exists_ok: bool = False, + ) -> None: + url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),) + +@@ -97,6 +98,10 @@ def request_registration( + r = requests.post(url, json=data) + + if r.status_code != 200: ++ response = r.json() ++ if exists_ok and response["errcode"] == "M_USER_IN_USE": ++ _print("User already exists. Skipping.") ++ return + _print("ERROR! Received %d %s" % (r.status_code, r.reason)) + if 400 <= r.status_code < 500: + try: +@@ -115,6 +120,7 @@ def register_new_user( + shared_secret: str, + admin: Optional[bool], + user_type: Optional[str], ++ exists_ok: bool = False, + ) -> None: + if not user: + try: +@@ -154,7 +160,13 @@ def register_new_user( + admin = False + + request_registration( +- user, password, server_location, shared_secret, bool(admin), user_type ++ user, ++ password, ++ server_location, ++ shared_secret, ++ bool(admin), ++ user_type, ++ exists_ok=exists_ok, + ) + + +@@ -173,6 +185,11 @@ def main() -> None: + default=None, + help="Local part of the new user. Will prompt if omitted.", + ) ++ parser.add_argument( ++ "--exists-ok", ++ action="store_true", ++ help="Do not fail if user already exists.", ++ ) + password_group = parser.add_mutually_exclusive_group() + password_group.add_argument( + "-p", +@@ -192,6 +209,7 @@ def main() -> None: + default=None, + help="User type as specified in synapse.api.constants.UserTypes", + ) ++ + admin_group = parser.add_mutually_exclusive_group() + admin_group.add_argument( + "-a", +@@ -281,7 +299,7 @@ def main() -> None: + if args.admin or args.no_admin: + admin = args.admin + +- register_new_user(args.user, password, server_url, secret, admin, args.user_type) ++ register_new_user(args.user, password, server_url, secret, admin, args.user_type, exists_ok=args.exists_ok) + + + def _read_file(file_path: Any, config_path: str) -> str: +-- +2.44.1 + diff --git a/clanModules/matrix-synapse/default.nix b/clanModules/matrix-synapse/default.nix index 3de37c61..f0ad82da 100644 --- a/clanModules/matrix-synapse/default.nix +++ b/clanModules/matrix-synapse/default.nix @@ -82,8 +82,11 @@ in pkgs.matrix-synapse.override { matrix-synapse-unwrapped = pkgs.matrix-synapse.unwrapped.overrideAttrs (_old: { doInstallCheck = false; # too slow, nixpkgs maintainer already run this. - # see: https://github.com/element-hq/synapse/pull/17294 - patches = [ ./0001-register_new_matrix_user-add-password-file-flag.patch ]; + patches = [ + # see: https://github.com/element-hq/synapse/pull/17304 + ./0001-register_new_matrix_user-add-password-file-flag.patch + ./0002-register-new-matrix-user-add-a-flag-to-ignore-alread.patch + ]; }); extras = wantedExtras; plugins = synapseCfg.plugins; @@ -102,6 +105,7 @@ in "turn:turn.matrix.org?transport=udp" "turn:turn.matrix.org?transport=tcp" ]; + registration_shared_secret_path = "/run/synapse-registration-shared-secret"; listeners = [ { port = 8008; @@ -122,11 +126,10 @@ in } ]; }; - extraConfigFiles = [ "/run/synapse-registration-shared-secret.yaml" ]; }; systemd.tmpfiles.settings."01-matrix" = { - "/run/synapse-registration-shared-secret.yaml" = { + "/run/synapse-registration-shared-secret" = { C.argument = config.clanCore.facts.services.matrix-synapse.secret.synapse-registration_shared_secret.path; z = { @@ -154,7 +157,7 @@ in pwgen ]; generator.script = '' - echo "registration_shared_secret: $(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret + echo -n "$(pwgen -s 32 1)" > "$secrets"/synapse-registration_shared_secret ''; }; } @@ -177,21 +180,12 @@ in if ! kill -0 "$MAINPID"; then exit 1; fi sleep 1; done - - headers=$(mktemp) - trap 'rm -f "$headers"' EXIT - - cat > "$headers" <&2; then - /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --password-file ${ - config.clanCore.facts.services."matrix-password-${user.name}".secret."matrix-password-${user.name}".path - } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"} - fi + /run/current-system/sw/bin/matrix-synapse-register_new_matrix_user --exists-ok --password-file ${ + config.clanCore.facts.services."matrix-password-${user.name}".secret."matrix-password-${user.name}".path + } --user "${user.name}" ${if user.admin then "--admin" else "--no-admin"} '') (lib.attrValues cfg.users); in { diff --git a/clanModules/postgresql/default.nix b/clanModules/postgresql/default.nix index b1e7621c..18592988 100644 --- a/clanModules/postgresql/default.nix +++ b/clanModules/postgresql/default.nix @@ -32,38 +32,7 @@ let runuser -u postgres -- pg_dump ${compression} --dbname=${db.name} -Fc -c > "${current}.tmp" mv "${current}.tmp" ${current} ''; - postRestoreCommand = '' - export PATH=${ - lib.makeBinPath [ - config.services.postgresql.package - config.systemd.package - pkgs.coreutils - pkgs.util-linux - pkgs.zstd - ] - } - while [[ "$(systemctl is-active postgresql)" == activating ]]; do - sleep 1 - done - echo "Waiting for postgres to be ready..." - while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do - if ! systemctl is-active postgresql; then exit 1; fi - sleep 0.1 - done - - if [[ -e "${current}" ]]; then - ( - systemctl stop ${lib.concatStringsSep " " db.restore.stopOnRestore} - trap "systemctl start ${lib.concatStringsSep " " db.restore.stopOnRestore}" EXIT - - mkdir -p "${folder}" - runuser -u postgres -- dropdb "${db.name}" - runuser -u postgres -- pg_restore -C -d postgres "${current}" - ) - else - echo No database backup found, skipping restore - fi - ''; + postRestoreCommand = "postgres-db-restore-command-${db.name}"; }; createDatabase = db: '' @@ -162,5 +131,49 @@ in clanCore.state = lib.mapAttrs' ( _: db: lib.nameValuePair "postgresql-${db.name}" (createDatatbaseState db) ) config.clan.postgresql.databases; + + environment.systemPackages = builtins.map ( + db: + let + folder = "/var/backup/postgres/${db.name}"; + current = "${folder}/pg-dump"; + in + pkgs.writeShellScriptBin "postgres-db-restore-command-${db.name}" '' + export PATH=${ + lib.makeBinPath [ + config.services.postgresql.package + config.systemd.package + pkgs.coreutils + pkgs.util-linux + pkgs.zstd + ] + } + while [[ "$(systemctl is-active postgresql)" == activating ]]; do + sleep 1 + done + echo "Waiting for postgres to be ready..." + while ! runuser -u postgres -- psql --port=${builtins.toString config.services.postgresql.settings.port} -d postgres -c "" ; do + if ! systemctl is-active postgresql; then exit 1; fi + sleep 0.1 + done + + if [[ -e "${current}" ]]; then + ( + ${ + lib.optionalString (db.restore.stopOnRestore != [ ]) '' + systemctl stop ${builtins.toString db.restore.stopOnRestore} + trap "systemctl start ${builtins.toString db.restore.stopOnRestore}" EXIT + '' + } + + mkdir -p "${folder}" + runuser -u postgres -- dropdb "${db.name}" + runuser -u postgres -- pg_restore -C -d postgres "${current}" + ) + else + echo No database backup found, skipping restore + fi + '' + ) (builtins.attrValues config.clan.postgresql.databases); }; }