Merge pull request 'switch to different hetzner machine' (#92) from Mic92-main into main

2023-09-05 15:02:11 +00:00 · 2023-09-05 15:02:11 +00:00 · 9154f39c7f
commit 9154f39c7f
parent 3c4419e8bc 08d5404f82
17 changed files with 376 additions and 84 deletions
--- a/flake.lock
+++ b/flake.lock
@ -206,11 +206,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1693468174,
-        "narHash": "sha256-anRxrOKI9nJ/ss8CvvIA5lDAI05ke44PZI1F6I2Zk3g=",
+        "lastModified": 1693923568,
+        "narHash": "sha256-NTwQIDIbX5cu50FMt1RHyb9G6rWhoTIax+rx9zQ24wM=",
        "owner": "numtide",
        "repo": "srvos",
-        "rev": "28f5734829676013fe72454df923c50c12077c1c",
+        "rev": "b47df74261b0217336b034c03289a10d5191dbca",
        "type": "github"
      },
      "original": {
--- a/modules/flake-module.nix
+++ b/modules/flake-module.nix
@ -19,10 +19,9 @@
      ./single-disk.nix
    ];

-    hetzner-ex101.imports = [
-      inputs.srvos.nixosModules.hardware-hetzner-online-intel
-      ./xfs-lvm-crypto-raid.nix
-      ./hetzner-ex101.nix
+    hetzner-ax102.imports = [
+      inputs.srvos.nixosModules.hardware-hetzner-online-amd
+      ./zfs-crypto-raid.nix
      ./initrd-networking.nix
    ];

--- a/modules/hetzner-ex101.nix
+++ b/modules/hetzner-ex101.nix
@ -1,13 +0,0 @@
-{ pkgs, ... }: {
-  # Enable raid support specifically, this will disable srvos's
-  # systemd-initrd as well, which currently is not compatible with mdraid.
-  boot.swraid.enable = true;
-  systemd.services.mdmonitor.enable = false;
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  # We are not limited by zfs, so we can use the latest kernel
-  boot.kernelPackages = pkgs.linuxPackages_latest;
-
-  powerManagement.cpuFreqGovernor = "powersave";
-}
--- a/modules/initrd-networking.nix
+++ b/modules/initrd-networking.nix
@ -61,6 +61,7 @@ in
    boot.initrd.kernelModules = [
      "e1000e" # older hetzner machines, 1 GbE nics
      "igc" # newer herzner machines, 2.5 GbE nics
+      "igb"
      # for debugging installation in vms
      "virtio_pci"
      "virtio_net"
--- a/modules/zfs-crypto-raid.nix
+++ b/modules/zfs-crypto-raid.nix
@ -0,0 +1,89 @@
+{ self, ... }:
+let
+  mirrorBoot = idx: {
+    type = "disk";
+    device = "/dev/nvme${idx}n1";
+    content = {
+      type = "gpt";
+      partitions = {
+        ESP = {
+          size = "1G";
+          type = "EF00";
+          content = {
+            type = "filesystem";
+            format = "vfat";
+            mountpoint = "/boot${idx}";
+          };
+        };
+        zfs = {
+          size = "100%";
+          content = {
+            type = "zfs";
+            pool = "zroot";
+          };
+        };
+      };
+    };
+  };
+in
+{
+  imports = [
+    self.inputs.disko.nixosModules.disko
+  ];
+
+  networking.hostId = "8425e349";
+
+  boot.initrd.postDeviceCommands = ''
+    while ! test -f /tmp/decrypted; do
+      echo "wait for zfs to be decrypted"
+      sleep 1
+    done
+  '';
+
+  boot.loader.grub = {
+    enable = true;
+    efiSupport = true;
+    efiInstallAsRemovable = true;
+    mirroredBoots = [
+      { path = "/boot0"; devices = [ "nodev" ]; }
+      { path = "/boot1"; devices = [ "nodev" ]; }
+    ];
+  };
+
+  disko.devices = {
+    disk = {
+      x = mirrorBoot "0";
+      y = mirrorBoot "1";
+    };
+    zpool = {
+      zroot = {
+        type = "zpool";
+        rootFsOptions = {
+          compression = "lz4";
+          "com.sun:auto-snapshot" = "true";
+        };
+        datasets = {
+          "root" = {
+            type = "zfs_fs";
+            options = {
+              mountpoint = "none";
+              encryption = "aes-256-gcm";
+              keyformat = "hex";
+              keylocation = "file:///tmp/secret.key";
+            };
+          };
+          "root/nixos" = {
+            type = "zfs_fs";
+            options.mountpoint = "/";
+            mountpoint = "/";
+          };
+          "root/home" = {
+            type = "zfs_fs";
+            options.mountpoint = "/home";
+            mountpoint = "/home";
+          };
+        };
+      };
+    };
+  };
+}
--- a/sops/secrets/zfs-key/groups/admins
+++ b/sops/secrets/zfs-key/groups/admins
@ -0,0 +1 @@
+../../../groups/admins
--- a/sops/secrets/zfs-key/machines/web01
+++ b/sops/secrets/zfs-key/machines/web01
@ -0,0 +1 @@
+../../../machines/web01
--- a/sops/secrets/zfs-key/secret
+++ b/sops/secrets/zfs-key/secret
@ -0,0 +1,32 @@
+{
+	"data": "ENC[AES256_GCM,data:nHDRxFqSwEzz/hAlKstrGLhYgidpDMDwrPTLyTzvez83DuUHdfPWEWVASxkLOrZMm2Qyxt/3ylrKnY9zVakal9I=,iv:WHLk4020GGg8m9upuptB6IZrUcumOkS18K6eJ1Oj4/A=,tag:zW1LA4ymBAuind/mK/hEdA==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": [
+			{
+				"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzNlVRVmtDZXVXV21icWpR\nZ1EvSTVBS05oOW1Kdlh5NzBQY1Fsa0EyVGdRCjZwc3ZLK1Q2d2RTTEZmSWQ5eTEr\naEk1VW9KdkdTQ1h6WDE4Mk1uZEFnNGsKLS0tIHhvWHlzeHdqeW10Q3VuMXRwQU91\nRk9COHRhMTlUR3VJaytDanZJNWsyVmMK+kanz0BW8TCjHw7322/fYTmOmK2Ru6rX\nPPh7vHnfXOuzbAS4Gt5Ci9v371WEMUloTltKQ2Jbmo9Fe/PwZTX4/Q==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0aUlZc0UvNUw4ZHpOSy95\nYTc2K0tzR01XWk9MclJVRFFBYnJrK0s3MW4wCkoyekJwNlM5SElQOEZQNnlFdG1Q\ndEhFbHVwUytCUFFXVXdjbGx2MkNvTm8KLS0tIG9aek9vaitBOHgrcThoY3BKRFlP\nUTBQM25Mc05IKzdzSDZWTDBWSmMzOWcKGXIYFBsz7HThc8Boy9uozkbtcuF6PaxP\n/3Sqgf+hhxPvqTt5JDPH+BdWURN40pdQ7PnpuepJRNYcCTVU8laq+g==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1eq0e6uhjj2tja8v338tkdz8ema2aw5anpuyaq2uru7rt4lq7msyqqut6m2",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrSTdrQmJNZ1NWT1M3LzRT\nVk45WVYwNVByK1pUZVVFa0srZTRIRktScXpnCkJLWnhlK3V0eDJCM1psdzg3WkNw\nMnB6L0xDOHZPencvUGZaNXV4TjR1THMKLS0tIGliclFzOURrRFJXem1qajJMaS9k\nenNXTDJzeXhmQ1d0L2ZCNTF3WTNzZzQKEq5zzmi52CaK+ZfL7enSQ/n3VDmDwje4\nuIg5KwvhhOFdYtqkZFC5XJE6DeHqgjWaACcAPvYGDEOFKWaOVSbOxg==\n-----END AGE ENCRYPTED FILE-----\n"
+			},
+			{
+				"recipient": "age1vphy2sr6uw4ptsua3gh9khrm2cqyt65t46tusmt44z98qa7q6ymq6prrdl",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUOUhXV0tNdEF1ZGRBVW5L\nWURVRmRHUTdKMXk5bE9BS2VxbXI2ZWp0NnhjCmI3UEltS3U0RG95MmFFSmZMT2hy\nZTF4dVU5SThmNU9ibVMveHgrMWxJNDgKLS0tIEEyQUJUVkMzanJBcWRRYTlIb2Uy\nRko4WmVrM0tsMUxLV1puWW41SWJlUFUKx78ZKKNXf3q+fwdPDVGZc4gjmISUwMwa\n8Fde5g74UDiB9TAbAIYoiAyZLnPibpvJzE1SihTLNmnaJ46KEgtGWA==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2023-09-05T13:34:30Z",
+		"mac": "ENC[AES256_GCM,data:Y8OTVsqwjxzPuSXUfPH/Dt6I5x77Y7wLXlP9r3qHnFwQsjEdaXx4ayijMgIf3aY72osRVgFOEDDTjV0RU/rMy5x/KFgaSAfH5CO2vATE3yBERGcURcReGC7OmQN+nGPD9A49CdY6RQyGOTdQ8aWCHc7dmPPY2fcWaLaE1TB76MI=,iv:acowBk0DDYvfhOCAuOsWxJS7w6pz3v40E9WSUZz3y5g=,tag:d4NuATRx2UGZKVZbPNrfng==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}
--- a/targets/web01/configuration.nix
+++ b/targets/web01/configuration.nix
@ -5,14 +5,14 @@ in
 {
  imports = [
    self.nixosModules.web01
-    self.nixosModules.hetzner-ex101
+    self.nixosModules.hetzner-ax102
  ];
  networking.hostName = "web01";
-  systemd.network.networks."10-uplink".networkConfig.Address = "2a01:4f9:3080:282a::1";
+  systemd.network.networks."10-uplink".networkConfig.Address = "2a01:4f9:3080:418b::1";
  users.users.root.openssh.authorizedKeys.keys = builtins.attrValues admins;

-  clan.networking.ipv4.address = "65.109.103.5";
-  clan.networking.ipv4.gateway = "65.109.103.1";
+  clan.networking.ipv4.address = "65.21.12.51";
+  clan.networking.ipv4.gateway = "65.21.12.1";
  clan.networking.ipv6.address = config.systemd.network.networks."10-uplink".networkConfig.Address;

  system.stateVersion = "23.05";
--- a/targets/web01/decrypt.sh
+++ b/targets/web01/decrypt.sh
@ -3,10 +3,7 @@

 set -euox pipefail

-HOST=clan.lol
-temp=$(mktemp -d)
-trap 'rm -rf $temp' EXIT
-sops --extract '["cryptsetup_key"]' -d secrets.yaml > "$temp/secret.key"
+HOST="clan.lol"

 while ! ping -4 -W 1 -c 1 "$HOST"; do
  sleep 1
@ -15,4 +12,4 @@ while ! timeout 4 ssh -p 2222 "root@$HOST" true; do
  sleep 1
 done

-ssh -p 2222 "root@$HOST" "cat > /crypt-ramfs/passphrase" < "$temp/secret.key"
+clan secrets get zfs-key | ssh -p 2222 "root@${HOST}" "zpool import -f -a; cat > /tmp/secret.key && zfs load-key -a && touch /tmp/decrypted"
--- a/targets/web01/nixos-vars.json
+++ b/targets/web01/nixos-vars.json
@ -1 +1 @@
-{"ipv6_address":"2a01:4f9:3080:282a::1"}
+{"ipv6_address":"2a01:4f9:3080:418b::1"}
--- a/targets/web01/secrets.auto.tfvars.sops.json
+++ b/targets/web01/secrets.auto.tfvars.sops.json
@ -1,5 +1,5 @@
 {
-	"hetznerdns_token": "ENC[AES256_GCM,data:QMMn/j2Lv0Mz/2PhaYQygBjxEoU6f6hL23D5DrderFo=,iv:lOeXBlx/Lb7adzK2SKDKELxXNjlDNWVWQtLp+Mn6YaI=,tag:zTBP/IFdum6T5zITk+WU9A==,type:str]",
+	"hetznerdns_token": "ENC[AES256_GCM,data:58TlsZWNJ569HfVCKoQKN0ea9XMLC0Y1+F9ltCmQP5A=,iv:uVvbmBuPce8VpCRxFWjOB5TIcUk3u66RflP54+GDPDw=,tag:Ah7zSDVst9WEogW+ejHRag==,type:str]",
 	"sops": {
 		"kms": null,
 		"gcp_kms": null,
@ -19,8 +19,8 @@
 				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQS002SkJjSEZyL3FEbG8r\nWnM1VW9CTy9yaVR5VlJqcnQ0Y2hnZ29QWWxjCnowbnRwUFRabngwWDhWUnp2S0Vt\naytMNEp6UGtnRVJJYzZEOWdUWEIwRUEKLS0tIEI2MUhocHJHb25DaWxYeTFaRzhT\nSGpOd2xXTjVOSUtuSkF0Z2o1QkhNcW8KukbPUN8NsCgCSzoIMnj+WT4WqLcwQSj/\nXh7+7fyZ+PCgtfc0peT8qZ/4sP8XrkXML2G+AFlaTVP7tdFF22En2A==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2023-07-13T15:46:42Z",
-		"mac": "ENC[AES256_GCM,data:TYlJZLdIvaWD96RQg5RnUJyNAR69bze0f0+Ai37BfA0G6VEWDZqvc537vRFk7dj4R8kYCe4q79w7yWmSt30UUZ+SXHSjVcUU9WijO4QprrUz/q4r9ezVZfQLe6disaUDdgsqhQvkQSh0AJ5eJtcr1uVChOViVfH/nk/FfJgUc7s=,iv:ulkInzkkD2ZG8uSQW3vrkAjVD1gWExtultU8zhs2+aU=,tag:bxNP152hKrLBh2zKeGM8KA==,type:str]",
+		"lastmodified": "2023-09-05T13:47:51Z",
+		"mac": "ENC[AES256_GCM,data:YI4Vx0lpqoqT9XrIlE6JfE55Cp0YIZzpl8c/hQHXdNJm6QX2qmN9ejtGMXG8DEQ3EAlnRDIFburKY0kZ+Dtv8mCzRVT4tiReWniSh20lT747ukX032PfhFsNYFOlhrd8rAV5GrVyklfo3mnu3pwduXyAo9RT+d9yOj8rGV3wsYk=,iv:fuX6BbTGaTcoaGV+3YbhqvKC8GQtVVKaUp8UvML1uAw=,tag:jT1ghf6j0sv19gxc13dy0Q==,type:str]",
 		"pgp": null,
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.7.3"
--- a/targets/web01/terraform.tf
+++ b/targets/web01/terraform.tf
@ -11,8 +11,8 @@ module "web01" {
  nixos_flake_attr  = ".#web01"
  nixos_vars_file   = "${path.module}/nixos-vars.json"
  hetznerdns_token  = var.hetznerdns_token
-  ipv4_address      = "65.109.103.5"
-  ipv6_address      = "2a01:4f9:3080:282a::1"
+  ipv4_address      = "65.21.12.51"
+  ipv6_address      = "2a01:4f9:3080:418b::1"
  sops_secrets_file = "${abspath(path.module)}/secrets.yaml"
  tags = {
    Terraform = "true"
--- a/targets/web01/terraform.tfstate
+++ b/targets/web01/terraform.tfstate
--- a/terraform/web01/decrypt-ssh-secrets.sh
+++ b/terraform/web01/decrypt-ssh-secrets.sh
@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+mkdir -p etc/ssh var/lib/secrets
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+umask 0177
+(cd "$SCRIPT_DIR" && clan secrets get initrd_ssh_key) > ./var/lib/secrets/initrd_ssh_key
+
+# restore umask
+umask 0022
+
+for keyname in ssh_host_rsa_key ssh_host_rsa_key.pub ssh_host_ed25519_key ssh_host_ed25519_key.pub; do
+  if [[ $keyname == *.pub ]]; then
+    umask 0133
+  else
+    umask 0177
+  fi
+  (cd "$SCRIPT_DIR" && clan secrets get "$keyname") >"./etc/ssh/$keyname"
+done
--- a/terraform/web01/decrypt-zfs-key.sh
+++ b/terraform/web01/decrypt-zfs-key.sh
@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$SCRIPT_DIR"
+clan secrets get zfs-key
--- a/terraform/web01/main.tf
+++ b/terraform/web01/main.tf
@ -1,28 +1,18 @@
 locals {
 }

-resource "null_resource" "nixos-anywhere" {
-  triggers = {
-    instance_id = var.ipv4_address
-  }
-  connection {
-    type = "ssh"
-    user = "root"
-    host = var.ipv4_address
-  }
-  provisioner "remote-exec" {
-    # needed because kexec is broken
-    # https://github.com/numtide/nixos-anywhere/issues/136
-    script = "${path.module}/nixosify.sh"
-  }
-  provisioner "local-exec" {
-    environment = {
-      HOST              = var.ipv4_address
-      FLAKE_ATTR        = var.nixos_flake_attr
-      SOPS_SECRETS_FILE = var.sops_secrets_file
-    }
-    command = "bash -x ${path.module}/install.sh"
-  }
+module "deploy" {
+  source                 = "github.com/numtide/nixos-anywhere//terraform/all-in-one?ref=extra-files"
+  nixos_system_attr      = ".#nixosConfigurations.web01.config.system.build.toplevel"
+  nixos_partitioner_attr = ".#nixosConfigurations.web01.config.system.build.diskoScript"
+  target_host            = var.ipv4_address
+  instance_id            = "web01"
+  debug_logging          = true
+  extra_files_script     = "${path.module}/decrypt-ssh-secrets.sh"
+  disk_encryption_key_scripts = [{
+    path   = "/tmp/secret.key"
+    script = "${path.module}/decrypt-zfs-key.sh"
+  }]
 }

 locals {