diff --git a/flake.nix b/flake.nix index 4f3f410..e6db5c1 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ nixConfig = { extra-substituters = [ "https://cache.clan.lol" ]; - extra-trusted-public-keys = [ "cache.clan.lol-1:j83TYLUVsrSXZvQdMoY+Ms81Xd/nO8GNuQQHqphzRSg=" ]; + extra-trusted-public-keys = [ "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" ]; }; inputs = { @@ -25,6 +25,7 @@ srvos.inputs.nixpkgs.follows = "nixpkgs"; clan-core.url = "git+https://git.clan.lol/clan/clan-core"; + #clan-core.url = "git+file:///home/joerg/work/clan/clan-core?ref=kvm-fix"; clan-core.inputs.flake-parts.follows = "flake-parts"; clan-core.inputs.nixpkgs.follows = "nixpkgs"; clan-core.inputs.treefmt-nix.follows = "treefmt-nix"; diff --git a/modules/web01/gitea/actions-runner.nix b/modules/web01/gitea/actions-runner.nix index 1ba2a1f..464bbba 100644 --- a/modules/web01/gitea/actions-runner.nix +++ b/modules/web01/gitea/actions-runner.nix @@ -12,8 +12,9 @@ let mkdir -p $out/etc/ssl/certs cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt ''; + numInstances = 10; in -{ +lib.mkMerge [{ # everything here has no dependencies on the store systemd.services.gitea-runner-nix-image = { wantedBy = [ "multi-user.target" ]; @@ -71,142 +72,145 @@ in isSystemUser = true; }; users.groups.nixuser = { }; - - systemd.services.gitea-runner-nix-token = { - wantedBy = [ "multi-user.target" ]; - after = [ "gitea.service" ]; - environment = { - GITEA_CUSTOM = "/var/lib/gitea/custom"; - GITEA_WORK_DIR = "/var/lib/gitea"; - }; - script = '' - set -euo pipefail - token=$(${lib.getExe self.packages.${pkgs.hostPlatform.system}.gitea} actions generate-runner-token) - echo "TOKEN=$token" > /var/lib/gitea-registration/token - ''; - unitConfig.ConditionPathExists = [ "!/var/lib/gitea-registration/token" ]; - serviceConfig = { - User = "gitea"; - Group = "gitea"; - StateDirectory = "gitea-registration"; - Type = "oneshot"; - RemainAfterExit = true; - }; - }; - - # Format of the token file: - virtualisation = { - podman.enable = true; - podman.extraPackages = [ pkgs.zfs ]; - }; - virtualisation.containers.storage.settings = { - storage.driver = "zfs"; - storage.graphroot = "/var/lib/containers/storage"; - storage.runroot = "/run/containers/storage"; - storage.options.zfs.fsname = "zroot/root/podman"; - }; - - virtualisation.containers.containersConf.settings = { - # podman seems to not work with systemd-resolved - containers.dns_servers = [ "8.8.8.8" "8.8.4.4" ]; - }; - - systemd.services.gitea-runner-nix = { - after = [ - "gitea-runner-nix-token.service" - "gitea-runner-nix-image.service" - ]; - requires = [ - "gitea-runner-nix-token.service" - "gitea-runner-nix-image.service" - ]; - - # TODO: systemd confinment - serviceConfig = { - # Hardening (may overlap with DynamicUser=) - # The following options are only for optimizing output of systemd-analyze - AmbientCapabilities = ""; - CapabilityBoundingSet = ""; - # ProtectClock= adds DeviceAllow=char-rtc r - DeviceAllow = ""; - NoNewPrivileges = true; - PrivateDevices = true; - PrivateMounts = true; - PrivateTmp = true; - PrivateUsers = true; - ProtectClock = true; - ProtectControlGroups = true; - ProtectHome = true; - ProtectHostname = true; - ProtectKernelLogs = true; - ProtectKernelModules = true; - ProtectKernelTunables = true; - ProtectSystem = "strict"; - RemoveIPC = true; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - UMask = "0066"; - ProtectProc = "invisible"; - SystemCallFilter = [ - "~@clock" - "~@cpu-emulation" - "~@module" - "~@mount" - "~@obsolete" - "~@raw-io" - "~@reboot" - "~@swap" - # needed by go? - #"~@resources" - "~@privileged" - "~capset" - "~setdomainname" - "~sethostname" - ]; - RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ]; - - # Needs network access - PrivateNetwork = false; - # Cannot be true due to Node - MemoryDenyWriteExecute = false; - - # The more restrictive "pid" option makes `nix` commands in CI emit - # "GC Warning: Couldn't read /proc/stat" - # You may want to set this to "pid" if not using `nix` commands - ProcSubset = "all"; - # Coverage programs for compiled code such as `cargo-tarpaulin` disable - # ASLR (address space layout randomization) which requires the - # `personality` syscall - # You may want to set this to `true` if not using coverage tooling on - # compiled code - LockPersonality = false; - - # Note that this has some interactions with the User setting; so you may - # want to consult the systemd docs if using both. - DynamicUser = true; - }; - }; - - services.gitea-actions-runner.instances.nix = { - enable = true; - name = "nix-runner"; - # take the git root url from the gitea config - # only possible if you've also configured your gitea though the same nix config - # otherwise you need to set it manually - url = config.services.gitea.settings.server.ROOT_URL; - # use your favourite nix secret manager to get a path for this - tokenFile = "/var/lib/gitea-registration/token"; - labels = [ "nix:docker://gitea-runner-nix" ]; - settings = { - container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm"; - # the default network that also respects our dns server settings - container.network = "podman"; - container.valid_volumes = [ - "/nix" - "${storeDeps}/bin" - "${storeDeps}/etc/ssl" - ]; - }; - }; } + { + systemd.services = lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}-token") numInstances) (name: { + wantedBy = [ "multi-user.target" ]; + after = [ "gitea.service" ]; + environment = { + GITEA_CUSTOM = "/var/lib/gitea/custom"; + GITEA_WORK_DIR = "/var/lib/gitea"; + }; + script = '' + set -euo pipefail + token=$(${lib.getExe self.packages.${pkgs.hostPlatform.system}.gitea} actions generate-runner-token) + echo "TOKEN=$token" > /var/lib/gitea-registration/${name} + ''; + unitConfig.ConditionPathExists = [ "!/var/lib/gitea-registration/${name}" ]; + serviceConfig = { + User = "gitea"; + Group = "gitea"; + StateDirectory = "gitea-registration"; + Type = "oneshot"; + RemainAfterExit = true; + }; + }); + + # Format of the token file: + virtualisation = { + podman.enable = true; + podman.extraPackages = [ pkgs.zfs ]; + }; + + virtualisation.containers.storage.settings = { + storage.driver = "zfs"; + storage.graphroot = "/var/lib/containers/storage"; + storage.runroot = "/run/containers/storage"; + storage.options.zfs.fsname = "zroot/root/podman"; + }; + + virtualisation.containers.containersConf.settings = { + # podman seems to not work with systemd-resolved + containers.dns_servers = [ "8.8.8.8" "8.8.4.4" ]; + }; + } + { + systemd.services = lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") numInstances) (name: { + after = [ + "${name}-token.service" + "gitea-runner-nix-image.service" + ]; + requires = [ + "${name}-token.service" + "gitea-runner-nix-image.service" + ]; + + # TODO: systemd confinment + serviceConfig = { + # Hardening (may overlap with DynamicUser=) + # The following options are only for optimizing output of systemd-analyze + AmbientCapabilities = ""; + CapabilityBoundingSet = ""; + # ProtectClock= adds DeviceAllow=char-rtc r + DeviceAllow = ""; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + PrivateUsers = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectSystem = "strict"; + RemoveIPC = true; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + UMask = "0066"; + ProtectProc = "invisible"; + SystemCallFilter = [ + "~@clock" + "~@cpu-emulation" + "~@module" + "~@mount" + "~@obsolete" + "~@raw-io" + "~@reboot" + "~@swap" + # needed by go? + #"~@resources" + "~@privileged" + "~capset" + "~setdomainname" + "~sethostname" + ]; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ]; + + # Needs network access + PrivateNetwork = false; + # Cannot be true due to Node + MemoryDenyWriteExecute = false; + + # The more restrictive "pid" option makes `nix` commands in CI emit + # "GC Warning: Couldn't read /proc/stat" + # You may want to set this to "pid" if not using `nix` commands + ProcSubset = "all"; + # Coverage programs for compiled code such as `cargo-tarpaulin` disable + # ASLR (address space layout randomization) which requires the + # `personality` syscall + # You may want to set this to `true` if not using coverage tooling on + # compiled code + LockPersonality = false; + + # Note that this has some interactions with the User setting; so you may + # want to consult the systemd docs if using both. + DynamicUser = true; + }; + }); + + services.gitea-actions-runner.instances = lib.genAttrs (builtins.genList (n: "nix${builtins.toString n}") numInstances) (name: { + enable = true; + name = "nix-runner"; + # take the git root url from the gitea config + # only possible if you've also configured your gitea though the same nix config + # otherwise you need to set it manually + url = config.services.gitea.settings.server.ROOT_URL; + # use your favourite nix secret manager to get a path for this + tokenFile = "/var/lib/gitea-registration/gitea-runner-${name}-token"; + labels = [ "nix:docker://gitea-runner-nix" ]; + settings = { + container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm"; + # the default network that also respects our dns server settings + container.network = "podman"; + container.valid_volumes = [ + "/nix" + "${storeDeps}/bin" + "${storeDeps}/etc/ssl" + ]; + }; + }); + }]