Merge pull request 'Add nixos module to import secrets automatically' (#154) from Mic92-mic92 into main

checks/flake-module.nix

@@ -1,5 +1,21 @@
-{
-  imports = [
-    ./schema.nix
-  ];
+{ self, ... }: {
+  perSystem = { pkgs, ... }: {
+    checks =
+      let
+        nixosTestArgs = {
+          # reference to nixpkgs for the current system
+          inherit pkgs;
+          # this gives us a reference to our flake but also all flake inputs
+          inherit self;
+        };
+        nixosTests = {
+          # import our test
+          secrets = import ./secrets nixosTestArgs;
+        };
+        schemaTests = pkgs.callPackages ./schemas.nix {
+          inherit self;
+        };
+      in
+      nixosTests // schemaTests;
+  };
 }

checks/lib/test-base.nix

@@ -0,0 +1,18 @@
+test:
+{ pkgs
+, self
+, ...
+}:
+let
+  inherit (pkgs) lib;
+  nixos-lib = import (pkgs.path + "/nixos/lib") { };
+in
+(nixos-lib.runTest {
+  hostPkgs = pkgs;
+  # speed-up evaluation
+  defaults.documentation.enable = lib.mkDefault false;
+  # to accept external dependencies such as disko
+  node.specialArgs.self = self;
+  imports = [ test ];
+}).config.result
+

checks/schemas.nix

@@ -0,0 +1,34 @@
+{ self, runCommand, check-jsonschema, pkgs, lib, ... }:
+let
+  clanModules = self.clanModules;
+
+  baseModule = {
+    imports =
+      (import (pkgs.path + "/nixos/modules/module-list.nix"))
+      ++ [{
+        nixpkgs.hostPlatform = "x86_64-linux";
+      }];
+  };
+
+  optionsFromModule = module:
+    let
+      evaled = lib.evalModules {
+        modules = [ module baseModule ];
+      };
+    in
+    evaled.options.clan.networking;
+
+  clanModuleSchemas = lib.mapAttrs (_: module: self.lib.jsonschema.parseOptions (optionsFromModule module)) clanModules;
+
+  mkTest = name: schema: runCommand "schema-${name}" { } ''
+    ${check-jsonschema}/bin/check-jsonschema \
+      --check-metaschema ${builtins.toFile "schema-${name}" (builtins.toJSON schema)}
+      touch $out
+  '';
+in
+lib.mapAttrs'
+  (name: schema: {
+    name = "schema-${name}";
+    value = mkTest name schema;
+  })
+  clanModuleSchemas

checks/secrets/clan-secrets

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux -o pipefail
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+export SOPS_AGE_KEY_FILE="${SCRIPT_DIR}/key.age"
+nix run .# -- secrets "$@"

checks/secrets/default.nix

@@ -0,0 +1,16 @@
+(import ../lib/test-base.nix) {
+  name = "secrets";
+
+  nodes.machine = { self, config, ... }: {
+    imports = [
+      self.nixosModules.secrets
+    ];
+    environment.etc."secret".source = config.sops.secrets.foo.path;
+    sops.age.keyFile = ./key.age;
+    clan.sops.sopsDirectory = ./sops;
+    networking.hostName = "machine";
+  };
+  testScript = ''
+    machine.succeed("cat /etc/secret >&2")
+  '';
+}

checks/secrets/key.age

@@ -0,0 +1 @@
+AGE-SECRET-KEY-1UCXEUJH6JXF8LFKWFHDM4N9AQE2CCGQZGXLUNV4TKR5KY0KC8FDQ2TY4NX

checks/secrets/sops/machines/machine/key.json

@@ -0,0 +1,4 @@
+{
+  "publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
+  "type": "age"
+}

checks/secrets/sops/secrets/foo/machines/machine

@@ -0,0 +1 @@
+../../../machines/machine

checks/secrets/sops/secrets/foo/secret

@@ -0,0 +1,20 @@
+{
+	"data": "ENC[AES256_GCM,data:bhxF,iv:iNs+IfSU/7EwssZ0GVTF2raxJkVlddfQEPGIBeUYAy8=,tag:JMOKTMW3/ic3UTj9eT9YFQ==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": [
+			{
+				"recipient": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxS0g4TEt4S09LQnFKdCtk\nZTlUQWhNUHZmcmZqdGtuZkhhTkMzZDVaWWdNCi9vNnZQeklNaFBBU2x0ditlUDR0\nNGJlRmFFb09WSUFGdEh5TGViTWtacFEKLS0tIE1OMWdQMHhGeFBwSlVEamtHUkcy\ndzI1VHRkZ1o4SStpekVNZmpQSnRkeUkKYmPS9sR6U0NHxd55DjRk29LNFINysOl6\nEM2MTrntLxOHFWZ1QgNx34l4rYIIXx97ONvR0SRpxN0ECL9VonQeZg==\n-----END AGE ENCRYPTED FILE-----\n"
+			}
+		],
+		"lastmodified": "2023-08-23T09:11:08Z",
+		"mac": "ENC[AES256_GCM,data:8z819mP4FJXE/ExWM1+/dhaXIXzCglhBuZwE6ikl/jNLUAnv3jYL9c9vPrPFl2by3wXSNzqB4AOiTKDQoxDx2SBQKxeWaUnOajD6hbzskoLqCCBfVx7qOHrk/BULcBvMSxBca4RnzXXoMFTwKs2A1fXqAPvSQd1X4gX6Xm9VXWM=,iv:3YxZX+gaEcRKDN0Kuf9y1oWL+sT/J5B/5CtCf4iur9Y=,tag:0dwyjpvjCqbm9vIrz6WSWQ==,type:str]",
+		"pgp": null,
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.3"
+	}
+}

checks/secrets/sops/secrets/foo/users/admin

@@ -0,0 +1 @@
+../../../users/admin

checks/secrets/sops/users/admin/key.json

@@ -0,0 +1,4 @@
+{
+  "publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
+  "type": "age"
+}

devShell.nix

@@ -2,13 +2,9 @@
   perSystem =
     { pkgs
     , self'
-    , config
     , ...
     }: {
       devShells.default = pkgs.mkShell {
-        inputsFrom = [
-          config.treefmt.build.devShell
-        ];
         packages = [
           pkgs.tea
           self'.packages.tea-create-pr

flake.lock

@@ -98,11 +98,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1692447944,
-        "narHash": "sha256-fkJGNjEmTPvqBs215EQU4r9ivecV5Qge5cF/QDLVn3U=",
+        "lastModified": 1692638711,
+        "narHash": "sha256-J0LgSFgJVGCC1+j5R2QndadWI1oumusg6hCtYAzLID4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "d680ded26da5cf104dd2735a51e88d2d8f487b4d",
+        "rev": "91a22f76cd1716f9d0149e8a5c68424bb691de15",
         "type": "github"
       },
       "original": {
@@ -119,9 +119,31 @@
         "floco": "floco",
         "nixos-generators": "nixos-generators",
         "nixpkgs": "nixpkgs",
+        "sops-nix": "sops-nix",
         "treefmt-nix": "treefmt-nix"
       }
     },
+    "sops-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "sops-nix"
+        ],
+        "nixpkgs-stable": []
+      },
+      "locked": {
+        "lastModified": 1692728678,
+        "narHash": "sha256-02MjG7Sb9k7eOi86CcC4GNWVOjT6gjmXFSqkRjZ8Xyk=",
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "rev": "1b7b3a32d65dbcd69c217d7735fdf0a6b2184f45",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Mic92",
+        "repo": "sops-nix",
+        "type": "github"
+      }
+    },
     "treefmt-nix": {
       "inputs": {
         "nixpkgs": [

flake.nix

@@ -7,6 +7,9 @@
     floco.inputs.nixpkgs.follows = "nixpkgs";
     disko.url = "github:nix-community/disko";
     disko.inputs.nixpkgs.follows = "nixpkgs";
+    sops-nix.url = "github:Mic92/sops-nix";
+    sops-nix.inputs.nixpkgs.follows = "sops-nix";
+    sops-nix.inputs.nixpkgs-stable.follows = "";
     nixos-generators.url = "github:nix-community/nixos-generators";
     nixos-generators.inputs.nixpkgs.follows = "nixpkgs";
     flake-parts.url = "github:hercules-ci/flake-parts";
@@ -32,9 +35,9 @@
         ./pkgs/flake-module.nix
 
         ./lib/flake-module.nix
+        ./nixosModules/flake-module.nix
         ({ self, lib, ... }: {
           flake.clanModules = lib.mapAttrs (_: nix: { imports = [ nix ]; }) (self.lib.findNixFiles ./clanModules);
-          flake.nixosModules = lib.mapAttrs (_: nix: { imports = [ nix ]; }) (self.lib.findNixFiles ./nixosModules);
         })
       ];
     });

formatter.nix

@@ -5,11 +5,17 @@
   imports = [
     inputs.treefmt-nix.flakeModule
   ];
-  perSystem = { pkgs, ... }: {
+  perSystem = { self', pkgs, ... }: {
     treefmt.projectRootFile = "flake.nix";
     treefmt.flakeCheck = true;
     treefmt.flakeFormatter = true;
     treefmt.programs.shellcheck.enable = true;
+
+    treefmt.programs.mypy.enable = true;
+    treefmt.programs.mypy.directories = {
+      "pkgs/clan-cli".extraPythonPackages = self'.packages.clan-cli.testDependencies;
+    };
+
     treefmt.settings.formatter.nix = {
       command = "sh";
       options = [
@@ -38,4 +44,3 @@
     };
   };
 }
-

nixosModules/flake-module.nix

@@ -0,0 +1,10 @@
+{ inputs, ... }: {
+  flake.nixosModules = {
+    hidden-ssh-announce.imports = [ ./hidden-ssh-announce.nix ];
+    installer.imports = [ ./installer ];
+    secrets.imports = [
+      inputs.sops-nix.nixosModules.sops
+      ./secrets
+    ];
+  };
+}

nixosModules/secrets/default.nix

@@ -0,0 +1,39 @@
+{ lib, config, ... }:
+let
+  encryptedForThisMachine = name: type:
+    let
+      symlink = config.clan.sops.sopsDirectory + "/secrets/${name}/machines/${config.clan.sops.machineName}";
+    in
+    # WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl
+    type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink");
+  secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.sops.sopsDirectory + "/secrets"));
+in
+{
+  imports = [
+  ];
+  options = {
+    clan.sops = {
+      machineName = lib.mkOption {
+        type = lib.types.str;
+        default = config.networking.hostName;
+        description = ''
+          Machine used to lookup secrets in the sops directory.
+        '';
+      };
+      sopsDirectory = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          Sops toplevel directory that stores users, machines, groups and secrets.
+        '';
+      };
+    };
+  };
+  config = {
+    sops.secrets = builtins.mapAttrs
+      (name: _: {
+        sopsFile = config.clan.sops.sopsDirectory + "/secrets/${name}/secret";
+        format = "binary";
+      })
+      secrets;
+  };
+}

pkgs/clan-cli/clan_cli/__init__.py

@@ -1,15 +1,17 @@
 import argparse
 import sys
+from types import ModuleType
+from typing import Optional
 
 from . import admin, config, secrets, update
 from .errors import ClanError
 from .ssh import cli as ssh_cli
 
-has_argcomplete = True
+argcomplete: Optional[ModuleType] = None
 try:
-    import argcomplete
+    import argcomplete  # type: ignore[no-redef]
 except ImportError:
-    has_argcomplete = False
+    pass
 
 
 # this will be the entrypoint under /bin/clan (see pyproject.toml config)
@@ -34,7 +36,7 @@ def main() -> None:
     )
     update.register_parser(parser_update)
 
-    if has_argcomplete:
+    if argcomplete:
         argcomplete.autocomplete(parser)
 
     if len(sys.argv) == 1:

pkgs/clan-cli/clan_cli/secrets/sops.py

@@ -43,7 +43,7 @@ def get_user_name(user: str) -> str:
     """Ask the user for their name until a unique one is provided."""
     while True:
         name = input(
-            f"Enter your user name for which your sops key will be stored in the repository [default: {user}]: "
+            f"Your key is not yet added to the repository. Enter your user name for which your sops key will be stored in the repository [default: {user}]: "
         )
         if name:
             user = name

pkgs/clan-cli/default.nix

@@ -3,7 +3,6 @@
 , black
 , bubblewrap
 , installShellFiles
-, mypy
 , nix
 , openssh
 , pytest
@@ -31,7 +30,6 @@ let
     pytest
     pytest-cov
     pytest-subprocess
-    mypy
     openssh
     stdenv.cc
   ];
@@ -60,27 +58,17 @@ python3.pkgs.buildPythonPackage {
   ];
   propagatedBuildInputs = dependencies;
 
-  passthru.tests = {
-    clan-mypy = runCommand "clan-mypy" { } ''
-      export CLAN_OPTIONS_FILE="${CLAN_OPTIONS_FILE}"
-      cp -r ${source} ./src
-      chmod +w -R ./src
-      cd ./src
-      ${checkPython}/bin/mypy .
-      touch $out
-    '';
-    clan-pytest = runCommand "clan-tests"
-      {
-        nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ];
-      } ''
-      export CLAN_OPTIONS_FILE="${CLAN_OPTIONS_FILE}"
-      cp -r ${source} ./src
-      chmod +w -R ./src
-      cd ./src
-      NIX_STATE_DIR=$TMPDIR/nix ${checkPython}/bin/python -m pytest -s ./tests
-      touch $out
-    '';
-  };
+  passthru.tests.clan-pytest = runCommand "clan-tests"
+    {
+      nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ];
+    } ''
+    export CLAN_OPTIONS_FILE="${CLAN_OPTIONS_FILE}"
+    cp -r ${source} ./src
+    chmod +w -R ./src
+    cd ./src
+    NIX_STATE_DIR=$TMPDIR/nix ${checkPython}/bin/python -m pytest -s ./tests
+    touch $out
+  '';
 
   passthru.devDependencies = [
     ruff
@@ -89,6 +77,8 @@ python3.pkgs.buildPythonPackage {
     wheel
   ] ++ testDependencies;
 
+  passthru.testDependencies = testDependencies;
+
   makeWrapperArgs = [
     "--set CLAN_FLAKE ${self}"
   ];