Merge branch 'main' into Qubasa-main
All checks were successful
checks-impure / test (pull_request) Successful in 2s
checks / test (pull_request) Successful in 2s

This commit is contained in:
Luis Hebendanz 2023-08-29 18:23:22 +02:00
commit 3df5bc4815
23 changed files with 411 additions and 179 deletions

View File

@ -1,29 +0,0 @@
{ config, lib, ... }:
{
options.clan.networking.zerotier = {
networkId = lib.mkOption {
type = lib.types.str;
description = ''
zerotier networking id
'';
};
};
config = {
systemd.network.networks.zerotier = {
matchConfig.Name = "zt*";
networkConfig = {
LLMNR = true;
LLDP = true;
MulticastDNS = true;
KeepConfiguration = "static";
};
};
networking.firewall.allowedUDPPorts = [ 9993 ];
networking.firewall.interfaces."zt+".allowedTCPPorts = [ 5353 ];
networking.firewall.interfaces."zt+".allowedUDPPorts = [ 5353 ];
services.zerotierone = {
enable = true;
joinNetworks = [ config.clan.networking.zerotier.networkId ];
};
};
}

View File

@ -98,11 +98,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1692638711,
"narHash": "sha256-J0LgSFgJVGCC1+j5R2QndadWI1oumusg6hCtYAzLID4=",
"lastModified": 1693003285,
"narHash": "sha256-5nm4yrEHKupjn62MibENtfqlP6pWcRTuSKrMiH9bLkc=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "91a22f76cd1716f9d0149e8a5c68424bb691de15",
"rev": "5690c4271f2998c304a45c91a0aeb8fb69feaea7",
"type": "github"
},
"original": {
@ -131,11 +131,11 @@
"nixpkgs-stable": []
},
"locked": {
"lastModified": 1692728678,
"narHash": "sha256-02MjG7Sb9k7eOi86CcC4GNWVOjT6gjmXFSqkRjZ8Xyk=",
"lastModified": 1693105804,
"narHash": "sha256-nlqNjW7dfucUJQqRGuG08MKPOSME8fLOCx/bd9hiEPs=",
"owner": "Mic92",
"repo": "sops-nix",
"rev": "1b7b3a32d65dbcd69c217d7735fdf0a6b2184f45",
"rev": "0618c8f0ed5255ad74ee08d1618841ff5af85c86",
"type": "github"
},
"original": {
@ -151,11 +151,11 @@
]
},
"locked": {
"lastModified": 1692524468,
"narHash": "sha256-wJffwu1deOgc3c/cBIZQ52dfWfPWBzjOamYBX121hcw=",
"lastModified": 1692972530,
"narHash": "sha256-LG+M7TjlLJ1lx2qbD1yaexvue1VAatpVandtHVEN5Lc=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "e2761d701581d8dcc4e0e88aecfde317ddf6f0cd",
"rev": "843e1e1b01ac7c9e858368fffd1692cbbdbe4a0e",
"type": "github"
},
"original": {

View File

@ -36,9 +36,7 @@
./lib/flake-module.nix
./nixosModules/flake-module.nix
({ self, lib, ... }: {
flake.clanModules = lib.mapAttrs (_: nix: { imports = [ nix ]; }) (self.lib.findNixFiles ./clanModules);
})
./nixosModules/clanCore/flake-module.nix
];
});
}

View File

@ -0,0 +1,28 @@
{ nixpkgs, clan, lib }:
{ directory # The directory containing the machines subdirectory
, specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available
, machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... }
}:
let
machinesDirs =
if builtins.pathExists (directory + /machines)
then builtins.readDir (directory + /machines)
else { };
machineSettings = machineName:
if builtins.pathExists (directory + /machines/${machineName}/settings.json)
then builtins.fromJSON (builtins.readFile (directory + /machines/${machineName}/settings.json))
else { };
nixosConfigurations = lib.mapAttrs
(name: _mod:
nixpkgs.lib.nixosSystem {
modules = [
(machineSettings name)
(machines.${name} or { })
] ++ lib.attrValues clan.clanModules;
specialArgs = specialArgs;
})
machinesDirs;
in
nixosConfigurations

View File

@ -1,4 +1,4 @@
{ lib, ... }:
{ lib, clan, nixpkgs, ... }:
{
findNixFiles = folder:
lib.mapAttrs'
@ -13,4 +13,6 @@
(builtins.readDir folder);
jsonschema = import ./jsonschema { inherit lib; };
buildClan = import ./build-clan { inherit lib clan nixpkgs; };
}

View File

@ -1,8 +1,12 @@
{ lib
, inputs
, ...
}: {
imports = [
./jsonschema/flake-module.nix
];
flake.lib = import ./default.nix { inherit lib; };
flake.lib = import ./default.nix {
inherit lib;
inherit (inputs) clan nixpkgs;
};
}

View File

@ -0,0 +1,32 @@
{ self, inputs, lib, ... }: {
flake.nixosModules.clanCore = { pkgs, ... }: {
options.clanCore = {
clanDir = lib.mkOption {
type = lib.types.str;
description = ''
the location of the flake repo, used to calculate the location of facts and secrets
'';
};
machineName = lib.mkOption {
type = lib.types.str;
description = ''
the name of the machine
'';
};
clanPkgs = lib.mkOption {
default = self.packages.${pkgs.system};
};
};
options.system.clan = lib.mkOption {
type = lib.types.lazyAttrsOf lib.types.raw;
description = ''
utility outputs for clan management of this machine
'';
};
imports = [
./secrets
./zerotier.nix
inputs.sops-nix.nixosModules.sops
];
};
}

View File

@ -0,0 +1,66 @@
{ config, lib, ... }:
{
options.clanCore.secrets = lib.mkOption {
type = lib.types.attrsOf
(lib.types.submodule (secret: {
options = {
name = lib.mkOption {
type = lib.types.str;
default = secret.config._module.args.name;
description = ''
namespace of the secret
'';
};
generator = lib.mkOption {
type = lib.types.nullOr lib.types.str;
description = ''
script to generate the secret.
can be set to null. then the user has to provide the secret via the clan cli
'';
};
secrets = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule (secret: {
options = {
name = lib.mkOption {
type = lib.types.str;
description = ''
name of the secret
'';
default = secret.config._module.args.name;
};
};
}));
description = ''
path where the secret is located in the filesystem
'';
};
facts = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule (fact: {
options = {
name = lib.mkOption {
type = lib.types.str;
description = ''
name of the fact
'';
default = fact.config._module.args.name;
};
path = lib.mkOption {
type = lib.types.str;
description = ''
path to a fact which is generated by the generator
'';
default = "${config.clanCore.clanDir}/facts/${config.clanCore.machineName}/${fact.config._module.args.name}";
};
value = lib.mkOption {
default = builtins.readFile fact.config.path;
};
};
}));
};
};
}));
};
imports = [
./sops.nix # for now we have only one implementation, thats why we import it here and not in clanModules
];
}

View File

@ -0,0 +1,63 @@
{ config, lib, pkgs, ... }:
{
config = {
system.clan.generateSecrets = pkgs.writeScript "generate_secrets" ''
#!/bin/sh
set -efu
set -x # remove for prod
PATH=$PATH:${lib.makeBinPath [
config.clanCore.clanPkgs.clan-cli
]}
# initialize secret store
if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
INITTMP=$(mktemp -d)
trap 'rm -rf "$INITTMP"' EXIT
${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
) fi
${lib.foldlAttrs (acc: n: v: ''
${acc}
# ${n}
# if any of the secrets are missing, we regenerate all connected facts/secrets
(if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
facts=$(mktemp -d)
trap "rm -rf $facts" EXIT
secrets=$(mktemp -d)
trap "rm -rf $secrets" EXIT
${v.generator}
${lib.concatMapStrings (fact: ''
mkdir -p "$(dirname ${fact.path})"
cp "$facts"/${fact.name} ${fact.path}
'') (lib.attrValues v.facts)}
${lib.concatMapStrings (secret: ''
cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
'') (lib.attrValues v.secrets)}
fi)
'') "" config.clanCore.secrets}
'';
sops.secrets =
let
encryptedForThisMachine = name: type:
let
symlink = config.clanCore.clanDir + "/sops/secrets/${name}/machines/${config.clanCore.machineName}";
in
# WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl
type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink");
secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clanCore.clanDir + "/sops/secrets"));
in
builtins.mapAttrs
(name: _: {
sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
format = "binary";
})
secrets;
};
}

View File

@ -0,0 +1,100 @@
{ config, lib, pkgs, ... }:
let
cfg = config.clan.networking.zerotier;
in
{
options.clan.networking.zerotier = {
networkId = lib.mkOption {
type = lib.types.str;
description = ''
zerotier networking id
'';
};
controller = {
enable = lib.mkEnableOption "turn this machine into the networkcontroller";
public = lib.mkOption {
type = lib.types.bool;
default = false;
description = ''
everyone can join a public network without having the administrator to accept
'';
};
};
};
config = {
systemd.network.networks.zerotier = {
matchConfig.Name = "zt*";
networkConfig = {
LLMNR = true;
LLDP = true;
MulticastDNS = true;
KeepConfiguration = "static";
};
};
networking.firewall.allowedUDPPorts = [ 9993 ];
networking.firewall.interfaces."zt+".allowedTCPPorts = [ 5353 ];
networking.firewall.interfaces."zt+".allowedUDPPorts = [ 5353 ];
services.zerotierone = {
enable = true;
joinNetworks = [ cfg.networkId ];
};
} // lib.mkIf cfg.controller.enable {
# only the controller needs to have the key in the repo, the other clients can be dynamic
# we generate the zerotier code manually for the controller, since it's part of the bootstrap command
clanCore.secrets.zerotier = {
facts."network.id" = { };
secrets."identity.secret" = { };
generator = ''
TMPDIR=$(mktemp -d)
trap 'rm -rf "$TMPDIR"' EXIT
${config.clanCore.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR"
cp "$TMPDIR"/network.id "$facts"/network.id
cp "$TMPDIR"/identity.secret "$secrets"/identity.secret
'';
};
systemd.tmpfiles.rules = [
"L+ /var/lib/zerotierone/controller.d/network/${cfg.networkId}.json - - - - ${pkgs.writeText "net.json" (builtins.toJSON {
authTokens = [
null
];
authorizationEndpoint = "";
capabilities = [];
clientId = "";
dns = [];
enableBroadcast = true;
id = cfg.networkId;
ipAssignmentPools = [];
mtu = 2800;
multicastLimit = 32;
name = "";
uwid = cfg.networkId;
objtype = "network";
private = true;
remoteTraceLevel = 0;
remoteTraceTarget = null;
revision = 1;
routes = [];
rules = [
{
not = false;
or = false;
type = "ACTION_ACCEPT";
}
];
rulesSource = "";
ssoEnabled = false;
tags = [];
v4AssignMode = {
zt = false;
};
v6AssignMode = {
"6plane" = false;
rfc4193 = false;
zt = false;
};
})}"
];
};
}

View File

@ -1,10 +1,6 @@
{ inputs, ... }: {
{ ... }: {
flake.nixosModules = {
hidden-ssh-announce.imports = [ ./hidden-ssh-announce.nix ];
installer.imports = [ ./installer ];
secrets.imports = [
inputs.sops-nix.nixosModules.sops
./secrets
];
};
}

View File

@ -1,39 +0,0 @@
{ lib, config, ... }:
let
encryptedForThisMachine = name: type:
let
symlink = config.clan.sops.sopsDirectory + "/secrets/${name}/machines/${config.clan.sops.machineName}";
in
# WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl
type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink");
secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.sops.sopsDirectory + "/secrets"));
in
{
imports = [
];
options = {
clan.sops = {
machineName = lib.mkOption {
type = lib.types.str;
default = config.networking.hostName;
description = ''
Machine used to lookup secrets in the sops directory.
'';
};
sopsDirectory = lib.mkOption {
type = lib.types.path;
description = ''
Sops toplevel directory that stores users, machines, groups and secrets.
'';
};
};
};
config = {
sops.secrets = builtins.mapAttrs
(name: _: {
sopsFile = config.clan.sops.sopsDirectory + "/secrets/${name}/secret";
format = "binary";
})
secrets;
};
}

View File

@ -4,7 +4,7 @@ import sys
from types import ModuleType
from typing import Optional
from . import admin, config, machines, secrets, webui
from . import admin, config, machines, secrets, webui, zerotier
from .errors import ClanError
from .ssh import cli as ssh_cli
@ -41,6 +41,9 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
parser_webui = subparsers.add_parser("webui", help="start webui")
webui.register_parser(parser_webui)
parser_zerotier = subparsers.add_parser("zerotier", help="create zerotier network")
zerotier.register_parser(parser_zerotier)
if argcomplete:
argcomplete.autocomplete(parser)

View File

@ -1,6 +1,7 @@
# !/usr/bin/env python3
import argparse
from .generate import register_generate_parser
from .groups import register_groups_parser
from .import_sops import register_import_sops_parser
from .machines import register_machines_parser
@ -29,4 +30,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
import_sops_parser = subparser.add_parser("import-sops", help="import a sops file")
register_import_sops_parser(import_sops_parser)
parser_generate = subparser.add_parser(
"generate", help="generate secrets for machines if they don't exist yet"
)
register_generate_parser(parser_generate)
register_secrets_parser(subparser)

View File

@ -0,0 +1,51 @@
import argparse
import subprocess
import sys
from clan_cli.errors import ClanError
def get_secret_script(machine: str) -> None:
proc = subprocess.run(
[
"nix",
"build",
"--impure",
"--print-out-paths",
"--expr",
"let f = builtins.getFlake (toString ./.); in "
f"(f.nixosConfigurations.{machine}.extendModules "
"{ modules = [{ clanCore.clanDir = toString ./.; }]; })"
".config.system.clan.generateSecrets",
],
check=True,
capture_output=True,
text=True,
)
if proc.returncode != 0:
print(proc.stderr, file=sys.stderr)
raise ClanError(f"failed to generate secrets:\n{proc.stderr}")
secret_generator_script = proc.stdout.strip()
print(secret_generator_script)
secret_generator = subprocess.run(
[secret_generator_script],
check=True,
)
if secret_generator.returncode != 0:
raise ClanError("failed to generate secrets")
else:
print("successfully generated secrets")
def generate_command(args: argparse.Namespace) -> None:
get_secret_script(args.machine)
def register_generate_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"machine",
help="The machine to generate secrets for",
)
parser.set_defaults(func=generate_command)

View File

@ -22,9 +22,12 @@ class SopsKey:
def get_public_key(privkey: str) -> str:
cmd = nix_shell(["age"], ["age-keygen", "-y"])
res = subprocess.run(
cmd, input=privkey, check=True, stdout=subprocess.PIPE, text=True
)
try:
res = subprocess.run(cmd, input=privkey, stdout=subprocess.PIPE, text=True)
except subprocess.CalledProcessError as e:
raise ClanError(
"Failed to get public key for age private key. Is the key malformed?"
) from e
return res.stdout.strip()
@ -100,7 +103,7 @@ def ensure_sops_key() -> SopsKey:
subprocess.run(cmd, check=True)
tty.info(
f"Generated age key at '{path}'. Please back it up on a secure location or you will lose access to your secrets."
f"Generated age key at '{path}' for your user. Please back it up on a secure location or you will lose access to your secrets."
)
return ensure_user(get_public_key(path.read_text()))

View File

@ -1,3 +1,4 @@
import argparse
import json
import socket
import subprocess
@ -42,7 +43,8 @@ class ZerotierController:
def __init__(self, port: int, home: Path) -> None:
self.port = port
self.home = home
self.secret = (home / "authtoken.secret").read_text()
self.authtoken = (home / "authtoken.secret").read_text()
self.secret = (home / "identity.secret").read_text()
def _http_request(
self,
@ -56,7 +58,7 @@ class ZerotierController:
if data is not None:
body = json.dumps(data).encode("ascii")
headers["Content-Type"] = "application/json"
headers["X-ZT1-AUTH"] = self.secret
headers["X-ZT1-AUTH"] = self.authtoken
url = f"http://127.0.0.1:{self.port}{path}"
req = urllib.request.Request(url, headers=headers, method=method, data=body)
resp = urllib.request.urlopen(req)
@ -75,11 +77,6 @@ class ZerotierController:
def get_network(self, id: str) -> dict[str, Any]:
return self._http_request(f"/controller/network/{id}")
def update_network(self, id: str, new_config: dict[str, Any]) -> dict[str, Any]:
return self._http_request(
f"/controller/network/{id}", method="POST", data=new_config
)
@contextmanager
def zerotier_controller() -> Iterator[ZerotierController]:
@ -117,6 +114,7 @@ def zerotier_controller() -> Iterator[ZerotierController]:
"/proc",
"--dev",
"/dev",
"--unshare-user",
"--uid",
"0",
"--gid",
@ -151,19 +149,28 @@ def zerotier_controller() -> Iterator[ZerotierController]:
p.wait()
class ZerotierNetwork:
def __init__(self, network_id: str) -> None:
self.network_id = network_id
# TODO: allow merging more network configuration here
def create_network(private: bool = False) -> ZerotierNetwork:
def create_network() -> dict:
with zerotier_controller() as controller:
network = controller.create_network()
network_id = network["nwid"]
network = controller.get_network(network_id)
network["private"] = private
network["v6AssignMode"]["rfc4193"] = True
controller.update_network(network_id, network)
# TODO: persist home into sops?
return ZerotierNetwork(network_id)
return {
"secret": controller.secret,
"networkid": network["nwid"],
}
def main(args: argparse.Namespace) -> None:
zerotier = create_network()
outpath = Path(args.outpath)
outpath.mkdir(parents=True, exist_ok=True)
with open(outpath / "network.id", "w+") as nwid_file:
nwid_file.write(zerotier["networkid"])
with open(outpath / "identity.secret", "w+") as secret_file:
secret_file.write(zerotier["secret"])
def register_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--outpath", help="directory to put the secret file to", required=True
)
parser.set_defaults(func=main)

View File

@ -49,7 +49,7 @@ let
cp -r ${./.} $out
chmod -R +w $out
rm $out/clan_cli/config/jsonschema
ln -s ${nixpkgs} $out/clan_cli/nixpkgs
ln -sTf ${nixpkgs} $out/clan_cli/nixpkgs
cp -r ${../../lib/jsonschema} $out/clan_cli/config/jsonschema
ln -s ${ui-assets} $out/clan_cli/webui/assets
'';
@ -106,7 +106,7 @@ python3.pkgs.buildPythonPackage {
passthru.testDependencies = dependencies ++ testDependencies;
postInstall = ''
ln -s ${nixpkgs} $out/${python3.sitePackages}/nixpkgs
ln -sTf ${nixpkgs} $out/${python3.sitePackages}/clan_cli/nixpkgs
installShellCompletion --bash --name clan \
<(${argcomplete}/bin/register-python-argcomplete --shell bash clan)
installShellCompletion --fish --name clan.fish \

View File

@ -3,4 +3,4 @@ from clan_cli.zerotier import create_network
def test_create_network() -> None:
network = create_network()
assert network.network_id
assert network["networkid"]

View File

@ -1,52 +0,0 @@
# AUTOMATICALLY GENERATED by clan
# TODO: move this code to a flakeModule in clan-core and import it via flake input
{ self, lib, inputs, ... }:
let
evalConfig = import (self.inputs.nixpkgs + /lib/eval-config.nix);
evalMachine = machineModule: evalConfig {
modules = [ machineModule ];
};
machinesDirs =
if builtins.pathExists ./machines
then builtins.readDir ./machines
else { };
machineSettings = machineName:
if builtins.pathExists ./machines/${machineName}/settings.json
then (builtins.fromJSON (builtins.readFile ./machines/${machineName}/settings.json))
else { };
# load all clan modules by default
machineModule = machineName: {
imports =
(lib.attrValues inputs.clan.clanModules)
++ [ (machineSettings machineName) ];
};
# autoamtically generate machine modules for all directories under ./machines
machineModulesGenerated =
lib.flip lib.mapAttrs' machinesDirs (name: _: {
name = "machine-${name}";
value = machineModule name;
});
# re-read the machine modules from the flake outputs, to allow amchines being
# defined by custom mechanisms
machineModules =
lib.filterAttrs (name: _: lib.hasPrefix "machine-" name) self.nixosModules;
in
{
imports =
let
relPaths = builtins.fromJSON (builtins.readFile ./imports.json);
paths = map (path: ./. + path) relPaths;
in
paths;
flake.nixosModules = machineModulesGenerated;
# generate nixosConfigurations for all machines under self.nixosModules.machines-{machine_name}
flake.nixosConfigurations =
lib.mapAttrs (_: mod: evalMachine mod) machineModules;
}

View File

@ -1,16 +1,11 @@
{
description = "<Put your description here>";
inputs = {
clan-core.url = "git+https://git.clan.lol/clan/clan-core";
flake-parts.url = "github:hercules-ci/flake-parts";
};
inputs.clan-core.url = "git+https://git.clan.lol/clan/clan-core";
outputs = inputs @ { flake-parts, ... }:
flake-parts.lib.mkFlake { inherit inputs; } {
systems = [ "x86_64-linux" ];
imports = [
./clan-flake-module.nix
];
outputs = { clan-core, ... }: {
nixosConfigurations = clan-core.lib.buildClan {
directory = ./.;
};
};
}

View File

@ -1 +0,0 @@
[]

View File

@ -1 +0,0 @@
["x86_64-linux"]