clan/clan-core
12
4
Fork 18
Code Issues 98 Pull Requests 4 Actions 1 Packages Projects 1 Releases Wiki Activity

Merge branch 'main' into Qubasa-main

Browse Source
This commit is contained in:
Luis Hebendanz 2023-08-30 15:41:42 +02:00
commit 47485395e5
18 changed files with 199 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
docs/quickstart.md Normal file
View File

@ -0,0 +1,112 @@
# Initializing a New Clan Project
## Clone the Clan Template
To start a new project, execute the following command to clone the Clan Core template:
```bash
$ nix flake init -t git+https://git.clan.lol/clan/clan-core
```
This action will generate two primary files: `flake.nix` and `.clan-flake`.
```bash
$ ls -la
drwx------ joerg users 5 B a minute ago ./
drwxrwxrwt root root 139 B 12 seconds ago ../
.rw-r--r-- joerg users 77 B a minute ago .clan-flake
.rw-r--r-- joerg users 4.8 KB a minute ago flake.lock
.rw-r--r-- joerg users 242 B a minute ago flake.nix
```
### Understanding the .clan-flake Marker File
The `.clan-flake` marker file serves an optional purpose: it helps the `clan-cli` utility locate the project's root directory.
If `.clan-flake` is missing, `clan-cli` will instead search for other indicators like `.git`, `.hg`, `.svn`, or `flake.nix` to identify the project root.
---
# Migrating Existing NixOS Configuration Flake
Absolutely, let's break down the migration step by step, explaining each action in detail:
#### Before You Begin
1. **Backup Your Current Configuration**: Always start by making a backup of your current NixOS configuration to ensure you can revert if needed.
```shell
cp -r /etc/nixos ~/nixos-backup
```
2. **Update Flake Inputs**: The patch adds a new input named `clan-core` to your `flake.nix`. This input points to a Git repository for Clan Core. Here's the addition:
```nix
inputs.clan-core = {
url = "git+https://git.clan.lol/clan/clan-core";
inputs.nixpkgs.follows = "nixpkgs";
};
```
- `url`: Specifies the Git repository URL for Clan Core.
- `inputs.nixpkgs.follows`: Tells Nix to use the same `nixpkgs` input as your main input (in this case, it follows `nixpkgs`).
3. **Update Outputs**: Then modify the `outputs` section of your `flake.nix` to adapt to Clan Core's new provisioning method. The key changes are as follows:
Add `clan-core` to the output
```diff
- outputs = { self, nixpkgs, }:
+ outputs = { self, nixpkgs, clan-core }:
```
Previous configuration:
```nix
nixosConfigurations.example-desktop = nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
./configuration.nix
];
[...]
};
```
After change:
```nix
nixosConfigurations = clan-core.lib.buildClan {
directory = ./.;
machines = {
example-desktop = {
nixpkgs.hostPlatform = "x86_64-linux";
imports = [
./configuration.nix
];
};
};
};
```
- `nixosConfigurations`: Defines NixOS configurations, using Clan Core’s `buildClan` function to manage the machines.
- Inside `machines`, a new machine configuration is defined (in this case, `example-desktop`).
- Inside `example-desktop` which is the target machine hostname, `nixpkgs.hostPlatform` specifies the host platform as `x86_64-linux`.
4. **Rebuild and Switch**: Rebuild your NixOS configuration using the updated flake:
```shell
sudo nixos-rebuild switch --flake .
```
- This command rebuilds and switches to the new configuration. Make sure to include the `--flake .` argument to use the current directory as the flake source.
5. **Test Configuration**: Before rebooting, verify that your new configuration builds without errors or warnings.
6. **Reboot**: If everything is fine, you can reboot your system to apply the changes:
```shell
sudo reboot
```
7. **Verify**: After the reboot, confirm that your system is running with the new configuration, and all services and applications are functioning as expected.
By following these steps, you've successfully migrated your NixOS Flake configuration to include the `clan-core` input and adapted the `outputs` section to work with Clan Core's new machine provisioning method.

3
flake.nix
View File

@ -24,6 +24,7 @@
"x86_64-linux"
"aarch64-linux"
];
flake.clanModules = { };
imports = [
./checks/flake-module.nix
./devShell.nix
@ -36,7 +37,7 @@
./lib/flake-module.nix
./nixosModules/flake-module.nix
./nixosModules/clanCore/flake-module.nix
./nixosModules/core/flake-module.nix
];
});
}

8
flakeModules/clan-config.nix
View File

@ -1,4 +1,4 @@
{ ... } @ clanCore: {
{ ... } @ core: {
flake.flakeModules.clan-config = { self, inputs, ... }:
let
@ -29,12 +29,12 @@
perSystem = { pkgs, ... }: {
devShells.clan-config = pkgs.mkShell {
packages = [
clanCore.config.flake.packages.${pkgs.system}.clan-cli
core.config.flake.packages.${pkgs.system}.clan-cli
];
shellHook = ''
export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions)
export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
export XDG_DATA_DIRS="${core.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="${core.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
'';
};
};

8
lib/build-clan/default.nix
View File

@ -1,4 +1,4 @@
{ nixpkgs, clan, lib }:
{ nixpkgs, lib }:
{ directory # The directory containing the machines subdirectory
, specialArgs ? { } # Extra arguments to pass to nixosSystem i.e. useful to make self available
, machines ? { } # allows to include machine-specific modules i.e. machines.${name} = { ... }
@ -15,14 +15,14 @@ let
else { };
nixosConfigurations = lib.mapAttrs
(name: _mod:
(name: _:
nixpkgs.lib.nixosSystem {
modules = [
(machineSettings name)
(machines.${name} or { })
] ++ lib.attrValues clan.clanModules;
];
specialArgs = specialArgs;
})
machinesDirs;
(machinesDirs // machines);
in
nixosConfigurations

4
lib/default.nix
View File

@ -1,4 +1,4 @@
{ lib, clan, nixpkgs, ... }:
{ lib, nixpkgs, ... }:
{
findNixFiles = folder:
lib.mapAttrs'
@ -14,5 +14,5 @@
jsonschema = import ./jsonschema { inherit lib; };
buildClan = import ./build-clan { inherit lib clan nixpkgs; };
buildClan = import ./build-clan { inherit lib nixpkgs; };
}

2
lib/flake-module.nix
View File

@ -7,6 +7,6 @@
];
flake.lib = import ./default.nix {
inherit lib;
inherit (inputs) clan nixpkgs;
inherit (inputs) nixpkgs;
};
}

4
nixosModules/clanCore/flake-module.nix → nixosModules/core/flake-module.nix
View File

@ -1,6 +1,6 @@
{ self, inputs, lib, ... }: {
flake.nixosModules.clanCore = { pkgs, ... }: {
options.clanCore = {
flake.nixosModules.clan.core = { pkgs, ... }: {
options.clan.core = {
clanDir = lib.mkOption {
type = lib.types.str;
description = ''

4
nixosModules/clanCore/secrets/default.nix → nixosModules/core/secrets/default.nix
View File

@ -1,6 +1,6 @@
{ config, lib, ... }:
{
options.clanCore.secrets = lib.mkOption {
options.clan.core.secrets = lib.mkOption {
type = lib.types.attrsOf
(lib.types.submodule (secret: {
options = {
@ -49,7 +49,7 @@
description = ''
path to a fact which is generated by the generator
'';
default = "${config.clanCore.clanDir}/facts/${config.clanCore.machineName}/${fact.config._module.args.name}";
default = "${config.clan.core.clanDir}/facts/${config.clan.core.machineName}/${fact.config._module.args.name}";
};
value = lib.mkOption {
default = builtins.readFile fact.config.path;

20
nixosModules/clanCore/secrets/sops.nix → nixosModules/core/secrets/sops.nix
View File

@ -7,24 +7,24 @@
set -x # remove for prod
PATH=$PATH:${lib.makeBinPath [
config.clanCore.clanPkgs.clan-cli
config.clan.core.clanPkgs.clan-cli
]}
# initialize secret store
if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
if ! clan secrets machines list | grep -q ${config.clan.core.machineName}; then (
INITTMP=$(mktemp -d)
trap 'rm -rf "$INITTMP"' EXIT
${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
clan secrets machines add ${config.clan.core.machineName} "$PUBKEY"
tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-age.key
) fi
${lib.foldlAttrs (acc: n: v: ''
${acc}
# ${n}
# if any of the secrets are missing, we regenerate all connected facts/secrets
(if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
(if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clan.core.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
facts=$(mktemp -d)
trap "rm -rf $facts" EXIT
@ -38,24 +38,24 @@
'') (lib.attrValues v.facts)}
${lib.concatMapStrings (secret: ''
cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clan.core.machineName} ${config.clan.core.machineName}-${secret.name}
'') (lib.attrValues v.secrets)}
fi)
'') "" config.clanCore.secrets}
'') "" config.clan.core.secrets}
'';
sops.secrets =
let
encryptedForThisMachine = name: type:
let
symlink = config.clanCore.clanDir + "/sops/secrets/${name}/machines/${config.clanCore.machineName}";
symlink = config.clan.core.clanDir + "/sops/secrets/${name}/machines/${config.clan.core.machineName}";
in
# WTF, nix bug, my symlink is in the nixos module detected as a directory also it works in the repl
type == "directory" && (builtins.readFileType symlink == "directory" || builtins.readFileType symlink == "symlink");
secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clanCore.clanDir + "/sops/secrets"));
secrets = lib.filterAttrs encryptedForThisMachine (builtins.readDir (config.clan.core.clanDir + "/sops/secrets"));
in
builtins.mapAttrs
(name: _: {
sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";
sopsFile = config.clan.core.clanDir + "/sops/secrets/${name}/secret";
format = "binary";
})
secrets;

4
nixosModules/clanCore/zerotier.nix → nixosModules/core/zerotier.nix
View File

@ -41,13 +41,13 @@ in
} // lib.mkIf cfg.controller.enable {
# only the controller needs to have the key in the repo, the other clients can be dynamic
# we generate the zerotier code manually for the controller, since it's part of the bootstrap command
clanCore.secrets.zerotier = {
clan.core.secrets.zerotier = {
facts."network.id" = { };
secrets."identity.secret" = { };
generator = ''
TMPDIR=$(mktemp -d)
trap 'rm -rf "$TMPDIR"' EXIT
${config.clanCore.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR"
${config.clan.core.clanPkgs.clan-cli}/bin/clan zerotier --outpath "$TMPDIR"
cp "$TMPDIR"/network.id "$facts"/network.id
cp "$TMPDIR"/identity.secret "$secrets"/identity.secret
'';

2
pkgs/clan-cli/.envrc
View File

@ -4,7 +4,9 @@ source_up
if type nix_direnv_watch_file &>/dev/null; then
nix_direnv_watch_file flake-module.nix
nix_direnv_watch_file default.nix
else
direnv watch flake-module.nix
direnv watch default.nix
fi
use flake .#clan-cli --builders ''

15
pkgs/clan-cli/README.md
View File

@ -27,3 +27,18 @@ To start a local developement environment instead, use the `--dev` flag:
```
This will spawn two webserver, a python one to for the api and a nodejs one that rebuilds the ui on the fly.
## Run locally single-threaded for debugging
By default tests run in parallel using pytest-parallel.
pytest-parallel however breaks `breakpoint()`. To disable it, use this:
```console
pytest --workers "" -s
```
You can also run a single test like this:
```console
pytest --workers "" -s tests/test_secrets_cli.py::test_users
```

2
pkgs/clan-cli/clan_cli/secrets/generate.py
View File

@ -15,7 +15,7 @@ def get_secret_script(machine: str) -> None:
"--expr",
"let f = builtins.getFlake (toString ./.); in "
f"(f.nixosConfigurations.{machine}.extendModules "
"{ modules = [{ clanCore.clanDir = toString ./.; }]; })"
"{ modules = [{ clan.core.clanDir = toString ./.; }]; })"
".config.system.clan.generateSecrets",
],
check=True,

21
pkgs/clan-cli/clan_cli/secrets/machines.py
View File

@ -3,11 +3,8 @@ import argparse
from ..machines.types import machine_name_type, validate_hostname
from . import secrets
from .folders import list_objects, remove_object, sops_machines_folder
from .sops import write_key
from .types import (
public_or_private_age_key_type,
secret_name_type,
)
from .sops import read_key, write_key
from .types import public_or_private_age_key_type, secret_name_type
def add_machine(name: str, key: str, force: bool) -> None:
@ -18,6 +15,10 @@ def remove_machine(name: str) -> None:
remove_object(sops_machines_folder(), name)
def get_machine(name: str) -> str:
return read_key(sops_machines_folder() / name)
def list_machines() -> list[str]:
return list_objects(sops_machines_folder(), lambda x: validate_hostname(x))
@ -42,6 +43,10 @@ def add_command(args: argparse.Namespace) -> None:
add_machine(args.machine, args.key, args.force)
def get_command(args: argparse.Namespace) -> None:
print(get_machine(args.machine))
def remove_command(args: argparse.Namespace) -> None:
remove_machine(args.machine)
@ -82,6 +87,12 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
)
add_parser.set_defaults(func=add_command)
get_parser = subparser.add_parser("get", help="get a machine public key")
get_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
get_parser.set_defaults(func=get_command)
remove_parser = subparser.add_parser("remove", help="remove a machine")
remove_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type

14
pkgs/clan-cli/clan_cli/secrets/users.py
View File

@ -2,7 +2,7 @@ import argparse
from . import secrets
from .folders import list_objects, remove_object, sops_users_folder
from .sops import write_key
from .sops import read_key, write_key
from .types import (
VALID_SECRET_NAME,
public_or_private_age_key_type,
@ -19,6 +19,10 @@ def remove_user(name: str) -> None:
remove_object(sops_users_folder(), name)
def get_user(name: str) -> str:
return read_key(sops_users_folder() / name)
def list_users() -> list[str]:
return list_objects(
sops_users_folder(), lambda n: VALID_SECRET_NAME.match(n) is not None
@ -43,6 +47,10 @@ def add_command(args: argparse.Namespace) -> None:
add_user(args.user, args.key, args.force)
def get_command(args: argparse.Namespace) -> None:
print(get_user(args.user))
def remove_command(args: argparse.Namespace) -> None:
remove_user(args.user)
@ -77,6 +85,10 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None:
)
add_parser.set_defaults(func=add_command)
get_parser = subparser.add_parser("get", help="get a user public key")
get_parser.add_argument("user", help="the name of the user", type=user_name_type)
get_parser.set_defaults(func=get_command)
remove_parser = subparser.add_parser("remove", help="remove a user")
remove_parser.add_argument("user", help="the name of the user", type=user_name_type)
remove_parser.set_defaults(func=remove_command)

7
pkgs/clan-cli/tests/test_secrets_cli.py
View File

@ -36,8 +36,13 @@ def _test_identities(
age_keys[0].privkey,
]
)
capsys.readouterr() # empty the buffer
capsys.readouterr() # empty the buffer
cli.run(["secrets", what, "get", "foo"])
out = capsys.readouterr() # empty the buffer
assert age_keys[0].pubkey in out.out
capsys.readouterr() # empty the buffer
cli.run(["secrets", what, "list"])
out = capsys.readouterr() # empty the buffer
assert "foo" in out.out

4
scripts/ci
View File

@ -26,7 +26,9 @@ fi
rc=0
for job in $(nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" | jq -r '. | @base64'); do
nix shell --inputs-from '.#' "nixpkgs#nix-eval-jobs" -c nix-eval-jobs "${args[@]}" > "jobs.json"
for job in $(jq -r '. | @base64' < "jobs.json"); do
job=$(echo "$job" | base64 -d)
attr=$(echo "$job" | jq -r .attr)
echo "### $attr"

3
templates/flake-module.nix
View File

@ -1,8 +1,9 @@
{
{ self, ... }: {
flake.templates = {
new-clan = {
description = "Initialize a new clan flake";
path = ./new-clan;
};
default = self.templates.new-clan;
};
}