schema improvements
Some checks failed
buildbot/nix-build .#checks.x86_64-linux.inventory-schema-checks Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-archlinux Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-apk Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-rpm Build done.
buildbot/nix-build .#checks.x86_64-linux.package-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-installer-deb Build done.
buildbot/nix-build .#checks.x86_64-linux.renderClanOptions Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-flash-installer Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-test-backup Build done.
buildbot/nix-build .#checks.aarch64-darwin.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-no-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.check-for-breakpoints Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-age Build done.
buildbot/nix-build .#checks.x86_64-linux.treefmt Build done.
buildbot/nix-build .#checks.aarch64-darwin.devShell-docs Build done.
buildbot/nix-build .#checks.aarch64-darwin.package-deploy-docs Build done.
buildbot/nix-build .#checks.aarch64-darwin.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-bash Build done.
buildbot/nix-build .#checks.aarch64-darwin.module-schema Build done.
buildbot/nix-build .#checks.aarch64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.aarch64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.aarch64-darwin.package-docs Build done.
buildbot/nix-build .#checks.aarch64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.aarch64-linux.package-docs Build done.
buildbot/nix-build .#checks.aarch64-darwin.package-module-schema Build done.
buildbot/nix-build .#checks.aarch64-linux.module-schema Build done.
buildbot/nix-build .#checks.aarch64-darwin.package-module-docs Build done.
buildbot/nix-build .#checks.aarch64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.aarch64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-e2fsprogs Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-fakeroot Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-git Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-nix Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-openssh Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.11-mypy" Build done.
buildbot/nix-build .#checks.x86_64-linux."clan-dep-python3.11-qemu" Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-rsync Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sops Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-sshpass Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-app-pytest Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-zbar Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-dep-tor Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-without-core Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-inventory-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.borgbackup Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-example-valid Build done.
buildbot/nix-build .#checks.aarch64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.container Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-flash-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.deltachat Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-test_install_machine Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-ts-api Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-app Build done.
buildbot/nix-build .#checks.x86_64-linux.package-default Build done.
buildbot/nix-build .#checks.x86_64-linux.package-deploy-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-editor Build done.
buildbot/nix-build .#checks.x86_64-linux.devShell-default Build done.
buildbot/nix-build .#checks.x86_64-linux.package-function-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.package-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.matrix-synapse Build done.
buildbot/nix-build .#checks.x86_64-linux.package-impure-checks Build done.
buildbot/nix-build .#checks.x86_64-linux.package-clan-cli Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-docs Build done.
buildbot/nix-build .#checks.x86_64-linux.package-moonlight-sunshine-accept Build done.
buildbot/nix-build .#checks.x86_64-linux.package-pending-reviews Build done.
buildbot/nix-build .#checks.x86_64-linux.package-webview-ui Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotier-members Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zerotierone Build done.
buildbot/nix-build .#checks.x86_64-linux.package-tea-create-pr Build done.
buildbot/nix-build .#checks.x86_64-linux.package-zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.package-module-schema Build done.
buildbot/nix-build .#checks.x86_64-linux.lib-jsonschema-nix-unit-tests Build done.
buildbot/nix-build .#checks.x86_64-linux.package-merge-after-ci Build done.
buildbot/nix-build .#checks.x86_64-linux.postgresql Build done.
buildbot/nix-build .#checks.x86_64-linux.nixos-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.zt-tcp-relay Build done.
buildbot/nix-build .#checks.x86_64-linux.secrets Build done.
buildbot/nix-build .#checks.x86_64-linux.template-minimal Build done.
buildbot/nix-build .#checks.x86_64-linux.wayland-proxy-virtwl Build done.
buildbot/nix-build .#checks.x86_64-linux.package-iso-installer Build done.
buildbot/nix-build .#checks.x86_64-linux.syncthing Build done.
buildbot/nix-build .#checks.x86_64-linux.package-gui-install-test-ubuntu-22-04 Build done.
buildbot/nix-build .#checks.x86_64-linux.clan-pytest-with-core Build done.
buildbot/nix-build .#checks.x86_64-linux.test-backups Build done.
buildbot/nix-build .#checks.x86_64-linux.flash Build done.
checks / checks-impure (pull_request) Successful in 2m11s
buildbot/nix-build .#checks.x86_64-linux.test-installation Build done.
buildbot/nix-eval Build done.

This commit is contained in:
Johannes Kirschbauer 2024-06-20 18:36:26 +02:00
parent a60b1bfafc
commit c1dfb2065f
Signed by: hsjobeki
SSH Key Fingerprint: SHA256:vX3utDqig7Ph5L0JPv87ZTPb/w7cMzREKVZzzLFg9qU
10 changed files with 244 additions and 135 deletions

View File

@ -3,23 +3,49 @@ let
clanDir = config.clan.core.clanDir; clanDir = config.clan.core.clanDir;
machineDir = clanDir + "/machines/"; machineDir = clanDir + "/machines/";
cfg = config.clan.borgbackup-static; # cfg.roles = config.clan.borgbackup-static;
# machine < machine_module < inventory
# nixos < borgbackup < borgbackup-static > UI
# metadata
# Developer User field descriptions
roles = config.clan.borgbackup-static.inventory.roles;
machine_name = config.clan.core.machineName; machine_name = config.clan.core.machineName;
in in
{ {
imports = [ ../borgbackup ]; imports = [ ../borgbackup ];
# imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ];
# Inventory / Interface.nix # Inventory / Interface.nix
# options.clan.inventory.borgbackup-static.description. # options.clan.inventory.borgbackup-static.description.
options.clan.borgbackup-static.roles = lib.mkOption { # options.clan.borgbackup-static.roles = lib.mkOption {
type = lib.types.attrsOf (lib.types.listOf lib.types.str); # type = lib.types.attrsOf (lib.types.listOf lib.types.str);
# };
# Can be used via inventory.json
#
# .borgbackup-static.inventory.roles
#
options.clan.borgbackup-static.inventory = lib.mkOption {
type = lib.types.submodule {
# imports = [./inventory/interface.nix];
# idea
# config.metadata = builtins.fromTOML ...
# config.defaultRoles = ["client"];
# -> interface.nix
options = {
roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); };
};
};
}; };
config.services.borgbackup.repos = config.services.borgbackup.repos =
let let
filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles); filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles);
borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub"; borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
machinesMaybeKey = builtins.map ( machinesMaybeKey = builtins.map (
@ -40,13 +66,13 @@ in
}; };
}) machinesWithKey; }) machinesWithKey;
in in
lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) ( lib.mkIf (builtins.elem "server" roles.${machine_name}) (
if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { } if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }
); );
config.clan.borgbackup.destinations = config.clan.borgbackup.destinations =
let let
servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles); servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles);
destinations = builtins.map (server_name: { destinations = builtins.map (server_name: {
name = server_name; name = server_name;
@ -55,5 +81,5 @@ in
}; };
}) servers; }) servers;
in in
lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations); lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations);
} }

View File

@ -1,5 +1,64 @@
# Inventory # Inventory
Questions:
- [x] Must roles be a list ?
-> Yes. In zerotier you can be "moon" and "controller" at the same time.
- [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different?
-> There are many roles. And they depend on the service.
- [x] Should we use the module name in the path of the service?
```json
// ${module_name}.${instance_name}
services.borgbackup-static.backup1 = {
}
```
Pro:
Easier to handle.
Better groups the module specific instances.
Contra:
More nesting in json
Neutral: Module name is hard to change. Exists anyways.
- [x] Should the machine specific service config be part of the service?
-> The config implements the schema of the module, which is declared in the service.
-> If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
Architecture
```
machine < machine_module < inventory
---------------------------------------------
nixos < borgbackup < borgbackup-static > UI
creates the config Maps from high level services to the borgbackup clan module
for ONE machine
```
- [ ] Why do we need 2 modules?
-> It is technically possible to have only 1 module.
Pros:
Simple to use/Easy to understand.
Less modules
Cons:
Harder to write a module. Because it must do 2 things.
One module should do only 1 thing.
```nix
clan.machines.${machine_name} = {
# "borgbackup.ssh.pub" = machineDir + machines + "/facts/borgbackup.ssh.pub";
facts = ...
};
clan.services.${instance} = {
# roles.server = [ "jon_machine" ]
# roles.${role_name} = [ ${machine_name} ];
};
```
This part provides a specification for the inventory. This part provides a specification for the inventory.
It is used for design phase and as validation helper. It is used for design phase and as validation helper.

View File

@ -1,16 +1,28 @@
{ inputs, self, ... }: { self, lib, ... }:
let let
clan-core = self; clan-core = self;
system = "x86_64-linux";
pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
# syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json); # syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json);
syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json); syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json);
machines = machinesFromInventory { machines = machinesFromInventory syncthing_inventory;
inherit clan-core;
lib = pkgs.lib; resolveGroups =
} syncthing_inventory; inventory: members:
lib.unique (
builtins.foldl' (
acc: currMember:
let
groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember;
groupMembers =
if inventory.groups.machines ? ${groupName} then
inventory.groups.machines.${groupName}
else
throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory.";
in
if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ]
) [ ] members
);
/* /*
Returns a NixOS configuration for every machine in the inventory. Returns a NixOS configuration for every machine in the inventory.
@ -18,48 +30,51 @@ let
machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration } machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
*/ */
machinesFromInventory = machinesFromInventory =
{ lib, clan-core, ... }:
inventory: inventory:
# For every machine in the inventory, build a NixOS configuration # For every machine in the inventory, build a NixOS configuration
# For each machine generate config, forEach service, if the machine is used. # For each machine generate config, forEach service, if the machine is used.
builtins.mapAttrs ( builtins.mapAttrs (
machine_name: _: machineName: _:
builtins.foldl' ( lib.foldlAttrs (
acc: service_name: # [ Modules ], String, { ${instance_name} :: ServiceConfig }
let acc: moduleName: serviceConfigs:
service_config = inventory.services.${service_name}; acc
isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig); # Collect service config
++ (lib.foldlAttrs (
# [ Modules ], String, ServiceConfig
acc2: instanceName: serviceConfig:
let
resolvedRoles = builtins.mapAttrs (
_roleName: members: resolveGroups inventory members
) serviceConfig.roles;
machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { }; isInService = builtins.any (members: builtins.elem machineName members) (
global_config = inventory.services.${service_name}.config; builtins.attrValues resolvedRoles
module_name = inventory.services.${service_name}.module; );
in
# Possible roles: "server", "client", "peer" machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
if globalConfig = serviceConfig.config;
builtins.trace '' in
isInService ${builtins.toJSON isInService}, if isInService then
${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)} acc2
'' isInService ++ [
then {
acc imports = [ clan-core.clanModules.${moduleName} ];
++ [ config.clan.${moduleName} = lib.mkMerge [
{ globalConfig
imports = [ clan-core.clanModules.${module_name} ]; machineServiceConfig
config.clan.${module_name} = lib.mkMerge [ ];
global_config }
machine_service_config {
]; config.clan.inventory.${instanceName} = {
} roles = resolvedRoles;
{ };
config.clan.${module_name} = { }
# TODO: filter, show only the roles that are needed by the machine ]
roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig; else
}; acc2
} ) [ ] serviceConfigs)
] ) [ ] inventory.services
else
acc
) [ ] (builtins.attrNames inventory.services)
) inventory.machines; ) inventory.machines;
in in
{ {
@ -81,9 +96,4 @@ in
}; };
}; };
intern = machines; intern = machines;
# inherit (clan) nixosConfigurations clanInternals;
# add the Clan cli tool to the dev shell
devShells.${system}.default = pkgs.mkShell {
packages = [ clan-core.packages.${system}.clan-cli ];
};
} }

View File

@ -1,6 +1,11 @@
{ inputs, self, ... }:
{ {
flake.inventory = import ./default.nix { inherit inputs self; }; inputs,
self,
lib,
...
}:
{
flake.inventory = import ./default.nix { inherit inputs self lib; };
perSystem = perSystem =
{ pkgs, config, ... }: { pkgs, config, ... }:
{ {

View File

@ -1,8 +0,0 @@
package machines
#machine: machines: [string]: {
name: string,
description?: string,
icon?: string
}

View File

@ -1,8 +1,7 @@
package inventory package inventory
import ( import (
"clan.lol/inventory/services" "clan.lol/inventory/schema"
"clan.lol/inventory/machines"
) )
@jsonschema(schema="http://json-schema.org/schema#") @jsonschema(schema="http://json-schema.org/schema#")
@ -16,9 +15,11 @@ import (
icon?: string icon?: string
} }
// A map of services // // A map of services
services.#service schema.#service
// A map of machines // // A map of machines
machines.#machine schema.#machine
schema.#groups
} }

View File

@ -1,22 +1,34 @@
package services package schema
#ServiceRole: "server" | "client" | "both" #groups: groups: {
// Machine groups
machines: {
// Group name mapped to list[machineName]
// "group1": ["machine1", "machine2"]
[string]: [...string]
}
}
#service: services: [string]: { #machine: machines: [string]: {
name: string,
description?: string,
icon?: string
}
#role: string
#service: services: [string]: [string]: {
// Required meta fields // Required meta fields
meta: { meta: {
name: string, name: string,
icon?: string icon?: string
description?: string, description?: string,
}, },
// Required module specifies the behavior of the service.
module: string,
// We moved the machine sepcific config to "machines". // We moved the machine sepcific config to "machines".
// It may be moved back depending on what makes more sense in the future. // It may be moved back depending on what makes more sense in the future.
machineConfig: { roles: [#role]: [...string],
machines: {
[string]: { [string]: {
roles?: [ ...#ServiceRole ],
config?: { config?: {
... ...
} }
@ -29,4 +41,4 @@ package services
// It declares the interface how the service can be configured. // It declares the interface how the service can be configured.
... ...
} }
} }

View File

@ -10,27 +10,32 @@
"name": "vi" "name": "vi"
} }
}, },
"groups": {
"machines": {
"laptops": ["camina_machine", "vi_machine"],
"all": ["camina_machine", "vi_machine", "vyr_machine"]
}
},
"meta": { "meta": {
"name": "kenjis clan" "name": "kenjis clan"
}, },
"services": { "services": {
"backup": { "borgbackup-static": {
"meta": { "instance_1": {
"name": "My backup" "meta": {
}, "name": "My backup"
"module": "borgbackup-static",
"machineConfig": {
"vyr_machine": {
"roles": ["server"]
}, },
"vi_machine": { "roles": {
"roles": ["client"] "server": ["vyr_machine"],
"client": ["group:laptops"]
}, },
"camina_machine": { "machines": {
"roles": ["client"] "vyr_machine": {},
} "vi_machine": {},
}, "camina_machine": {}
"config": {} },
"config": {}
}
} }
} }
} }

View File

@ -14,29 +14,29 @@
"name": "kenjis clan" "name": "kenjis clan"
}, },
"services": { "services": {
"sync_files": { "syncthing-static-peers": {
"meta": { "instance_1": {
"name": "My sync" "meta": {
}, "name": "My sync"
"module": "syncthing-static-peers", },
"machineConfig": { "roles": {
"vyr_machine": {}, "peer": ["vyr_machine", "vi_machine", "camina_machine"]
"vi_machine": {}, },
"camina_machine": {} "machines": {},
}, "config": {
"config": { "folders": {
"folders": { "test": {
"test": { "path": "~/data/docs",
"path": "~/data/docs", "devices": ["camina_machine", "vyr_machine", "vi_machine"]
"devices": ["camina_machine", "vyr_machine", "vi_machine"] },
}, "videos": {
"videos": { "path": "~/data/videos",
"path": "~/data/videos", "devices": ["camina_machine", "vyr_machine"]
"devices": ["camina_machine", "vyr_machine"] },
}, "playlist": {
"playlist": { "path": "~/data/playlist",
"path": "~/data/playlist", "devices": ["camina_machine", "vi_machine"]
"devices": ["camina_machine", "vi_machine"] }
} }
} }
} }

View File

@ -14,23 +14,22 @@
"name": "kenjis clan" "name": "kenjis clan"
}, },
"services": { "services": {
"backup": { "zerotier-static": {
"meta": { "instance_1": {
"name": "My backup" "meta": {
}, "name": "My Network"
"module": "borbackup-static",
"machineConfig": {
"vyr_machine": {
"roles": ["server"]
}, },
"vi_machine": { "roles": {
"roles": ["peer"] "server": ["vyr_machine"],
"peer": ["vi_machine", "camina_machine"]
}, },
"camina_machine": { "machines": {
"roles": ["peer"] "vyr_machine": {
} "config": {}
}, }
"config": {} },
"config": {}
}
} }
} }
} }