Inventory: init first implementation #1638
@ -3,23 +3,49 @@ let
|
|||||||
clanDir = config.clan.core.clanDir;
|
clanDir = config.clan.core.clanDir;
|
||||||
machineDir = clanDir + "/machines/";
|
machineDir = clanDir + "/machines/";
|
||||||
|
|
||||||
cfg = config.clan.borgbackup-static;
|
# cfg.roles = config.clan.borgbackup-static;
|
||||||
|
|
||||||
|
# machine < machine_module < inventory
|
||||||
|
# nixos < borgbackup < borgbackup-static > UI
|
||||||
|
# metadata
|
||||||
|
# Developer User field descriptions
|
||||||
|
|
||||||
|
roles = config.clan.borgbackup-static.inventory.roles;
|
||||||
|
|
||||||
machine_name = config.clan.core.machineName;
|
machine_name = config.clan.core.machineName;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
imports = [ ../borgbackup ];
|
imports = [ ../borgbackup ];
|
||||||
|
# imports = if myRole == "server" then [ ../borgbackup/roles/server.nix ];
|
||||||
# Inventory / Interface.nix
|
# Inventory / Interface.nix
|
||||||
# options.clan.inventory.borgbackup-static.description.
|
# options.clan.inventory.borgbackup-static.description.
|
||||||
options.clan.borgbackup-static.roles = lib.mkOption {
|
# options.clan.borgbackup-static.roles = lib.mkOption {
|
||||||
type = lib.types.attrsOf (lib.types.listOf lib.types.str);
|
# type = lib.types.attrsOf (lib.types.listOf lib.types.str);
|
||||||
|
# };
|
||||||
|
|
||||||
|
# Can be used via inventory.json
|
||||||
|
#
|
||||||
|
# .borgbackup-static.inventory.roles
|
||||||
|
#
|
||||||
|
options.clan.borgbackup-static.inventory = lib.mkOption {
|
||||||
|
type = lib.types.submodule {
|
||||||
|
# imports = [./inventory/interface.nix];
|
||||||
|
|
||||||
|
# idea
|
||||||
|
# config.metadata = builtins.fromTOML ...
|
||||||
|
# config.defaultRoles = ["client"];
|
||||||
|
|
||||||
|
# -> interface.nix
|
||||||
|
options = {
|
||||||
|
roles = lib.mkOption { type = lib.types.attrsOf (lib.types.listOf lib.types.str); };
|
||||||
|
};
|
||||||
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
config.services.borgbackup.repos =
|
config.services.borgbackup.repos =
|
||||||
let
|
let
|
||||||
|
|
||||||
filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) cfg.roles);
|
filteredMachines = builtins.attrNames (lib.filterAttrs (_: v: builtins.elem "client" v) roles);
|
||||||
|
|
||||||
borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
|
borgbackupIpMachinePath = machines: machineDir + machines + "/facts/borgbackup.ssh.pub";
|
||||||
machinesMaybeKey = builtins.map (
|
machinesMaybeKey = builtins.map (
|
||||||
@ -40,13 +66,13 @@ in
|
|||||||
};
|
};
|
||||||
}) machinesWithKey;
|
}) machinesWithKey;
|
||||||
in
|
in
|
||||||
lib.mkIf (builtins.elem "server" cfg.roles.${machine_name}) (
|
lib.mkIf (builtins.elem "server" roles.${machine_name}) (
|
||||||
if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }
|
if (builtins.listToAttrs hosts) != null then builtins.listToAttrs hosts else { }
|
||||||
);
|
);
|
||||||
|
|
||||||
config.clan.borgbackup.destinations =
|
config.clan.borgbackup.destinations =
|
||||||
let
|
let
|
||||||
servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) cfg.roles);
|
servers = builtins.attrNames (lib.filterAttrs (_n: v: (builtins.elem "server" v)) roles);
|
||||||
|
|
||||||
destinations = builtins.map (server_name: {
|
destinations = builtins.map (server_name: {
|
||||||
name = server_name;
|
name = server_name;
|
||||||
@ -55,5 +81,5 @@ in
|
|||||||
};
|
};
|
||||||
}) servers;
|
}) servers;
|
||||||
in
|
in
|
||||||
lib.mkIf (builtins.elem "client" cfg.roles.${machine_name}) (builtins.listToAttrs destinations);
|
lib.mkIf (builtins.elem "client" roles.${machine_name}) (builtins.listToAttrs destinations);
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,64 @@
|
|||||||
# Inventory
|
# Inventory
|
||||||
|
|
||||||
|
Questions:
|
||||||
|
|
||||||
|
- [x] Must roles be a list ?
|
||||||
|
-> Yes. In zerotier you can be "moon" and "controller" at the same time.
|
||||||
|
|
||||||
|
- [x] Is role client different from peer ? Do we have one example where we use client and peer together and they are different?
|
||||||
|
-> There are many roles. And they depend on the service.
|
||||||
|
|
||||||
|
- [x] Should we use the module name in the path of the service?
|
||||||
|
```json
|
||||||
|
// ${module_name}.${instance_name}
|
||||||
|
services.borgbackup-static.backup1 = {
|
||||||
|
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Pro:
|
||||||
|
Easier to handle.
|
||||||
|
Better groups the module specific instances.
|
||||||
|
Contra:
|
||||||
|
More nesting in json
|
||||||
|
|
||||||
|
Neutral: Module name is hard to change. Exists anyways.
|
||||||
|
|
||||||
|
- [x] Should the machine specific service config be part of the service?
|
||||||
|
-> The config implements the schema of the module, which is declared in the service.
|
||||||
|
-> If the config is placed in the machine, it becomes unclear that the scope is ONLY the service and NOT the global nixos config.
|
||||||
|
|
||||||
|
Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
machine < machine_module < inventory
|
||||||
|
---------------------------------------------
|
||||||
|
nixos < borgbackup < borgbackup-static > UI
|
||||||
|
|
||||||
|
creates the config Maps from high level services to the borgbackup clan module
|
||||||
|
for ONE machine
|
||||||
|
```
|
||||||
|
|
||||||
|
- [ ] Why do we need 2 modules?
|
||||||
|
-> It is technically possible to have only 1 module.
|
||||||
|
Pros:
|
||||||
|
Simple to use/Easy to understand.
|
||||||
|
Less modules
|
||||||
|
Cons:
|
||||||
|
Harder to write a module. Because it must do 2 things.
|
||||||
|
One module should do only 1 thing.
|
||||||
|
|
||||||
|
```nix
|
||||||
|
clan.machines.${machine_name} = {
|
||||||
|
# "borgbackup.ssh.pub" = machineDir + machines + "/facts/borgbackup.ssh.pub";
|
||||||
|
facts = ...
|
||||||
|
};
|
||||||
|
clan.services.${instance} = {
|
||||||
|
# roles.server = [ "jon_machine" ]
|
||||||
|
# roles.${role_name} = [ ${machine_name} ];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
This part provides a specification for the inventory.
|
This part provides a specification for the inventory.
|
||||||
|
|
||||||
It is used for design phase and as validation helper.
|
It is used for design phase and as validation helper.
|
||||||
|
@ -1,16 +1,28 @@
|
|||||||
{ inputs, self, ... }:
|
{ self, lib, ... }:
|
||||||
let
|
let
|
||||||
clan-core = self;
|
clan-core = self;
|
||||||
system = "x86_64-linux";
|
|
||||||
pkgs = clan-core.inputs.nixpkgs.legacyPackages.${system};
|
|
||||||
|
|
||||||
# syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json);
|
# syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/syncthing.json);
|
||||||
syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json);
|
syncthing_inventory = builtins.fromJSON (builtins.readFile ./src/tests/borgbackup.json);
|
||||||
|
|
||||||
machines = machinesFromInventory {
|
machines = machinesFromInventory syncthing_inventory;
|
||||||
inherit clan-core;
|
|
||||||
lib = pkgs.lib;
|
resolveGroups =
|
||||||
} syncthing_inventory;
|
inventory: members:
|
||||||
|
lib.unique (
|
||||||
|
builtins.foldl' (
|
||||||
|
acc: currMember:
|
||||||
|
let
|
||||||
|
groupName = builtins.substring 6 (builtins.stringLength currMember - 6) currMember;
|
||||||
|
groupMembers =
|
||||||
|
if inventory.groups.machines ? ${groupName} then
|
||||||
|
inventory.groups.machines.${groupName}
|
||||||
|
else
|
||||||
|
throw "Machine group ${currMember} not found. Key: groups.machines.${groupName} not in inventory.";
|
||||||
|
in
|
||||||
|
if lib.hasPrefix "group:" currMember then (acc ++ groupMembers) else acc ++ [ currMember ]
|
||||||
|
) [ ] members
|
||||||
|
);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Returns a NixOS configuration for every machine in the inventory.
|
Returns a NixOS configuration for every machine in the inventory.
|
||||||
@ -18,48 +30,51 @@ let
|
|||||||
machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
|
machinesFromInventory :: Inventory -> { ${machine_name} :: NixOSConfiguration }
|
||||||
*/
|
*/
|
||||||
machinesFromInventory =
|
machinesFromInventory =
|
||||||
{ lib, clan-core, ... }:
|
|
||||||
inventory:
|
inventory:
|
||||||
# For every machine in the inventory, build a NixOS configuration
|
# For every machine in the inventory, build a NixOS configuration
|
||||||
# For each machine generate config, forEach service, if the machine is used.
|
# For each machine generate config, forEach service, if the machine is used.
|
||||||
builtins.mapAttrs (
|
builtins.mapAttrs (
|
||||||
machine_name: _:
|
machineName: _:
|
||||||
builtins.foldl' (
|
lib.foldlAttrs (
|
||||||
acc: service_name:
|
# [ Modules ], String, { ${instance_name} :: ServiceConfig }
|
||||||
let
|
acc: moduleName: serviceConfigs:
|
||||||
service_config = inventory.services.${service_name};
|
|
||||||
isInService = builtins.elem machine_name (builtins.attrNames service_config.machineConfig);
|
|
||||||
|
|
||||||
machine_service_config = (service_config.machineConfig.${machine_name} or { }).config or { };
|
|
||||||
global_config = inventory.services.${service_name}.config;
|
|
||||||
module_name = inventory.services.${service_name}.module;
|
|
||||||
in
|
|
||||||
# Possible roles: "server", "client", "peer"
|
|
||||||
if
|
|
||||||
builtins.trace ''
|
|
||||||
isInService ${builtins.toJSON isInService},
|
|
||||||
${builtins.toJSON machine_name} ${builtins.toJSON (builtins.attrNames service_config.machineConfig)}
|
|
||||||
'' isInService
|
|
||||||
then
|
|
||||||
acc
|
acc
|
||||||
|
# Collect service config
|
||||||
|
++ (lib.foldlAttrs (
|
||||||
|
# [ Modules ], String, ServiceConfig
|
||||||
|
acc2: instanceName: serviceConfig:
|
||||||
|
let
|
||||||
|
resolvedRoles = builtins.mapAttrs (
|
||||||
|
_roleName: members: resolveGroups inventory members
|
||||||
|
) serviceConfig.roles;
|
||||||
|
|
||||||
|
isInService = builtins.any (members: builtins.elem machineName members) (
|
||||||
|
builtins.attrValues resolvedRoles
|
||||||
|
);
|
||||||
|
|
||||||
|
machineServiceConfig = (serviceConfig.machines.${machineName} or { }).config or { };
|
||||||
|
globalConfig = serviceConfig.config;
|
||||||
|
in
|
||||||
|
if isInService then
|
||||||
|
acc2
|
||||||
++ [
|
++ [
|
||||||
{
|
{
|
||||||
imports = [ clan-core.clanModules.${module_name} ];
|
imports = [ clan-core.clanModules.${moduleName} ];
|
||||||
config.clan.${module_name} = lib.mkMerge [
|
config.clan.${moduleName} = lib.mkMerge [
|
||||||
global_config
|
globalConfig
|
||||||
machine_service_config
|
machineServiceConfig
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
config.clan.${module_name} = {
|
config.clan.inventory.${instanceName} = {
|
||||||
# TODO: filter, show only the roles that are needed by the machine
|
roles = resolvedRoles;
|
||||||
roles = builtins.mapAttrs (_m: c: c.roles) service_config.machineConfig;
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
else
|
else
|
||||||
acc
|
acc2
|
||||||
) [ ] (builtins.attrNames inventory.services)
|
) [ ] serviceConfigs)
|
||||||
|
) [ ] inventory.services
|
||||||
) inventory.machines;
|
) inventory.machines;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
@ -81,9 +96,4 @@ in
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
intern = machines;
|
intern = machines;
|
||||||
# inherit (clan) nixosConfigurations clanInternals;
|
|
||||||
# add the Clan cli tool to the dev shell
|
|
||||||
devShells.${system}.default = pkgs.mkShell {
|
|
||||||
packages = [ clan-core.packages.${system}.clan-cli ];
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,11 @@
|
|||||||
{ inputs, self, ... }:
|
|
||||||
{
|
{
|
||||||
flake.inventory = import ./default.nix { inherit inputs self; };
|
inputs,
|
||||||
|
self,
|
||||||
|
lib,
|
||||||
|
...
|
||||||
|
}:
|
||||||
|
{
|
||||||
|
flake.inventory = import ./default.nix { inherit inputs self lib; };
|
||||||
perSystem =
|
perSystem =
|
||||||
{ pkgs, config, ... }:
|
{ pkgs, config, ... }:
|
||||||
{
|
{
|
||||||
|
@ -1,8 +0,0 @@
|
|||||||
package machines
|
|
||||||
|
|
||||||
|
|
||||||
#machine: machines: [string]: {
|
|
||||||
name: string,
|
|
||||||
description?: string,
|
|
||||||
icon?: string
|
|
||||||
}
|
|
@ -1,8 +1,7 @@
|
|||||||
package inventory
|
package inventory
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"clan.lol/inventory/services"
|
"clan.lol/inventory/schema"
|
||||||
"clan.lol/inventory/machines"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@jsonschema(schema="http://json-schema.org/schema#")
|
@jsonschema(schema="http://json-schema.org/schema#")
|
||||||
@ -16,9 +15,11 @@ import (
|
|||||||
icon?: string
|
icon?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
// A map of services
|
// // A map of services
|
||||||
services.#service
|
schema.#service
|
||||||
|
|
||||||
// A map of machines
|
// // A map of machines
|
||||||
machines.#machine
|
schema.#machine
|
||||||
|
|
||||||
|
schema.#groups
|
||||||
}
|
}
|
||||||
|
@ -1,22 +1,34 @@
|
|||||||
package services
|
package schema
|
||||||
|
|
||||||
#ServiceRole: "server" | "client" | "both"
|
#groups: groups: {
|
||||||
|
// Machine groups
|
||||||
|
machines: {
|
||||||
|
// Group name mapped to list[machineName]
|
||||||
|
// "group1": ["machine1", "machine2"]
|
||||||
|
[string]: [...string]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#service: services: [string]: {
|
#machine: machines: [string]: {
|
||||||
|
name: string,
|
||||||
|
description?: string,
|
||||||
|
icon?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
#role: string
|
||||||
|
|
||||||
|
#service: services: [string]: [string]: {
|
||||||
// Required meta fields
|
// Required meta fields
|
||||||
meta: {
|
meta: {
|
||||||
name: string,
|
name: string,
|
||||||
icon?: string
|
icon?: string
|
||||||
description?: string,
|
description?: string,
|
||||||
},
|
},
|
||||||
// Required module specifies the behavior of the service.
|
|
||||||
module: string,
|
|
||||||
|
|
||||||
// We moved the machine sepcific config to "machines".
|
// We moved the machine sepcific config to "machines".
|
||||||
// It may be moved back depending on what makes more sense in the future.
|
// It may be moved back depending on what makes more sense in the future.
|
||||||
machineConfig: {
|
roles: [#role]: [...string],
|
||||||
|
machines: {
|
||||||
[string]: {
|
[string]: {
|
||||||
roles?: [ ...#ServiceRole ],
|
|
||||||
config?: {
|
config?: {
|
||||||
...
|
...
|
||||||
}
|
}
|
@ -10,27 +10,32 @@
|
|||||||
"name": "vi"
|
"name": "vi"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"groups": {
|
||||||
|
"machines": {
|
||||||
|
"laptops": ["camina_machine", "vi_machine"],
|
||||||
|
"all": ["camina_machine", "vi_machine", "vyr_machine"]
|
||||||
|
}
|
||||||
|
},
|
||||||
"meta": {
|
"meta": {
|
||||||
"name": "kenjis clan"
|
"name": "kenjis clan"
|
||||||
},
|
},
|
||||||
"services": {
|
"services": {
|
||||||
"backup": {
|
"borgbackup-static": {
|
||||||
|
"instance_1": {
|
||||||
"meta": {
|
"meta": {
|
||||||
"name": "My backup"
|
"name": "My backup"
|
||||||
},
|
},
|
||||||
"module": "borgbackup-static",
|
"roles": {
|
||||||
"machineConfig": {
|
"server": ["vyr_machine"],
|
||||||
"vyr_machine": {
|
"client": ["group:laptops"]
|
||||||
"roles": ["server"]
|
|
||||||
},
|
},
|
||||||
"vi_machine": {
|
"machines": {
|
||||||
"roles": ["client"]
|
"vyr_machine": {},
|
||||||
},
|
"vi_machine": {},
|
||||||
"camina_machine": {
|
"camina_machine": {}
|
||||||
"roles": ["client"]
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"config": {}
|
"config": {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
@ -14,16 +14,15 @@
|
|||||||
"name": "kenjis clan"
|
"name": "kenjis clan"
|
||||||
},
|
},
|
||||||
"services": {
|
"services": {
|
||||||
"sync_files": {
|
"syncthing-static-peers": {
|
||||||
|
"instance_1": {
|
||||||
"meta": {
|
"meta": {
|
||||||
"name": "My sync"
|
"name": "My sync"
|
||||||
},
|
},
|
||||||
"module": "syncthing-static-peers",
|
"roles": {
|
||||||
"machineConfig": {
|
"peer": ["vyr_machine", "vi_machine", "camina_machine"]
|
||||||
"vyr_machine": {},
|
|
||||||
"vi_machine": {},
|
|
||||||
"camina_machine": {}
|
|
||||||
},
|
},
|
||||||
|
"machines": {},
|
||||||
"config": {
|
"config": {
|
||||||
"folders": {
|
"folders": {
|
||||||
"test": {
|
"test": {
|
||||||
@ -43,3 +42,4 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
@ -14,23 +14,22 @@
|
|||||||
"name": "kenjis clan"
|
"name": "kenjis clan"
|
||||||
},
|
},
|
||||||
"services": {
|
"services": {
|
||||||
"backup": {
|
"zerotier-static": {
|
||||||
|
"instance_1": {
|
||||||
"meta": {
|
"meta": {
|
||||||
"name": "My backup"
|
"name": "My Network"
|
||||||
},
|
},
|
||||||
"module": "borbackup-static",
|
"roles": {
|
||||||
"machineConfig": {
|
"server": ["vyr_machine"],
|
||||||
|
"peer": ["vi_machine", "camina_machine"]
|
||||||
|
},
|
||||||
|
"machines": {
|
||||||
"vyr_machine": {
|
"vyr_machine": {
|
||||||
"roles": ["server"]
|
"config": {}
|
||||||
},
|
|
||||||
"vi_machine": {
|
|
||||||
"roles": ["peer"]
|
|
||||||
},
|
|
||||||
"camina_machine": {
|
|
||||||
"roles": ["peer"]
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"config": {}
|
"config": {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user