Inventory: generate exact schema for validation & documentation

This commit is contained in:
Johannes Kirschbauer 2024-07-14 13:30:02 +02:00
parent 3034b9ef92
commit e54101165f
Signed by: hsjobeki
SSH Key Fingerprint: SHA256:vX3utDqig7Ph5L0JPv87ZTPb/w7cMzREKVZzzLFg9qU
8 changed files with 209 additions and 205 deletions

View File

@ -32,6 +32,9 @@ let
# - Each service role must reference a valid machine after all machines are merged
mergedInventory =
(lib.evalModules {
specialArgs = {
inherit clan-core;
};
modules = [
clan-core.lib.inventory.interface
{ inherit meta; }

View File

@ -15,7 +15,7 @@ let
};
machineRef = lib.mkOptionType {
name = "machineRef";
name = "str";
description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]";
check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines);
merge = lib.mergeEqualOption;
@ -29,20 +29,85 @@ let
);
tagRef = lib.mkOptionType {
name = "tagRef";
name = "str";
description = "Tags :: [${builtins.concatStringsSep " | " allTags}]";
check = v: lib.isString v && builtins.elem v allTags;
merge = lib.mergeEqualOption;
};
moduleConfig = lib.mkOption {
default = { };
type = t.attrsOf t.anything;
};
in
{
options.assertions = lib.mkOption {
type = t.listOf t.unspecified;
internal = true;
default = [ ];
options = {
assertions = lib.mkOption {
type = t.listOf t.unspecified;
internal = true;
visible = false;
default = [ ];
};
meta = metaOptions;
machines = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options = {
inherit (metaOptions) name description icon;
tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf t.str;
};
system = lib.mkOption {
default = null;
type = t.nullOr t.str;
};
};
}
);
};
services = lib.mkOption {
default = { };
type = t.attrsOf (
t.attrsOf (
t.submodule {
options.meta = metaOptions;
options.config = moduleConfig;
options.machines = lib.mkOption {
default = { };
type = t.attrsOf (t.submodule { options.config = moduleConfig; });
};
options.roles = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options.machines = lib.mkOption {
default = [ ];
type = t.listOf machineRef;
};
options.tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf tagRef;
};
}
);
};
}
)
);
};
};
# Smoke validation of the inventory
config.assertions =
let
# Inventory assertions
# - All referenced machines must exist in the top-level machines
serviceAssertions = lib.foldlAttrs (
ass1: serviceName: c:
ass1
@ -60,8 +125,11 @@ in
ass2 ++ assertions
) [ ] c
) [ ] config.services;
# Machine assertions
# - A machine must define their host system
machineAssertions = map (
{ name, value }:
{ name }:
{
assertion = true;
message = "Machine ${name} should define its host system in the inventory. ()";
@ -69,68 +137,4 @@ in
) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines));
in
machineAssertions ++ serviceAssertions;
options.meta = metaOptions;
options.machines = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options = {
inherit (metaOptions) name description icon;
tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf t.str;
};
system = lib.mkOption {
default = null;
type = t.nullOr t.str;
};
};
}
);
};
options.services = lib.mkOption {
default = { };
type = t.attrsOf (
t.attrsOf (
t.submodule {
options.meta = metaOptions;
options.config = lib.mkOption {
default = { };
type = t.anything;
};
options.machines = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options.config = lib.mkOption {
default = { };
type = t.anything;
};
}
);
};
options.roles = lib.mkOption {
default = { };
type = t.attrsOf (
t.submodule {
options.machines = lib.mkOption {
default = [ ];
type = t.listOf machineRef;
};
options.tags = lib.mkOption {
default = [ ];
apply = lib.unique;
type = t.listOf tagRef;
};
}
);
};
}
)
);
};
}

View File

@ -1,47 +0,0 @@
{
"machines": {
"camina_machine": {
"name": "camina"
},
"vyr_machine": {
"name": "vyr"
},
"vi_machine": {
"name": "vi"
}
},
"meta": {
"name": "kenjis clan"
},
"services": {
"syncthing": {
"instance_1": {
"meta": {
"name": "My sync"
},
"roles": {
"peer": {
"machines": ["vyr_machine", "vi_machine", "camina_machine"]
}
},
"machines": {},
"config": {
"folders": {
"test": {
"path": "~/data/docs",
"devices": ["camina_machine", "vyr_machine", "vi_machine"]
},
"videos": {
"path": "~/data/videos",
"devices": ["camina_machine", "vyr_machine"]
},
"playlist": {
"path": "~/data/playlist",
"devices": ["camina_machine", "vi_machine"]
}
}
}
}
}
}
}

View File

@ -1,36 +0,0 @@
{
"machines": {
"camina_machine": {
"name": "camina"
},
"vyr_machine": {
"name": "vyr"
},
"vi_machine": {
"name": "vi"
}
},
"meta": {
"name": "kenjis clan"
},
"services": {
"zerotier": {
"instance_1": {
"meta": {
"name": "My Network"
},
"roles": {
"controller": { "machines": ["vyr_machine"] },
"moon": { "machines": ["vyr_machine"] },
"peer": { "machines": ["vi_machine", "camina_machine"] }
},
"machines": {
"vyr_machine": {
"config": {}
}
},
"config": {}
}
}
}
}

View File

@ -21,16 +21,113 @@ in
clan-core = self;
inherit lib;
};
optionsFromModule =
mName:
let
eval = self.lib.evalClanModules [ mName ];
in
if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
modulesSchema = lib.mapAttrs (
moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { }
) self.clanModules;
jsonLib = self.lib.jsonschema {
# includeDefaults = false;
};
jsonLib' = self.lib.jsonschema {
# includeDefaults = false;
header = { };
};
inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix);
getRoles =
modulePath:
let
rolesDir = "${modulePath}/roles";
in
if builtins.pathExists rolesDir then
lib.pipe rolesDir [
builtins.readDir
(lib.filterAttrs (_n: v: v == "regular"))
lib.attrNames
(map (fileName: lib.removeSuffix ".nix" fileName))
]
else
null;
schema = inventorySchema // {
properties = inventorySchema.properties // {
services = {
type = "object";
additionalProperties = false;
properties = lib.mapAttrs (moduleName: moduleSchema: {
type = "object";
additionalProperties = {
type = "object";
additionalProperties = false;
properties = {
meta =
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta;
config = moduleSchema;
roles = {
type = "object";
additionalProperties = false;
required = [ ];
properties = lib.listToAttrs (
map
(role: {
name = role;
value =
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties;
})
(
let
roles = getRoles self.clanModules.${moduleName};
in
if roles == null then [ ] else roles
)
);
};
machines =
lib.recursiveUpdate
inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines
{ additionalProperties.properties.config = moduleSchema; };
};
};
}) modulesSchema;
};
};
};
in
{
legacyPackages.inventorySchema = schema;
devShells.inventory-schema = pkgs.mkShell {
inputsFrom = with config.checks; [
lib-inventory-examples-cue
lib-inventory-schema
lib-inventory-eval
self'.devShells.default
];
};
# Inventory schema with concrete module implementations
packages.inventory-schema = pkgs.stdenv.mkDerivation {
name = "inventory-schema";
buildInputs = [ pkgs.cue ];
src = ./.;
buildPhase = ''
export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchema)}
cp $SCHEMA schema.json
cue import -f -p compose -l '#Root:' schema.json
mkdir $out
cp schema.cue $out
cp schema.json $out
'';
};
# Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests
legacyPackages.evalTests-inventory = import ./tests {
inherit buildInventory;
@ -38,32 +135,21 @@ in
};
checks = {
lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
export HOME="$(realpath .)"
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-inventory
touch $out
'';
lib-inventory-schema = pkgs.stdenv.mkDerivation {
lib-inventory-examples-cue = pkgs.stdenv.mkDerivation {
name = "inventory-schema-checks";
src = ./.;
buildInputs = [ pkgs.cue ];
buildPhase = ''
echo "Running inventory tests..."
# Cue is easier to run in the same directory as the schema
cd spec
cp ${self'.packages.inventory-schema}/schema.cue root.cue
echo "Export cue as json-schema..."
cue export --out openapi root.cue
ls -la .
echo "Validate test/*.json against inventory-schema..."
cat root.cue
test_dir="../examples"
test_dir="./examples"
for file in "$test_dir"/*; do
# Check if the item is a file
if [ -f "$file" ]; then
@ -78,6 +164,16 @@ in
touch $out
'';
};
lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } ''
export HOME="$(realpath .)"
nix-unit --eval-store "$HOME" \
--extra-experimental-features flakes \
${inputOverrides} \
--flake ${self}#legacyPackages.${system}.evalTests-inventory
touch $out
'';
};
};
}

View File

@ -1,9 +1,16 @@
{
lib ? import <nixpkgs/lib>,
}:
{
excludedTypes ? [
"functionTo"
"package"
],
includeDefaults ? true,
header ? {
"$schema" = "http://json-schema.org/draft-07/schema#";
},
specialArgs ? { },
}:
let
# remove _module attribute from options
@ -40,12 +47,14 @@ let
];
in
rec {
# parses a nixos module to a jsonschema
parseModule =
module:
let
evaled = lib.evalModules { modules = [ module ]; };
evaled = lib.evalModules {
modules = [ module ];
inherit specialArgs;
};
in
parseOptions evaled.options { };
@ -57,9 +66,6 @@ rec {
{
# The top-level header object should specify at least the schema version
# Can be customized if needed
header ? {
"$schema" = "http://json-schema.org/draft-07/schema#";
},
# By default the header is not added to the schema
addHeader ? true,
}:
@ -86,7 +92,7 @@ rec {
parseOption =
option:
let
default = lib.optionalAttrs (option ? default) { inherit (option) default; };
default = lib.optionalAttrs (option ? default && includeDefaults) { inherit (option) default; };
example = lib.optionalAttrs (option ? example) {
examples =
if (builtins.typeOf option.example) == "list" then option.example else [ option.example ];
@ -95,7 +101,6 @@ rec {
description = option.description.text or option.description;
};
in
# either type
# TODO: if all nested options are excluded, the parent should be excluded too
if
@ -117,16 +122,13 @@ rec {
];
optionsList = filterExcluded optionsList';
in
default // example // description // { anyOf = map parseOption optionsList; }
default // example // description // { oneOf = map parseOption optionsList; }
# handle nested options (not a submodule)
else if !option ? _type then
parseOptions' option
# throw if not an option
else if option._type != "option" && option._type != "option-type" then
throw "parseOption: not an option"
# parse nullOr
else if
option.type.name == "nullOr"
@ -143,32 +145,28 @@ rec {
// example
// description
// {
anyOf = [
oneOf = [
{ type = "null"; }
] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption));
}
# parse bool
else if
option.type.name == "bool"
# return jsonschema property definition for bool
then
default // example // description // { type = "boolean"; }
# parse float
else if
option.type.name == "float"
# return jsonschema property definition for float
then
default // example // description // { type = "number"; }
# parse int
else if
(option.type.name == "int" || option.type.name == "positiveInt")
# return jsonschema property definition for int
then
default // example // description // { type = "integer"; }
# TODO: Add support for intMatching in jsonschema
# parse port type aka. "unsignedInt16"
else if
@ -178,7 +176,6 @@ rec {
|| option.type.name == "intBetween"
then
default // example // description // { type = "integer"; }
# parse string
# TODO: parse more precise string types
else if
@ -189,51 +186,43 @@ rec {
# return jsonschema property definition for string
then
default // example // description // { type = "string"; }
# TODO: Add support for stringMatching in jsonschema
# parse stringMatching
else if lib.strings.hasPrefix "strMatching" option.type.name then
default // example // description // { type = "string"; }
# TODO: Add support for separatedString in jsonschema
else if lib.strings.hasPrefix "separatedString" option.type.name then
default // example // description // { type = "string"; }
# parse string
else if
option.type.name == "path"
# return jsonschema property definition for path
then
default // example // description // { type = "string"; }
# parse anything
else if
option.type.name == "anything"
# return jsonschema property definition for anything
then
default // example // description // { type = allBasicTypes; }
# parse unspecified
else if
option.type.name == "unspecified"
# return jsonschema property definition for unspecified
then
default // example // description // { type = allBasicTypes; }
# parse raw
else if
option.type.name == "raw"
# return jsonschema property definition for raw
then
default // example // description // { type = allBasicTypes; }
# parse enum
else if
option.type.name == "enum"
# return jsonschema property definition for enum
then
default // example // description // { enum = option.type.functor.payload; }
# parse listOf submodule
else if
option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule"
@ -246,7 +235,6 @@ rec {
type = "array";
items = parseOptions' (option.type.functor.wrapped.getSubOptions option.loc);
}
# parse list
else if
(option.type.name == "listOf")
@ -266,14 +254,12 @@ rec {
type = "array";
}
// (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; })
# parse list of unspecified
else if
(option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified")
# return jsonschema property definition for list
then
default // example // description // { type = "array"; }
# parse attrsOf submodule
else if
option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule"
@ -286,7 +272,6 @@ rec {
type = "object";
additionalProperties = parseOptions' (option.type.nestedTypes.elemType.getSubOptions option.loc);
}
# parse attrs
else if
option.type.name == "attrs"
@ -299,7 +284,6 @@ rec {
type = "object";
additionalProperties = true;
}
# parse attrsOf
# TODO: if nested option is excluded, the parent sould be excluded too
else if
@ -328,7 +312,6 @@ rec {
else
false;
}
# parse submodule
else if
option.type.name == "submodule"
@ -336,7 +319,6 @@ rec {
# then (lib.attrNames (option.type.getSubOptions option.loc).opt)
then
parseOptions' (option.type.getSubOptions option.loc)
# throw error if option type is not supported
else
notSupported option;

View File

@ -1,6 +1,6 @@
{ options, lib, ... }:
let
jsonschema = import ../../lib/jsonschema { inherit lib; };
jsonschema = import ../../lib/jsonschema { inherit lib; } { };
in
{
options.clanSchema = lib.mkOption {

View File

@ -5,6 +5,8 @@
let
clanModules = self.clanModules;
jsonLib = self.lib.jsonschema { };
# Uncomment if you only want one module to be available
# clanModules = {
# borgbackup = self.clanModules.borgbackup;
@ -18,13 +20,13 @@
if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { };
clanModuleSchemas = lib.mapAttrs (
modulename: _: self.lib.jsonschema.parseOptions (optionsFromModule modulename) { }
modulename: _: jsonLib.parseOptions (optionsFromModule modulename) { }
) clanModules;
clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: {
name = modulename;
description = self.lib.modules.getShortDescription modulename;
parameters = self.lib.jsonschema.parseOptions (optionsFromModule modulename) { };
parameters = jsonLib.parseOptions (optionsFromModule modulename) { };
}) clanModules;
in
rec {