diff --git a/lib/inventory/build-inventory/interface.nix b/lib/inventory/build-inventory/interface.nix index 88663ce9..89acedb3 100644 --- a/lib/inventory/build-inventory/interface.nix +++ b/lib/inventory/build-inventory/interface.nix @@ -15,7 +15,7 @@ let }; machineRef = lib.mkOptionType { - name = "machineRef"; + name = "str"; description = "Machine :: [${builtins.concatStringsSep " | " (builtins.attrNames config.machines)}]"; check = v: lib.isString v && builtins.elem v (builtins.attrNames config.machines); merge = lib.mergeEqualOption; @@ -29,20 +29,85 @@ let ); tagRef = lib.mkOptionType { - name = "tagRef"; + name = "str"; description = "Tags :: [${builtins.concatStringsSep " | " allTags}]"; check = v: lib.isString v && builtins.elem v allTags; merge = lib.mergeEqualOption; }; + + moduleConfig = lib.mkOption { + default = { }; + type = t.attrsOf t.anything; + }; in { - options.assertions = lib.mkOption { - type = t.listOf t.unspecified; - internal = true; - default = [ ]; + options = { + assertions = lib.mkOption { + type = t.listOf t.unspecified; + internal = true; + visible = false; + default = [ ]; + }; + meta = metaOptions; + + machines = lib.mkOption { + default = { }; + type = t.attrsOf ( + t.submodule { + options = { + inherit (metaOptions) name description icon; + tags = lib.mkOption { + default = [ ]; + apply = lib.unique; + type = t.listOf t.str; + }; + system = lib.mkOption { + default = null; + type = t.nullOr t.str; + }; + }; + } + ); + }; + + services = lib.mkOption { + default = { }; + type = t.attrsOf ( + t.attrsOf ( + t.submodule { + options.meta = metaOptions; + options.config = moduleConfig; + options.machines = lib.mkOption { + default = { }; + type = t.attrsOf (t.submodule { options.config = moduleConfig; }); + }; + options.roles = lib.mkOption { + default = { }; + type = t.attrsOf ( + t.submodule { + options.machines = lib.mkOption { + default = [ ]; + type = t.listOf machineRef; + }; + options.tags = lib.mkOption { + default = [ ]; + apply = lib.unique; + type = t.listOf tagRef; + }; + } + ); + }; + } + ) + ); + }; }; + + # Smoke validation of the inventory config.assertions = let + # Inventory assertions + # - All referenced machines must exist in the top-level machines serviceAssertions = lib.foldlAttrs ( ass1: serviceName: c: ass1 @@ -60,8 +125,11 @@ in ass2 ++ assertions ) [ ] c ) [ ] config.services; + + # Machine assertions + # - A machine must define their host system machineAssertions = map ( - { name, value }: + { name, ... }: { assertion = true; message = "Machine ${name} should define its host system in the inventory. ()"; @@ -69,68 +137,4 @@ in ) (lib.attrsToList (lib.filterAttrs (_n: v: v.system or null == null) config.machines)); in machineAssertions ++ serviceAssertions; - - options.meta = metaOptions; - - options.machines = lib.mkOption { - default = { }; - type = t.attrsOf ( - t.submodule { - options = { - inherit (metaOptions) name description icon; - tags = lib.mkOption { - default = [ ]; - apply = lib.unique; - type = t.listOf t.str; - }; - system = lib.mkOption { - default = null; - type = t.nullOr t.str; - }; - }; - } - ); - }; - - options.services = lib.mkOption { - default = { }; - type = t.attrsOf ( - t.attrsOf ( - t.submodule { - options.meta = metaOptions; - options.config = lib.mkOption { - default = { }; - type = t.anything; - }; - options.machines = lib.mkOption { - default = { }; - type = t.attrsOf ( - t.submodule { - options.config = lib.mkOption { - default = { }; - type = t.anything; - }; - } - ); - }; - options.roles = lib.mkOption { - default = { }; - type = t.attrsOf ( - t.submodule { - options.machines = lib.mkOption { - default = [ ]; - type = t.listOf machineRef; - }; - options.tags = lib.mkOption { - default = [ ]; - apply = lib.unique; - type = t.listOf tagRef; - }; - } - ); - }; - } - ) - ); - }; } diff --git a/lib/inventory/examples/syncthing.json b/lib/inventory/examples/syncthing.json deleted file mode 100644 index e7ed8af6..00000000 --- a/lib/inventory/examples/syncthing.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "machines": { - "camina_machine": { - "name": "camina" - }, - "vyr_machine": { - "name": "vyr" - }, - "vi_machine": { - "name": "vi" - } - }, - "meta": { - "name": "kenjis clan" - }, - "services": { - "syncthing": { - "instance_1": { - "meta": { - "name": "My sync" - }, - "roles": { - "peer": { - "machines": ["vyr_machine", "vi_machine", "camina_machine"] - } - }, - "machines": {}, - "config": { - "folders": { - "test": { - "path": "~/data/docs", - "devices": ["camina_machine", "vyr_machine", "vi_machine"] - }, - "videos": { - "path": "~/data/videos", - "devices": ["camina_machine", "vyr_machine"] - }, - "playlist": { - "path": "~/data/playlist", - "devices": ["camina_machine", "vi_machine"] - } - } - } - } - } - } -} diff --git a/lib/inventory/examples/zerotier.json b/lib/inventory/examples/zerotier.json deleted file mode 100644 index f35e2ab4..00000000 --- a/lib/inventory/examples/zerotier.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "machines": { - "camina_machine": { - "name": "camina" - }, - "vyr_machine": { - "name": "vyr" - }, - "vi_machine": { - "name": "vi" - } - }, - "meta": { - "name": "kenjis clan" - }, - "services": { - "zerotier": { - "instance_1": { - "meta": { - "name": "My Network" - }, - "roles": { - "controller": { "machines": ["vyr_machine"] }, - "moon": { "machines": ["vyr_machine"] }, - "peer": { "machines": ["vi_machine", "camina_machine"] } - }, - "machines": { - "vyr_machine": { - "config": {} - } - }, - "config": {} - } - } - } -} diff --git a/lib/inventory/flake-module.nix b/lib/inventory/flake-module.nix index a723c171..fec26ce0 100644 --- a/lib/inventory/flake-module.nix +++ b/lib/inventory/flake-module.nix @@ -21,16 +21,112 @@ in clan-core = self; inherit lib; }; + + optionsFromModule = + mName: + let + eval = self.lib.evalClanModules [ mName ]; + in + if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { }; + + modulesSchema = lib.mapAttrs ( + moduleName: _: jsonLib'.parseOptions (optionsFromModule moduleName) { } + ) self.clanModules; + + jsonLib = self.lib.jsonschema { + # includeDefaults = false; + }; + jsonLib' = self.lib.jsonschema { + # includeDefaults = false; + header = { }; + }; + inventorySchema = jsonLib.parseModule (import ./build-inventory/interface.nix); + + getRoles = + modulePath: + let + rolesDir = "${modulePath}/roles"; + in + if builtins.pathExists rolesDir then + lib.pipe rolesDir [ + builtins.readDir + (lib.filterAttrs (_n: v: v == "regular")) + lib.attrNames + (map (fileName: lib.removeSuffix ".nix" fileName)) + ] + else + null; + + schema = inventorySchema // { + properties = inventorySchema.properties // { + services = { + type = "object"; + additionalProperties = false; + properties = lib.mapAttrs (moduleName: moduleSchema: { + type = "object"; + additionalProperties = { + type = "object"; + additionalProperties = false; + properties = { + meta = + inventorySchema.properties.services.additionalProperties.additionalProperties.properties.meta; + config = moduleSchema; + roles = { + type = "object"; + additionalProperties = false; + required = [ ]; + properties = lib.listToAttrs ( + map + (role: { + name = role; + value = + inventorySchema.properties.services.additionalProperties.additionalProperties.properties.roles.additionalProperties; + }) + ( + let + roles = getRoles self.clanModules.${moduleName}; + in + if roles == null then [ ] else roles + ) + ); + }; + machines = + lib.recursiveUpdate + inventorySchema.properties.services.additionalProperties.additionalProperties.properties.machines + { additionalProperties.properties.config = moduleSchema; }; + }; + }; + }) modulesSchema; + }; + }; + }; in { + legacyPackages.inventorySchema = schema; + devShells.inventory-schema = pkgs.mkShell { inputsFrom = with config.checks; [ - lib-inventory-schema + lib-inventory-examples-cue lib-inventory-eval self'.devShells.default ]; }; + # Inventory schema with concrete module implementations + packages.inventory-schema = pkgs.stdenv.mkDerivation { + name = "inventory-schema"; + buildInputs = [ pkgs.cue ]; + src = ./.; + buildPhase = '' + export SCHEMA=${builtins.toFile "inventory-schema.json" (builtins.toJSON self'.legacyPackages.inventorySchema)} + cp $SCHEMA schema.json + cue import -f -p compose -l '#Root:' schema.json + mkdir $out + cp schema.cue $out + cp schema.json $out + ''; + }; + # Run: nix-unit --extra-experimental-features flakes --flake .#legacyPackages.x86_64-linux.evalTests legacyPackages.evalTests-inventory = import ./tests { inherit buildInventory; @@ -38,32 +134,21 @@ in }; checks = { - lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' - export HOME="$(realpath .)" - - nix-unit --eval-store "$HOME" \ - --extra-experimental-features flakes \ - ${inputOverrides} \ - --flake ${self}#legacyPackages.${system}.evalTests-inventory - - touch $out - ''; - - lib-inventory-schema = pkgs.stdenv.mkDerivation { + lib-inventory-examples-cue = pkgs.stdenv.mkDerivation { name = "inventory-schema-checks"; src = ./.; buildInputs = [ pkgs.cue ]; buildPhase = '' echo "Running inventory tests..." # Cue is easier to run in the same directory as the schema - cd spec + cp ${self'.packages.inventory-schema}/schema.cue root.cue - echo "Export cue as json-schema..." - cue export --out openapi root.cue + ls -la . echo "Validate test/*.json against inventory-schema..." + cat root.cue - test_dir="../examples" + test_dir="./examples" for file in "$test_dir"/*; do # Check if the item is a file if [ -f "$file" ]; then @@ -78,6 +163,16 @@ in touch $out ''; }; + lib-inventory-eval = pkgs.runCommand "tests" { nativeBuildInputs = [ pkgs.nix-unit ]; } '' + export HOME="$(realpath .)" + + nix-unit --eval-store "$HOME" \ + --extra-experimental-features flakes \ + ${inputOverrides} \ + --flake ${self}#legacyPackages.${system}.evalTests-inventory + + touch $out + ''; }; }; } diff --git a/lib/inventory/spec/cue.mod/module.cue b/lib/inventory/spec/cue.mod/module.cue deleted file mode 100644 index 773cae9d..00000000 --- a/lib/inventory/spec/cue.mod/module.cue +++ /dev/null @@ -1,2 +0,0 @@ -module: "clan.lol/inventory" -language: version: "v0.8.2" \ No newline at end of file diff --git a/lib/inventory/spec/root.cue b/lib/inventory/spec/root.cue deleted file mode 100644 index da64c1b2..00000000 --- a/lib/inventory/spec/root.cue +++ /dev/null @@ -1,23 +0,0 @@ -package inventory - -import ( - "clan.lol/inventory/schema" -) - -@jsonschema(schema="http://json-schema.org/schema#") -#Root: { - meta: { - // A name of the clan (primarily shown by the UI) - name: string - // A description of the clan - description?: string - // The icon path - icon?: string - } - - // // A map of services - schema.#service - - // // A map of machines - schema.#machine -} diff --git a/lib/inventory/spec/schema/schema.cue b/lib/inventory/spec/schema/schema.cue deleted file mode 100644 index eded84e2..00000000 --- a/lib/inventory/spec/schema/schema.cue +++ /dev/null @@ -1,40 +0,0 @@ -package schema - -#machine: machines: [string]: { - name: string, - description?: string, - icon?: string - tags: [...string] - system?: string -} - -#role: string - -#service: services: [string]: [string]: { - // Required meta fields - meta: { - name: string, - icon?: string - description?: string, - }, - // We moved the machine sepcific config to "machines". - // It may be moved back depending on what makes more sense in the future. - roles: [#role]: { - machines: [...string], - tags: [...string], - } - machines?: { - [string]: { - config?: { - ... - } - } - }, - - // Global Configuration for the service - config?: { - // Schema depends on the module. - // It declares the interface how the service can be configured. - ... - } -} \ No newline at end of file diff --git a/lib/jsonschema/default.nix b/lib/jsonschema/default.nix index 0a92da9b..631ef016 100644 --- a/lib/jsonschema/default.nix +++ b/lib/jsonschema/default.nix @@ -1,9 +1,16 @@ { lib ? import , +}: +{ excludedTypes ? [ "functionTo" "package" ], + includeDefaults ? true, + header ? { + "$schema" = "http://json-schema.org/draft-07/schema#"; + }, + specialArgs ? { }, }: let # remove _module attribute from options @@ -40,29 +47,41 @@ let ]; in rec { - # parses a nixos module to a jsonschema parseModule = module: let - evaled = lib.evalModules { modules = [ module ]; }; + evaled = lib.evalModules { + modules = [ module ]; + inherit specialArgs; + }; in - { "$schema" = "http://json-schema.org/draft-07/schema#"; } // parseOptions evaled.options; + parseOptions evaled.options { }; + + parseOptions' = lib.flip parseOptions { addHeader = false; }; # parses a set of evaluated nixos options to a jsonschema parseOptions = - options': + options: + { + # The top-level header object should specify at least the schema version + # Can be customized if needed + # By default the header is not added to the schema + addHeader ? true, + }: let - options = filterInvisibleOpts (filterExcludedAttrs (clean options')); + options' = filterInvisibleOpts (filterExcludedAttrs (clean options)); # parse options to jsonschema properties - properties = lib.mapAttrs (_name: option: parseOption option) options; + properties = lib.mapAttrs (_name: option: parseOption option) options'; # TODO: figure out how to handle if prop.anyOf is used isRequired = prop: !(prop ? default || prop.type or null == "object"); requiredProps = lib.filterAttrs (_: prop: isRequired prop) properties; required = lib.optionalAttrs (requiredProps != { }) { required = lib.attrNames requiredProps; }; + header' = if addHeader then header else { }; in # return jsonschema - required + header' + // required // { type = "object"; inherit properties; @@ -73,7 +92,7 @@ rec { parseOption = option: let - default = lib.optionalAttrs (option ? default) { inherit (option) default; }; + default = lib.optionalAttrs (option ? default && includeDefaults) { inherit (option) default; }; example = lib.optionalAttrs (option ? example) { examples = if (builtins.typeOf option.example) == "list" then option.example else [ option.example ]; @@ -82,7 +101,6 @@ rec { description = option.description.text or option.description; }; in - # either type # TODO: if all nested options are excluded, the parent should be excluded too if @@ -104,16 +122,13 @@ rec { ]; optionsList = filterExcluded optionsList'; in - default // example // description // { anyOf = map parseOption optionsList; } - + default // example // description // { oneOf = map parseOption optionsList; } # handle nested options (not a submodule) else if !option ? _type then - parseOptions option - + parseOptions' option # throw if not an option else if option._type != "option" && option._type != "option-type" then throw "parseOption: not an option" - # parse nullOr else if option.type.name == "nullOr" @@ -130,32 +145,28 @@ rec { // example // description // { - anyOf = [ + oneOf = [ { type = "null"; } ] ++ (lib.optional (!isExcludedOption nestedOption) (parseOption nestedOption)); } - # parse bool else if option.type.name == "bool" # return jsonschema property definition for bool then default // example // description // { type = "boolean"; } - # parse float else if option.type.name == "float" # return jsonschema property definition for float then default // example // description // { type = "number"; } - # parse int else if (option.type.name == "int" || option.type.name == "positiveInt") # return jsonschema property definition for int then default // example // description // { type = "integer"; } - # TODO: Add support for intMatching in jsonschema # parse port type aka. "unsignedInt16" else if @@ -165,7 +176,6 @@ rec { || option.type.name == "intBetween" then default // example // description // { type = "integer"; } - # parse string # TODO: parse more precise string types else if @@ -176,51 +186,43 @@ rec { # return jsonschema property definition for string then default // example // description // { type = "string"; } - # TODO: Add support for stringMatching in jsonschema # parse stringMatching else if lib.strings.hasPrefix "strMatching" option.type.name then default // example // description // { type = "string"; } - # TODO: Add support for separatedString in jsonschema else if lib.strings.hasPrefix "separatedString" option.type.name then default // example // description // { type = "string"; } - # parse string else if option.type.name == "path" # return jsonschema property definition for path then default // example // description // { type = "string"; } - # parse anything else if option.type.name == "anything" # return jsonschema property definition for anything then default // example // description // { type = allBasicTypes; } - # parse unspecified else if option.type.name == "unspecified" # return jsonschema property definition for unspecified then default // example // description // { type = allBasicTypes; } - # parse raw else if option.type.name == "raw" # return jsonschema property definition for raw then default // example // description // { type = allBasicTypes; } - # parse enum else if option.type.name == "enum" # return jsonschema property definition for enum then default // example // description // { enum = option.type.functor.payload; } - # parse listOf submodule else if option.type.name == "listOf" && option.type.functor.wrapped.name == "submodule" @@ -231,9 +233,8 @@ rec { // description // { type = "array"; - items = parseOptions (option.type.functor.wrapped.getSubOptions option.loc); + items = parseOptions' (option.type.functor.wrapped.getSubOptions option.loc); } - # parse list else if (option.type.name == "listOf") @@ -253,14 +254,12 @@ rec { type = "array"; } // (lib.optionalAttrs (!isExcludedOption nestedOption) { items = parseOption nestedOption; }) - # parse list of unspecified else if (option.type.name == "listOf") && (option.type.functor.wrapped.name == "unspecified") # return jsonschema property definition for list then default // example // description // { type = "array"; } - # parse attrsOf submodule else if option.type.name == "attrsOf" && option.type.nestedTypes.elemType.name == "submodule" @@ -271,9 +270,8 @@ rec { // description // { type = "object"; - additionalProperties = parseOptions (option.type.nestedTypes.elemType.getSubOptions option.loc); + additionalProperties = parseOptions' (option.type.nestedTypes.elemType.getSubOptions option.loc); } - # parse attrs else if option.type.name == "attrs" @@ -286,7 +284,6 @@ rec { type = "object"; additionalProperties = true; } - # parse attrsOf # TODO: if nested option is excluded, the parent sould be excluded too else if @@ -315,15 +312,13 @@ rec { else false; } - # parse submodule else if option.type.name == "submodule" # return jsonschema property definition for submodule # then (lib.attrNames (option.type.getSubOptions option.loc).opt) then - parseOptions (option.type.getSubOptions option.loc) - + parseOptions' (option.type.getSubOptions option.loc) # throw error if option type is not supported else notSupported option; diff --git a/lib/jsonschema/test.nix b/lib/jsonschema/test.nix index adf8ab00..56e69d6a 100644 --- a/lib/jsonschema/test.nix +++ b/lib/jsonschema/test.nix @@ -1,7 +1,7 @@ # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib, - slib ? import ./. { inherit lib; }, + slib ? (import ./. { inherit lib; } { }), }: { parseOption = import ./test_parseOption.nix { inherit lib slib; }; diff --git a/lib/jsonschema/test_parseOption.nix b/lib/jsonschema/test_parseOption.nix index 930eee44..1a9004de 100644 --- a/lib/jsonschema/test_parseOption.nix +++ b/lib/jsonschema/test_parseOption.nix @@ -2,7 +2,7 @@ # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib, - slib ? import ./. { inherit lib; }, + slib ? (import ./. { inherit lib; } { }), }: let description = "Test Description"; @@ -236,7 +236,7 @@ in { expr = slib.parseOption (evalType (lib.types.nullOr lib.types.bool) default); expected = { - anyOf = [ + oneOf = [ { type = "null"; } { type = "boolean"; } ]; @@ -251,10 +251,10 @@ in { expr = slib.parseOption (evalType (lib.types.nullOr (lib.types.nullOr lib.types.bool)) default); expected = { - anyOf = [ + oneOf = [ { type = "null"; } { - anyOf = [ + oneOf = [ { type = "null"; } { type = "boolean"; } ]; @@ -386,7 +386,7 @@ in { expr = slib.parseOption (evalType (lib.types.either lib.types.bool lib.types.str) default); expected = { - anyOf = [ + oneOf = [ { type = "boolean"; } { type = "string"; } ]; diff --git a/lib/jsonschema/test_parseOptions.nix b/lib/jsonschema/test_parseOptions.nix index 9467160f..f637999a 100644 --- a/lib/jsonschema/test_parseOptions.nix +++ b/lib/jsonschema/test_parseOptions.nix @@ -2,7 +2,7 @@ # run these tests via `nix-unit ./test.nix` { lib ? (import { }).lib, - slib ? import ./. { inherit lib; }, + slib ? (import ./. { inherit lib; } { }), }: { testParseOptions = { @@ -17,8 +17,9 @@ }; in { - expr = slib.parseOptions evaled.options; + expr = slib.parseOptions evaled.options { }; expected = { + "$schema" = "http://json-schema.org/draft-07/schema#"; additionalProperties = false; properties = { foo = { diff --git a/nixosModules/clanCore/schema.nix b/nixosModules/clanCore/schema.nix index 4b5ad59a..8b3e40be 100644 --- a/nixosModules/clanCore/schema.nix +++ b/nixosModules/clanCore/schema.nix @@ -1,6 +1,6 @@ { options, lib, ... }: let - jsonschema = import ../../lib/jsonschema { inherit lib; }; + jsonschema = import ../../lib/jsonschema { inherit lib; } { }; in { options.clanSchema = lib.mkOption { diff --git a/pkgs/clan-cli/clan_cli/config/schema.py b/pkgs/clan-cli/clan_cli/config/schema.py index fc41f8b2..bebce02c 100644 --- a/pkgs/clan-cli/clan_cli/config/schema.py +++ b/pkgs/clan-cli/clan_cli/config/schema.py @@ -94,8 +94,8 @@ def machine_schema( ++ (map (name: clan-core.clanModules.${{name}}) config.clanImports or []); }}; options = fakeMachine.options{"." + ".".join(option_path) if option_path else ""}; - jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }}; - jsonschema = jsonschemaLib.parseOptions options; + jsonschemaLib = import {Path(__file__).parent / "jsonschema"} {{ inherit lib; }} {{}}; + jsonschema = jsonschemaLib.parseOptions options {{}}; in jsonschema """, diff --git a/pkgs/schemas/flake-module.nix b/pkgs/schemas/flake-module.nix index 59c25d01..8b81a15f 100644 --- a/pkgs/schemas/flake-module.nix +++ b/pkgs/schemas/flake-module.nix @@ -5,6 +5,8 @@ let clanModules = self.clanModules; + jsonLib = self.lib.jsonschema { }; + # Uncomment if you only want one module to be available # clanModules = { # borgbackup = self.clanModules.borgbackup; @@ -18,13 +20,13 @@ if (eval.options.clan ? "${mName}") then eval.options.clan.${mName} else { }; clanModuleSchemas = lib.mapAttrs ( - modulename: _: self.lib.jsonschema.parseOptions (optionsFromModule modulename) + modulename: _: jsonLib.parseOptions (optionsFromModule modulename) { } ) clanModules; clanModuleFunctionSchemas = lib.mapAttrsFlatten (modulename: _: { name = modulename; description = self.lib.modules.getShortDescription modulename; - parameters = self.lib.jsonschema.parseOptions (optionsFromModule modulename); + parameters = jsonLib.parseOptions (optionsFromModule modulename) { }; }) clanModules; in rec {