forked from clan/clan-infra
Compare commits
141 Commits
flake-upda
...
main
Author | SHA1 | Date | |
---|---|---|---|
641de3a69c | |||
02130659bf | |||
baed0e788a | |||
4fccd9ebac | |||
15c073be42 | |||
7580f900d4 | |||
9896565854 | |||
b1ebf8a38f | |||
72cdef29f4 | |||
74ecdd572a | |||
72a9e4cf36 | |||
86bb7c163e | |||
ab5310a96a | |||
bb3f4bcf5b | |||
f14cdada31 | |||
fc62ed3108 | |||
d6bf1ae1a9 | |||
cda3428f62 | |||
92b6881740 | |||
8dc1280552 | |||
0cade52b1b | |||
906bfbbade | |||
333ec475d5 | |||
af2c85e3e2 | |||
39cb8663e8 | |||
3737b2eae5 | |||
821c78c0e0 | |||
41478ddf8c | |||
72e8ebe7a2 | |||
334a6b4644 | |||
05cae050e5 | |||
d4a6ee151a | |||
674720b414 | |||
4bff8ab685 | |||
3f4381d604 | |||
7d1e49806d | |||
|
5fb6689f1e | ||
2866613a88 | |||
9a5f5b1f10 | |||
6015b3bed6 | |||
5fa3aa98b2 | |||
fef5bada95 | |||
04b3a89b0b | |||
02dbc30d63 | |||
f845fa8525 | |||
0a91fe28ed | |||
8228bbcfe2 | |||
a94db37b0b | |||
56e464a83a | |||
e2be04ddba | |||
c93f912ad7 | |||
eca556a5e4 | |||
dc5575bf93 | |||
db27f36daf | |||
|
e0ff1cd916 | ||
10bd57de32 | |||
c36d176982 | |||
ade9c76e6b | |||
b5bf567f1e | |||
e00e337121 | |||
97debd3628 | |||
d404814920 | |||
b83068be43 | |||
1709564fb6 | |||
91ff330472 | |||
711ae663ae | |||
7a3fddf088 | |||
83bcdc0edb | |||
102b25ddc2 | |||
|
6108e3ee2c | ||
83927d980a | |||
85d3dc9613 | |||
039e26c28f | |||
efe12541fe | |||
b99cbbaa4c | |||
813982b2d6 | |||
34a284d191 | |||
670a5c3e3f | |||
f4c8243e9c | |||
aa5e6c7c7c | |||
ad07fec05f | |||
b663547364 | |||
ed2d910832 | |||
8f79b31ac3 | |||
64ddd5e574 | |||
0075e822a0 | |||
7f17bb53ed | |||
2ca09ea549 | |||
ded75bf7ab | |||
6ade243260 | |||
abb431276f | |||
0037d7d218 | |||
6533ed7b6e | |||
07075cc45f | |||
562dcb9133 | |||
6fe30e619b | |||
27f4204a5f | |||
1f93db72bc | |||
fa2ede9f82 | |||
|
4e94427a6b | ||
28c5fb0459 | |||
b2d0830e14 | |||
e0b7df1590 | |||
4393cef4e1 | |||
9f38cf0417 | |||
a17e2fe029 | |||
b700a79e26 | |||
988a38d1a3 | |||
d8be26269b | |||
bf86ecbc38 | |||
07659fa6f3 | |||
18914f9ab1 | |||
7be3d162e6 | |||
|
b9da343e5d | ||
d3e9877bd7 | |||
e7613d611a | |||
3339ebadfa | |||
d3f9ea2cea | |||
5962d1226d | |||
81351953fc | |||
1fbd237a91 | |||
9751dc648e | |||
b2e55511af | |||
6d09907117 | |||
4e182bec1d | |||
ab6c39c77e | |||
ae636eb267 | |||
0b6f47f25d | |||
9ea0945374 | |||
1c6b7b61d5 | |||
1dc9adebf1 | |||
ac170ab190 | |||
6593b52d04 | |||
6977384cb0 | |||
f12e6ac3b9 | |||
1b7c3b44f8 | |||
7e39d50ebe | |||
fb452856d7 | |||
a200ad5f62 | |||
585a48c931 | |||
dc29c0d63a |
@ -1,2 +1,3 @@
|
||||
# shellcheck shell=bash
|
||||
# https://console.hetzner.cloud/projects/2447141/security/tokens
|
||||
export HCLOUD_TOKEN=<your-hetzner-token>
|
||||
export HCLOUD_TOKEN="<your-hetzner-token>"
|
||||
|
28
.sops.yaml
28
.sops.yaml
@ -13,21 +13,21 @@ keys:
|
||||
creation_rules:
|
||||
- path_regex: targets/.*/(terraform.tfstate|secrets.auto.tfvars.sops.json)$
|
||||
key_groups:
|
||||
- age:
|
||||
- *joerg
|
||||
- *lassulus
|
||||
- *dave
|
||||
- age:
|
||||
- *joerg
|
||||
- *lassulus
|
||||
- *dave
|
||||
- path_regex: targets/web01/secrets.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- *joerg
|
||||
- *lassulus
|
||||
- *dave
|
||||
- *web01
|
||||
- age:
|
||||
- *joerg
|
||||
- *lassulus
|
||||
- *dave
|
||||
- *web01
|
||||
- path_regex: targets/web01-new/secrets.yaml$
|
||||
key_groups:
|
||||
- age:
|
||||
- *joerg
|
||||
- *lassulus
|
||||
- *dave
|
||||
- *web01
|
||||
- age:
|
||||
- *joerg
|
||||
- *lassulus
|
||||
- *dave
|
||||
- *web01
|
||||
|
17
LICENSE.md
17
LICENSE.md
@ -1,7 +1,18 @@
|
||||
Copyright 2023 Clan contributers
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
10
README.md
10
README.md
@ -4,8 +4,10 @@ This repository contains nixos modules and terraform code that powers clan.lol.
|
||||
The website and git hosting is currently on [hetzner](https://www.hetzner.com/).
|
||||
|
||||
## Servers
|
||||
|
||||
- web01:
|
||||
- Instance type: [ex101](https://www.hetzner.com/de/dedicated-rootserver/ex101)
|
||||
- Instance type:
|
||||
[ex101](https://www.hetzner.com/de/dedicated-rootserver/ex101)
|
||||
- CPU: Intel Core i9-13900 (24 cores / 32 threads)
|
||||
- RAM: 64GB DDR5
|
||||
- Drives: 2 x 1.92 TB NVME
|
||||
@ -26,5 +28,7 @@ $ ./tf.sh apply
|
||||
|
||||
## To add a new project to CI
|
||||
|
||||
1. Add the 'buildbot-clan' topic to the repository using the "Manage topics" button below the project description
|
||||
2. Go to https://buildbot.clan.lol/#/builders/2 and press "Update projects" after you have logged in.
|
||||
1. Add the 'buildbot-clan' topic to the repository using the "Manage topics"
|
||||
button below the project description
|
||||
2. Go to https://buildbot.clan.lol/#/builders/2 and press "Update projects"
|
||||
after you have logged in.
|
||||
|
@ -1,14 +1,19 @@
|
||||
{
|
||||
perSystem =
|
||||
{ inputs'
|
||||
, pkgs
|
||||
, lib
|
||||
, ...
|
||||
{
|
||||
config,
|
||||
inputs',
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
convert2Tofu = provider: provider.override (prev: {
|
||||
homepage = builtins.replaceStrings [ "registry.terraform.io/providers" ] [ "registry.opentofu.org" ] prev.homepage;
|
||||
});
|
||||
convert2Tofu =
|
||||
provider:
|
||||
provider.override (prev: {
|
||||
homepage = builtins.replaceStrings [ "registry.terraform.io/providers" ] [
|
||||
"registry.opentofu.org"
|
||||
] prev.homepage;
|
||||
});
|
||||
in
|
||||
{
|
||||
devShells.default = pkgs.mkShellNoCC {
|
||||
@ -16,18 +21,25 @@
|
||||
pkgs.bashInteractive
|
||||
pkgs.sops
|
||||
|
||||
pkgs.nixVersions.latest
|
||||
|
||||
inputs'.clan-core.packages.tea-create-pr
|
||||
inputs'.clan-core.packages.merge-after-ci
|
||||
inputs'.clan-core.packages.clan-cli
|
||||
|
||||
(pkgs.opentofu.withPlugins (p: builtins.map convert2Tofu [
|
||||
p.hetznerdns
|
||||
p.hcloud
|
||||
p.null
|
||||
p.external
|
||||
p.local
|
||||
]))
|
||||
];
|
||||
inputsFrom = [
|
||||
inputs'.clan-core.devShells.default
|
||||
# treefmt with config defined in ./flake.nix
|
||||
config.treefmt.build.wrapper
|
||||
|
||||
(pkgs.opentofu.withPlugins (
|
||||
p:
|
||||
builtins.map convert2Tofu [
|
||||
p.hetznerdns
|
||||
p.hcloud
|
||||
p.null
|
||||
p.external
|
||||
p.local
|
||||
]
|
||||
))
|
||||
];
|
||||
};
|
||||
};
|
||||
|
238
flake.lock
238
flake.lock
@ -1,5 +1,21 @@
|
||||
{
|
||||
"nodes": {
|
||||
"blobs": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1604995301,
|
||||
"narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
|
||||
"owner": "simple-nixos-mailserver",
|
||||
"repo": "blobs",
|
||||
"rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
|
||||
"type": "gitlab"
|
||||
},
|
||||
"original": {
|
||||
"owner": "simple-nixos-mailserver",
|
||||
"repo": "blobs",
|
||||
"type": "gitlab"
|
||||
}
|
||||
},
|
||||
"buildbot-nix": {
|
||||
"inputs": {
|
||||
"flake-parts": [
|
||||
@ -13,11 +29,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717897980,
|
||||
"narHash": "sha256-CR85YGXFUaskmVRLa3WbAnD9+PgYle0TGkQMnEshuHQ=",
|
||||
"lastModified": 1725761443,
|
||||
"narHash": "sha256-RX3qnLYaFxlvOAYL6WsM5nGjNnMZQIgKIpIxigPmiAU=",
|
||||
"owner": "Mic92",
|
||||
"repo": "buildbot-nix",
|
||||
"rev": "0d88c6776110ecf6705e9bfe1b777e6be6277da2",
|
||||
"rev": "ade5f42d7e56c8298d729aa0e804c8062e7a77ac",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -32,22 +48,23 @@
|
||||
"flake-parts": [
|
||||
"flake-parts"
|
||||
],
|
||||
"nixos-generators": "nixos-generators",
|
||||
"nixos-facter-modules": "nixos-facter-modules",
|
||||
"nixos-images": "nixos-images",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"sops-nix": "sops-nix",
|
||||
"systems": "systems",
|
||||
"treefmt-nix": [
|
||||
"treefmt-nix"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717937354,
|
||||
"narHash": "sha256-qms0yCxEPvF/Vz0K8g5sBvPJlfXkYEmZuNT+hL7KYIY=",
|
||||
"rev": "1eaf6cec391232a0b1f655fb4bf28380b89f7799",
|
||||
"lastModified": 1726048703,
|
||||
"narHash": "sha256-47AY9tRlFiwfD0tCXNX8435f1UbwlEyJ5t5TMV/iI70=",
|
||||
"rev": "8d27e0412dac90398e37e129312460f644df9aff",
|
||||
"type": "tarball",
|
||||
"url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/1eaf6cec391232a0b1f655fb4bf28380b89f7799.tar.gz"
|
||||
"url": "https://git.clan.lol/api/v1/repos/clan/clan-core/archive/8d27e0412dac90398e37e129312460f644df9aff.tar.gz"
|
||||
},
|
||||
"original": {
|
||||
"type": "tarball",
|
||||
@ -62,11 +79,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717177033,
|
||||
"narHash": "sha256-G3CZJafCO8WDy3dyA2EhpUJEmzd5gMJ2IdItAg0Hijw=",
|
||||
"lastModified": 1725377834,
|
||||
"narHash": "sha256-tqoAO8oT6zEUDXte98cvA1saU9+1dLJQe3pMKLXv8ps=",
|
||||
"owner": "nix-community",
|
||||
"repo": "disko",
|
||||
"rev": "0274af4c92531ebfba4a5bd493251a143bc51f3c",
|
||||
"rev": "e55f9a8678adc02024a4877c2a403e3f6daf24fe",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -75,6 +92,21 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"locked": {
|
||||
"lastModified": 1696426674,
|
||||
"narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "edolstra",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
@ -82,11 +114,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717285511,
|
||||
"narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
|
||||
"lastModified": 1725234343,
|
||||
"narHash": "sha256-+ebgonl3NbiKD2UD0x4BszCZQ6sTfL4xioaM49o5B3Y=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
|
||||
"rev": "567b938d64d4b4112ee253b9274472dc3a346eb6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -95,73 +127,56 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixlib": {
|
||||
"locked": {
|
||||
"lastModified": 1712450863,
|
||||
"narHash": "sha256-K6IkdtMtq9xktmYPj0uaYc8NsIqHuaAoRBaMgu9Fvrw=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixpkgs.lib",
|
||||
"rev": "3c62b6a12571c9a7f65ab037173ee153d539905f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nixpkgs.lib",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-2311": {
|
||||
"locked": {
|
||||
"lastModified": 1717017538,
|
||||
"narHash": "sha256-S5kltvDDfNQM3xx9XcvzKEOyN2qk8Sa+aSOLqZ+1Ujc=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "64e468fd2652105710d86cd2ae3e65a5a6d58dec",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "release-23.11",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-generators": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"nixlib": "nixlib",
|
||||
"nixpkgs": [
|
||||
"clan-core",
|
||||
"nixpkgs"
|
||||
]
|
||||
"systems": "systems_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1716210724,
|
||||
"narHash": "sha256-iqQa3omRcHGpWb1ds75jS9ruA5R39FTmAkeR3J+ve1w=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-generators",
|
||||
"rev": "d14b286322c7f4f897ca4b1726ce38cb68596c94",
|
||||
"lastModified": 1710146030,
|
||||
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-generators",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-facter-modules": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1725379040,
|
||||
"narHash": "sha256-yJIy595vpcdAYznxamszZhteQCIZM3OJUOIKeU4cIq4=",
|
||||
"owner": "numtide",
|
||||
"repo": "nixos-facter-modules",
|
||||
"rev": "a389bf16ed7db1f7e6a5e9dc4b5547e927498803",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "numtide",
|
||||
"repo": "nixos-facter-modules",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixos-images": {
|
||||
"inputs": {
|
||||
"nixos-2311": "nixos-2311",
|
||||
"nixos-stable": [
|
||||
"clan-core"
|
||||
],
|
||||
"nixos-unstable": [
|
||||
"clan-core",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717040312,
|
||||
"narHash": "sha256-yI/en4IxuCEClIUpIs3QTyYCCtmSPLOhwLJclfNwdeg=",
|
||||
"lastModified": 1725726968,
|
||||
"narHash": "sha256-sqkJZDj4NjPFhWCDoSyXJoBRfRfXIyvhfssGqLvzgW8=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-images",
|
||||
"rev": "47bfb55316e105390dd761e0b6e8e0be09462b67",
|
||||
"rev": "51c98a9f328e7aad81d8aa048f38e4e5c86d3389",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -170,27 +185,56 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"nixos-mailserver": {
|
||||
"inputs": {
|
||||
"blobs": "blobs",
|
||||
"flake-compat": [
|
||||
"flake-compat"
|
||||
],
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
],
|
||||
"nixpkgs-24_05": []
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717868076,
|
||||
"narHash": "sha256-c83Y9t815Wa34khrux81j8K8ET94ESmCuwORSKm2bQY=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "cd18e2ae9ab8e2a0a8d715b60c91b54c0ac35ff9",
|
||||
"type": "github"
|
||||
"lastModified": 1722877200,
|
||||
"narHash": "sha256-qgKDNJXs+od+1UbRy62uk7dYal3h98I4WojfIqMoGcg=",
|
||||
"owner": "simple-nixos-mailserver",
|
||||
"repo": "nixos-mailserver",
|
||||
"rev": "af7d3bf5daeba3fc28089b015c0dd43f06b176f2",
|
||||
"type": "gitlab"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
"owner": "simple-nixos-mailserver",
|
||||
"repo": "nixos-mailserver",
|
||||
"type": "gitlab"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1726033636,
|
||||
"narHash": "sha256-U5BSY461QUg9x0fatmPSczjVpszOJTdJWQVmFTxt9LU=",
|
||||
"ref": "nixos-unstable-small",
|
||||
"rev": "62f0186c41a3f1398e3e025f7dc0ccc603482d5a",
|
||||
"shallow": true,
|
||||
"type": "git",
|
||||
"url": "https://github.com/NixOS/nixpkgs"
|
||||
},
|
||||
"original": {
|
||||
"ref": "nixos-unstable-small",
|
||||
"shallow": true,
|
||||
"type": "git",
|
||||
"url": "https://github.com/NixOS/nixpkgs"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"buildbot-nix": "buildbot-nix",
|
||||
"clan-core": "clan-core",
|
||||
"flake-compat": "flake-compat",
|
||||
"flake-parts": "flake-parts",
|
||||
"flake-utils": "flake-utils",
|
||||
"nixos-mailserver": "nixos-mailserver",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"srvos": "srvos",
|
||||
"treefmt-nix": "treefmt-nix"
|
||||
@ -207,11 +251,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717297459,
|
||||
"narHash": "sha256-cZC2f68w5UrJ1f+2NWGV9Gx0dEYmxwomWN2B0lx0QRA=",
|
||||
"lastModified": 1725765163,
|
||||
"narHash": "sha256-rfd2c47iVSFI6bRYy5l8wRijRBaYDeU7dM8XCDUGqlA=",
|
||||
"owner": "Mic92",
|
||||
"repo": "sops-nix",
|
||||
"rev": "ab2a43b0d21d1d37d4d5726a892f714eaeb4b075",
|
||||
"rev": "b68757cd2c3fa66d6ccaa0d046ce42a9324e0070",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -227,11 +271,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717807544,
|
||||
"narHash": "sha256-djHfn29HdlfWdmyeu3rqlVS8k5q/xRh2P0mX2RAafb0=",
|
||||
"lastModified": 1725909399,
|
||||
"narHash": "sha256-4+SWOnHF0ccWW83bRwNdCoRT1guUP0NFb9MjmUAtL/0=",
|
||||
"owner": "numtide",
|
||||
"repo": "srvos",
|
||||
"rev": "64ae31cb29923128f27a503a550ee4fb1631c4c6",
|
||||
"rev": "e7022e399408e7d1be6abdd16fa4c041755df14b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@ -240,6 +284,36 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"treefmt-nix": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
@ -247,11 +321,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1717850719,
|
||||
"narHash": "sha256-npYqVg+Wk4oxnWrnVG7416fpfrlRhp/lQ6wQ4DHI8YE=",
|
||||
"lastModified": 1725271838,
|
||||
"narHash": "sha256-VcqxWT0O/gMaeWTTjf1r4MOyG49NaNxW4GHTO3xuThE=",
|
||||
"owner": "numtide",
|
||||
"repo": "treefmt-nix",
|
||||
"rev": "4fc1c45a5f50169f9f29f6a98a438fb910b834ed",
|
||||
"rev": "9fb342d14b69aefdf46187f6bb80a4a0d97007cd",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
118
flake.nix
118
flake.nix
@ -7,13 +7,21 @@
|
||||
#};
|
||||
|
||||
inputs = {
|
||||
nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
|
||||
|
||||
nixpkgs.url = "git+https://github.com/NixOS/nixpkgs?ref=nixos-unstable-small&shallow=1";
|
||||
flake-utils.url = "github:numtide/flake-utils";
|
||||
flake-compat.url = "github:edolstra/flake-compat";
|
||||
flake-parts.url = "github:hercules-ci/flake-parts";
|
||||
flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs";
|
||||
treefmt-nix.url = "github:numtide/treefmt-nix";
|
||||
treefmt-nix.inputs.nixpkgs.follows = "nixpkgs";
|
||||
|
||||
nixos-mailserver = {
|
||||
url = "gitlab:simple-nixos-mailserver/nixos-mailserver";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.nixpkgs-24_05.follows = "";
|
||||
inputs.flake-compat.follows = "flake-compat";
|
||||
};
|
||||
|
||||
srvos.url = "github:numtide/srvos";
|
||||
# Use the version of nixpkgs that has been tested to work with SrvOS
|
||||
srvos.inputs.nixpkgs.follows = "nixpkgs";
|
||||
@ -29,39 +37,75 @@
|
||||
buildbot-nix.inputs.treefmt-nix.follows = "treefmt-nix";
|
||||
};
|
||||
|
||||
outputs = inputs@{ flake-parts, ... }:
|
||||
flake-parts.lib.mkFlake { inherit inputs; } ({ self, ... }: {
|
||||
systems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
];
|
||||
imports = [
|
||||
inputs.treefmt-nix.flakeModule
|
||||
./devShells/flake-module.nix
|
||||
./targets/flake-module.nix
|
||||
./modules/flake-module.nix
|
||||
./pkgs/flake-module.nix
|
||||
];
|
||||
perSystem = ({ lib, self', system, ... }: {
|
||||
treefmt = {
|
||||
projectRootFile = ".git/config";
|
||||
programs.hclfmt.enable = true;
|
||||
programs.nixpkgs-fmt.enable = true;
|
||||
settings.formatter.nixpkgs-fmt.excludes = [
|
||||
# generated files
|
||||
"node-env.nix"
|
||||
"node-packages.nix"
|
||||
"composition.nix"
|
||||
];
|
||||
};
|
||||
checks =
|
||||
let
|
||||
nixosMachines = lib.mapAttrs' (name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel) ((lib.filterAttrs (_: config: config.pkgs.system == system)) self.nixosConfigurations);
|
||||
packages = lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages;
|
||||
devShells = lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells;
|
||||
homeConfigurations = lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (self'.legacyPackages.homeConfigurations or { });
|
||||
in
|
||||
nixosMachines // packages // devShells // homeConfigurations;
|
||||
});
|
||||
});
|
||||
outputs =
|
||||
inputs@{ flake-parts, ... }:
|
||||
flake-parts.lib.mkFlake { inherit inputs; } (
|
||||
{ self, ... }:
|
||||
{
|
||||
systems = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
];
|
||||
imports = [
|
||||
inputs.clan-core.flakeModules.default
|
||||
inputs.treefmt-nix.flakeModule
|
||||
./devShells/flake-module.nix
|
||||
./targets/flake-module.nix
|
||||
./modules/flake-module.nix
|
||||
./pkgs/flake-module.nix
|
||||
];
|
||||
perSystem = (
|
||||
{
|
||||
lib,
|
||||
self',
|
||||
system,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
treefmt = {
|
||||
projectRootFile = ".git/config";
|
||||
programs.terraform.enable = true;
|
||||
programs.shellcheck.enable = true;
|
||||
|
||||
programs.deno.enable = true;
|
||||
|
||||
programs.ruff.check = true;
|
||||
programs.ruff.format = true;
|
||||
programs.yamlfmt.enable = true;
|
||||
|
||||
settings.global.excludes = [
|
||||
# generated files
|
||||
"sops/*"
|
||||
"terraform.tfstate"
|
||||
"*.tfvars.sops.json"
|
||||
"*nixos-vars.json"
|
||||
"secrets.yaml"
|
||||
];
|
||||
|
||||
programs.nixfmt.enable = true;
|
||||
programs.nixfmt.package = pkgs.nixfmt-rfc-style;
|
||||
settings.formatter.nixfmt.excludes = [
|
||||
# generated files
|
||||
"node-env.nix"
|
||||
"node-packages.nix"
|
||||
"composition.nix"
|
||||
];
|
||||
};
|
||||
checks =
|
||||
let
|
||||
nixosMachines = lib.mapAttrs' (
|
||||
name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel
|
||||
) ((lib.filterAttrs (_: config: config.pkgs.system == system)) self.nixosConfigurations);
|
||||
packages = lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages;
|
||||
devShells = lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells;
|
||||
homeConfigurations = lib.mapAttrs' (
|
||||
name: config: lib.nameValuePair "home-manager-${name}" config.activation-script
|
||||
) (self'.legacyPackages.homeConfigurations or { });
|
||||
in
|
||||
nixosMachines // packages // devShells // homeConfigurations;
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
1
machines/web01/facts/borgbackup.ssh.pub
Normal file
1
machines/web01/facts/borgbackup.ssh.pub
Normal file
@ -0,0 +1 @@
|
||||
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHS2PvT2e04pqbt1EFFN2y1za9nNmr8rcfnXq9kG5RS2 nixbld@turingmachine
|
1
machines/web01/facts/nginx-acme-email
Normal file
1
machines/web01/facts/nginx-acme-email
Normal file
@ -0,0 +1 @@
|
||||
joerg@thalheim.io
|
@ -41,7 +41,10 @@ in
|
||||
extraGroups = [ "wheel" ];
|
||||
shell = "/run/current-system/sw/bin/zsh";
|
||||
uid = 1004;
|
||||
openssh.authorizedKeys.keys = [ admins.kenji admins.kenji-remote ];
|
||||
openssh.authorizedKeys.keys = [
|
||||
admins.kenji
|
||||
admins.kenji-remote
|
||||
];
|
||||
};
|
||||
johannes = {
|
||||
isNormalUser = true;
|
||||
@ -51,6 +54,18 @@ in
|
||||
uid = 1005;
|
||||
openssh.authorizedKeys.keys = [ admins.johannes ];
|
||||
};
|
||||
flokli = {
|
||||
isNormalUser = true;
|
||||
home = "/home/flokli";
|
||||
extraGroups = [ "wheel" ];
|
||||
shell = "/run/current-system/sw/bin/zsh";
|
||||
uid = 1006;
|
||||
openssh.authorizedKeys.keys = [
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTVTXOutUZZjXLB0lUSgeKcSY/8mxKkC0ingGK1whD2 flokli"
|
||||
"sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIP7rdJ1klzK8nx74QQA8jYdFwznM1klLS0C7M5lHiu+IAAAABHNzaDo= flokli 20240617 28772765"
|
||||
"sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIA34k0FVKDGNdJ8uk0Ytbvh6J8v+H86F4t6BXAIoW/7xAAAABHNzaDo= flokli 20240704 14321691"
|
||||
];
|
||||
};
|
||||
|
||||
root.openssh.authorizedKeys.keys = builtins.attrValues admins;
|
||||
};
|
||||
|
@ -36,7 +36,7 @@
|
||||
|
||||
# optional nix-eval-jobs settings
|
||||
evalWorkerCount = 10; # limit number of concurrent evaluations
|
||||
evalMaxMemorySize = "4096"; # limit memory usage per evaluation
|
||||
evalMaxMemorySize = 4096; # limit memory usage per evaluation
|
||||
};
|
||||
|
||||
# Optional: Enable acme/TLS in nginx (recommended)
|
||||
|
@ -15,6 +15,9 @@
|
||||
pkgs.tig
|
||||
pkgs.tmux
|
||||
pkgs.direnv
|
||||
|
||||
# for flokli
|
||||
pkgs.kitty.terminfo
|
||||
];
|
||||
|
||||
programs.bash = {
|
||||
|
@ -1,4 +1,5 @@
|
||||
{ self, inputs, ... }: {
|
||||
{ self, inputs, ... }:
|
||||
{
|
||||
flake.nixosModules = {
|
||||
server.imports = [
|
||||
inputs.srvos.nixosModules.server
|
||||
@ -26,7 +27,10 @@
|
||||
self.nixosModules.buildbot
|
||||
inputs.srvos.nixosModules.mixins-nginx
|
||||
inputs.srvos.nixosModules.mixins-nix-experimental
|
||||
./matrix-bot.nix
|
||||
./web01
|
||||
inputs.nixos-mailserver.nixosModules.mailserver
|
||||
./mailserver.nix
|
||||
];
|
||||
};
|
||||
}
|
||||
|
@ -1,31 +1,22 @@
|
||||
{ config
|
||||
, lib
|
||||
, ...
|
||||
}:
|
||||
with lib; let
|
||||
cfg = config.clan.networking;
|
||||
{ config, lib, ... }:
|
||||
let
|
||||
cfg = config.clan-infra.networking;
|
||||
in
|
||||
{
|
||||
options = {
|
||||
clan.networking.ipv4.address = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
clan-infra.networking.ipv4.address = lib.mkOption { type = lib.types.str; };
|
||||
|
||||
clan.networking.ipv4.cidr = mkOption {
|
||||
type = types.str;
|
||||
clan-infra.networking.ipv4.cidr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "26";
|
||||
};
|
||||
|
||||
clan.networking.ipv4.gateway = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
clan-infra.networking.ipv4.gateway = lib.mkOption { type = lib.types.str; };
|
||||
|
||||
clan.networking.ipv6.address = mkOption {
|
||||
type = types.str;
|
||||
};
|
||||
clan-infra.networking.ipv6.address = lib.mkOption { type = lib.types.str; };
|
||||
|
||||
clan.networking.ipv6.cidr = mkOption {
|
||||
type = types.str;
|
||||
clan-infra.networking.ipv6.cidr = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "64";
|
||||
};
|
||||
};
|
||||
|
59
modules/mailserver.nix
Normal file
59
modules/mailserver.nix
Normal file
@ -0,0 +1,59 @@
|
||||
{ config, pkgs, ... }:
|
||||
let
|
||||
mailPassword =
|
||||
{ service }:
|
||||
{
|
||||
secret."${service}-password" = { };
|
||||
secret."${service}-password-hash" = { };
|
||||
generator.path = with pkgs; [
|
||||
coreutils
|
||||
xkcdpass
|
||||
mkpasswd
|
||||
];
|
||||
generator.script = ''
|
||||
xkcdpass -n 4 -d - > $secrets/${service}-password
|
||||
cat $secrets/${service}-password | mkpasswd -s -m bcrypt > $secrets/${service}-password-hash
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
mailserver = {
|
||||
enable = true;
|
||||
fqdn = "mail.clan.lol";
|
||||
domains = [ "clan.lol" ];
|
||||
enablePop3 = true;
|
||||
certificateScheme = "acme-nginx";
|
||||
# kresd sucks unfortunally (fails when one NS server is not working, instead of trying other ones)
|
||||
localDnsResolver = false;
|
||||
|
||||
loginAccounts."golem@clan.lol".hashedPasswordFile =
|
||||
config.clan.core.facts.services.golem-mail.secret.golem-password-hash.path;
|
||||
|
||||
loginAccounts."w@clan.lol".hashedPasswordFile =
|
||||
config.clan.core.facts.services.w-mail.secret.w-password-hash.path;
|
||||
|
||||
loginAccounts."gitea@clan.lol".hashedPasswordFile =
|
||||
config.clan.core.facts.services.gitea-mail.secret.gitea-password-hash.path;
|
||||
};
|
||||
|
||||
services.unbound = {
|
||||
enable = true;
|
||||
settings.server = {
|
||||
prefetch = "yes";
|
||||
prefetch-key = true;
|
||||
qname-minimisation = true;
|
||||
# Too many broken dnssec setups even at big companies such as amazon.
|
||||
# Breaks my email setup. Better rely on tls for security.
|
||||
val-permissive-mode = "yes";
|
||||
};
|
||||
};
|
||||
|
||||
# use local unbound as dns resolver
|
||||
networking.nameservers = [ "127.0.0.1" ];
|
||||
|
||||
security.acme.acceptTerms = true;
|
||||
|
||||
clan.core.facts.services.golem-mail = mailPassword { service = "golem"; };
|
||||
clan.core.facts.services.w-mail = mailPassword { service = "w"; };
|
||||
clan.core.facts.services.gitea-mail = mailPassword { service = "gitea"; };
|
||||
}
|
49
modules/matrix-bot.nix
Normal file
49
modules/matrix-bot.nix
Normal file
@ -0,0 +1,49 @@
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
self,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
name = "matrix-bot";
|
||||
in
|
||||
{
|
||||
users.groups.matrix-bot-user = { };
|
||||
users.users.matrix-bot-user = {
|
||||
group = "matrix-bot-user";
|
||||
isSystemUser = true;
|
||||
description = "User for matrix-bot service";
|
||||
home = "/var/lib/matrix-bot";
|
||||
createHome = true;
|
||||
};
|
||||
|
||||
systemd.services.${name} = {
|
||||
path = [ self.packages.${pkgs.system}.matrix-bot ];
|
||||
description = "Matrix bot for changelog and reviews";
|
||||
after = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
environment = {
|
||||
MATRIX_PASSWORD_FILE = "%d/MATRIX_PASSWORD_FILE";
|
||||
OPENAI_API_KEY_FILE = "%d/OPENAI_API_KEY_FILE";
|
||||
HOME = "/var/lib/${name}";
|
||||
};
|
||||
|
||||
serviceConfig = {
|
||||
LoadCredential = [
|
||||
"MATRIX_PASSWORD_FILE:${config.sops.secrets.web01-matrix-password-clan-bot.path}"
|
||||
"OPENAI_API_KEY_FILE:${config.sops.secrets.qubasas-openai-api-key.path}"
|
||||
];
|
||||
User = "matrix-bot-user";
|
||||
Group = "matrix-bot-user";
|
||||
WorkingDirectory = "/var/lib/${name}";
|
||||
RuntimeDirectory = "/var/lib/${name}";
|
||||
};
|
||||
|
||||
script = ''
|
||||
set -euxo pipefail
|
||||
|
||||
mbot --changelog-room "!FdCwyKsRlfooNYKYzx:matrix.org" --review-room "!tmSRJlbsVXFUKAddiM:gchq.icu"
|
||||
'';
|
||||
};
|
||||
}
|
@ -1,26 +1,21 @@
|
||||
{ config, ... }: {
|
||||
{ config, self, ... }:
|
||||
{
|
||||
imports = [ self.inputs.clan-core.clanModules.borgbackup ];
|
||||
|
||||
# 100GB storagebox is under the nix-community hetzner account
|
||||
|
||||
systemd.services.borgbackup-job-clan-lol.serviceConfig.ReadWritePaths = [
|
||||
"/var/log/telegraf"
|
||||
];
|
||||
|
||||
# Run this from the hetzner network:
|
||||
# ssh-keyscan -p 23 u359378.your-storagebox.de
|
||||
programs.ssh.knownHosts = {
|
||||
storagebox-ecdsa.hostNames = [ "[u359378.your-storagebox.de]:23" ];
|
||||
storagebox-ecdsa.publicKey = "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAGK0po6usux4Qv2d8zKZN1dDvbWjxKkGsx7XwFdSUCnF19Q8psHEUWR7C/LtSQ5crU/g+tQVRBtSgoUcE8T+FWp5wBxKvWG2X9gD+s9/4zRmDeSJR77W6gSA/+hpOZoSE+4KgNdnbYSNtbZH/dN74EG7GLb/gcIpbUUzPNXpfKl7mQitw==";
|
||||
|
||||
storagebox-rsa.hostNames = [ "[u359378.your-storagebox.de]:23" ];
|
||||
storagebox-rsa.publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
|
||||
clan.borgbackup.destinations.${config.networking.hostName} = {
|
||||
repo = "u366395@u366395.your-storagebox.de:/./borgbackup";
|
||||
rsh = "ssh -oPort=23 -i ${config.clan.core.facts.services.borgbackup.secret."borgbackup.ssh".path}";
|
||||
};
|
||||
|
||||
services.borgbackup.jobs.clan-lol = {
|
||||
paths = [
|
||||
"/home"
|
||||
"/var"
|
||||
"/root"
|
||||
];
|
||||
clan.core.state.system.folders = [
|
||||
"/home"
|
||||
"/etc"
|
||||
"/var"
|
||||
"/root"
|
||||
];
|
||||
|
||||
services.borgbackup.jobs.${config.networking.hostName} = {
|
||||
exclude = [
|
||||
"*.pyc"
|
||||
"/home/*/.direnv"
|
||||
@ -41,32 +36,20 @@
|
||||
"/var/tmp"
|
||||
"/var/log"
|
||||
];
|
||||
# $ ssh-keygen -y -f /run/secrets/hetzner-borgbackup-ssh > /tmp/hetzner-borgbackup-ssh.pub
|
||||
# $ cat /tmp/hetzner-borgbackup-ssh.pub | ssh -p23 u366395@u366395.your-storagebox.de install-ssh-key
|
||||
repo = "u366395@u366395.your-storagebox.de:/./borgbackup";
|
||||
|
||||
# Disaster recovery:
|
||||
# get the backup passphrase and ssh key from the sops and store them in /tmp
|
||||
# $ export BORG_PASSCOMMAND='cat /tmp/hetzner-borgbackup-passphrase'
|
||||
# $ export BORG_REPO='u359378@u359378.your-storagebox.de:/./borgbackup'
|
||||
# $ export BORG_RSH='ssh -oPort=23 -i /tmp/hetzner-borgbackup-ssh'
|
||||
# $ export BORG_RSH='ssh -oPort=23 -i /tmp/hetzner-borgbackup-ssh'
|
||||
# $ borg list
|
||||
# web01-clan-lol-2023-07-21T14:12:22 Fri, 2023-07-21 14:12:27 [539b1037669ffd0d3f50020f439bbe2881b7234910e405eafc333125383351bc]
|
||||
# $ borg mount u359378@u359378.your-storagebox.de:/./borgbackup::web01-clan-lol-2023-07-21T14:12:22 /tmp/backup
|
||||
doInit = true;
|
||||
encryption = {
|
||||
mode = "repokey-blake2";
|
||||
# $ nix run nixpkgs#xkcdpass -- -d '-' -n 3 -C capitalize "$@"
|
||||
passCommand = "cat ${config.sops.secrets.hetzner-borgbackup-passphrase.path}";
|
||||
};
|
||||
compression = "auto,zstd";
|
||||
startAt = "daily";
|
||||
|
||||
# Also enable ssh support in the storagebox web interface.
|
||||
# By default the storage box is only accessible from the hetzner network.
|
||||
# $ ssh-keygen -t ed25519 -N "" -f /tmp/ssh_host_ed25519_key
|
||||
# $ cat /tmp/ssh_host_ed25519_key.pub | ssh -p23 u359378@u359378.your-storagebox.de install-ssh-key
|
||||
environment.BORG_RSH = "ssh -oPort=23 -i ${config.sops.secrets.hetzner-borgbackup-ssh.path}";
|
||||
# $ clan facts generate
|
||||
# $ clan facts list web01 | jq .borgbackup.ssh.pub | ssh -p23 u359378@u359378.your-storagebox.de install-ssh-key
|
||||
preHook = ''
|
||||
set -x
|
||||
'';
|
||||
@ -76,12 +59,19 @@
|
||||
task,frequency=daily last_run=$(date +%s)i,state="$([[ $exitStatus == 0 ]] && echo ok || echo fail)"
|
||||
EOF
|
||||
'';
|
||||
};
|
||||
|
||||
prune.keep = {
|
||||
within = "1d"; # Keep all archives from the last day
|
||||
daily = 7;
|
||||
weekly = 4;
|
||||
monthly = 0;
|
||||
};
|
||||
systemd.services."borgbackup-job-${config.networking.hostName}".serviceConfig.ReadWritePaths = [
|
||||
"/var/log/telegraf"
|
||||
];
|
||||
|
||||
# Run this from the hetzner network:
|
||||
# ssh-keyscan -p 23 u359378.your-storagebox.de
|
||||
programs.ssh.knownHosts = {
|
||||
storagebox-ecdsa.hostNames = [ "[u359378.your-storagebox.de]:23" ];
|
||||
storagebox-ecdsa.publicKey = "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAGK0po6usux4Qv2d8zKZN1dDvbWjxKkGsx7XwFdSUCnF19Q8psHEUWR7C/LtSQ5crU/g+tQVRBtSgoUcE8T+FWp5wBxKvWG2X9gD+s9/4zRmDeSJR77W6gSA/+hpOZoSE+4KgNdnbYSNtbZH/dN74EG7GLb/gcIpbUUzPNXpfKl7mQitw==";
|
||||
|
||||
storagebox-rsa.hostNames = [ "[u359378.your-storagebox.de]:23" ];
|
||||
storagebox-rsa.publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
|
||||
};
|
||||
}
|
||||
|
@ -1,10 +1,18 @@
|
||||
{ config, self, pkgs, ... }: {
|
||||
{
|
||||
config,
|
||||
self,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
# service to for automatic merge bot
|
||||
systemd.services.clan-merge = {
|
||||
description = "Merge clan.lol PRs automatically";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network.target" ];
|
||||
environment = { GITEA_TOKEN_FILE = "%d/GITEA_TOKEN_FILE"; };
|
||||
environment = {
|
||||
GITEA_TOKEN_FILE = "%d/GITEA_TOKEN_FILE";
|
||||
};
|
||||
serviceConfig = {
|
||||
LoadCredential = [ "GITEA_TOKEN_FILE:${config.sops.secrets.merge-bot-gitea-token.path}" ];
|
||||
Restart = "on-failure";
|
||||
@ -14,7 +22,7 @@
|
||||
while sleep 10; do
|
||||
${self.packages.${pkgs.system}.clan-merge}/bin/clan-merge \
|
||||
--bot-name clan-bot \
|
||||
--repos clan-infra clan-core clan-homepage
|
||||
--repos clan-infra clan-core clan-homepage data-mesher
|
||||
done
|
||||
'';
|
||||
};
|
||||
|
@ -1,4 +1,5 @@
|
||||
{ self, ... }: {
|
||||
{ self, ... }:
|
||||
{
|
||||
imports = [
|
||||
./borgbackup.nix
|
||||
./clan-merge.nix
|
||||
@ -8,7 +9,7 @@
|
||||
./homepage.nix
|
||||
./postfix.nix
|
||||
./jobs.nix
|
||||
#./matrix-synapse.nix
|
||||
./matrix-synapse.nix
|
||||
../dev.nix
|
||||
self.inputs.clan-core.clanModules.zt-tcp-relay
|
||||
];
|
||||
|
@ -1,8 +1,26 @@
|
||||
{ config, self, pkgs, lib, ... }:
|
||||
{
|
||||
config,
|
||||
self,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
storeDeps = pkgs.runCommand "store-deps" { } ''
|
||||
mkdir -p $out/bin
|
||||
for dir in ${toString [ pkgs.coreutils pkgs.findutils pkgs.gnugrep pkgs.gawk pkgs.git pkgs.nix pkgs.bash pkgs.jq pkgs.nodejs ]}; do
|
||||
for dir in ${
|
||||
toString [
|
||||
pkgs.coreutils
|
||||
pkgs.findutils
|
||||
pkgs.gnugrep
|
||||
pkgs.gawk
|
||||
pkgs.git
|
||||
pkgs.nix
|
||||
pkgs.bash
|
||||
pkgs.jq
|
||||
pkgs.nodejs
|
||||
]
|
||||
}; do
|
||||
for bin in "$dir"/bin/*; do
|
||||
ln -s "$bin" "$out/bin/$(basename "$bin")"
|
||||
done
|
||||
@ -14,87 +32,95 @@ let
|
||||
'';
|
||||
numInstances = 2;
|
||||
in
|
||||
lib.mkMerge [{
|
||||
# everything here has no dependencies on the store
|
||||
systemd.services.gitea-runner-nix-image = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "podman.service" ];
|
||||
requires = [ "podman.service" ];
|
||||
path = [ config.virtualisation.podman.package pkgs.gnutar pkgs.shadow pkgs.getent ];
|
||||
# we also include etc here because the cleanup job also wants the nixuser to be present
|
||||
script = ''
|
||||
set -eux -o pipefail
|
||||
mkdir -p etc/nix
|
||||
|
||||
# Create an unpriveleged user that we can use also without the run-as-user.sh script
|
||||
touch etc/passwd etc/group
|
||||
groupid=$(cut -d: -f3 < <(getent group nixuser))
|
||||
userid=$(cut -d: -f3 < <(getent passwd nixuser))
|
||||
groupadd --prefix $(pwd) --gid "$groupid" nixuser
|
||||
emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
|
||||
useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
|
||||
|
||||
cat <<NIX_CONFIG > etc/nix/nix.conf
|
||||
accept-flake-config = true
|
||||
experimental-features = nix-command flakes
|
||||
NIX_CONFIG
|
||||
|
||||
cat <<NSSWITCH > etc/nsswitch.conf
|
||||
passwd: files mymachines systemd
|
||||
group: files mymachines systemd
|
||||
shadow: files
|
||||
|
||||
hosts: files mymachines dns myhostname
|
||||
networks: files
|
||||
|
||||
ethers: files
|
||||
services: files
|
||||
protocols: files
|
||||
rpc: files
|
||||
NSSWITCH
|
||||
|
||||
# list the content as it will be imported into the container
|
||||
tar -cv . | tar -tvf -
|
||||
tar -cv . | podman import - gitea-runner-nix
|
||||
'';
|
||||
serviceConfig = {
|
||||
RuntimeDirectory = "gitea-runner-nix-image";
|
||||
WorkingDirectory = "/run/gitea-runner-nix-image";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
};
|
||||
|
||||
users.users.nixuser = {
|
||||
group = "nixuser";
|
||||
description = "Used for running nix ci jobs";
|
||||
home = "/var/empty";
|
||||
isSystemUser = true;
|
||||
};
|
||||
users.groups.nixuser = { };
|
||||
}
|
||||
lib.mkMerge [
|
||||
{
|
||||
systemd.services = lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}-token") numInstances) (name: {
|
||||
# everything here has no dependencies on the store
|
||||
systemd.services.gitea-runner-nix-image = {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "gitea.service" ];
|
||||
environment = {
|
||||
GITEA_CUSTOM = "/var/lib/gitea/custom";
|
||||
GITEA_WORK_DIR = "/var/lib/gitea";
|
||||
};
|
||||
after = [ "podman.service" ];
|
||||
requires = [ "podman.service" ];
|
||||
path = [
|
||||
config.virtualisation.podman.package
|
||||
pkgs.gnutar
|
||||
pkgs.shadow
|
||||
pkgs.getent
|
||||
];
|
||||
# we also include etc here because the cleanup job also wants the nixuser to be present
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
token=$(${lib.getExe self.packages.${pkgs.hostPlatform.system}.gitea} actions generate-runner-token)
|
||||
echo "TOKEN=$token" > /var/lib/gitea-registration/${name}
|
||||
set -eux -o pipefail
|
||||
mkdir -p etc/nix
|
||||
|
||||
# Create an unpriveleged user that we can use also without the run-as-user.sh script
|
||||
touch etc/passwd etc/group
|
||||
groupid=$(cut -d: -f3 < <(getent group nixuser))
|
||||
userid=$(cut -d: -f3 < <(getent passwd nixuser))
|
||||
groupadd --prefix $(pwd) --gid "$groupid" nixuser
|
||||
emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
|
||||
useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nixuser nixuser
|
||||
|
||||
cat <<NIX_CONFIG > etc/nix/nix.conf
|
||||
accept-flake-config = true
|
||||
experimental-features = nix-command flakes
|
||||
NIX_CONFIG
|
||||
|
||||
cat <<NSSWITCH > etc/nsswitch.conf
|
||||
passwd: files mymachines systemd
|
||||
group: files mymachines systemd
|
||||
shadow: files
|
||||
|
||||
hosts: files mymachines dns myhostname
|
||||
networks: files
|
||||
|
||||
ethers: files
|
||||
services: files
|
||||
protocols: files
|
||||
rpc: files
|
||||
NSSWITCH
|
||||
|
||||
# list the content as it will be imported into the container
|
||||
tar -cv . | tar -tvf -
|
||||
tar -cv . | podman import - gitea-runner-nix
|
||||
'';
|
||||
unitConfig.ConditionPathExists = [ "!/var/lib/gitea-registration/${name}" ];
|
||||
serviceConfig = {
|
||||
User = "gitea";
|
||||
Group = "gitea";
|
||||
StateDirectory = "gitea-registration";
|
||||
RuntimeDirectory = "gitea-runner-nix-image";
|
||||
WorkingDirectory = "/run/gitea-runner-nix-image";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
users.users.nixuser = {
|
||||
group = "nixuser";
|
||||
description = "Used for running nix ci jobs";
|
||||
home = "/var/empty";
|
||||
isSystemUser = true;
|
||||
};
|
||||
users.groups.nixuser = { };
|
||||
}
|
||||
{
|
||||
systemd.services =
|
||||
lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}-token") numInstances)
|
||||
(name: {
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "gitea.service" ];
|
||||
environment = {
|
||||
GITEA_CUSTOM = "/var/lib/gitea/custom";
|
||||
GITEA_WORK_DIR = "/var/lib/gitea";
|
||||
};
|
||||
script = ''
|
||||
set -euo pipefail
|
||||
token=$(${lib.getExe self.packages.${pkgs.hostPlatform.system}.gitea} actions generate-runner-token)
|
||||
echo "TOKEN=$token" > /var/lib/gitea-registration/${name}
|
||||
'';
|
||||
unitConfig.ConditionPathExists = [ "!/var/lib/gitea-registration/${name}" ];
|
||||
serviceConfig = {
|
||||
User = "gitea";
|
||||
Group = "gitea";
|
||||
StateDirectory = "gitea-registration";
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
});
|
||||
|
||||
# Format of the token file:
|
||||
virtualisation = {
|
||||
@ -111,106 +137,119 @@ lib.mkMerge [{
|
||||
|
||||
virtualisation.containers.containersConf.settings = {
|
||||
# podman seems to not work with systemd-resolved
|
||||
containers.dns_servers = [ "8.8.8.8" "8.8.4.4" ];
|
||||
containers.dns_servers = [
|
||||
"8.8.8.8"
|
||||
"8.8.4.4"
|
||||
];
|
||||
};
|
||||
}
|
||||
{
|
||||
systemd.services = lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") numInstances) (name: {
|
||||
after = [
|
||||
"${name}-token.service"
|
||||
"gitea-runner-nix-image.service"
|
||||
];
|
||||
requires = [
|
||||
"${name}-token.service"
|
||||
"gitea-runner-nix-image.service"
|
||||
];
|
||||
systemd.services =
|
||||
lib.genAttrs (builtins.genList (n: "gitea-runner-nix${builtins.toString n}") numInstances)
|
||||
(name: {
|
||||
after = [
|
||||
"${name}-token.service"
|
||||
"gitea-runner-nix-image.service"
|
||||
];
|
||||
requires = [
|
||||
"${name}-token.service"
|
||||
"gitea-runner-nix-image.service"
|
||||
];
|
||||
|
||||
# TODO: systemd confinment
|
||||
serviceConfig = {
|
||||
# Hardening (may overlap with DynamicUser=)
|
||||
# The following options are only for optimizing output of systemd-analyze
|
||||
AmbientCapabilities = "";
|
||||
CapabilityBoundingSet = "";
|
||||
# ProtectClock= adds DeviceAllow=char-rtc r
|
||||
DeviceAllow = "";
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateMounts = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
UMask = "0066";
|
||||
ProtectProc = "invisible";
|
||||
SystemCallFilter = [
|
||||
"~@clock"
|
||||
"~@cpu-emulation"
|
||||
"~@module"
|
||||
"~@mount"
|
||||
"~@obsolete"
|
||||
"~@raw-io"
|
||||
"~@reboot"
|
||||
"~@swap"
|
||||
# needed by go?
|
||||
#"~@resources"
|
||||
"~@privileged"
|
||||
"~capset"
|
||||
"~setdomainname"
|
||||
"~sethostname"
|
||||
];
|
||||
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
|
||||
# TODO: systemd confinment
|
||||
serviceConfig = {
|
||||
# Hardening (may overlap with DynamicUser=)
|
||||
# The following options are only for optimizing output of systemd-analyze
|
||||
AmbientCapabilities = "";
|
||||
CapabilityBoundingSet = "";
|
||||
# ProtectClock= adds DeviceAllow=char-rtc r
|
||||
DeviceAllow = "";
|
||||
NoNewPrivileges = true;
|
||||
PrivateDevices = true;
|
||||
PrivateMounts = true;
|
||||
PrivateTmp = true;
|
||||
PrivateUsers = true;
|
||||
ProtectClock = true;
|
||||
ProtectControlGroups = true;
|
||||
ProtectHome = true;
|
||||
ProtectHostname = true;
|
||||
ProtectKernelLogs = true;
|
||||
ProtectKernelModules = true;
|
||||
ProtectKernelTunables = true;
|
||||
ProtectSystem = "strict";
|
||||
RemoveIPC = true;
|
||||
RestrictNamespaces = true;
|
||||
RestrictRealtime = true;
|
||||
RestrictSUIDSGID = true;
|
||||
UMask = "0066";
|
||||
ProtectProc = "invisible";
|
||||
SystemCallFilter = [
|
||||
"~@clock"
|
||||
"~@cpu-emulation"
|
||||
"~@module"
|
||||
"~@mount"
|
||||
"~@obsolete"
|
||||
"~@raw-io"
|
||||
"~@reboot"
|
||||
"~@swap"
|
||||
# needed by go?
|
||||
#"~@resources"
|
||||
"~@privileged"
|
||||
"~capset"
|
||||
"~setdomainname"
|
||||
"~sethostname"
|
||||
];
|
||||
RestrictAddressFamilies = [
|
||||
"AF_INET"
|
||||
"AF_INET6"
|
||||
"AF_UNIX"
|
||||
"AF_NETLINK"
|
||||
];
|
||||
|
||||
# Needs network access
|
||||
PrivateNetwork = false;
|
||||
# Cannot be true due to Node
|
||||
MemoryDenyWriteExecute = false;
|
||||
# Needs network access
|
||||
PrivateNetwork = false;
|
||||
# Cannot be true due to Node
|
||||
MemoryDenyWriteExecute = false;
|
||||
|
||||
# The more restrictive "pid" option makes `nix` commands in CI emit
|
||||
# "GC Warning: Couldn't read /proc/stat"
|
||||
# You may want to set this to "pid" if not using `nix` commands
|
||||
ProcSubset = "all";
|
||||
# Coverage programs for compiled code such as `cargo-tarpaulin` disable
|
||||
# ASLR (address space layout randomization) which requires the
|
||||
# `personality` syscall
|
||||
# You may want to set this to `true` if not using coverage tooling on
|
||||
# compiled code
|
||||
LockPersonality = false;
|
||||
# The more restrictive "pid" option makes `nix` commands in CI emit
|
||||
# "GC Warning: Couldn't read /proc/stat"
|
||||
# You may want to set this to "pid" if not using `nix` commands
|
||||
ProcSubset = "all";
|
||||
# Coverage programs for compiled code such as `cargo-tarpaulin` disable
|
||||
# ASLR (address space layout randomization) which requires the
|
||||
# `personality` syscall
|
||||
# You may want to set this to `true` if not using coverage tooling on
|
||||
# compiled code
|
||||
LockPersonality = false;
|
||||
|
||||
# Note that this has some interactions with the User setting; so you may
|
||||
# want to consult the systemd docs if using both.
|
||||
DynamicUser = true;
|
||||
};
|
||||
});
|
||||
# Note that this has some interactions with the User setting; so you may
|
||||
# want to consult the systemd docs if using both.
|
||||
DynamicUser = true;
|
||||
};
|
||||
});
|
||||
|
||||
services.gitea-actions-runner.instances = lib.genAttrs (builtins.genList (n: "nix${builtins.toString n}") numInstances) (name: {
|
||||
enable = true;
|
||||
name = "nix-runner";
|
||||
# take the git root url from the gitea config
|
||||
# only possible if you've also configured your gitea though the same nix config
|
||||
# otherwise you need to set it manually
|
||||
url = config.services.gitea.settings.server.ROOT_URL;
|
||||
# use your favourite nix secret manager to get a path for this
|
||||
tokenFile = "/var/lib/gitea-registration/gitea-runner-${name}-token";
|
||||
labels = [ "nix:docker://gitea-runner-nix" ];
|
||||
settings = {
|
||||
container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm";
|
||||
# the default network that also respects our dns server settings
|
||||
container.network = "host";
|
||||
container.valid_volumes = [
|
||||
"/nix"
|
||||
"${storeDeps}/bin"
|
||||
"${storeDeps}/etc/ssl"
|
||||
];
|
||||
};
|
||||
});
|
||||
}]
|
||||
services.gitea-actions-runner.instances =
|
||||
lib.genAttrs (builtins.genList (n: "nix${builtins.toString n}") numInstances)
|
||||
(name: {
|
||||
enable = true;
|
||||
name = "nix-runner";
|
||||
# take the git root url from the gitea config
|
||||
# only possible if you've also configured your gitea though the same nix config
|
||||
# otherwise you need to set it manually
|
||||
url = config.services.gitea.settings.server.ROOT_URL;
|
||||
# use your favourite nix secret manager to get a path for this
|
||||
tokenFile = "/var/lib/gitea-registration/gitea-runner-${name}-token";
|
||||
labels = [ "nix:docker://gitea-runner-nix" ];
|
||||
settings = {
|
||||
container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nixuser --device=/dev/kvm";
|
||||
# the default network that also respects our dns server settings
|
||||
container.network = "host";
|
||||
container.valid_volumes = [
|
||||
"/nix"
|
||||
"${storeDeps}/bin"
|
||||
"${storeDeps}/etc/ssl"
|
||||
];
|
||||
};
|
||||
});
|
||||
}
|
||||
]
|
||||
|
@ -1,12 +1,22 @@
|
||||
{ config, pkgs, lib, self, ... }:
|
||||
{
|
||||
pkgs,
|
||||
lib,
|
||||
self,
|
||||
config,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
# make the logs for this host "public" so that they show up in e.g. metrics
|
||||
publog = vhost: lib.attrsets.unionOfDisjoint vhost {
|
||||
extraConfig = (vhost.extraConfig or "") + ''
|
||||
access_log /var/log/nginx/public.log vcombined;
|
||||
'';
|
||||
};
|
||||
publog =
|
||||
vhost:
|
||||
lib.attrsets.unionOfDisjoint vhost {
|
||||
extraConfig =
|
||||
(vhost.extraConfig or "")
|
||||
+ ''
|
||||
access_log /var/log/nginx/public.log vcombined;
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
|
||||
@ -27,13 +37,17 @@ in
|
||||
package = self.packages.${pkgs.hostPlatform.system}.gitea;
|
||||
|
||||
settings.actions.ENABLED = true;
|
||||
|
||||
mailerPasswordFile = config.clan.core.facts.services.gitea-mail.secret.gitea-password.path;
|
||||
|
||||
settings.mailer = {
|
||||
ENABLED = true;
|
||||
FROM = "gitea@clan.lol";
|
||||
SMTP_ADDR = "localhost";
|
||||
SMTP_PORT = 25;
|
||||
PROTOCOL = "smtps";
|
||||
USER = "gitea@clan.lol";
|
||||
SMTP_ADDR = "mail.clan.lol";
|
||||
SMTP_PORT = "587";
|
||||
};
|
||||
|
||||
settings.log.LEVEL = "Error";
|
||||
settings.service.DISABLE_REGISTRATION = false;
|
||||
settings.metrics.ENABLED = true;
|
||||
@ -49,11 +63,15 @@ in
|
||||
settings.session.COOKIE_SECURE = true;
|
||||
};
|
||||
|
||||
sops.secrets.web01-gitea-password.owner = config.systemd.services.gitea.serviceConfig.User;
|
||||
|
||||
services.nginx.virtualHosts."git.clan.lol" = publog {
|
||||
forceSSL = true;
|
||||
enableACME = true;
|
||||
locations."/".extraConfig = ''
|
||||
proxy_pass http://localhost:3002;
|
||||
'';
|
||||
|
||||
locations."= /robots.txt".alias = ./robots.txt;
|
||||
};
|
||||
}
|
||||
|
@ -1,4 +1,5 @@
|
||||
{ pkgs, ... }: {
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
services.postgresql.enable = true;
|
||||
services.postgresql.package = pkgs.postgresql_14;
|
||||
services.postgresql.settings = {
|
||||
|
114
modules/web01/gitea/robots.txt
Normal file
114
modules/web01/gitea/robots.txt
Normal file
@ -0,0 +1,114 @@
|
||||
# sane defaults for a Forgejo instance
|
||||
# contributing sources:
|
||||
# - https://code.forgejo.org/robots.txt
|
||||
# - https://github.com/robots.txt
|
||||
# - https://codeberg.org/robots.txt
|
||||
|
||||
User-agent: *
|
||||
Disallow: /api/*
|
||||
Disallow: /avatars
|
||||
Disallow: /user/*
|
||||
Disallow: /*/*/src/commit/*
|
||||
Disallow: /*/*/commit/*
|
||||
Disallow: /*/*/*/refs/*
|
||||
Disallow: /*/*/*/star
|
||||
Disallow: /*/*/*/watch
|
||||
Disallow: /*/*/labels
|
||||
Disallow: /*/*/activity/*
|
||||
Disallow: /vendor/*
|
||||
Disallow: /swagger.*.json
|
||||
|
||||
# Language spam
|
||||
Disallow: /*?lang=
|
||||
|
||||
Disallow: /explore/*?*
|
||||
Disallow: /*q=
|
||||
|
||||
Disallow: /repo/create
|
||||
Disallow: /repo/migrate
|
||||
Disallow: /org/create
|
||||
|
||||
Disallow: /*/*/activity
|
||||
Disallow: /*/*/projects
|
||||
Disallow: /*/*/forks
|
||||
Disallow: /*/*/commits/
|
||||
Disallow: /*/*/branches
|
||||
Disallow: /*/*/contributors
|
||||
Disallow: /*/*/tags
|
||||
Disallow: /*/*/stargazers
|
||||
Disallow: /*/*/watchers
|
||||
Disallow: /*/*/network
|
||||
Disallow: /*/*/graphs
|
||||
Disallow: /*/*/compare
|
||||
|
||||
Disallow: /*/*/issues/new
|
||||
Disallow: /*/*/issues/?*
|
||||
Disallow: /*/*/pulls/?*
|
||||
|
||||
Disallow: /*/tree/
|
||||
Disallow: /*/download
|
||||
Disallow: /*/revisions
|
||||
Disallow: /*/commits/*?author
|
||||
Disallow: /*/commits/*?path
|
||||
Disallow: /*/comments
|
||||
Disallow: /*/blame/
|
||||
Disallow: /*/raw/
|
||||
Disallow: /*/cache/
|
||||
Disallow: /.git/
|
||||
Disallow: */.git/
|
||||
Disallow: /*.git
|
||||
Disallow: /*.atom
|
||||
Disallow: /*.rss
|
||||
|
||||
Disallow: /*/*/archive/
|
||||
Disallow: *.bundle
|
||||
Disallow: */commit/*.pacth
|
||||
Disallow: */commit/*.diff
|
||||
|
||||
Disallow: /*source=*
|
||||
Disallow: /*ref_cta=*
|
||||
Disallow: /*plan=*
|
||||
Disallow: /*return_to=*
|
||||
Disallow: /*ref_loc=*
|
||||
Disallow: /*setup_organization=*
|
||||
Disallow: /*source_repo=*
|
||||
Disallow: /*ref_page=*
|
||||
Disallow: /*source=*
|
||||
Disallow: /*referrer=*
|
||||
Disallow: /*report=*
|
||||
Disallow: /*author=*
|
||||
Disallow: /*since=*
|
||||
Disallow: /*until=*
|
||||
Disallow: /*commits?author=*
|
||||
Disallow: /*tab=*
|
||||
Disallow: /*q=*
|
||||
Disallow: /*repo-search-archived=*
|
||||
|
||||
Crawl-delay: 2
|
||||
|
||||
## ChatGPT - See: https://platform.openai.com/docs/plugins/bot
|
||||
User-agent: GPTBot
|
||||
Disallow: /
|
||||
|
||||
## Google AI Bots - See: https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers?hl=en#google-extended
|
||||
User-agent: Google-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: CCBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: Omgilibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Omgili
|
||||
Disallow: /
|
||||
|
||||
User-agent: FacebookBot
|
||||
Disallow: /
|
||||
|
||||
## Amazon, e.g. Alexa - See: https://developer.amazon.com/amazonbot
|
||||
User-agent: Amazonbot
|
||||
Disallow: /
|
@ -1,4 +1,4 @@
|
||||
{ stdenv, lib, pkgs, ... }:
|
||||
{ pkgs, ... }:
|
||||
|
||||
let
|
||||
domain = "metrics.clan.lol";
|
||||
@ -38,14 +38,13 @@ in
|
||||
"d ${pub_goaccess} 0755 goaccess nginx -"
|
||||
];
|
||||
|
||||
|
||||
# --browsers-file=/etc/goaccess/browsers.list
|
||||
# https://raw.githubusercontent.com/allinurl/goaccess/master/config/browsers.list
|
||||
systemd.services.goaccess = {
|
||||
description = "GoAccess server monitoring";
|
||||
preStart = ''
|
||||
rm -f ${pub_goaccess}/index.html
|
||||
'';
|
||||
rm -f ${pub_goaccess}/index.html
|
||||
'';
|
||||
serviceConfig = {
|
||||
User = "goaccess";
|
||||
Group = "nginx";
|
||||
@ -83,7 +82,11 @@ in
|
||||
ProtectSystem = "strict";
|
||||
SystemCallFilter = "~@clock @cpu-emulation @debug @keyring @memlock @module @mount @obsolete @privileged @reboot @resources @setuid @swap @raw-io";
|
||||
ReadOnlyPaths = "/";
|
||||
ReadWritePaths = [ "/proc/self" "${pub_goaccess}" "${priv_goaccess}" ];
|
||||
ReadWritePaths = [
|
||||
"/proc/self"
|
||||
"${pub_goaccess}"
|
||||
"${priv_goaccess}"
|
||||
];
|
||||
PrivateDevices = "yes";
|
||||
ProtectKernelModules = "yes";
|
||||
ProtectKernelTunables = "yes";
|
||||
@ -92,7 +95,6 @@ in
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
};
|
||||
|
||||
|
||||
services.nginx.virtualHosts."${domain}" = {
|
||||
addSSL = true;
|
||||
enableACME = true;
|
||||
|
@ -1,17 +1,18 @@
|
||||
{ config, pkgs, ... }: {
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
services.harmonia.enable = true;
|
||||
# $ nix-store --generate-binary-cache-key cache.yourdomain.tld-1 harmonia.secret harmonia.pub
|
||||
services.harmonia.signKeyPath = config.sops.secrets.harmonia-secret.path;
|
||||
|
||||
services.nginx = {
|
||||
package = pkgs.nginxStable.override {
|
||||
modules = [ pkgs.nginxModules.zstd ];
|
||||
};
|
||||
package = pkgs.nginxStable.override { modules = [ pkgs.nginxModules.zstd ]; };
|
||||
};
|
||||
|
||||
# trust our own cache
|
||||
nix.settings.trusted-substituters = [ "https://cache.clan.lol" ];
|
||||
nix.settings.trusted-public-keys = [ "cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28=" ];
|
||||
nix.settings.trusted-public-keys = [
|
||||
"cache.clan.lol-1:3KztgSAB5R1M+Dz7vzkBGzXdodizbgLXGXKXlcQLA28="
|
||||
];
|
||||
|
||||
services.nginx.virtualHosts."cache.clan.lol" = {
|
||||
forceSSL = true;
|
||||
|
@ -1,18 +1,15 @@
|
||||
{ config, lib, pkgs, self, ... }:
|
||||
{ config, ... }:
|
||||
|
||||
{
|
||||
security.acme.defaults.email = "admins@clan.lol";
|
||||
security.acme.acceptTerms = true;
|
||||
clan.nginx.acme.email = "admins@clan.lol";
|
||||
|
||||
# www user to push website artifacts via ssh
|
||||
users.users.www = {
|
||||
openssh.authorizedKeys.keys =
|
||||
config.users.users.root.openssh.authorizedKeys.keys
|
||||
++ [
|
||||
# ssh-homepage-key
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMxZ3Av30M6Sh6NU1mnCskB16bYtNP8vskc/+ud0AU1C ssh-homepage-key"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBuYyfSuETSrwqCsWHeeClqjcsFlMEmiJN6Rr8/DwrU0 gitea-ci"
|
||||
];
|
||||
openssh.authorizedKeys.keys = config.users.users.root.openssh.authorizedKeys.keys ++ [
|
||||
# ssh-homepage-key
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMxZ3Av30M6Sh6NU1mnCskB16bYtNP8vskc/+ud0AU1C ssh-homepage-key"
|
||||
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBuYyfSuETSrwqCsWHeeClqjcsFlMEmiJN6Rr8/DwrU0 gitea-ci"
|
||||
];
|
||||
isSystemUser = true;
|
||||
shell = "/run/current-system/sw/bin/bash";
|
||||
group = "www";
|
||||
@ -20,9 +17,7 @@
|
||||
users.groups.www = { };
|
||||
|
||||
# ensure /var/www can be accessed by nginx and www user
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/www 0755 www nginx"
|
||||
];
|
||||
systemd.tmpfiles.rules = [ "d /var/www 0755 www nginx" ];
|
||||
|
||||
services.nginx = {
|
||||
|
||||
@ -75,6 +70,9 @@
|
||||
locations."^~ /blog".extraConfig = ''
|
||||
rewrite ^/blog(.*)$ https://docs.clan.lol/blog permanent;
|
||||
'';
|
||||
locations."^~ /wclan".extraConfig = ''
|
||||
rewrite ^/wclan(.*)$ https://clan.lol/what-is-clan.html permanent;
|
||||
'';
|
||||
locations."/thaigersprint".return = "307 https://pad.lassul.us/s/clan-thaigersprint";
|
||||
};
|
||||
|
||||
|
@ -1,4 +1,10 @@
|
||||
{ config, self, pkgs, lib, ... }:
|
||||
{
|
||||
config,
|
||||
self,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
configForJob = name: {
|
||||
systemd.timers.${name} = {
|
||||
@ -46,9 +52,13 @@ let
|
||||
};
|
||||
in
|
||||
{
|
||||
config = lib.mkMerge (map configForJob [
|
||||
"job-flake-update-clan-core"
|
||||
"job-flake-update-clan-homepage"
|
||||
"job-flake-update-clan-infra"
|
||||
]);
|
||||
config = lib.mkMerge (
|
||||
map configForJob [
|
||||
"job-flake-update-clan-core"
|
||||
"job-flake-update-clan-core-individual"
|
||||
"job-flake-update-clan-homepage"
|
||||
"job-flake-update-clan-infra"
|
||||
"job-flake-update-data-mesher"
|
||||
]
|
||||
);
|
||||
}
|
||||
|
@ -1,6 +1,31 @@
|
||||
{ self, ... }:
|
||||
{
|
||||
imports = [ self.inputs.clan-core.clanModules.matrix-synapse ];
|
||||
clan.matrix-synapse.enable = true;
|
||||
clan.matrix-synapse.domain = "clan.lol";
|
||||
clan.matrix-synapse.server_tld = "clan.lol";
|
||||
clan.matrix-synapse.app_domain = "matrix.clan.lol";
|
||||
|
||||
clan.matrix-synapse.users.admin = {
|
||||
admin = true;
|
||||
};
|
||||
clan.matrix-synapse.users.monitoring = { };
|
||||
clan.matrix-synapse.users.clan-bot = { };
|
||||
|
||||
# Rate limiting settings
|
||||
# we need to up this to be able to support matrix bots
|
||||
services.matrix-synapse.settings = {
|
||||
rc_login = {
|
||||
address = {
|
||||
per_second = 20;
|
||||
burst_count = 200;
|
||||
};
|
||||
account = {
|
||||
per_second = 20;
|
||||
burst_count = 200;
|
||||
};
|
||||
failed_attempts = {
|
||||
per_second = 3;
|
||||
burst_count = 15;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
@ -1,40 +1,41 @@
|
||||
{ config, ... }:
|
||||
{ }
|
||||
|
||||
let
|
||||
domain = "clan.lol";
|
||||
in
|
||||
{
|
||||
services.opendkim.enable = true;
|
||||
services.opendkim.domains = domain;
|
||||
services.opendkim.selector = "v1";
|
||||
services.opendkim.user = config.services.postfix.user;
|
||||
services.opendkim.group = config.services.postfix.group;
|
||||
|
||||
# postfix configuration for sending emails only
|
||||
services.postfix = {
|
||||
enable = true;
|
||||
hostname = "mail.${domain}";
|
||||
inherit domain;
|
||||
|
||||
config = {
|
||||
smtp_tls_note_starttls_offer = "yes";
|
||||
|
||||
smtp_dns_support_level = "dnssec";
|
||||
smtp_tls_security_level = "dane";
|
||||
|
||||
tls_medium_cipherlist = "AES128+EECDH:AES128+EDH";
|
||||
|
||||
smtpd_relay_restrictions = "permit_mynetworks permit_sasl_authenticated defer_unauth_destination";
|
||||
mydestination = "localhost.$mydomain, localhost, $myhostname";
|
||||
myorigin = "$mydomain";
|
||||
|
||||
milter_default_action = "accept";
|
||||
milter_protocol = "6";
|
||||
smtpd_milters = "unix:/run/opendkim/opendkim.sock";
|
||||
non_smtpd_milters = "unix:/run/opendkim/opendkim.sock";
|
||||
|
||||
inet_interfaces = "loopback-only";
|
||||
inet_protocols = "all";
|
||||
};
|
||||
};
|
||||
}
|
||||
#{ config, ... }:
|
||||
#let
|
||||
# domain = "clan.lol";
|
||||
#in
|
||||
#{
|
||||
# services.opendkim.enable = true;
|
||||
# services.opendkim.domains = domain;
|
||||
# services.opendkim.selector = "v1";
|
||||
# services.opendkim.user = config.services.postfix.user;
|
||||
# services.opendkim.group = config.services.postfix.group;
|
||||
#
|
||||
# # postfix configuration for sending emails only
|
||||
# services.postfix = {
|
||||
# enable = true;
|
||||
# hostname = "mail.${domain}";
|
||||
# inherit domain;
|
||||
#
|
||||
# config = {
|
||||
# smtp_tls_note_starttls_offer = "yes";
|
||||
#
|
||||
# smtp_dns_support_level = "dnssec";
|
||||
# smtp_tls_security_level = "dane";
|
||||
#
|
||||
# tls_medium_cipherlist = "AES128+EECDH:AES128+EDH";
|
||||
#
|
||||
# smtpd_relay_restrictions = "permit_mynetworks permit_sasl_authenticated defer_unauth_destination";
|
||||
# mydestination = "localhost.$mydomain, localhost, $myhostname";
|
||||
# myorigin = "$mydomain";
|
||||
#
|
||||
# milter_default_action = "accept";
|
||||
# milter_protocol = "6";
|
||||
# smtpd_milters = "unix:/run/opendkim/opendkim.sock";
|
||||
# non_smtpd_milters = "unix:/run/opendkim/opendkim.sock";
|
||||
#
|
||||
# inet_interfaces = "loopback-only";
|
||||
# inet_protocols = "all";
|
||||
# };
|
||||
# };
|
||||
#}
|
||||
|
@ -1,4 +1,3 @@
|
||||
{ self, ... }:
|
||||
let
|
||||
mirrorBoot = idx: {
|
||||
type = "disk";
|
||||
@ -41,8 +40,14 @@ in
|
||||
efiSupport = true;
|
||||
efiInstallAsRemovable = true;
|
||||
mirroredBoots = [
|
||||
{ path = "/boot0"; devices = [ "nodev" ]; }
|
||||
{ path = "/boot1"; devices = [ "nodev" ]; }
|
||||
{
|
||||
path = "/boot0";
|
||||
devices = [ "nodev" ];
|
||||
}
|
||||
{
|
||||
path = "/boot1";
|
||||
devices = [ "nodev" ];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
|
@ -1,10 +1,19 @@
|
||||
{ bash
|
||||
, coreutils
|
||||
, git
|
||||
, tea
|
||||
, openssh
|
||||
, writePureShellScriptBin
|
||||
{
|
||||
bash,
|
||||
coreutils,
|
||||
git,
|
||||
tea,
|
||||
openssh,
|
||||
writePureShellScriptBin,
|
||||
}:
|
||||
writePureShellScriptBin "action-create-pr" [ bash coreutils git tea openssh ] ''
|
||||
bash ${./script.sh} "$@"
|
||||
''
|
||||
writePureShellScriptBin "action-create-pr"
|
||||
[
|
||||
bash
|
||||
coreutils
|
||||
git
|
||||
tea
|
||||
openssh
|
||||
]
|
||||
''
|
||||
bash ${./script.sh} "$@"
|
||||
''
|
||||
|
@ -1,8 +1,15 @@
|
||||
{ bash
|
||||
, coreutils
|
||||
, tea
|
||||
, writePureShellScriptBin
|
||||
{
|
||||
bash,
|
||||
coreutils,
|
||||
tea,
|
||||
writePureShellScriptBin,
|
||||
}:
|
||||
writePureShellScriptBin "action-ensure-tea-login" [ bash coreutils tea ] ''
|
||||
bash ${./script.sh}
|
||||
''
|
||||
writePureShellScriptBin "action-ensure-tea-login"
|
||||
[
|
||||
bash
|
||||
coreutils
|
||||
tea
|
||||
]
|
||||
''
|
||||
bash ${./script.sh}
|
||||
''
|
||||
|
@ -8,5 +8,5 @@ fi
|
||||
GITEA_TOKEN="${GITEA_TOKEN:-"$(cat "$GITEA_TOKEN_FILE")"}"
|
||||
|
||||
tea login add \
|
||||
--token $GITEA_TOKEN \
|
||||
--url $GITEA_URL
|
||||
--token "$GITEA_TOKEN" \
|
||||
--url "$GITEA_URL"
|
||||
|
29
pkgs/action-flake-update-pr-clan-individual/default.nix
Normal file
29
pkgs/action-flake-update-pr-clan-individual/default.nix
Normal file
@ -0,0 +1,29 @@
|
||||
{
|
||||
bash,
|
||||
coreutils,
|
||||
git,
|
||||
gnugrep,
|
||||
jq,
|
||||
nix,
|
||||
openssh,
|
||||
action-ensure-tea-login,
|
||||
action-create-pr,
|
||||
action-flake-update,
|
||||
writePureShellScriptBin,
|
||||
}:
|
||||
writePureShellScriptBin "action-flake-update-pr-clan-individual"
|
||||
[
|
||||
bash
|
||||
coreutils
|
||||
git
|
||||
gnugrep
|
||||
jq
|
||||
nix
|
||||
openssh
|
||||
action-create-pr
|
||||
action-ensure-tea-login
|
||||
action-flake-update
|
||||
]
|
||||
''
|
||||
bash ${./script.sh}
|
||||
''
|
47
pkgs/action-flake-update-pr-clan-individual/script.sh
Normal file
47
pkgs/action-flake-update-pr-clan-individual/script.sh
Normal file
@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# prevent these variables from being unset by writePureShellScript
|
||||
export KEEP_VARS="GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GITEA_URL GITEA_USER PR_TITLE REMOTE_BRANCH REPO_DIR${KEEP_VARS:+ $KEEP_VARS}"
|
||||
|
||||
# configure variables for actions
|
||||
today=$(date --iso-8601)
|
||||
today_minutes=$(date --iso-8601=minutes)
|
||||
export REPO_DIR=$TMPDIR/repo
|
||||
export GIT_AUTHOR_NAME="Clan Merge Bot"
|
||||
export GIT_AUTHOR_EMAIL="clan-bot@git.clan.lol"
|
||||
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
|
||||
export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_NAME"
|
||||
export GITEA_USER="clan-bot"
|
||||
export GITEA_URL="https://git.clan.lol"
|
||||
|
||||
|
||||
git clone --depth 1 --branch main "$REPO" "$REPO_DIR"
|
||||
cd "$REPO_DIR"
|
||||
|
||||
inputs=$(nix flake metadata --json | jq '.locks.nodes | keys[]' --raw-output | grep -v "root")
|
||||
|
||||
for input in $inputs;
|
||||
do
|
||||
target_branch="update-${input}"
|
||||
|
||||
echo "updating input: ${input}"
|
||||
echo "checking out: git checkout main"
|
||||
git checkout main
|
||||
git checkout -b "$target_branch"
|
||||
echo "checking out: git checkout -b update-${input}"
|
||||
export PR_TITLE="Automatic flake update - ${input} - ${today_minutes}"
|
||||
export REMOTE_BRANCH="flake-update-${input}-${today}"
|
||||
echo "action-ensure-tea-login"
|
||||
action-ensure-tea-login
|
||||
echo "action-flake-update: ${input}"
|
||||
action-flake-update "$input"
|
||||
echo "check diff"
|
||||
if git diff --quiet main.."$target_branch" --;then
|
||||
echo "No lockfile changes for input: ${input}"
|
||||
else
|
||||
echo "action-create-pr"
|
||||
action-create-pr --assignees clan-bot
|
||||
fi
|
||||
done
|
||||
|
@ -1,20 +1,23 @@
|
||||
{ bash
|
||||
, coreutils
|
||||
, git
|
||||
, openssh
|
||||
, action-ensure-tea-login
|
||||
, action-create-pr
|
||||
, action-flake-update
|
||||
, writePureShellScriptBin
|
||||
{
|
||||
bash,
|
||||
coreutils,
|
||||
git,
|
||||
openssh,
|
||||
action-ensure-tea-login,
|
||||
action-create-pr,
|
||||
action-flake-update,
|
||||
writePureShellScriptBin,
|
||||
}:
|
||||
writePureShellScriptBin "action-flake-update-pr-clan" [
|
||||
bash
|
||||
coreutils
|
||||
git
|
||||
openssh
|
||||
action-ensure-tea-login
|
||||
action-create-pr
|
||||
action-flake-update
|
||||
] ''
|
||||
bash ${./script.sh}
|
||||
''
|
||||
writePureShellScriptBin "action-flake-update-pr-clan"
|
||||
[
|
||||
bash
|
||||
coreutils
|
||||
git
|
||||
openssh
|
||||
action-ensure-tea-login
|
||||
action-create-pr
|
||||
action-flake-update
|
||||
]
|
||||
''
|
||||
bash ${./script.sh}
|
||||
''
|
||||
|
@ -5,8 +5,10 @@ set -euo pipefail
|
||||
export KEEP_VARS="GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GITEA_URL GITEA_USER PR_TITLE REMOTE_BRANCH REPO_DIR${KEEP_VARS:+ $KEEP_VARS}"
|
||||
|
||||
# configure variables for actions
|
||||
export PR_TITLE="Automatic flake update - $(date --iso-8601=minutes)"
|
||||
export REMOTE_BRANCH="flake-update-$(date --iso-8601)"
|
||||
today=$(date --iso-8601)
|
||||
today_minutes=$(date --iso-8601=minutes)
|
||||
export PR_TITLE="Automatic flake update - ${today_minutes}"
|
||||
export REMOTE_BRANCH="flake-update-${today}"
|
||||
export REPO_DIR=$TMPDIR/repo
|
||||
export GIT_AUTHOR_NAME="Clan Merge Bot"
|
||||
export GIT_AUTHOR_EMAIL="clan-bot@git.clan.lol"
|
||||
|
@ -1,9 +1,17 @@
|
||||
{ bash
|
||||
, coreutils
|
||||
, git
|
||||
, nix
|
||||
, writePureShellScriptBin
|
||||
{
|
||||
bash,
|
||||
coreutils,
|
||||
git,
|
||||
nix,
|
||||
writePureShellScriptBin,
|
||||
}:
|
||||
writePureShellScriptBin "action-flake-update" [ bash coreutils git nix ] ''
|
||||
bash ${./script.sh}
|
||||
''
|
||||
writePureShellScriptBin "action-flake-update"
|
||||
[
|
||||
bash
|
||||
coreutils
|
||||
git
|
||||
nix
|
||||
]
|
||||
''
|
||||
bash ${./script.sh} "$@"
|
||||
''
|
||||
|
@ -1,7 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
COMMIT_MSG="update flake lock - $(date --iso-8601=minutes)"
|
||||
NIX_VERSION=$(nix --version)
|
||||
echo "Nix version: $NIX_VERSION"
|
||||
|
||||
if [ -z "${*}" ];then
|
||||
COMMIT_MSG="update flake lock - $(date --iso-8601=minutes)"
|
||||
nix --experimental-features "nix-command flakes" \
|
||||
flake update --commit-lock-file --commit-lockfile-summary "$COMMIT_MSG"
|
||||
else
|
||||
# Support for ancient nix versions
|
||||
COMMIT_MSG="update flake lock - ${*} - $(date --iso-8601=minutes)"
|
||||
nix --experimental-features "nix-command flakes" \
|
||||
flake lock --commit-lock-file --commit-lockfile-summary "$COMMIT_MSG" --update-input "${@}"
|
||||
fi
|
||||
|
@ -1 +1,2 @@
|
||||
# shellcheck shell=bash
|
||||
use flake .#clan-merge
|
||||
|
@ -1,7 +1,7 @@
|
||||
import argparse
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from os import environ
|
||||
from typing import Optional
|
||||
|
||||
@ -38,6 +38,7 @@ def is_ci_green(pr: dict) -> bool:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_org_member(user: str, token: str) -> bool:
|
||||
url = "https://git.clan.lol/api/v1/orgs/clan/members/" + user + f"?token={token}"
|
||||
try:
|
||||
@ -50,7 +51,6 @@ def is_org_member(user: str, token: str) -> bool:
|
||||
raise
|
||||
|
||||
|
||||
|
||||
def merge_allowed(pr: dict, bot_name: str, token: str) -> bool:
|
||||
assignees = pr["assignees"] if pr["assignees"] else []
|
||||
if (
|
||||
|
@ -1,9 +1,9 @@
|
||||
{ pkgs ? import <nixpkgs> { }
|
||||
, lib ? pkgs.lib
|
||||
, python3 ? pkgs.python3
|
||||
, ruff ? pkgs.ruff
|
||||
, runCommand ? pkgs.runCommand
|
||||
,
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
lib ? pkgs.lib,
|
||||
python3 ? pkgs.python3,
|
||||
ruff ? pkgs.ruff,
|
||||
runCommand ? pkgs.runCommand,
|
||||
}:
|
||||
let
|
||||
pyproject = builtins.fromTOML (builtins.readFile ./pyproject.toml);
|
||||
@ -32,13 +32,11 @@ let
|
||||
package = python3.pkgs.buildPythonPackage {
|
||||
inherit name src;
|
||||
format = "pyproject";
|
||||
nativeBuildInputs = [
|
||||
python3.pkgs.setuptools
|
||||
];
|
||||
propagatedBuildInputs =
|
||||
dependencies
|
||||
++ [ ];
|
||||
passthru.tests = { inherit check; };
|
||||
nativeBuildInputs = [ python3.pkgs.setuptools ];
|
||||
propagatedBuildInputs = dependencies ++ [ ];
|
||||
passthru.tests = {
|
||||
inherit check;
|
||||
};
|
||||
passthru.devDependencies = devDependencies;
|
||||
};
|
||||
|
||||
|
@ -1,5 +1,6 @@
|
||||
{
|
||||
perSystem = { pkgs, ... }:
|
||||
perSystem =
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
package = pkgs.callPackage ./default.nix { inherit pkgs; };
|
||||
in
|
||||
|
@ -1,16 +1,11 @@
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
}:
|
||||
let
|
||||
inherit (pkgs) lib python3;
|
||||
package = import ./default.nix {
|
||||
inherit lib pkgs python3;
|
||||
};
|
||||
package = import ./default.nix { inherit lib pkgs python3; };
|
||||
pythonWithDeps = python3.withPackages (
|
||||
ps:
|
||||
package.propagatedBuildInputs
|
||||
++ package.devDependencies
|
||||
++ [
|
||||
ps.pip
|
||||
]
|
||||
ps: package.propagatedBuildInputs ++ package.devDependencies ++ [ ps.pip ]
|
||||
);
|
||||
checkScript = pkgs.writeScriptBin "check" ''
|
||||
nix build -f . tests -L "$@"
|
||||
|
@ -112,4 +112,6 @@ def test_list_prs_to_merge(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
assignees=[dict(login=bot_name)],
|
||||
),
|
||||
]
|
||||
assert clan_merge.list_prs_to_merge(prs, bot_name=bot_name, gitea_token="test") == [prs[0]]
|
||||
assert clan_merge.list_prs_to_merge(prs, bot_name=bot_name, gitea_token="test") == [
|
||||
prs[0]
|
||||
]
|
||||
|
@ -1,33 +1,46 @@
|
||||
{
|
||||
imports = [
|
||||
./clan-merge/flake-module.nix
|
||||
./matrix-bot/flake-module.nix
|
||||
];
|
||||
perSystem = { pkgs, config, ... }: {
|
||||
packages =
|
||||
let
|
||||
writers = pkgs.callPackage ./writers.nix { };
|
||||
in
|
||||
{
|
||||
inherit (pkgs.callPackage ./renovate { }) renovate;
|
||||
gitea = pkgs.callPackage ./gitea { };
|
||||
perSystem =
|
||||
{ pkgs, config, ... }:
|
||||
{
|
||||
packages =
|
||||
let
|
||||
writers = pkgs.callPackage ./writers.nix { };
|
||||
in
|
||||
{
|
||||
gitea = pkgs.callPackage ./gitea { };
|
||||
|
||||
action-create-pr = pkgs.callPackage ./action-create-pr {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
action-create-pr = pkgs.callPackage ./action-create-pr {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
};
|
||||
action-ensure-tea-login = pkgs.callPackage ./action-ensure-tea-login {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
};
|
||||
action-flake-update = pkgs.callPackage ./action-flake-update {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
};
|
||||
action-flake-update-pr-clan = pkgs.callPackage ./action-flake-update-pr-clan {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
inherit (config.packages) action-ensure-tea-login action-create-pr action-flake-update;
|
||||
};
|
||||
action-flake-update-pr-clan-individual = pkgs.callPackage ./action-flake-update-pr-clan-individual {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
inherit (config.packages) action-ensure-tea-login action-create-pr action-flake-update;
|
||||
};
|
||||
inherit
|
||||
(pkgs.callPackages ./job-flake-updates {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
inherit (config.packages) action-flake-update-pr-clan action-flake-update-pr-clan-individual;
|
||||
})
|
||||
job-flake-update-clan-core
|
||||
job-flake-update-clan-core-individual
|
||||
job-flake-update-clan-homepage
|
||||
job-flake-update-clan-infra
|
||||
job-flake-update-data-mesher
|
||||
;
|
||||
};
|
||||
action-ensure-tea-login = pkgs.callPackage ./action-ensure-tea-login {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
};
|
||||
action-flake-update = pkgs.callPackage ./action-flake-update {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
};
|
||||
action-flake-update-pr-clan = pkgs.callPackage ./action-flake-update-pr-clan {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
inherit (config.packages) action-ensure-tea-login action-create-pr action-flake-update;
|
||||
};
|
||||
inherit (pkgs.callPackages ./job-flake-updates {
|
||||
inherit (writers) writePureShellScriptBin;
|
||||
inherit (config.packages) action-flake-update-pr-clan;
|
||||
}) job-flake-update-clan-core job-flake-update-clan-homepage job-flake-update-clan-infra;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
@ -1,120 +0,0 @@
|
||||
From dd2ccf4ff923757b81088e27e362e3fdb222c9d3 Mon Sep 17 00:00:00 2001
|
||||
From: Jade Lovelace <software@lfcode.ca>
|
||||
Date: Tue, 28 May 2024 16:36:25 +0200
|
||||
Subject: [PATCH] Add an immutable tarball link to archive download headers for
|
||||
Nix
|
||||
MIME-Version: 1.0
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
Content-Transfer-Encoding: 8bit
|
||||
|
||||
This allows `nix flake metadata` and nix in general to lock a *branch*
|
||||
tarball link in a manner that causes it to fetch the correct commit even
|
||||
if the branch is updated with a newer version.
|
||||
|
||||
For further context, Nix flakes are a feature that, among other things,
|
||||
allows for "inputs" that are "github:someuser/somerepo",
|
||||
"https://some-tarball-service/some-tarball.tar.gz",
|
||||
"sourcehut:~meow/nya" or similar. This feature allows our users to fetch
|
||||
tarballs of git-based inputs to their builds rather than using git to
|
||||
fetch them, saving significant download time.
|
||||
|
||||
There is presently no gitea or forgejo specific fetcher in Nix, and we
|
||||
don't particularly wish to have one. Ideally (as a developer on a Nix
|
||||
implementation myself) we could just use the generic tarball fetcher and
|
||||
not add specific forgejo support, but to do so, we need additional
|
||||
metadata to know which commit a given *branch* tarball represents, which
|
||||
is the purpose of the Link header added here.
|
||||
|
||||
The result of this patch is that a Nix user can specify `inputs.something.url =
|
||||
"https://forgejo-host/some/project/archive/main.tar.gz"` in flake.nix
|
||||
and get a link to some concrete tarball for the actual commit in the
|
||||
lock file, then when they run `nix flake update` in the future, they
|
||||
will get the latest commit in that branch.
|
||||
|
||||
Example of it working locally:
|
||||
|
||||
» nix flake metadata --refresh 'http://localhost:3000/api/v1/repos/jade/cats/archive/main.tar.gz?dir=configs/nix'
|
||||
Resolved URL: http://localhost:3000/api/v1/repos/jade/cats/archive/main.tar.gz?dir=configs/nix
|
||||
Locked URL: http://localhost:3000/api/v1/repos/jade/cats/archive/804ede182b6b66469b23ea4d21eece52766b7a06.tar.gz?dir=configs
|
||||
/nix&narHash=sha256-yP7KkDVfuixZzs0fsqhSETXFC0y8m6nmPLw2GrAMxKQ%3D
|
||||
Description: Computers with the nixos
|
||||
Path: /nix/store/s856c6yqghyan4v0zy6jj19ksv0q22nx-source
|
||||
Revision: 804ede182b6b66469b23ea4d21eece52766b7a06
|
||||
Last modified: 2024-05-02 00:48:32
|
||||
|
||||
For details on the header value, see:
|
||||
https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
|
||||
|
||||
Signed-off-by: Jörg Thalheim <joerg@thalheim.io>
|
||||
---
|
||||
routers/api/v1/repo/file.go | 6 ++++++
|
||||
routers/web/repo/repo.go | 6 ++++++
|
||||
tests/integration/api_repo_archive_test.go | 11 +++++++++++
|
||||
3 files changed, 23 insertions(+)
|
||||
|
||||
diff --git a/routers/api/v1/repo/file.go b/routers/api/v1/repo/file.go
|
||||
index 156033f58a..b7ad63af08 100644
|
||||
--- a/routers/api/v1/repo/file.go
|
||||
+++ b/routers/api/v1/repo/file.go
|
||||
@@ -319,6 +319,12 @@ func archiveDownload(ctx *context.APIContext) {
|
||||
func download(ctx *context.APIContext, archiveName string, archiver *repo_model.RepoArchiver) {
|
||||
downloadName := ctx.Repo.Repository.Name + "-" + archiveName
|
||||
|
||||
+ // Add nix format link header so tarballs lock correctly:
|
||||
+ // https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
|
||||
+ ctx.Resp.Header().Add("Link", fmt.Sprintf("<%s/archive/%s.tar.gz?rev=%s>; rel=\"immutable\"",
|
||||
+ ctx.Repo.Repository.APIURL(),
|
||||
+ archiver.CommitID, archiver.CommitID))
|
||||
+
|
||||
rPath := archiver.RelativePath()
|
||||
if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
|
||||
// If we have a signed url (S3, object storage), redirect to this directly.
|
||||
diff --git a/routers/web/repo/repo.go b/routers/web/repo/repo.go
|
||||
index 71c582b5f9..bb6349658f 100644
|
||||
--- a/routers/web/repo/repo.go
|
||||
+++ b/routers/web/repo/repo.go
|
||||
@@ -484,6 +484,12 @@ func Download(ctx *context.Context) {
|
||||
func download(ctx *context.Context, archiveName string, archiver *repo_model.RepoArchiver) {
|
||||
downloadName := ctx.Repo.Repository.Name + "-" + archiveName
|
||||
|
||||
+ // Add nix format link header so tarballs lock correctly:
|
||||
+ // https://github.com/nixos/nix/blob/56763ff918eb308db23080e560ed2ea3e00c80a7/doc/manual/src/protocols/tarball-fetcher.md
|
||||
+ ctx.Resp.Header().Add("Link", fmt.Sprintf("<%s/archive/%s.tar.gz?rev=%s>; rel=\"immutable\"",
|
||||
+ ctx.Repo.Repository.APIURL(),
|
||||
+ archiver.CommitID, archiver.CommitID))
|
||||
+
|
||||
rPath := archiver.RelativePath()
|
||||
if setting.RepoArchive.Storage.MinioConfig.ServeDirect {
|
||||
// If we have a signed url (S3, object storage), redirect to this directly.
|
||||
diff --git a/tests/integration/api_repo_archive_test.go b/tests/integration/api_repo_archive_test.go
|
||||
index 57d3abfe84..340ff03961 100644
|
||||
--- a/tests/integration/api_repo_archive_test.go
|
||||
+++ b/tests/integration/api_repo_archive_test.go
|
||||
@@ -8,6 +8,7 @@
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
+ "regexp"
|
||||
"testing"
|
||||
|
||||
auth_model "code.gitea.io/gitea/models/auth"
|
||||
@@ -39,6 +40,16 @@ func TestAPIDownloadArchive(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, bs, 266)
|
||||
|
||||
+ // Must return a link to a commit ID as the "immutable" archive link
|
||||
+ linkHeaderRe := regexp.MustCompile(`<(?P<url>https?://.*/api/v1/repos/user2/repo1/archive/[a-f0-9]+\.tar\.gz.*)>; rel="immutable"`)
|
||||
+ m := linkHeaderRe.FindStringSubmatch(resp.Header().Get("Link"))
|
||||
+ assert.NotEmpty(t, m[1])
|
||||
+ resp = MakeRequest(t, NewRequest(t, "GET", m[1]).AddTokenAuth(token), http.StatusOK)
|
||||
+ bs2, err := io.ReadAll(resp.Body)
|
||||
+ assert.NoError(t, err)
|
||||
+ // The locked URL should give the same bytes as the non-locked one
|
||||
+ assert.EqualValues(t, bs, bs2)
|
||||
+
|
||||
link, _ = url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/archive/master.bundle", user2.Name, repo.Name))
|
||||
resp = MakeRequest(t, NewRequest(t, "GET", link.String()).AddTokenAuth(token), http.StatusOK)
|
||||
bs, err = io.ReadAll(resp.Body)
|
||||
--
|
||||
2.44.1
|
||||
|
@ -1,8 +1,5 @@
|
||||
{ gitea }:
|
||||
|
||||
gitea.overrideAttrs (old: {
|
||||
patches = old.patches ++ [
|
||||
./0001-add-bot-check.patch
|
||||
./0001-Add-an-immutable-tarball-link-to-archive-download-he.patch
|
||||
];
|
||||
patches = old.patches ++ [ ./0001-add-bot-check.patch ];
|
||||
})
|
||||
|
@ -1,16 +1,32 @@
|
||||
{ action-flake-update-pr-clan
|
||||
, writePureShellScriptBin
|
||||
{
|
||||
action-flake-update-pr-clan,
|
||||
action-flake-update-pr-clan-individual,
|
||||
writePureShellScriptBin,
|
||||
}:
|
||||
let
|
||||
job-flake-update = repo: writePureShellScriptBin "job-flake-update-${repo}" [ action-flake-update-pr-clan ] ''
|
||||
export REPO="gitea@git.clan.lol:clan/${repo}.git"
|
||||
export KEEP_VARS="REPO''${KEEP_VARS:+ $KEEP_VARS}"
|
||||
job-flake-update =
|
||||
repo:
|
||||
writePureShellScriptBin "job-flake-update-${repo}" [ action-flake-update-pr-clan ] ''
|
||||
export REPO="gitea@git.clan.lol:clan/${repo}.git"
|
||||
export KEEP_VARS="REPO''${KEEP_VARS:+ $KEEP_VARS}"
|
||||
|
||||
action-flake-update-pr-clan
|
||||
'';
|
||||
action-flake-update-pr-clan
|
||||
'';
|
||||
job-flake-update-individual =
|
||||
repo:
|
||||
writePureShellScriptBin "job-flake-update-${repo}-individual"
|
||||
[ action-flake-update-pr-clan-individual ]
|
||||
''
|
||||
export REPO="gitea@git.clan.lol:clan/${repo}.git"
|
||||
export KEEP_VARS="REPO''${KEEP_VARS:+ $KEEP_VARS}"
|
||||
|
||||
action-flake-update-pr-clan-individual
|
||||
'';
|
||||
in
|
||||
{
|
||||
job-flake-update-clan-core = job-flake-update "clan-core";
|
||||
job-flake-update-clan-core-individual = job-flake-update-individual "clan-core";
|
||||
job-flake-update-clan-homepage = job-flake-update "clan-homepage";
|
||||
job-flake-update-clan-infra = job-flake-update "clan-infra";
|
||||
job-flake-update-data-mesher = job-flake-update "data-mesher";
|
||||
}
|
||||
|
7
pkgs/matrix-bot/.envrc
Normal file
7
pkgs/matrix-bot/.envrc
Normal file
@ -0,0 +1,7 @@
|
||||
# shellcheck shell=bash
|
||||
source_up
|
||||
|
||||
watch_file flake-module.nix shell.nix default.nix
|
||||
|
||||
# Because we depend on nixpkgs sources, uploading to builders takes a long time
|
||||
use flake .#matrix-bot --builders ''
|
3
pkgs/matrix-bot/.gitignore
vendored
Normal file
3
pkgs/matrix-bot/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
*.json
|
||||
**/data
|
||||
**/__pycache__
|
13
pkgs/matrix-bot/bin/mbot
Executable file
13
pkgs/matrix-bot/bin/mbot
Executable file
@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(
|
||||
0, os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
|
||||
)
|
||||
|
||||
from matrix_bot import main # NOQA
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
44
pkgs/matrix-bot/default.nix
Normal file
44
pkgs/matrix-bot/default.nix
Normal file
@ -0,0 +1,44 @@
|
||||
{
|
||||
python3,
|
||||
setuptools,
|
||||
matrix-nio,
|
||||
aiofiles,
|
||||
aiohttp,
|
||||
markdown2,
|
||||
git,
|
||||
tiktoken,
|
||||
...
|
||||
}:
|
||||
|
||||
let
|
||||
|
||||
pythonDependencies = [
|
||||
matrix-nio
|
||||
aiofiles
|
||||
aiohttp
|
||||
markdown2
|
||||
tiktoken
|
||||
];
|
||||
|
||||
runtimeDependencies = [ git ];
|
||||
|
||||
testDependencies = pythonDependencies ++ runtimeDependencies ++ [ ];
|
||||
in
|
||||
python3.pkgs.buildPythonApplication {
|
||||
name = "matrix-bot";
|
||||
src = ./.;
|
||||
format = "pyproject";
|
||||
|
||||
nativeBuildInputs = [ setuptools ];
|
||||
|
||||
propagatedBuildInputs = pythonDependencies ++ runtimeDependencies;
|
||||
|
||||
passthru.testDependencies = testDependencies;
|
||||
|
||||
# Clean up after the package to avoid leaking python packages into a devshell
|
||||
postFixup = ''
|
||||
rm $out/nix-support/propagated-build-inputs
|
||||
'';
|
||||
|
||||
meta.mainProgram = "matrix-bot";
|
||||
}
|
14
pkgs/matrix-bot/flake-module.nix
Normal file
14
pkgs/matrix-bot/flake-module.nix
Normal file
@ -0,0 +1,14 @@
|
||||
{ ... }:
|
||||
{
|
||||
perSystem =
|
||||
{ self', pkgs, ... }:
|
||||
{
|
||||
|
||||
devShells.matrix-bot = pkgs.callPackage ./shell.nix { inherit (self'.packages) matrix-bot; };
|
||||
packages = {
|
||||
matrix-bot = pkgs.python3.pkgs.callPackage ./default.nix { };
|
||||
};
|
||||
|
||||
checks = { };
|
||||
};
|
||||
}
|
214
pkgs/matrix-bot/matrix_bot/__init__.py
Normal file
214
pkgs/matrix-bot/matrix_bot/__init__.py
Normal file
@ -0,0 +1,214 @@
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from os import environ
|
||||
from pathlib import Path
|
||||
|
||||
from matrix_bot.bot_conf import BotConfig
|
||||
from matrix_bot.custom_logger import setup_logging
|
||||
from matrix_bot.gitea import GiteaData
|
||||
from matrix_bot.main import bot_main
|
||||
from matrix_bot.matrix import MatrixData
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
curr_dir = Path(__file__).parent
|
||||
data_dir = Path(os.getcwd()) / "data"
|
||||
|
||||
|
||||
def create_parser(prog: str | None = None) -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog=prog,
|
||||
description="A gitea bot for matrix",
|
||||
formatter_class=argparse.RawTextHelpFormatter,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--debug",
|
||||
help="Enable debug logging",
|
||||
action="store_true",
|
||||
default=False,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--server",
|
||||
help="The matrix server to connect to",
|
||||
default="https://matrix.clan.lol",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--admin",
|
||||
help="The matrix user to ping on error",
|
||||
default="@qubasa:gchq.icu",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--user",
|
||||
help="The matrix user to connect as",
|
||||
default="@clan-bot:clan.lol",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--avatar",
|
||||
help="The path to the image to use as the avatar",
|
||||
default=curr_dir / "avatar.png",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--repo-owner",
|
||||
help="The owner of gitea the repository",
|
||||
default="clan",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--repo-name",
|
||||
help="The name of the repository",
|
||||
default="clan-core",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--changelog-room",
|
||||
help="The matrix room to join for the changelog bot",
|
||||
default="#bot-test:gchq.icu",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--review-room",
|
||||
help="The matrix room to join for the review bot",
|
||||
default="#bot-test:gchq.icu",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--changelog-frequency",
|
||||
help="The frequency to check for changelog updates in days",
|
||||
default=7,
|
||||
type=int,
|
||||
)
|
||||
|
||||
def valid_weekday(value: str) -> str:
|
||||
days = [
|
||||
"Monday",
|
||||
"Tuesday",
|
||||
"Wednesday",
|
||||
"Thursday",
|
||||
"Friday",
|
||||
"Saturday",
|
||||
"Sunday",
|
||||
]
|
||||
if value not in days:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"{value} is not a valid weekday. Choose from {', '.join(days)}"
|
||||
)
|
||||
return value
|
||||
|
||||
parser.add_argument(
|
||||
"--publish-day",
|
||||
help="The day of the week to publish the changelog. Ignored if changelog-frequency is less than 7 days.",
|
||||
default="Wednesday",
|
||||
type=valid_weekday,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--gitea-url",
|
||||
help="The gitea url to connect to",
|
||||
default="https://git.clan.lol",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--data-dir",
|
||||
help="The directory to store data",
|
||||
default=data_dir,
|
||||
type=Path,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--poll-frequency",
|
||||
help="The frequency to poll for new reviews in minutes",
|
||||
default=10,
|
||||
type=float,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--disable-pr-bot",
|
||||
help="Disable the PR bot",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-changelog-bot",
|
||||
help="Disable the changelog bot",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pull-mention-every-update",
|
||||
help="Mention the user every time the PR is updated",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mention-labels",
|
||||
help="The labels that trigger a mention",
|
||||
nargs="+",
|
||||
default=["needs-review"],
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def matrix_password() -> str:
|
||||
matrix_password = environ.get("MATRIX_PASSWORD")
|
||||
if matrix_password is not None:
|
||||
return matrix_password
|
||||
matrix_password_file = environ.get("MATRIX_PASSWORD_FILE", default=None)
|
||||
if matrix_password_file is None:
|
||||
raise Exception("MATRIX_PASSWORD_FILE environment variable is not set")
|
||||
with open(matrix_password_file) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = create_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.debug:
|
||||
setup_logging(logging.DEBUG, root_log_name=__name__.split(".")[0])
|
||||
log.debug("Debug log activated")
|
||||
else:
|
||||
setup_logging(logging.INFO, root_log_name=__name__.split(".")[0])
|
||||
|
||||
bot_conf = BotConfig(
|
||||
data_dir=args.data_dir,
|
||||
disable_pr_bot=args.disable_pr_bot,
|
||||
disable_changelog_bot=args.disable_changelog_bot,
|
||||
)
|
||||
|
||||
matrix = MatrixData(
|
||||
server=args.server,
|
||||
user=args.user,
|
||||
avatar=args.avatar,
|
||||
changelog_room=args.changelog_room,
|
||||
changelog_frequency=args.changelog_frequency,
|
||||
publish_day=args.publish_day,
|
||||
review_room=args.review_room,
|
||||
password=matrix_password(),
|
||||
admin=args.admin,
|
||||
)
|
||||
|
||||
gitea = GiteaData(
|
||||
url=args.gitea_url,
|
||||
owner=args.repo_owner,
|
||||
repo=args.repo_name,
|
||||
access_token=os.getenv("GITEA_ACCESS_TOKEN"),
|
||||
poll_frequency=args.poll_frequency,
|
||||
mention_on_update=args.pull_mention_every_update,
|
||||
mention_labels=args.mention_labels,
|
||||
)
|
||||
|
||||
bot_conf.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
asyncio.run(bot_main(matrix, gitea, bot_conf))
|
||||
except KeyboardInterrupt:
|
||||
print("User Interrupt", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
4
pkgs/matrix-bot/matrix_bot/__main__.py
Normal file
4
pkgs/matrix-bot/matrix_bot/__main__.py
Normal file
@ -0,0 +1,4 @@
|
||||
from . import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
BIN
pkgs/matrix-bot/matrix_bot/avatar.png
Normal file
BIN
pkgs/matrix-bot/matrix_bot/avatar.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 105 KiB |
9
pkgs/matrix-bot/matrix_bot/bot_conf.py
Normal file
9
pkgs/matrix-bot/matrix_bot/bot_conf.py
Normal file
@ -0,0 +1,9 @@
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass
|
||||
class BotConfig:
|
||||
data_dir: Path
|
||||
disable_pr_bot: bool
|
||||
disable_changelog_bot: bool
|
288
pkgs/matrix-bot/matrix_bot/changelog_bot.py
Normal file
288
pkgs/matrix-bot/matrix_bot/changelog_bot.py
Normal file
@ -0,0 +1,288 @@
|
||||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
from nio import (
|
||||
AsyncClient,
|
||||
JoinResponse,
|
||||
)
|
||||
|
||||
from matrix_bot.gitea import (
|
||||
GiteaData,
|
||||
)
|
||||
|
||||
from . import BotConfig
|
||||
from .locked_open import read_locked_file, write_locked_file
|
||||
from .matrix import MatrixData, send_message
|
||||
from .openai import create_jsonl_data, upload_and_process_files
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def last_ndays_to_today(ndays: int) -> tuple[str, str]:
|
||||
# Get today's date
|
||||
today = datetime.datetime.now()
|
||||
|
||||
# Calculate the date one week ago
|
||||
last_week = today - datetime.timedelta(days=ndays)
|
||||
|
||||
# Format both dates to "YYYY-MM-DD"
|
||||
todate = today.strftime("%Y-%m-%d")
|
||||
fromdate = last_week.strftime("%Y-%m-%d")
|
||||
|
||||
return (fromdate, todate)
|
||||
|
||||
|
||||
def write_file_with_date_prefix(
|
||||
content: str, directory: Path, *, ndays: int, suffix: str
|
||||
) -> Path:
|
||||
"""
|
||||
Write content to a file with the current date as filename prefix.
|
||||
|
||||
:param content: The content to write to the file.
|
||||
:param directory: The directory where the file will be saved.
|
||||
:return: The path to the created file.
|
||||
"""
|
||||
# Ensure the directory exists
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Get the current date
|
||||
fromdate, todate = last_ndays_to_today(ndays)
|
||||
|
||||
# Create the filename
|
||||
filename = f"{fromdate}__{todate}_{suffix}.txt"
|
||||
file_path = directory / filename
|
||||
|
||||
# Write the content to the file
|
||||
with open(file_path, "w") as file:
|
||||
file.write(content)
|
||||
|
||||
return file_path
|
||||
|
||||
|
||||
async def git_pull(repo_path: Path) -> None:
|
||||
cmd = ["git", "pull"]
|
||||
log.debug(f"Running command: {shlex.join(cmd)}")
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
cwd=str(repo_path),
|
||||
)
|
||||
await process.wait()
|
||||
|
||||
|
||||
async def git_log(repo_path: Path, ndays: int) -> str:
|
||||
cmd = [
|
||||
"git",
|
||||
"log",
|
||||
f"--since={ndays} days ago",
|
||||
"--pretty=format:%h - %an, %ar : %s",
|
||||
"--stat",
|
||||
"--patch",
|
||||
]
|
||||
log.debug(f"Running command: {shlex.join(cmd)}")
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
cwd=str(repo_path),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
raise Exception(
|
||||
f"Command '{' '.join(cmd)}' failed with exit code {process.returncode}"
|
||||
)
|
||||
|
||||
return stdout.decode()
|
||||
|
||||
|
||||
async def changelog_bot(
|
||||
client: AsyncClient,
|
||||
http: aiohttp.ClientSession,
|
||||
matrix: MatrixData,
|
||||
gitea: GiteaData,
|
||||
bot_conf: BotConfig,
|
||||
) -> None:
|
||||
last_run_path = bot_conf.data_dir / "last_changelog_run.json"
|
||||
last_run = read_locked_file(last_run_path)
|
||||
|
||||
today = datetime.datetime.now()
|
||||
today_weekday = today.strftime("%A")
|
||||
|
||||
if today_weekday != matrix.publish_day:
|
||||
log.debug(f"Changelog not due yet. Due on {matrix.publish_day}")
|
||||
return
|
||||
|
||||
if last_run == {}:
|
||||
log.debug(f"First run. Setting last_run to {last_run}")
|
||||
else:
|
||||
last_date = datetime.datetime.strptime(last_run["todate"], "%Y-%m-%d")
|
||||
upper_bound = datetime.timedelta(days=matrix.changelog_frequency)
|
||||
delta = today - last_date
|
||||
if delta <= upper_bound:
|
||||
log.debug(
|
||||
f"Changelog not due yet. Due in {upper_bound.days - delta.days} days"
|
||||
)
|
||||
return
|
||||
|
||||
fromdate, todate = last_ndays_to_today(matrix.changelog_frequency)
|
||||
last_run = {
|
||||
"fromdate": fromdate,
|
||||
"todate": todate,
|
||||
"ndays": matrix.changelog_frequency,
|
||||
}
|
||||
|
||||
# If you made a new room and haven't joined as that user, you can use
|
||||
room: JoinResponse = await client.join(matrix.changelog_room)
|
||||
|
||||
if not room.transport_response.ok:
|
||||
log.error("This can happen if the room doesn't exist or the bot isn't invited")
|
||||
raise Exception(f"Failed to join room {room}")
|
||||
|
||||
repo_path = bot_conf.data_dir / gitea.repo
|
||||
|
||||
if not repo_path.exists():
|
||||
cmd = [
|
||||
"git",
|
||||
"clone",
|
||||
f"{gitea.url}/{gitea.owner}/{gitea.repo}.git",
|
||||
gitea.repo,
|
||||
]
|
||||
subprocess.run(cmd, cwd=bot_conf.data_dir, check=True)
|
||||
|
||||
# git pull
|
||||
await git_pull(repo_path)
|
||||
|
||||
# git log
|
||||
diff = await git_log(repo_path, matrix.changelog_frequency)
|
||||
|
||||
fromdate, todate = last_ndays_to_today(matrix.changelog_frequency)
|
||||
log.info(f"Generating changelog from {fromdate} to {todate}")
|
||||
|
||||
# Write the last run to the file before processing the changelog
|
||||
# This ensures that the changelog is only generated once per period
|
||||
# even if openai fails
|
||||
write_locked_file(last_run_path, last_run)
|
||||
|
||||
system_prompt = f"""
|
||||
Follow these guidelines:
|
||||
|
||||
- Follow the pull request format: "scope: message (#number1, #number2)"
|
||||
- Don't use the commit messages tied to a pull request as is and instead explain the change in a user-friendly way
|
||||
- Link pull requests as: '{gitea.url}/{gitea.owner}/{gitea.repo}/pulls/<number>'
|
||||
- Use markdown links to make the pull request number clickable
|
||||
- Mention each pull request number at most once
|
||||
- Group similar changes / pull requests together
|
||||
- Explain changes in a user-friendly way (be detailed if necessary)
|
||||
- Always use four '#' for headings never less than that. Example: `####New Features`
|
||||
- WRITE IN THE STYLE OF THE NEW YORK TIMES, PLEASE!
|
||||
---
|
||||
Example Changelog:
|
||||
#### Changelog:
|
||||
For the last {matrix.changelog_frequency} days from {fromdate} to {todate}
|
||||
#### New Features
|
||||
- `secrets`: [#1679]({gitea.url}/{gitea.owner}/{gitea.repo}/pulls/1679)
|
||||
> Users can now generate secrets and manage settings in the new submodules
|
||||
> This feature is available to all users with the 'admin' role
|
||||
- `sshd`: [#1674]({gitea.url}/{gitea.owner}/{gitea.repo}/pulls/1674)
|
||||
> A workaround has been added to mitigate the security vulnerability in the sshd module
|
||||
> This workaround is temporary and will be replaced with a permanent fix in the next release
|
||||
...
|
||||
#### Refactoring
|
||||
...
|
||||
#### Documentation
|
||||
...
|
||||
#### Bug Fixes
|
||||
...
|
||||
#### Additional Notes
|
||||
...
|
||||
---
|
||||
#### Changelog:
|
||||
For the last {matrix.changelog_frequency} days from {fromdate} to {todate}
|
||||
#### New Features
|
||||
"""
|
||||
|
||||
# Step 1: Create the JSONL file
|
||||
jsonl_files = await create_jsonl_data(user_prompt=diff, system_prompt=system_prompt)
|
||||
|
||||
# Step 2: Upload the JSONL file and process it
|
||||
results = await upload_and_process_files(session=http, jsonl_files=jsonl_files)
|
||||
|
||||
# Join responses together
|
||||
all_changelogs = []
|
||||
for result in results:
|
||||
choices = result["response"]["body"]["choices"]
|
||||
changelog = "\n".join(choice["message"]["content"] for choice in choices)
|
||||
all_changelogs.append(changelog)
|
||||
full_changelog = "\n\n".join(all_changelogs)
|
||||
|
||||
log.debug(f"Changelog generated:\n{full_changelog}")
|
||||
|
||||
if len(results) == 1:
|
||||
# Write the results to a file in the changelogs directory
|
||||
new_result_file = write_file_with_date_prefix(
|
||||
json.dumps(results, indent=4),
|
||||
bot_conf.data_dir / "changelogs",
|
||||
ndays=matrix.changelog_frequency,
|
||||
suffix="result",
|
||||
)
|
||||
log.info(f"LLM result written to: {new_result_file}")
|
||||
|
||||
await send_message(client, room, full_changelog)
|
||||
return
|
||||
|
||||
combine_prompt = """
|
||||
Please combine the following changelogs into a single markdown changelog.
|
||||
- Merge duplicates sections.
|
||||
- Make sure the changelog is concise and easy to read.
|
||||
- Always use four '#' for headings never less than that. Example: `####New Features`
|
||||
- WRITE IN THE STYLE OF THE NEW YORK TIMES, PLEASE!
|
||||
---
|
||||
Example Changelog:
|
||||
#### Changelog:
|
||||
For the last {matrix.changelog_frequency} days from {fromdate} to {todate}
|
||||
#### New Features
|
||||
...
|
||||
#### Refactoring
|
||||
...
|
||||
#### Documentation
|
||||
...
|
||||
#### Bug Fixes
|
||||
...
|
||||
#### Additional Notes
|
||||
...
|
||||
---
|
||||
#### Changelog:
|
||||
For the last {matrix.changelog_frequency} days from {fromdate} to {todate}
|
||||
#### New Features
|
||||
"""
|
||||
new_jsonl_files = await create_jsonl_data(
|
||||
user_prompt=full_changelog, system_prompt=combine_prompt
|
||||
)
|
||||
new_results = await upload_and_process_files(
|
||||
session=http, jsonl_files=new_jsonl_files
|
||||
)
|
||||
|
||||
new_all_changelogs = []
|
||||
for result in new_results:
|
||||
choices = result["response"]["body"]["choices"]
|
||||
changelog = "\n".join(choice["message"]["content"] for choice in choices)
|
||||
new_all_changelogs.append(changelog)
|
||||
new_full_changelog = "\n\n".join(new_all_changelogs)
|
||||
log.info(f"Changelog generated:\n{new_full_changelog}")
|
||||
|
||||
# Write the results to a file in the changelogs directory
|
||||
new_result_file = write_file_with_date_prefix(
|
||||
json.dumps(new_results, indent=4),
|
||||
bot_conf.data_dir / "changelogs",
|
||||
ndays=matrix.changelog_frequency,
|
||||
suffix="result",
|
||||
)
|
||||
log.info(f"LLM result written to: {new_result_file}")
|
||||
|
||||
await send_message(client, room, new_full_changelog)
|
97
pkgs/matrix-bot/matrix_bot/custom_logger.py
Normal file
97
pkgs/matrix-bot/matrix_bot/custom_logger.py
Normal file
@ -0,0 +1,97 @@
|
||||
import inspect
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
grey = "\x1b[38;20m"
|
||||
yellow = "\x1b[33;20m"
|
||||
red = "\x1b[31;20m"
|
||||
bold_red = "\x1b[31;1m"
|
||||
green = "\u001b[32m"
|
||||
blue = "\u001b[34m"
|
||||
|
||||
|
||||
def get_formatter(color: str) -> Callable[[logging.LogRecord, bool], logging.Formatter]:
|
||||
def myformatter(
|
||||
record: logging.LogRecord, with_location: bool
|
||||
) -> logging.Formatter:
|
||||
reset = "\x1b[0m"
|
||||
|
||||
try:
|
||||
filepath = Path(record.pathname).resolve()
|
||||
filepath = Path("~", filepath.relative_to(Path.home()))
|
||||
except Exception:
|
||||
filepath = Path(record.pathname)
|
||||
|
||||
if not with_location:
|
||||
return logging.Formatter(f"{color}%(levelname)s{reset}: %(message)s")
|
||||
|
||||
return logging.Formatter(
|
||||
f"{color}%(levelname)s{reset}: %(message)s\nLocation: {filepath}:%(lineno)d::%(funcName)s\n"
|
||||
)
|
||||
|
||||
return myformatter
|
||||
|
||||
|
||||
FORMATTER = {
|
||||
logging.DEBUG: get_formatter(blue),
|
||||
logging.INFO: get_formatter(green),
|
||||
logging.WARNING: get_formatter(yellow),
|
||||
logging.ERROR: get_formatter(red),
|
||||
logging.CRITICAL: get_formatter(bold_red),
|
||||
}
|
||||
|
||||
|
||||
class CustomFormatter(logging.Formatter):
|
||||
def __init__(self, log_locations: bool) -> None:
|
||||
super().__init__()
|
||||
self.log_locations = log_locations
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
return FORMATTER[record.levelno](record, self.log_locations).format(record)
|
||||
|
||||
|
||||
class ThreadFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
return FORMATTER[record.levelno](record, False).format(record)
|
||||
|
||||
|
||||
def get_caller() -> str:
|
||||
frame = inspect.currentframe()
|
||||
if frame is None:
|
||||
return "unknown"
|
||||
caller_frame = frame.f_back
|
||||
if caller_frame is None:
|
||||
return "unknown"
|
||||
caller_frame = caller_frame.f_back
|
||||
if caller_frame is None:
|
||||
return "unknown"
|
||||
frame_info = inspect.getframeinfo(caller_frame)
|
||||
|
||||
try:
|
||||
filepath = Path(frame_info.filename).resolve()
|
||||
filepath = Path("~", filepath.relative_to(Path.home()))
|
||||
except Exception:
|
||||
filepath = Path(frame_info.filename)
|
||||
|
||||
ret = f"{filepath}:{frame_info.lineno}::{frame_info.function}"
|
||||
return ret
|
||||
|
||||
|
||||
def setup_logging(level: Any, root_log_name: str = __name__.split(".")[0]) -> None:
|
||||
# Get the root logger and set its level
|
||||
main_logger = logging.getLogger(root_log_name)
|
||||
main_logger.setLevel(level)
|
||||
|
||||
# Create and add the default handler
|
||||
default_handler = logging.StreamHandler()
|
||||
|
||||
# Create and add your custom handler
|
||||
default_handler.setLevel(level)
|
||||
default_handler.setFormatter(CustomFormatter(str(level) == str(logging.DEBUG)))
|
||||
main_logger.addHandler(default_handler)
|
||||
|
||||
# Set logging level for other modules used by this module
|
||||
logging.getLogger("asyncio").setLevel(logging.INFO)
|
||||
logging.getLogger("httpx").setLevel(level=logging.WARNING)
|
91
pkgs/matrix-bot/matrix_bot/gitea.py
Normal file
91
pkgs/matrix-bot/matrix_bot/gitea.py
Normal file
@ -0,0 +1,91 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
import aiohttp
|
||||
|
||||
|
||||
@dataclass
|
||||
class GiteaData:
|
||||
url: str
|
||||
owner: str
|
||||
repo: str
|
||||
poll_frequency: float
|
||||
mention_on_update: bool
|
||||
mention_labels: list[dict]
|
||||
access_token: str | None
|
||||
|
||||
|
||||
def endpoint_url(gitea: GiteaData, endpoint: str) -> str:
|
||||
return f"{gitea.url}/api/v1/repos/{gitea.owner}/{gitea.repo}/{endpoint}"
|
||||
|
||||
|
||||
async def fetch_repo_labels(
|
||||
gitea: GiteaData,
|
||||
session: aiohttp.ClientSession,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Fetch labels from a Gitea repository.
|
||||
|
||||
Returns:
|
||||
list: List of labels in the repository.
|
||||
"""
|
||||
url = endpoint_url(gitea, "labels")
|
||||
headers = {"Accept": "application/vnd.github.v3+json"}
|
||||
if gitea.access_token:
|
||||
headers["Authorization"] = f"token {gitea.access_token}"
|
||||
|
||||
async with session.get(url, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
labels = await response.json()
|
||||
return labels
|
||||
else:
|
||||
# You may want to handle different statuses differently
|
||||
raise Exception(
|
||||
f"Failed to fetch labels: {response.status}, {await response.text()}"
|
||||
)
|
||||
|
||||
|
||||
class PullState(Enum):
|
||||
OPEN = "open"
|
||||
CLOSED = "closed"
|
||||
ALL = "all"
|
||||
|
||||
|
||||
async def fetch_pull_requests(
|
||||
gitea: GiteaData,
|
||||
session: aiohttp.ClientSession,
|
||||
*,
|
||||
limit: int,
|
||||
state: PullState,
|
||||
label_ids: list[int] = [],
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Fetch pull requests from a Gitea repository.
|
||||
|
||||
Returns:
|
||||
list: List of pull requests.
|
||||
"""
|
||||
# You can use the same pattern as fetch_repo_labels
|
||||
url = endpoint_url(gitea, "pulls")
|
||||
params = {
|
||||
"state": state.value,
|
||||
"sort": "recentupdate",
|
||||
"limit": limit,
|
||||
"labels": label_ids,
|
||||
}
|
||||
headers = {"accept": "application/json"}
|
||||
|
||||
async with session.get(url, params=params, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
labels = await response.json()
|
||||
return labels
|
||||
else:
|
||||
# You may want to handle different statuses differently
|
||||
raise Exception(
|
||||
f"Failed to fetch labels: {response.status}, {await response.text()}"
|
||||
)
|
31
pkgs/matrix-bot/matrix_bot/locked_open.py
Normal file
31
pkgs/matrix-bot/matrix_bot/locked_open.py
Normal file
@ -0,0 +1,31 @@
|
||||
import fcntl
|
||||
import json
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@contextmanager
|
||||
def locked_open(filename: str | Path, mode: str = "r") -> Generator:
|
||||
"""
|
||||
This is a context manager that provides an advisory write lock on the file specified by `filename` when entering the context, and releases the lock when leaving the context. The lock is acquired using the `fcntl` module's `LOCK_EX` flag, which applies an exclusive write lock to the file.
|
||||
"""
|
||||
with open(filename, mode) as fd:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX)
|
||||
yield fd
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def write_locked_file(path: Path, data: dict[str, Any]) -> None:
|
||||
with locked_open(path, "w+") as f:
|
||||
f.write(json.dumps(data, indent=4))
|
||||
|
||||
|
||||
def read_locked_file(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
with locked_open(path, "r") as f:
|
||||
content: str = f.read()
|
||||
parsed: dict[str, Any] = json.loads(content)
|
||||
return parsed
|
71
pkgs/matrix-bot/matrix_bot/main.py
Normal file
71
pkgs/matrix-bot/matrix_bot/main.py
Normal file
@ -0,0 +1,71 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
import aiohttp
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
from nio import AsyncClient, ClientConfig, ProfileGetAvatarResponse, RoomMessageText
|
||||
|
||||
from . import BotConfig
|
||||
from .changelog_bot import changelog_bot
|
||||
from .gitea import GiteaData
|
||||
from .matrix import MatrixData, set_avatar, upload_image
|
||||
from .review_bot import message_callback, review_requested_bot, send_error
|
||||
|
||||
|
||||
async def bot_main(
|
||||
matrix: MatrixData,
|
||||
gitea: GiteaData,
|
||||
bot_conf: BotConfig,
|
||||
) -> None:
|
||||
# Setup client configuration to handle encryption
|
||||
client_config = ClientConfig(
|
||||
encryption_enabled=False,
|
||||
)
|
||||
|
||||
log.info(f"Connecting to {matrix.server} as {matrix.user}")
|
||||
client = AsyncClient(matrix.server, matrix.user, config=client_config)
|
||||
client.add_event_callback(message_callback, RoomMessageText)
|
||||
|
||||
result = await client.login(matrix.password)
|
||||
if not result.transport_response.ok:
|
||||
log.critical(f"Failed to login: {result}")
|
||||
exit(1)
|
||||
log.info(f"Logged in as {result}")
|
||||
|
||||
avatar: ProfileGetAvatarResponse = await client.get_avatar()
|
||||
if not avatar.avatar_url:
|
||||
mxc_url = await upload_image(client, matrix.avatar)
|
||||
log.info(f"Uploaded avatar to {mxc_url}")
|
||||
await set_avatar(client, mxc_url)
|
||||
else:
|
||||
log.info(f"Bot already has an avatar {avatar.avatar_url}")
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
while True:
|
||||
if not bot_conf.disable_changelog_bot:
|
||||
try:
|
||||
await changelog_bot(client, session, matrix, gitea, bot_conf)
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
await send_error(client, matrix, f"Changelog bot failed: {e}")
|
||||
|
||||
if not bot_conf.disable_pr_bot:
|
||||
try:
|
||||
await review_requested_bot(
|
||||
client, session, matrix, gitea, bot_conf
|
||||
)
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
await send_error(
|
||||
client, matrix, f"Review requested bot failed: {e}"
|
||||
)
|
||||
|
||||
log.debug(f"Sleeping for {60 * gitea.poll_frequency / 60} minutes")
|
||||
await asyncio.sleep(60 * gitea.poll_frequency)
|
||||
except Exception as e:
|
||||
log.exception(e)
|
||||
finally:
|
||||
await client.close()
|
89
pkgs/matrix-bot/matrix_bot/matrix.py
Normal file
89
pkgs/matrix-bot/matrix_bot/matrix.py
Normal file
@ -0,0 +1,89 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
from dataclasses import dataclass
|
||||
|
||||
from markdown2 import markdown
|
||||
from nio import (
|
||||
AsyncClient,
|
||||
JoinedMembersResponse,
|
||||
JoinResponse,
|
||||
ProfileSetAvatarResponse,
|
||||
RoomMember,
|
||||
RoomSendResponse,
|
||||
UploadResponse,
|
||||
)
|
||||
|
||||
|
||||
async def upload_image(client: AsyncClient, image_path: Path) -> str:
|
||||
with open(image_path, "rb") as image_file:
|
||||
response: UploadResponse
|
||||
response, _ = await client.upload(image_file, content_type="image/png")
|
||||
if not response.transport_response.ok:
|
||||
raise Exception(f"Failed to upload image {response}")
|
||||
return response.content_uri # This is the MXC URL
|
||||
|
||||
|
||||
async def set_avatar(client: AsyncClient, mxc_url: str) -> None:
|
||||
response: ProfileSetAvatarResponse
|
||||
response = await client.set_avatar(mxc_url)
|
||||
if not response.transport_response.ok:
|
||||
raise Exception(f"Failed to set avatar {response}")
|
||||
|
||||
|
||||
async def get_room_members(client: AsyncClient, room: JoinResponse) -> list[RoomMember]:
|
||||
users: JoinedMembersResponse = await client.joined_members(room.room_id)
|
||||
|
||||
if not users.transport_response.ok:
|
||||
raise Exception(f"Failed to get users {users}")
|
||||
return users.members
|
||||
|
||||
|
||||
async def send_message(
|
||||
client: AsyncClient,
|
||||
room: JoinResponse,
|
||||
message: str,
|
||||
user_ids: list[str] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Send a message in a Matrix room, optionally mentioning users.
|
||||
"""
|
||||
|
||||
# If user_ids are provided, format the message to mention them
|
||||
formatted_message = markdown(message)
|
||||
if user_ids:
|
||||
mention_list = ", ".join(
|
||||
[
|
||||
f"<a href='https://matrix.to/#/{user_id}'>{user_id}</a>"
|
||||
for user_id in user_ids
|
||||
]
|
||||
)
|
||||
formatted_message = f"{mention_list}: {formatted_message}"
|
||||
|
||||
content = {
|
||||
"msgtype": "m.text" if user_ids else "m.notice",
|
||||
"format": "org.matrix.custom.html",
|
||||
"body": message,
|
||||
"formatted_body": formatted_message,
|
||||
}
|
||||
|
||||
res: RoomSendResponse = await client.room_send(
|
||||
room_id=room.room_id, message_type="m.room.message", content=content
|
||||
)
|
||||
|
||||
if not res.transport_response.ok:
|
||||
raise Exception(f"Failed to send message {res}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MatrixData:
|
||||
server: str
|
||||
user: str
|
||||
avatar: Path
|
||||
password: str
|
||||
changelog_room: str
|
||||
review_room: str
|
||||
changelog_frequency: int
|
||||
publish_day: str
|
||||
admin: str
|
196
pkgs/matrix-bot/matrix_bot/openai.py
Normal file
196
pkgs/matrix-bot/matrix_bot/openai.py
Normal file
@ -0,0 +1,196 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from os import environ
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
import tiktoken
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# The URL to which the request is sent
|
||||
url: str = "https://api.openai.com/v1/chat/completions"
|
||||
|
||||
|
||||
def get_api_key() -> str:
|
||||
openapi_key = environ.get("OPENAI_API_KEY")
|
||||
if openapi_key is not None:
|
||||
return openapi_key
|
||||
|
||||
openai_key_file = environ.get("OPENAI_API_KEY_FILE", default=None)
|
||||
if openai_key_file is None:
|
||||
raise Exception("OPENAI_API_KEY_FILE environment variable is not set")
|
||||
with open(openai_key_file) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
async def create_jsonl_data(
|
||||
*,
|
||||
user_prompt: str,
|
||||
system_prompt: str,
|
||||
model: str = "gpt-4o",
|
||||
max_response_tokens: int = 4096,
|
||||
) -> list[bytes]:
|
||||
def split_message(content: str, max_tokens: int) -> list[str]:
|
||||
# Split the content into chunks of max_tokens
|
||||
content_tokens = encoder.encode(content)
|
||||
chunks = []
|
||||
for i in range(0, len(content_tokens), max_tokens):
|
||||
chunk = content_tokens[i : i + max_tokens]
|
||||
chunks.append(encoder.decode(chunk))
|
||||
log.debug(f"Chunk {i/max_tokens}: {len(chunk)} tokens")
|
||||
return chunks
|
||||
|
||||
encoder = tiktoken.encoding_for_model(model)
|
||||
max_message_tokens = 127_000 - max_response_tokens
|
||||
|
||||
# Split user_prompt into multiple user messages if it exceeds the max_message_tokens
|
||||
user_messages = []
|
||||
for message_chunk in split_message(user_prompt, max_message_tokens):
|
||||
if len(message_chunk) == 0:
|
||||
raise Exception("Empty message chunk")
|
||||
user_messages.append({"role": "user", "content": message_chunk})
|
||||
|
||||
## count number of tokens for every user message
|
||||
count_tokens: int = 0
|
||||
for i, message in enumerate(user_messages):
|
||||
count_tokens = len(encoder.encode(message["content"]))
|
||||
log.debug(f"Number of tokens in the user messages: {count_tokens}")
|
||||
if count_tokens > max_message_tokens:
|
||||
raise Exception(f"Too many tokens in the user message[{i}] {count_tokens}")
|
||||
|
||||
batch_jobs: list[bytes] = []
|
||||
for message in user_messages:
|
||||
summary_request: dict[str, Any] = {
|
||||
"custom_id": "request-1",
|
||||
"method": "POST",
|
||||
"url": "/v1/chat/completions",
|
||||
"body": {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
message,
|
||||
],
|
||||
"max_tokens": max_response_tokens,
|
||||
},
|
||||
}
|
||||
|
||||
dumped = json.dumps(summary_request)
|
||||
batch_jobs.append(dumped.encode("utf-8"))
|
||||
|
||||
return batch_jobs
|
||||
|
||||
|
||||
async def upload_and_process_files(
|
||||
*,
|
||||
session: aiohttp.ClientSession,
|
||||
jsonl_files: list[bytes],
|
||||
api_key: str | None = None,
|
||||
completion_window: str = "24h",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Upload multiple JSONL files to OpenAI's Batch API and process them asynchronously.
|
||||
"""
|
||||
if api_key is None:
|
||||
api_key = get_api_key()
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
}
|
||||
|
||||
log.debug(
|
||||
f"Uploading {len(jsonl_files)} files to OpenAI, completion window: {completion_window}"
|
||||
)
|
||||
|
||||
async def upload_file(jsonl_data: bytes) -> str:
|
||||
upload_url = "https://api.openai.com/v1/files"
|
||||
|
||||
data = aiohttp.FormData()
|
||||
data.add_field(
|
||||
"file",
|
||||
jsonl_data,
|
||||
filename="changelog.jsonl",
|
||||
content_type="application/jsonl",
|
||||
)
|
||||
data.add_field("purpose", "batch")
|
||||
|
||||
async with session.post(upload_url, headers=headers, data=data) as response:
|
||||
if response.status != 200:
|
||||
raise Exception(
|
||||
f"File upload failed with status code {response.status}"
|
||||
)
|
||||
upload_response = await response.json()
|
||||
file_id = upload_response.get("id")
|
||||
|
||||
if not file_id:
|
||||
raise Exception("File ID not returned from upload")
|
||||
|
||||
return file_id
|
||||
|
||||
async def create_batch(file_id: str) -> str:
|
||||
batch_url = "https://api.openai.com/v1/batches"
|
||||
batch_data = {
|
||||
"input_file_id": file_id,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": f"{completion_window}",
|
||||
}
|
||||
|
||||
async with session.post(
|
||||
batch_url, headers=headers, json=batch_data
|
||||
) as response:
|
||||
if response.status != 200:
|
||||
raise Exception(
|
||||
f"Batch creation failed with status code {response.status}"
|
||||
)
|
||||
batch_response = await response.json()
|
||||
batch_id = batch_response.get("id")
|
||||
|
||||
if not batch_id:
|
||||
raise Exception("Batch ID not returned from creation")
|
||||
|
||||
return batch_id
|
||||
|
||||
async def check_batch_status(batch_id: str) -> str:
|
||||
status_url = f"https://api.openai.com/v1/batches/{batch_id}"
|
||||
while True:
|
||||
async with session.get(status_url, headers=headers) as response:
|
||||
if response.status != 200:
|
||||
raise Exception(
|
||||
f"Failed to check batch status with status code {response.status}"
|
||||
)
|
||||
status_response = await response.json()
|
||||
status = status_response.get("status")
|
||||
if status in ["completed", "failed", "expired"]:
|
||||
if status != "completed":
|
||||
raise Exception(
|
||||
f"Batch processing failed with status: {status}"
|
||||
)
|
||||
return status_response.get("output_file_id")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
async def retrieve_results(output_file_id: str) -> list[dict[str, Any]]:
|
||||
output_url = f"https://api.openai.com/v1/files/{output_file_id}/content"
|
||||
async with session.get(output_url, headers=headers) as response:
|
||||
if response.status != 200:
|
||||
raise Exception(
|
||||
f"Failed to retrieve batch results with status code {response.status} reason {response.reason}"
|
||||
)
|
||||
content = await response.text()
|
||||
results = [json.loads(line) for line in content.splitlines()]
|
||||
return results
|
||||
|
||||
file_ids = await asyncio.gather(
|
||||
*[upload_file(jsonl_data) for jsonl_data in jsonl_files]
|
||||
)
|
||||
batch_ids = await asyncio.gather(*[create_batch(file_id) for file_id in file_ids])
|
||||
output_file_ids = await asyncio.gather(
|
||||
*[check_batch_status(batch_id) for batch_id in batch_ids]
|
||||
)
|
||||
all_results = await asyncio.gather(
|
||||
*[retrieve_results(output_file_id) for output_file_id in output_file_ids]
|
||||
)
|
||||
|
||||
# Flatten the list of results
|
||||
combined_results = [item for sublist in all_results for item in sublist]
|
||||
|
||||
return combined_results
|
131
pkgs/matrix-bot/matrix_bot/review_bot.py
Normal file
131
pkgs/matrix-bot/matrix_bot/review_bot.py
Normal file
@ -0,0 +1,131 @@
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
import datetime
|
||||
import time
|
||||
|
||||
import aiohttp
|
||||
from nio import (
|
||||
AsyncClient,
|
||||
JoinResponse,
|
||||
MatrixRoom,
|
||||
RoomMessageText,
|
||||
)
|
||||
|
||||
from matrix_bot.gitea import (
|
||||
GiteaData,
|
||||
PullState,
|
||||
fetch_pull_requests,
|
||||
)
|
||||
|
||||
from . import BotConfig
|
||||
from .locked_open import read_locked_file, write_locked_file
|
||||
from .matrix import MatrixData, get_room_members, send_message
|
||||
|
||||
|
||||
async def message_callback(room: MatrixRoom, event: RoomMessageText) -> None:
|
||||
log.debug(
|
||||
f"Message received in room {room.display_name}\n"
|
||||
f"{room.user_name(event.sender)} | {event.body}"
|
||||
)
|
||||
|
||||
|
||||
async def send_error(client: AsyncClient, matrix: MatrixData, msg: str) -> None:
|
||||
# If you made a new room and haven't joined as that user, you can use
|
||||
room: JoinResponse = await client.join(matrix.review_room)
|
||||
if not room.transport_response.ok:
|
||||
log.error("This can happen if the room doesn't exist or the bot isn't invited")
|
||||
raise Exception(f"Failed to join room {room}")
|
||||
await send_message(client, room, msg, user_ids=[matrix.admin])
|
||||
|
||||
|
||||
async def review_requested_bot(
|
||||
client: AsyncClient,
|
||||
http: aiohttp.ClientSession,
|
||||
matrix: MatrixData,
|
||||
gitea: GiteaData,
|
||||
bot_conf: BotConfig,
|
||||
) -> None:
|
||||
# If you made a new room and haven't joined as that user, you can use
|
||||
room: JoinResponse = await client.join(matrix.review_room)
|
||||
|
||||
if not room.transport_response.ok:
|
||||
log.error("This can happen if the room doesn't exist or the bot isn't invited")
|
||||
raise Exception(f"Failed to join room {room}")
|
||||
|
||||
# Get the members of the room
|
||||
room_users = await get_room_members(client, room)
|
||||
|
||||
# Fetch the pull requests
|
||||
tstart = time.time()
|
||||
pulls = await fetch_pull_requests(gitea, http, limit=50, state=PullState.ALL)
|
||||
|
||||
# Read the last updated pull request
|
||||
ping_hist_path = bot_conf.data_dir / "last_review_run.json"
|
||||
ping_hist = read_locked_file(ping_hist_path)
|
||||
|
||||
# Check if the pull request is mergeable and needs review
|
||||
# and if the pull request is newer than the last updated pull request
|
||||
for pull in pulls:
|
||||
requested_reviewers = pull["requested_reviewers"]
|
||||
assigned_users = pull["assignees"]
|
||||
mentioned_users = []
|
||||
if assigned_users:
|
||||
mentioned_users.extend(assigned_users)
|
||||
if requested_reviewers:
|
||||
mentioned_users.extend(requested_reviewers)
|
||||
|
||||
mentioned_users = list(map(lambda x: x["login"].lower(), mentioned_users))
|
||||
mentioned_users = list(
|
||||
filter(lambda name: name not in matrix.user, mentioned_users)
|
||||
)
|
||||
pull_id = str(pull["id"])
|
||||
needs_review_label = any(
|
||||
x["name"] in gitea.mention_labels for x in pull["labels"]
|
||||
)
|
||||
if (
|
||||
len(mentioned_users) > 0
|
||||
and pull["mergeable"]
|
||||
or needs_review_label
|
||||
and pull["mergeable"]
|
||||
):
|
||||
# Mention the pull request again if it has been updated
|
||||
if gitea.mention_on_update:
|
||||
last_time_updated = ping_hist.get(pull_id, {}).get(
|
||||
"updated_at", datetime.datetime.min.isoformat()
|
||||
)
|
||||
if ping_hist == {} or pull["updated_at"] > last_time_updated:
|
||||
ping_hist[pull_id] = pull
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
if ping_hist == {} or pull_id not in ping_hist:
|
||||
ping_hist[pull_id] = pull
|
||||
else:
|
||||
continue
|
||||
|
||||
# Check if the requested reviewers are in the room
|
||||
ping_users = []
|
||||
for user in room_users:
|
||||
user_name = user.display_name.lower()
|
||||
if any(
|
||||
user_name in mentioned_user or mentioned_user in user_name
|
||||
for mentioned_user in mentioned_users
|
||||
):
|
||||
ping_users.append(user.user_id)
|
||||
|
||||
# Send a message to the room and mention the users
|
||||
log.info(f"Pull request {pull['title']} needs review")
|
||||
log.debug(
|
||||
f"Mentioned users: {mentioned_users}, has needs-review label: {needs_review_label}"
|
||||
)
|
||||
message = f"Review Requested:\n[{pull['title']}]({pull['html_url']})"
|
||||
await send_message(client, room, message, user_ids=ping_users)
|
||||
|
||||
# Write the new last updated pull request
|
||||
write_locked_file(ping_hist_path, ping_hist)
|
||||
|
||||
# Time taken
|
||||
tend = time.time()
|
||||
tdiff = round(tend - tstart)
|
||||
log.debug(f"Time taken: {tdiff}s")
|
68
pkgs/matrix-bot/pyproject.toml
Normal file
68
pkgs/matrix-bot/pyproject.toml
Normal file
@ -0,0 +1,68 @@
|
||||
[build-system]
|
||||
requires = ["setuptools"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "matrix-bot"
|
||||
description = "matrix bot for release messages from git commits"
|
||||
dynamic = ["version"]
|
||||
scripts = { mbot = "matrix_bot:main" }
|
||||
license = {text = "MIT"}
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://clan.lol/"
|
||||
Documentation = "https://docs.clan.lol/"
|
||||
Repository = "https://git.clan.lol/clan/clan-core"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
exclude = ["result"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
matrix_bot = ["py.typed"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = "tests"
|
||||
faulthandler_timeout = 60
|
||||
log_level = "DEBUG"
|
||||
log_format = "%(levelname)s: %(message)s\n %(pathname)s:%(lineno)d::%(funcName)s"
|
||||
addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --durations 5 --color=yes --new-first" # Add --pdb for debugging
|
||||
norecursedirs = "tests/helpers"
|
||||
markers = ["impure", "with_core"]
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.11"
|
||||
warn_redundant_casts = true
|
||||
disallow_untyped_calls = true
|
||||
disallow_untyped_defs = true
|
||||
no_implicit_optional = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "argcomplete.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "ipdb.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "pytest.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "setuptools.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "nio.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
[[tool.mypy.overrides]]
|
||||
module = "markdown2.*"
|
||||
ignore_missing_imports = true
|
||||
|
||||
|
||||
[tool.ruff]
|
||||
target-version = "py311"
|
||||
line-length = 88
|
||||
lint.select = [ "E", "F", "I", "U", "N", "RUF", "ANN", "A" ]
|
||||
lint.ignore = ["E501", "E402", "E731", "ANN101", "ANN401", "A003"]
|
31
pkgs/matrix-bot/shell.nix
Normal file
31
pkgs/matrix-bot/shell.nix
Normal file
@ -0,0 +1,31 @@
|
||||
{
|
||||
matrix-bot,
|
||||
mkShell,
|
||||
ruff,
|
||||
python3,
|
||||
}:
|
||||
let
|
||||
devshellTestDeps =
|
||||
matrix-bot.passthru.testDependencies
|
||||
++ (with python3.pkgs; [
|
||||
rope
|
||||
setuptools
|
||||
mypy
|
||||
wheel
|
||||
ipdb
|
||||
pip
|
||||
]);
|
||||
in
|
||||
mkShell {
|
||||
buildInputs = [ ruff ] ++ devshellTestDeps;
|
||||
|
||||
PYTHONBREAKPOINT = "ipdb.set_trace";
|
||||
|
||||
shellHook = ''
|
||||
export GIT_ROOT="$(git rev-parse --show-toplevel)"
|
||||
export PKG_ROOT="$GIT_ROOT/pkgs/matrix-bot"
|
||||
|
||||
# Add clan command to PATH
|
||||
export PATH="$PKG_ROOT/bin":"$PATH"
|
||||
'';
|
||||
}
|
@ -1,17 +0,0 @@
|
||||
# This file has been generated by node2nix 1.11.1. Do not edit!
|
||||
|
||||
{pkgs ? import <nixpkgs> {
|
||||
inherit system;
|
||||
}, system ? builtins.currentSystem, nodejs ? pkgs."nodejs_18"}:
|
||||
|
||||
let
|
||||
nodeEnv = import ./node-env.nix {
|
||||
inherit (pkgs) stdenv lib python2 runCommand writeTextFile writeShellScript;
|
||||
inherit pkgs nodejs;
|
||||
libtool = if pkgs.stdenv.isDarwin then pkgs.darwin.cctools else null;
|
||||
};
|
||||
in
|
||||
import ./node-packages.nix {
|
||||
inherit (pkgs) fetchurl nix-gitignore stdenv lib fetchgit;
|
||||
inherit nodeEnv;
|
||||
}
|
@ -1,8 +0,0 @@
|
||||
{ pkgs, system, nodejs-18_x, makeWrapper }:
|
||||
let
|
||||
nodePackages = import ./composition.nix {
|
||||
inherit pkgs system;
|
||||
nodejs = nodejs-18_x;
|
||||
};
|
||||
in
|
||||
nodePackages
|
@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#! nix-shell -i bash -p nodePackages.node2nix
|
||||
|
||||
rm -f node-env.nix
|
||||
node2nix -18 -i node-packages.json -o node-packages.nix -c composition.nix
|
@ -1,689 +0,0 @@
|
||||
# This file originates from node2nix
|
||||
|
||||
{lib, stdenv, nodejs, python2, pkgs, libtool, runCommand, writeTextFile, writeShellScript}:
|
||||
|
||||
let
|
||||
# Workaround to cope with utillinux in Nixpkgs 20.09 and util-linux in Nixpkgs master
|
||||
utillinux = if pkgs ? utillinux then pkgs.utillinux else pkgs.util-linux;
|
||||
|
||||
python = if nodejs ? python then nodejs.python else python2;
|
||||
|
||||
# Create a tar wrapper that filters all the 'Ignoring unknown extended header keyword' noise
|
||||
tarWrapper = runCommand "tarWrapper" {} ''
|
||||
mkdir -p $out/bin
|
||||
|
||||
cat > $out/bin/tar <<EOF
|
||||
#! ${stdenv.shell} -e
|
||||
$(type -p tar) "\$@" --warning=no-unknown-keyword --delay-directory-restore
|
||||
EOF
|
||||
|
||||
chmod +x $out/bin/tar
|
||||
'';
|
||||
|
||||
# Function that generates a TGZ file from a NPM project
|
||||
buildNodeSourceDist =
|
||||
{ name, version, src, ... }:
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "node-tarball-${name}-${version}";
|
||||
inherit src;
|
||||
buildInputs = [ nodejs ];
|
||||
buildPhase = ''
|
||||
export HOME=$TMPDIR
|
||||
tgzFile=$(npm pack | tail -n 1) # Hooks to the pack command will add output (https://docs.npmjs.com/misc/scripts)
|
||||
'';
|
||||
installPhase = ''
|
||||
mkdir -p $out/tarballs
|
||||
mv $tgzFile $out/tarballs
|
||||
mkdir -p $out/nix-support
|
||||
echo "file source-dist $out/tarballs/$tgzFile" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
};
|
||||
|
||||
# Common shell logic
|
||||
installPackage = writeShellScript "install-package" ''
|
||||
installPackage() {
|
||||
local packageName=$1 src=$2
|
||||
|
||||
local strippedName
|
||||
|
||||
local DIR=$PWD
|
||||
cd $TMPDIR
|
||||
|
||||
unpackFile $src
|
||||
|
||||
# Make the base dir in which the target dependency resides first
|
||||
mkdir -p "$(dirname "$DIR/$packageName")"
|
||||
|
||||
if [ -f "$src" ]
|
||||
then
|
||||
# Figure out what directory has been unpacked
|
||||
packageDir="$(find . -maxdepth 1 -type d | tail -1)"
|
||||
|
||||
# Restore write permissions to make building work
|
||||
find "$packageDir" -type d -exec chmod u+x {} \;
|
||||
chmod -R u+w "$packageDir"
|
||||
|
||||
# Move the extracted tarball into the output folder
|
||||
mv "$packageDir" "$DIR/$packageName"
|
||||
elif [ -d "$src" ]
|
||||
then
|
||||
# Get a stripped name (without hash) of the source directory.
|
||||
# On old nixpkgs it's already set internally.
|
||||
if [ -z "$strippedName" ]
|
||||
then
|
||||
strippedName="$(stripHash $src)"
|
||||
fi
|
||||
|
||||
# Restore write permissions to make building work
|
||||
chmod -R u+w "$strippedName"
|
||||
|
||||
# Move the extracted directory into the output folder
|
||||
mv "$strippedName" "$DIR/$packageName"
|
||||
fi
|
||||
|
||||
# Change to the package directory to install dependencies
|
||||
cd "$DIR/$packageName"
|
||||
}
|
||||
'';
|
||||
|
||||
# Bundle the dependencies of the package
|
||||
#
|
||||
# Only include dependencies if they don't exist. They may also be bundled in the package.
|
||||
includeDependencies = {dependencies}:
|
||||
lib.optionalString (dependencies != []) (
|
||||
''
|
||||
mkdir -p node_modules
|
||||
cd node_modules
|
||||
''
|
||||
+ (lib.concatMapStrings (dependency:
|
||||
''
|
||||
if [ ! -e "${dependency.packageName}" ]; then
|
||||
${composePackage dependency}
|
||||
fi
|
||||
''
|
||||
) dependencies)
|
||||
+ ''
|
||||
cd ..
|
||||
''
|
||||
);
|
||||
|
||||
# Recursively composes the dependencies of a package
|
||||
composePackage = { name, packageName, src, dependencies ? [], ... }@args:
|
||||
builtins.addErrorContext "while evaluating node package '${packageName}'" ''
|
||||
installPackage "${packageName}" "${src}"
|
||||
${includeDependencies { inherit dependencies; }}
|
||||
cd ..
|
||||
${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
|
||||
'';
|
||||
|
||||
pinpointDependencies = {dependencies, production}:
|
||||
let
|
||||
pinpointDependenciesFromPackageJSON = writeTextFile {
|
||||
name = "pinpointDependencies.js";
|
||||
text = ''
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
|
||||
function resolveDependencyVersion(location, name) {
|
||||
if(location == process.env['NIX_STORE']) {
|
||||
return null;
|
||||
} else {
|
||||
var dependencyPackageJSON = path.join(location, "node_modules", name, "package.json");
|
||||
|
||||
if(fs.existsSync(dependencyPackageJSON)) {
|
||||
var dependencyPackageObj = JSON.parse(fs.readFileSync(dependencyPackageJSON));
|
||||
|
||||
if(dependencyPackageObj.name == name) {
|
||||
return dependencyPackageObj.version;
|
||||
}
|
||||
} else {
|
||||
return resolveDependencyVersion(path.resolve(location, ".."), name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function replaceDependencies(dependencies) {
|
||||
if(typeof dependencies == "object" && dependencies !== null) {
|
||||
for(var dependency in dependencies) {
|
||||
var resolvedVersion = resolveDependencyVersion(process.cwd(), dependency);
|
||||
|
||||
if(resolvedVersion === null) {
|
||||
process.stderr.write("WARNING: cannot pinpoint dependency: "+dependency+", context: "+process.cwd()+"\n");
|
||||
} else {
|
||||
dependencies[dependency] = resolvedVersion;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Read the package.json configuration */
|
||||
var packageObj = JSON.parse(fs.readFileSync('./package.json'));
|
||||
|
||||
/* Pinpoint all dependencies */
|
||||
replaceDependencies(packageObj.dependencies);
|
||||
if(process.argv[2] == "development") {
|
||||
replaceDependencies(packageObj.devDependencies);
|
||||
}
|
||||
else {
|
||||
packageObj.devDependencies = {};
|
||||
}
|
||||
replaceDependencies(packageObj.optionalDependencies);
|
||||
replaceDependencies(packageObj.peerDependencies);
|
||||
|
||||
/* Write the fixed package.json file */
|
||||
fs.writeFileSync("package.json", JSON.stringify(packageObj, null, 2));
|
||||
'';
|
||||
};
|
||||
in
|
||||
''
|
||||
node ${pinpointDependenciesFromPackageJSON} ${if production then "production" else "development"}
|
||||
|
||||
${lib.optionalString (dependencies != [])
|
||||
''
|
||||
if [ -d node_modules ]
|
||||
then
|
||||
cd node_modules
|
||||
${lib.concatMapStrings (dependency: pinpointDependenciesOfPackage dependency) dependencies}
|
||||
cd ..
|
||||
fi
|
||||
''}
|
||||
'';
|
||||
|
||||
# Recursively traverses all dependencies of a package and pinpoints all
|
||||
# dependencies in the package.json file to the versions that are actually
|
||||
# being used.
|
||||
|
||||
pinpointDependenciesOfPackage = { packageName, dependencies ? [], production ? true, ... }@args:
|
||||
''
|
||||
if [ -d "${packageName}" ]
|
||||
then
|
||||
cd "${packageName}"
|
||||
${pinpointDependencies { inherit dependencies production; }}
|
||||
cd ..
|
||||
${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
|
||||
fi
|
||||
'';
|
||||
|
||||
# Extract the Node.js source code which is used to compile packages with
|
||||
# native bindings
|
||||
nodeSources = runCommand "node-sources" {} ''
|
||||
tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
|
||||
mv node-* $out
|
||||
'';
|
||||
|
||||
# Script that adds _integrity fields to all package.json files to prevent NPM from consulting the cache (that is empty)
|
||||
addIntegrityFieldsScript = writeTextFile {
|
||||
name = "addintegrityfields.js";
|
||||
text = ''
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
|
||||
function augmentDependencies(baseDir, dependencies) {
|
||||
for(var dependencyName in dependencies) {
|
||||
var dependency = dependencies[dependencyName];
|
||||
|
||||
// Open package.json and augment metadata fields
|
||||
var packageJSONDir = path.join(baseDir, "node_modules", dependencyName);
|
||||
var packageJSONPath = path.join(packageJSONDir, "package.json");
|
||||
|
||||
if(fs.existsSync(packageJSONPath)) { // Only augment packages that exist. Sometimes we may have production installs in which development dependencies can be ignored
|
||||
console.log("Adding metadata fields to: "+packageJSONPath);
|
||||
var packageObj = JSON.parse(fs.readFileSync(packageJSONPath));
|
||||
|
||||
if(dependency.integrity) {
|
||||
packageObj["_integrity"] = dependency.integrity;
|
||||
} else {
|
||||
packageObj["_integrity"] = "sha1-000000000000000000000000000="; // When no _integrity string has been provided (e.g. by Git dependencies), add a dummy one. It does not seem to harm and it bypasses downloads.
|
||||
}
|
||||
|
||||
if(dependency.resolved) {
|
||||
packageObj["_resolved"] = dependency.resolved; // Adopt the resolved property if one has been provided
|
||||
} else {
|
||||
packageObj["_resolved"] = dependency.version; // Set the resolved version to the version identifier. This prevents NPM from cloning Git repositories.
|
||||
}
|
||||
|
||||
if(dependency.from !== undefined) { // Adopt from property if one has been provided
|
||||
packageObj["_from"] = dependency.from;
|
||||
}
|
||||
|
||||
fs.writeFileSync(packageJSONPath, JSON.stringify(packageObj, null, 2));
|
||||
}
|
||||
|
||||
// Augment transitive dependencies
|
||||
if(dependency.dependencies !== undefined) {
|
||||
augmentDependencies(packageJSONDir, dependency.dependencies);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(fs.existsSync("./package-lock.json")) {
|
||||
var packageLock = JSON.parse(fs.readFileSync("./package-lock.json"));
|
||||
|
||||
if(![1, 2].includes(packageLock.lockfileVersion)) {
|
||||
process.stderr.write("Sorry, I only understand lock file versions 1 and 2!\n");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if(packageLock.dependencies !== undefined) {
|
||||
augmentDependencies(".", packageLock.dependencies);
|
||||
}
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# Reconstructs a package-lock file from the node_modules/ folder structure and package.json files with dummy sha1 hashes
|
||||
reconstructPackageLock = writeTextFile {
|
||||
name = "reconstructpackagelock.js";
|
||||
text = ''
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
|
||||
var packageObj = JSON.parse(fs.readFileSync("package.json"));
|
||||
|
||||
var lockObj = {
|
||||
name: packageObj.name,
|
||||
version: packageObj.version,
|
||||
lockfileVersion: 2,
|
||||
requires: true,
|
||||
packages: {
|
||||
"": {
|
||||
name: packageObj.name,
|
||||
version: packageObj.version,
|
||||
license: packageObj.license,
|
||||
bin: packageObj.bin,
|
||||
dependencies: packageObj.dependencies,
|
||||
engines: packageObj.engines,
|
||||
optionalDependencies: packageObj.optionalDependencies
|
||||
}
|
||||
},
|
||||
dependencies: {}
|
||||
};
|
||||
|
||||
function augmentPackageJSON(filePath, packages, dependencies) {
|
||||
var packageJSON = path.join(filePath, "package.json");
|
||||
if(fs.existsSync(packageJSON)) {
|
||||
var packageObj = JSON.parse(fs.readFileSync(packageJSON));
|
||||
packages[filePath] = {
|
||||
version: packageObj.version,
|
||||
integrity: "sha1-000000000000000000000000000=",
|
||||
dependencies: packageObj.dependencies,
|
||||
engines: packageObj.engines,
|
||||
optionalDependencies: packageObj.optionalDependencies
|
||||
};
|
||||
dependencies[packageObj.name] = {
|
||||
version: packageObj.version,
|
||||
integrity: "sha1-000000000000000000000000000=",
|
||||
dependencies: {}
|
||||
};
|
||||
processDependencies(path.join(filePath, "node_modules"), packages, dependencies[packageObj.name].dependencies);
|
||||
}
|
||||
}
|
||||
|
||||
function processDependencies(dir, packages, dependencies) {
|
||||
if(fs.existsSync(dir)) {
|
||||
var files = fs.readdirSync(dir);
|
||||
|
||||
files.forEach(function(entry) {
|
||||
var filePath = path.join(dir, entry);
|
||||
var stats = fs.statSync(filePath);
|
||||
|
||||
if(stats.isDirectory()) {
|
||||
if(entry.substr(0, 1) == "@") {
|
||||
// When we encounter a namespace folder, augment all packages belonging to the scope
|
||||
var pkgFiles = fs.readdirSync(filePath);
|
||||
|
||||
pkgFiles.forEach(function(entry) {
|
||||
if(stats.isDirectory()) {
|
||||
var pkgFilePath = path.join(filePath, entry);
|
||||
augmentPackageJSON(pkgFilePath, packages, dependencies);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
augmentPackageJSON(filePath, packages, dependencies);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
processDependencies("node_modules", lockObj.packages, lockObj.dependencies);
|
||||
|
||||
fs.writeFileSync("package-lock.json", JSON.stringify(lockObj, null, 2));
|
||||
'';
|
||||
};
|
||||
|
||||
# Script that links bins defined in package.json to the node_modules bin directory
|
||||
# NPM does not do this for top-level packages itself anymore as of v7
|
||||
linkBinsScript = writeTextFile {
|
||||
name = "linkbins.js";
|
||||
text = ''
|
||||
var fs = require('fs');
|
||||
var path = require('path');
|
||||
|
||||
var packageObj = JSON.parse(fs.readFileSync("package.json"));
|
||||
|
||||
var nodeModules = Array(packageObj.name.split("/").length).fill("..").join(path.sep);
|
||||
|
||||
if(packageObj.bin !== undefined) {
|
||||
fs.mkdirSync(path.join(nodeModules, ".bin"))
|
||||
|
||||
if(typeof packageObj.bin == "object") {
|
||||
Object.keys(packageObj.bin).forEach(function(exe) {
|
||||
if(fs.existsSync(packageObj.bin[exe])) {
|
||||
console.log("linking bin '" + exe + "'");
|
||||
fs.symlinkSync(
|
||||
path.join("..", packageObj.name, packageObj.bin[exe]),
|
||||
path.join(nodeModules, ".bin", exe)
|
||||
);
|
||||
}
|
||||
else {
|
||||
console.log("skipping non-existent bin '" + exe + "'");
|
||||
}
|
||||
})
|
||||
}
|
||||
else {
|
||||
if(fs.existsSync(packageObj.bin)) {
|
||||
console.log("linking bin '" + packageObj.bin + "'");
|
||||
fs.symlinkSync(
|
||||
path.join("..", packageObj.name, packageObj.bin),
|
||||
path.join(nodeModules, ".bin", packageObj.name.split("/").pop())
|
||||
);
|
||||
}
|
||||
else {
|
||||
console.log("skipping non-existent bin '" + packageObj.bin + "'");
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(packageObj.directories !== undefined && packageObj.directories.bin !== undefined) {
|
||||
fs.mkdirSync(path.join(nodeModules, ".bin"))
|
||||
|
||||
fs.readdirSync(packageObj.directories.bin).forEach(function(exe) {
|
||||
if(fs.existsSync(path.join(packageObj.directories.bin, exe))) {
|
||||
console.log("linking bin '" + exe + "'");
|
||||
fs.symlinkSync(
|
||||
path.join("..", packageObj.name, packageObj.directories.bin, exe),
|
||||
path.join(nodeModules, ".bin", exe)
|
||||
);
|
||||
}
|
||||
else {
|
||||
console.log("skipping non-existent bin '" + exe + "'");
|
||||
}
|
||||
})
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
prepareAndInvokeNPM = {packageName, bypassCache, reconstructLock, npmFlags, production}:
|
||||
let
|
||||
forceOfflineFlag = if bypassCache then "--offline" else "--registry http://www.example.com";
|
||||
in
|
||||
''
|
||||
# Pinpoint the versions of all dependencies to the ones that are actually being used
|
||||
echo "pinpointing versions of dependencies..."
|
||||
source $pinpointDependenciesScriptPath
|
||||
|
||||
# Patch the shebangs of the bundled modules to prevent them from
|
||||
# calling executables outside the Nix store as much as possible
|
||||
patchShebangs .
|
||||
|
||||
# Deploy the Node.js package by running npm install. Since the
|
||||
# dependencies have been provided already by ourselves, it should not
|
||||
# attempt to install them again, which is good, because we want to make
|
||||
# it Nix's responsibility. If it needs to install any dependencies
|
||||
# anyway (e.g. because the dependency parameters are
|
||||
# incomplete/incorrect), it fails.
|
||||
#
|
||||
# The other responsibilities of NPM are kept -- version checks, build
|
||||
# steps, postprocessing etc.
|
||||
|
||||
export HOME=$TMPDIR
|
||||
cd "${packageName}"
|
||||
runHook preRebuild
|
||||
|
||||
${lib.optionalString bypassCache ''
|
||||
${lib.optionalString reconstructLock ''
|
||||
if [ -f package-lock.json ]
|
||||
then
|
||||
echo "WARNING: Reconstruct lock option enabled, but a lock file already exists!"
|
||||
echo "This will most likely result in version mismatches! We will remove the lock file and regenerate it!"
|
||||
rm package-lock.json
|
||||
else
|
||||
echo "No package-lock.json file found, reconstructing..."
|
||||
fi
|
||||
|
||||
node ${reconstructPackageLock}
|
||||
''}
|
||||
|
||||
node ${addIntegrityFieldsScript}
|
||||
''}
|
||||
|
||||
npm ${forceOfflineFlag} --nodedir=${nodeSources} ${npmFlags} ${lib.optionalString production "--production"} rebuild
|
||||
|
||||
runHook postRebuild
|
||||
|
||||
if [ "''${dontNpmInstall-}" != "1" ]
|
||||
then
|
||||
# NPM tries to download packages even when they already exist if npm-shrinkwrap is used.
|
||||
rm -f npm-shrinkwrap.json
|
||||
|
||||
npm ${forceOfflineFlag} --nodedir=${nodeSources} --no-bin-links --ignore-scripts ${npmFlags} ${lib.optionalString production "--production"} install
|
||||
fi
|
||||
|
||||
# Link executables defined in package.json
|
||||
node ${linkBinsScript}
|
||||
'';
|
||||
|
||||
# Builds and composes an NPM package including all its dependencies
|
||||
buildNodePackage =
|
||||
{ name
|
||||
, packageName
|
||||
, version ? null
|
||||
, dependencies ? []
|
||||
, buildInputs ? []
|
||||
, production ? true
|
||||
, npmFlags ? ""
|
||||
, dontNpmInstall ? false
|
||||
, bypassCache ? false
|
||||
, reconstructLock ? false
|
||||
, preRebuild ? ""
|
||||
, dontStrip ? true
|
||||
, unpackPhase ? "true"
|
||||
, buildPhase ? "true"
|
||||
, meta ? {}
|
||||
, ... }@args:
|
||||
|
||||
let
|
||||
extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "preRebuild" "unpackPhase" "buildPhase" "meta" ];
|
||||
in
|
||||
stdenv.mkDerivation ({
|
||||
name = "${name}${if version == null then "" else "-${version}"}";
|
||||
buildInputs = [ tarWrapper python nodejs ]
|
||||
++ lib.optional (stdenv.isLinux) utillinux
|
||||
++ lib.optional (stdenv.isDarwin) libtool
|
||||
++ buildInputs;
|
||||
|
||||
inherit nodejs;
|
||||
|
||||
inherit dontStrip; # Stripping may fail a build for some package deployments
|
||||
inherit dontNpmInstall preRebuild unpackPhase buildPhase;
|
||||
|
||||
compositionScript = composePackage args;
|
||||
pinpointDependenciesScript = pinpointDependenciesOfPackage args;
|
||||
|
||||
passAsFile = [ "compositionScript" "pinpointDependenciesScript" ];
|
||||
|
||||
installPhase = ''
|
||||
source ${installPackage}
|
||||
|
||||
# Create and enter a root node_modules/ folder
|
||||
mkdir -p $out/lib/node_modules
|
||||
cd $out/lib/node_modules
|
||||
|
||||
# Compose the package and all its dependencies
|
||||
source $compositionScriptPath
|
||||
|
||||
${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }}
|
||||
|
||||
# Create symlink to the deployed executable folder, if applicable
|
||||
if [ -d "$out/lib/node_modules/.bin" ]
|
||||
then
|
||||
ln -s $out/lib/node_modules/.bin $out/bin
|
||||
|
||||
# Fixup all executables
|
||||
ls $out/bin/* | while read i
|
||||
do
|
||||
file="$(readlink -f "$i")"
|
||||
chmod u+rwx "$file"
|
||||
if isScript "$file"
|
||||
then
|
||||
sed -i 's/\r$//' "$file" # convert crlf to lf
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Create symlinks to the deployed manual page folders, if applicable
|
||||
if [ -d "$out/lib/node_modules/${packageName}/man" ]
|
||||
then
|
||||
mkdir -p $out/share
|
||||
for dir in "$out/lib/node_modules/${packageName}/man/"*
|
||||
do
|
||||
mkdir -p $out/share/man/$(basename "$dir")
|
||||
for page in "$dir"/*
|
||||
do
|
||||
ln -s $page $out/share/man/$(basename "$dir")
|
||||
done
|
||||
done
|
||||
fi
|
||||
|
||||
# Run post install hook, if provided
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# default to Node.js' platforms
|
||||
platforms = nodejs.meta.platforms;
|
||||
} // meta;
|
||||
} // extraArgs);
|
||||
|
||||
# Builds a node environment (a node_modules folder and a set of binaries)
|
||||
buildNodeDependencies =
|
||||
{ name
|
||||
, packageName
|
||||
, version ? null
|
||||
, src
|
||||
, dependencies ? []
|
||||
, buildInputs ? []
|
||||
, production ? true
|
||||
, npmFlags ? ""
|
||||
, dontNpmInstall ? false
|
||||
, bypassCache ? false
|
||||
, reconstructLock ? false
|
||||
, dontStrip ? true
|
||||
, unpackPhase ? "true"
|
||||
, buildPhase ? "true"
|
||||
, ... }@args:
|
||||
|
||||
let
|
||||
extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" ];
|
||||
in
|
||||
stdenv.mkDerivation ({
|
||||
name = "node-dependencies-${name}${if version == null then "" else "-${version}"}";
|
||||
|
||||
buildInputs = [ tarWrapper python nodejs ]
|
||||
++ lib.optional (stdenv.isLinux) utillinux
|
||||
++ lib.optional (stdenv.isDarwin) libtool
|
||||
++ buildInputs;
|
||||
|
||||
inherit dontStrip; # Stripping may fail a build for some package deployments
|
||||
inherit dontNpmInstall unpackPhase buildPhase;
|
||||
|
||||
includeScript = includeDependencies { inherit dependencies; };
|
||||
pinpointDependenciesScript = pinpointDependenciesOfPackage args;
|
||||
|
||||
passAsFile = [ "includeScript" "pinpointDependenciesScript" ];
|
||||
|
||||
installPhase = ''
|
||||
source ${installPackage}
|
||||
|
||||
mkdir -p $out/${packageName}
|
||||
cd $out/${packageName}
|
||||
|
||||
source $includeScriptPath
|
||||
|
||||
# Create fake package.json to make the npm commands work properly
|
||||
cp ${src}/package.json .
|
||||
chmod 644 package.json
|
||||
${lib.optionalString bypassCache ''
|
||||
if [ -f ${src}/package-lock.json ]
|
||||
then
|
||||
cp ${src}/package-lock.json .
|
||||
chmod 644 package-lock.json
|
||||
fi
|
||||
''}
|
||||
|
||||
# Go to the parent folder to make sure that all packages are pinpointed
|
||||
cd ..
|
||||
${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
|
||||
|
||||
${prepareAndInvokeNPM { inherit packageName bypassCache reconstructLock npmFlags production; }}
|
||||
|
||||
# Expose the executables that were installed
|
||||
cd ..
|
||||
${lib.optionalString (builtins.substring 0 1 packageName == "@") "cd .."}
|
||||
|
||||
mv ${packageName} lib
|
||||
ln -s $out/lib/node_modules/.bin $out/bin
|
||||
'';
|
||||
} // extraArgs);
|
||||
|
||||
# Builds a development shell
|
||||
buildNodeShell =
|
||||
{ name
|
||||
, packageName
|
||||
, version ? null
|
||||
, src
|
||||
, dependencies ? []
|
||||
, buildInputs ? []
|
||||
, production ? true
|
||||
, npmFlags ? ""
|
||||
, dontNpmInstall ? false
|
||||
, bypassCache ? false
|
||||
, reconstructLock ? false
|
||||
, dontStrip ? true
|
||||
, unpackPhase ? "true"
|
||||
, buildPhase ? "true"
|
||||
, ... }@args:
|
||||
|
||||
let
|
||||
nodeDependencies = buildNodeDependencies args;
|
||||
extraArgs = removeAttrs args [ "name" "dependencies" "buildInputs" "dontStrip" "dontNpmInstall" "unpackPhase" "buildPhase" ];
|
||||
in
|
||||
stdenv.mkDerivation ({
|
||||
name = "node-shell-${name}${if version == null then "" else "-${version}"}";
|
||||
|
||||
buildInputs = [ python nodejs ] ++ lib.optional (stdenv.isLinux) utillinux ++ buildInputs;
|
||||
buildCommand = ''
|
||||
mkdir -p $out/bin
|
||||
cat > $out/bin/shell <<EOF
|
||||
#! ${stdenv.shell} -e
|
||||
$shellHook
|
||||
exec ${stdenv.shell}
|
||||
EOF
|
||||
chmod +x $out/bin/shell
|
||||
'';
|
||||
|
||||
# Provide the dependencies in a development shell through the NODE_PATH environment variable
|
||||
inherit nodeDependencies;
|
||||
shellHook = lib.optionalString (dependencies != []) ''
|
||||
export NODE_PATH=${nodeDependencies}/lib/node_modules
|
||||
export PATH="${nodeDependencies}/bin:$PATH"
|
||||
'';
|
||||
} // extraArgs);
|
||||
in
|
||||
{
|
||||
buildNodeSourceDist = lib.makeOverridable buildNodeSourceDist;
|
||||
buildNodePackage = lib.makeOverridable buildNodePackage;
|
||||
buildNodeDependencies = lib.makeOverridable buildNodeDependencies;
|
||||
buildNodeShell = lib.makeOverridable buildNodeShell;
|
||||
}
|
@ -1,3 +0,0 @@
|
||||
[
|
||||
"renovate"
|
||||
]
|
File diff suppressed because it is too large
Load Diff
@ -1,12 +1,13 @@
|
||||
{ lib
|
||||
, bash
|
||||
, coreutils
|
||||
, gawk
|
||||
, path
|
||||
, # nixpkgs path
|
||||
writeScript
|
||||
, writeScriptBin
|
||||
, ...
|
||||
{
|
||||
lib,
|
||||
bash,
|
||||
coreutils,
|
||||
gawk,
|
||||
path,
|
||||
# nixpkgs path
|
||||
writeScript,
|
||||
writeScriptBin,
|
||||
...
|
||||
}:
|
||||
let
|
||||
# Create a script that runs in a `pure` environment, in the sense that:
|
||||
@ -18,12 +19,12 @@ let
|
||||
# - all environment variables are unset, except:
|
||||
# - the ones listed in `keepVars` defined in ./default.nix
|
||||
# - the ones listed via the `KEEP_VARS` variable
|
||||
writePureShellScript = PATH: script:
|
||||
writeScript "script.sh" (mkScript PATH script);
|
||||
writePureShellScript = PATH: script: writeScript "script.sh" (mkScript PATH script);
|
||||
|
||||
# Creates a script in a `bin/` directory in the output; suitable for use with `lib.makeBinPath`, etc.
|
||||
# See {option}`writers.writePureShellScript`
|
||||
writePureShellScriptBin = binName: PATH: script:
|
||||
writePureShellScriptBin =
|
||||
binName: PATH: script:
|
||||
writeScriptBin binName (mkScript PATH script);
|
||||
|
||||
mkScript = PATH: scriptText: ''
|
||||
@ -91,8 +92,5 @@ let
|
||||
'';
|
||||
in
|
||||
{
|
||||
inherit
|
||||
writePureShellScript
|
||||
writePureShellScriptBin
|
||||
;
|
||||
inherit writePureShellScript writePureShellScriptBin;
|
||||
}
|
||||
|
1
sops/secrets/qubasas-openai-api-key/machines/web01
Symbolic link
1
sops/secrets/qubasas-openai-api-key/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/qubasas-openai-api-key/secret
Normal file
24
sops/secrets/qubasas-openai-api-key/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:iJTjs8bG2GLGnGp/Hf4Egtorrk87rkgh9Yn+gPuWAJ61wIAtN3g9SU3vyYpvRrIqHVUyLObGbrWYi3Ol07M=,iv:YTOctq9aw4tc9xwoOO4UbR2cYPHV0ZmuE1FRWn13sgk=,tag:zU3HFqxwZcn/9S02bj3/fA==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUa1FIbWt2aHduZVBlcGpq\nOUsybTRmR1I4M1JZY3A4ZUFpZEN3dlZCTkZRCmdVajFUcDMyeEdJMThVZElJdmlJ\naEhvSk9sYThXdkxoaXVLem15dlJMcm8KLS0tIEJsdFY4L0M3Q2cwdzFOdy9LN0k3\nOEdCM09PUWlZbE91U2ZYNmVHeU43bUUKC+z+6XZCiVfwGQQCAHoB+WGE5Mm3qJZq\nuyD5r3Ra6MAvvwIhnqbwadRoxVH1HcdIB6hJsNREE/x6YNLxi3T7nw==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age1zwte859d9nvg6wy5dugjkf38dqe8w8qkt2as7xcc5pw3285833xs797uan",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFOHBub0J3U2FuOFNuS2xl\nRkExYVEvcmowWDlyUGlpYko3N1dIcmN5dmxJCkRtRm9qVVNQK3FQcTB1U3g3OHhS\ncU9XaHUxNWVlL2tpblpZUHF1UWt4am8KLS0tIGJOZkJ4eDJ6WWx1d2R1VG1qODZS\ncHhXMVhEUHdLZjIvNUYxRmduZkpjaU0KqZKUb9KYpSvwxaJRAbYhkuOdnzsU3p9Q\nU2WO5TIwS762yNqWTzyYdxb9YxvTOatW7uWTorRXZu1yqCTMTuq+1Q==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-07-03T09:47:40Z",
|
||||
"mac": "ENC[AES256_GCM,data:dpdmkhedaqivzIlxhoWb+u77JmfWRo94iWDolAa9UKvnjBo1QE5sHbqWasCH81wjO0wPBPRUqnj9JQ7kG9AFp24Fad+gAp74Gwx5M/PSx1dsd6xkcxt6PJ8sFXGb0H3lYduCaNfDGgsJTVoDcbk8rgYzjo5+mxs2pqrrn10t4iU=,iv:MGMIq2rF4+hr89/dppi2JDVbpAShscYTMM9viHPepIY=,tag:Dj9B6qvAkmiUmgRvZ6B94Q==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/qubasas-openai-api-key/users/qubasa
Symbolic link
1
sops/secrets/qubasas-openai-api-key/users/qubasa
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/qubasa
|
1
sops/secrets/web01-borgbackup.repokey/machines/web01
Symbolic link
1
sops/secrets/web01-borgbackup.repokey/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-borgbackup.repokey/secret
Normal file
24
sops/secrets/web01-borgbackup.repokey/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:WW0RmSs3k81jSgYLt8dHEiJOxlncPWl3QWvRtmNgtIxvup7h,iv:nw7SP15EVWfS78dJE37msnxAZ/goYb7rGqAKNzhXFP4=,tag:yxVyGUMFczq8cGuU4V/FzA==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBiM0FBVkhPc2luMjlpSW1R\nN0NlUU9ZQkxIOVAwa3hlMVg3bFluTjRlRUdRCmMxSkMvZjg2ckUyUThhSC9VOW1H\nZExFY2owcHQ5NzJtUW5pbDFjd2oyaEUKLS0tIG1Fd25acHdYWEdlQkMxajhRQXNw\nTGxJUDdPMlRrQ0t3SkVSaWdZZXJGT0EK7WfQ+6jVzOBToqO9wJby/qaF6kM00hMh\n+Y4A08X/ItLzyfCc5LQ97GQ2VlwXK5+HoD7jNnn//3xeH6YC1VBdkg==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhNlpOQ1QydEJuM3pCbEpK\nSDUzVlppNkFnSDJLSU1ITEdWWCtaUEE0THcwCmljSUl0amx2OTBVZXBPMFNGbjJP\nakNQcWlad1R3cDZYWWZpQkJkQmEvUEEKLS0tIFpJOU1GUnNaTnlaL25GRkdxZnhs\nUEhIVEpNWjNOV2FTSmVnRkVCWm90MDgKMvz6QdPRoYb2bPjS9oSOVA5gTfwrgn4q\nIyboQIMV3oAaAs9LSUcUMBvERzQ31JXnHRzrnqtdiNX0NLbIrN47yg==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-06T16:09:34Z",
|
||||
"mac": "ENC[AES256_GCM,data:7iKDT5577mLLeNyi46JHa4AUumqbQm65V3DXqNdNyLWccpIcML8n7jgFNxuK9gTqV2LM6bG18qS1orBJtPdawKnvxJwUaFb3Mo06C2+LVnWG4fT6MV+5eF8y6SM3IngT9BPk7IhTTGWe8lGJ6HTlg+9/f4/cq5NSKfeRgTkDEcE=,iv:T8wjeq2D1J8krhWeQJbVCOPY5sr05z/wMJqvr9onQK8=,tag:XgDTOTa2zv4NiBFN0b3rqA==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-borgbackup.repokey/users/joerg
Symbolic link
1
sops/secrets/web01-borgbackup.repokey/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
1
sops/secrets/web01-borgbackup.ssh/machines/web01
Symbolic link
1
sops/secrets/web01-borgbackup.ssh/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-borgbackup.ssh/secret
Normal file
24
sops/secrets/web01-borgbackup.ssh/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:ybX1/Uc+LqfgUoZQqCURgPfsTyzlsO+Xn7z8/0H9v5kyfJYX7PI1VlXVFBgR3Xh+2iuTF+v98PQyYeJOYLk3NTWggZayQQ4ivt0DLdhgG+DRFbeN8GMiqV5NWNhnL2tgLBu9DZViBSpgcbg9aHfI2cagboJnCSqyS2w1i/anvKaEgKa5YucrS4jywVxhBbvON6Oa2v8Hb0f/R8Ldl9HSqMM6o3pQEaYOsTNieNy63h9C4ERP/jIhKSajggpeHENdnuQC7Kavz54faL9xaz0jwRHb1fd+IGTM7fxqbyB5702nKEGytDwKzH0fh6q1HJNHbhWeWyCmGFKOkqywaQjcpJsczP2FIwkZmoui0juTEluNk1KzugP0gxtsuwjUiJlZeJxtZEgsnifLPpHyCaN99jzPjhd1TknT7MWZMVJT/R14bdD+QdwvR8rHK8IMctMGrNsqu3+Crdwu3WSfDH9jEM1zAZQNvLUT13azRABz1rpJvNFnvhDTBwDbUJlLpIQcPOPtVKO0IQeM5EUnCr+oCfBrP/To3mqo82s3,iv:jbY6WK0BcyLlU3Sbo7qNOHfCGU4TjUqTiww546Tyq20=,tag:VklHST51z5XI9+UiASBO9g==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlenFrWW45UXcyd3pwaTE0\nY3ROWWFPdzIwK1JuU2h2NCsyanF1VzhoM1NjCndscHZpTUNmcWc0TDd4V2xHM0lh\ndVNZQW1jMGFNeDdxbDhwdFB1Z3AwdVUKLS0tIGNmbVNMRUZGb1lPcnlrdkhnZ3JX\nM2szRHVydldGN3haV2lhZlFMeUgzcU0KDDwWVSjsua4DKXlqqk2Ns2e1zkzJK2Y2\n8+r8bXkBLJyXqQCQteXBrc5U+0n1KfHVkkvPmuBI3BmcAiVVmr/RxQ==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJdHEyT3U0TzJQUW9wM2Z3\nYVdTYmRSeUwrZy9iNTVpcmZJMnhOeHlGRGp3CnVYbk15NWxadk9waFdJVFBKa0Vx\nYUtkYWZuYmhabk1xREtDdzFGdUcyaW8KLS0tIGYxcDVuQXZwdk1rVHJOOHljTmVl\nMXNXTHdSam12djE3Z0UyU2dSZDcrRGcKaQnrYuUpSTjOYYHH0EsqnTLHkU5Md4Ro\nUpeJX1GmAoIAUGruB/8jPbMaDQQXbjNLDCCalStlbqMgbgz/Ty4ukQ==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-06T16:09:35Z",
|
||||
"mac": "ENC[AES256_GCM,data:2RptHkE/k4JfqdybmnI3sbeEDaaD4bUtEPLuBcpltZjR5EHFYLsEB1Woxlzj2rLqq+8Wr6kWZtsG3uJSxsColUbazJd1CoVJxHpm6tAnM47Mv1YG5PdLwqpwJWji4AI5lAer4ZMfuGDpNbrwvbO3qB8R55r5SYay4b4Yc49wQXA=,iv:ESimFSybysRrgEj+27ECUi6kIklv1IunWVclTjX7C5g=,tag:LONP+cUm5NVcBvgVStZnwQ==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-borgbackup.ssh/users/joerg
Symbolic link
1
sops/secrets/web01-borgbackup.ssh/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
1
sops/secrets/web01-gitea-password-hash/machines/web01
Symbolic link
1
sops/secrets/web01-gitea-password-hash/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-gitea-password-hash/secret
Normal file
24
sops/secrets/web01-gitea-password-hash/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:zCWFFE6+923po+i6g+ehKgC3FdAEhbmFDTbc6VZIXdBqNO7qvC8K1Q34aZVzQ3HaE6l/p5V7Ax0U0xRypQ==,iv:NJhOMcGg55fznrpM6bSqNvr/lOYAsUUVtfK8eJRs0Iw=,tag:6jadN151/70a7BBXsqMClg==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHcHNaTjJlejlJYy90eGFT\nVTloRFV3OVV4enI1OENaeGVpcXpCV0dUenlBCkdONUE3eXhlY1JMRko5Q0VJVFN6\nMkdSR1krYjlJRyswOExRSW9UeUI2czAKLS0tIEJWRDZwRWp1U3V4S0NLOXJDS0ZZ\ncXRFNGxnNXZHNHpvOUpVcTYvM3RoNU0KPgJoJ/22jyUtqGeXfO+DInB3zIwrB+OP\ncjw6Dt7mPYT/OUG6Cq12D6+xMYCm+r4jswtkvWaPhnzGcIOcqMJHwg==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUa1IvdklYcENvUzlwdnNi\nbnlidGVvMzZLRS9EU1RzZ0VzMUtvOGRGR25zCklqVTA4T2FIR3l2MER2RjRsbkZH\nRWlxUkYyUjIwSzl5SWJHblMvclZwOGsKLS0tICtaYW83M3lXakJsMFNEc0FjYWdC\nU3ZDUEplYk1tOFRiUUpXTVA0NTUyaHMKdtR+rqRz+Jjf4BfCd5B7ygRLYKTDDRJk\nq0eSNG+i+Xjz/kLWsMpmO4Cevhp0SPyLZV2g2CiDo5vXZQ5Qiy8pSQ==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-10T09:11:22Z",
|
||||
"mac": "ENC[AES256_GCM,data:D+NLO8U8mXc4wzQC1OHoba5t+i92P3ZeZy7M8nPhBvnWFznhWBmHRLTI55c8+Q3tkNJI0rBt43+XjC7X1ij36eSza/8O6dh5+jM4UkvFBBJG8ZTPSqakISmPBN1k80qm6G15ELgRrJc0+DNAuuZVuBAwVNUFmaZNx6FmX/G4nRU=,iv:RlhgqQoXAeNFTLRJubVzFJq0wbZwZOeAyZs2nD7IHfg=,tag:6zgWakwYjf93qyMwKlSG9g==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-gitea-password-hash/users/joerg
Symbolic link
1
sops/secrets/web01-gitea-password-hash/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
1
sops/secrets/web01-gitea-password/machines/web01
Symbolic link
1
sops/secrets/web01-gitea-password/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-gitea-password/secret
Normal file
24
sops/secrets/web01-gitea-password/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:bcYm9Jx6NS5T2085GmeUJJeLdD1ZtGSfMtXNWcNkeL7F,iv:jR8k0EMO20ZiBXmb1ddJS5x0c95y9vEPvMig0Y0iXBg=,tag:wZBLbCe8ucQSIGrNOjN1jg==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSVWx6TzM4MEpmZ3ExczZo\nNS9kU1Z5NEl1aDdwSzUwSy93anlPQXdOVVVNClBqMENEWUhLVml6dkRZaVk4OU1V\nNjBNV0p2MjFLMDI1c3paOUU0Zndsd28KLS0tIEJZVFA4akVLMzVSanJMcWwweCtE\nZ2h2NE1mdWJNd1VWZDFyT0tvTmlrV0kKfsW5qG12wP+hI/ZCcZNsjv5ububSITLp\n4SzzyeTzpDrGlu/h52szD0VYnB0w3/fF2Ar/lvBYN0y9MXXYUQGdRA==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArdmZUaThzQzQ0em9reXJM\nTkNNRlhKRWoxR3dVTXc0TEdXV2pNQytXK3lrCkk1Z1g2d2R6V002d2lXNWtFMmo5\nT2tiTGpyRTE4WXk4c0hYOGdFejBITWsKLS0tIFdib0UzL2dNbXRjZHFYOEVGSWVU\nTDlNN0xSQWgzdFVhV21SSE9JNkM0OGcK2icnV6pvh7PMVp5r51b+Ukgl95XiiTHG\nDjj3M24jEh9UX2bYraGyRNnLh3piQe7Jim3/ZAHSOzl105GulapU5g==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-10T09:11:20Z",
|
||||
"mac": "ENC[AES256_GCM,data:Ie9j/N4dB6qKtpzPrQROPbsGQCfzYL8dhtptOB0XQw+mh19vpcvWyzLqYOorM1eBKrUWYob6ZHe27KXxN+9RtPe+KFABlFAQRENfPBVPi9Y7/XxMiMQ2gL6JQkvN47Aou/jWhPIOeuCXuEqr4VEOa0F6jPLmS9aPPc95MV/cHxo=,iv:/R67c5rBG3nIm6iAJedPdXL8R+b1RGez/ejzBDW4tf4=,tag:2A9njvLHsAzda+kh8PYj5w==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-gitea-password/users/joerg
Symbolic link
1
sops/secrets/web01-gitea-password/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
1
sops/secrets/web01-golem-password-hash/machines/web01
Symbolic link
1
sops/secrets/web01-golem-password-hash/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-golem-password-hash/secret
Normal file
24
sops/secrets/web01-golem-password-hash/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:U1NXeka1c0Fe55r8D6lAQiujSHbOW6zLjZ85dmtk02q9Szcjj79A6v/jFezqjbQjTtBvBs7tn39/MhQ6CQ==,iv:WPd7Jl4qldLztNUfErlF0dlMo4fe96aJUpiJk0GJePM=,tag:ruIMTtbVOYE7Y4XXhoBSww==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnWHlzTzYwckQwU2VKSDhW\nQmdQdWRSOFgrYk1ZamtDK3JPdkQvcFhrUWhjCjVWbWJYZFUyWnloM1Bram1Rbm1Q\nclZ4NExNOTVCZURFRVhqbGpvNEh5WG8KLS0tIFFkT1ZEOUoxS2NlcFZ0NTFjQmp6\ncUNHOFM1ZWJFaVk4SzJQUzMzbXFXTlEKDUDq9ErdYGm0KYWoXaG8/mVRuW/Sy7hW\nUIzOJ4gdPfB8BxGN5y/Nb0dX+lHN/M4qebcW9KXXPI6Pa3Y6aXCP8g==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEMThwK1FPZUdaNnFmZnJm\nZmx6cWs4QTlwVEd2UFBhdHpIYWdZNW5BMEE0Cjg3bTMxbTFWSGF0UG0rTktrdHpG\ncFBvNDJnY1hWbmxKUUhpRHVpRndhMVEKLS0tIE9BemM0ck5MQWw0YTBRUHpIVjI1\nQTI1c1B3T0FOdkc5MVZZSEhzUFNiNncKuTDwqvXvUcXSX0q8aqlKHr4YewKuL82v\nf/6Mow2JDODVJXtdG36ZBUGQWfCcrSDHVrZjlcoTGyxiHXYh49Y8hQ==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-10T09:11:23Z",
|
||||
"mac": "ENC[AES256_GCM,data:ELrw2J+ar72JSJVWN2qJl3SvmtZUIDaeannl75UJN1Z/HZ70F6HDfasu8gtfRraAc5uKuBviyKm83eElwXELV5ZHz5IMkEvFNYOJsAp65YBzfEZuAMoPMFsBYE9U0MTJeYuN62/j13X8Lyld2JPDyPy6INgozFr5XgWfLgkHfrA=,iv:W51r68thFudKRgl9yaSClSG9ByRMfDzFETIWAycBNHw=,tag:8oRyvy53Cvn1u7UH4DuhMA==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-golem-password-hash/users/joerg
Symbolic link
1
sops/secrets/web01-golem-password-hash/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
1
sops/secrets/web01-golem-password/machines/web01
Symbolic link
1
sops/secrets/web01-golem-password/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-golem-password/secret
Normal file
24
sops/secrets/web01-golem-password/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:Nx5x4US7N7vKqAhnn2NFwsBiuh9tnAWCBrc6pbNCDQ==,iv:ijhwJFzxggDFPdXVPwKKG0vI8HA8m21xkdFUhHIvCBk=,tag:p1QbTFm/TTyUaGI1s73MIQ==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4dDF3ZEVtTVlyMTRLQ2c2\nOTB6WUxvMXdJZ2xrMnlFOElpYmlEMnorb0NrCkN5MzFmMG9GbTc2N0pvbGtTZFdp\nNjI2YmlodlhSaXMyTENjMG44UkxxYUEKLS0tIEhPZEJhWGozdVBMVWM1QkV5cDAx\nYWRBL3VGU0RFY29HVWtTVjJQZVpIdnMKAftERIDtOMw8k3fbMo+KZJ4JYc5UyL3S\n+16m0hWK1BCXkeL2XFGujkzmrGXJF1bxFXCegdH4fnW2+IMESZZO6w==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB4WmJobW8xTU40RUYyYlVj\nclk4UndKRTBMSDNFSmc4RGx1OGJyd0poTFNjClhQV3BQdlVEeU8rME5OUUtlTmYr\nTzZhR2srYnlzL3l5NUZlVmhFV3BOcXcKLS0tIFVMUm5tTVBXckRsVHVsc0ZrSzB1\nWE02MVJZNWtYc201ZDBrc1d2SUptcW8KPSqT5mBQymSksUv3j1y6vgnMuwQKbiXW\nCtzVtF05hv2Z21L+XIV3LOpJ98GGUoJu2uq7qjKIM4CYX+Jj/GS9Nw==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-10T09:11:22Z",
|
||||
"mac": "ENC[AES256_GCM,data:6EeQBxukfz2iNypbkasgDSqb8vMiRaORrA8OvYP5+YNUUguF+jCmSpOUHOM6d2KMF6vGSPLiG15e5IxW7x0QIotMf91Bj46FquzT8PS1hcPTe4WIcg/FHAlLNYqQUgZ9ZlojekkYqs13P8NvFW9pY+MSeYMRQFQLrXvaakcYDHs=,iv:xXALlG13aSaiKiAFUAE/8cZnjh5DaKlinKemoM5tl9E=,tag:x3xVmQwufZav5Yhwxp8cUw==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-golem-password/users/joerg
Symbolic link
1
sops/secrets/web01-golem-password/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
1
sops/secrets/web01-matrix-password-admin/machines/web01
Symbolic link
1
sops/secrets/web01-matrix-password-admin/machines/web01
Symbolic link
@ -0,0 +1 @@
|
||||
../../../machines/web01
|
24
sops/secrets/web01-matrix-password-admin/secret
Normal file
24
sops/secrets/web01-matrix-password-admin/secret
Normal file
@ -0,0 +1,24 @@
|
||||
{
|
||||
"data": "ENC[AES256_GCM,data:vef3eK79BP0R8KS3Ycal0HOfcVTZkB9whZIqjpmgQw==,iv:3i37rWIn5kh6jWQqGFRu0yxyT1Bfa99espOT1DpYB/E=,tag:u90TEhOKTQ8Y8iPvnToeXA==,type:str]",
|
||||
"sops": {
|
||||
"kms": null,
|
||||
"gcp_kms": null,
|
||||
"azure_kv": null,
|
||||
"hc_vault": null,
|
||||
"age": [
|
||||
{
|
||||
"recipient": "age17n64ahe3wesh8l8lj0zylf4nljdmqn28hvqns2g7hgm9mdkhlsvsjuvkxz",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1K2tkSUE0d0UzYi9XSVdi\nWE5BMVBSTFdLRXJkbk5FaEVvZTUwVjBGT3pRCnVOY3oyK0xIcm4zeXpSb2lzdUZ6\nWDJyUDBDdTdVSmJsYzlBL01qNGtBaW8KLS0tIFoybmZ4cXpZWnhjM2JsMDhNdFVH\nNHhRZTUveDhvRHFGNjhUL0hjNmV2TU0KU9pt2aRKN77uQW5Mq6l/g21YEpokW8Rn\nH0jmBc+n9pPkphojl7VUhm52aBMsUxU94Chko3oHUnTWJXjaS36LAw==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
},
|
||||
{
|
||||
"recipient": "age17xuvz0fqtynzdmf8rfh4g3e46tx8w3mc6zgytrmuj5v9dhnldgxs7ue7ct",
|
||||
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBER0c3VCtzb29MUVVvU3p1\naXJXS01uT2NrcmNIT1RyOGJFazA3bFlTNnpvCjVzWmtieVR4ZUV1cWhaR3p3bVNP\ncXp4aWlrZE85cjFRY3RzcThJTlE3SkUKLS0tIGc5UWRINkhhV3lKMjBqYlJDUE5O\ncTY4V05qaGxWM1RYWjFKTXdiM0ZPc00K5nIxr2jMrBdtXJIWjwORM2jXjk7Xcvxu\nIc2KKoOOaQwo7SpAdf/GQm8BoHh2TJcUevfpM0GDII/DyMenhPf1pw==\n-----END AGE ENCRYPTED FILE-----\n"
|
||||
}
|
||||
],
|
||||
"lastmodified": "2024-06-11T13:38:24Z",
|
||||
"mac": "ENC[AES256_GCM,data:B2gQaMJIFpJuKpPPdaa6Gw0K9DN4FcoEOybSkA+2kMAqMv5cTPVnG7HV/XY21bQKvmGkAvKa6PhouIp46QyajTmUrUlAGrZrt9W1tBwctJlsHiY7O6she6S02NPpKn9CWurh5XIz8NxcZVcMEwWhc3wQdld47puubUOgHpmfqFU=,iv:Bi3CL4Zb9EZoEmA0e3Qg0B6Kwhc5pvlFlioCcX8Fnco=,tag:gk7VaDhZlbfnv8Xrx6mypA==,type:str]",
|
||||
"pgp": null,
|
||||
"unencrypted_suffix": "_unencrypted",
|
||||
"version": "3.8.1"
|
||||
}
|
||||
}
|
1
sops/secrets/web01-matrix-password-admin/users/joerg
Symbolic link
1
sops/secrets/web01-matrix-password-admin/users/joerg
Symbolic link
@ -0,0 +1 @@
|
||||
../../../users/joerg
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user