diff options
24 files changed, 951 insertions, 174 deletions
diff --git a/_sources/generated.json b/_sources/generated.json index 78285ff5..dcbde8b1 100644 --- a/_sources/generated.json +++ b/_sources/generated.json | |||
@@ -64,11 +64,11 @@ | |||
64 | "pinned": false, | 64 | "pinned": false, |
65 | "src": { | 65 | "src": { |
66 | "name": null, | 66 | "name": null, |
67 | "sha256": "sha256-eOq2PYvLB6ueIjC8Rif/p7HJKW8AkbEjf1La9/HUaC8=", | 67 | "sha256": "sha256-9Gx7Cwb5UWE1NaSA0iun6FY/TwjT0/jjcAR98SLLFjc=", |
68 | "type": "url", | 68 | "type": "url", |
69 | "url": "https://github.com/wofr06/lesspipe/archive/refs/tags/v2.05.tar.gz" | 69 | "url": "https://github.com/wofr06/lesspipe/archive/refs/tags/v2.06.tar.gz" |
70 | }, | 70 | }, |
71 | "version": "2.05" | 71 | "version": "2.06" |
72 | }, | 72 | }, |
73 | "mpv-autosave": { | 73 | "mpv-autosave": { |
74 | "cargoLocks": null, | 74 | "cargoLocks": null, |
@@ -101,11 +101,11 @@ | |||
101 | "name": null, | 101 | "name": null, |
102 | "owner": "po5", | 102 | "owner": "po5", |
103 | "repo": "chapterskip", | 103 | "repo": "chapterskip", |
104 | "rev": "f4c5da3e7661212eb491cc1d85beafbf951e32f0", | 104 | "rev": "b26825316e3329882206ae78dc903ebc4613f039", |
105 | "sha256": "sha256-ZK64WdhXMubzfkKFVg7rX5dPc4IVHDwN0T1X9tXwsJI=", | 105 | "sha256": "sha256-OTrLQE3rYvPQamEX23D6HttNjx3vafWdTMxTiWpDy90=", |
106 | "type": "github" | 106 | "type": "github" |
107 | }, | 107 | }, |
108 | "version": "f4c5da3e7661212eb491cc1d85beafbf951e32f0" | 108 | "version": "b26825316e3329882206ae78dc903ebc4613f039" |
109 | }, | 109 | }, |
110 | "mpv-createchapter": { | 110 | "mpv-createchapter": { |
111 | "cargoLocks": null, | 111 | "cargoLocks": null, |
@@ -139,11 +139,11 @@ | |||
139 | "name": null, | 139 | "name": null, |
140 | "owner": "hoyon", | 140 | "owner": "hoyon", |
141 | "repo": "mpv-mpris", | 141 | "repo": "mpv-mpris", |
142 | "rev": "0.8.1", | 142 | "rev": "0.9", |
143 | "sha256": "sha256-ugEiQZA1vQCVwyv3ViM84Qz8lhRvy17vcxjayYevTAs=", | 143 | "sha256": "sha256-leW7oCWTnJuprVnJJ+iXd3nuB2VXl3fw8FmPxv7d6rA=", |
144 | "type": "github" | 144 | "type": "github" |
145 | }, | 145 | }, |
146 | "version": "0.8.1" | 146 | "version": "0.9" |
147 | }, | 147 | }, |
148 | "mpv-reload": { | 148 | "mpv-reload": { |
149 | "cargoLocks": null, | 149 | "cargoLocks": null, |
@@ -172,11 +172,25 @@ | |||
172 | "pinned": false, | 172 | "pinned": false, |
173 | "src": { | 173 | "src": { |
174 | "name": null, | 174 | "name": null, |
175 | "sha256": "sha256-snvUmKZVckDNt2nnFOEa4cbGLtm825UgvA3cBpoNGLw=", | 175 | "sha256": "sha256-3vB6krsP6G25bviG27QI+9NyJN2YKOOmM5KhKUclJPc=", |
176 | "type": "url", | 176 | "type": "url", |
177 | "url": "https://github.com/Snawoot/postfix-mta-sts-resolver/archive/refs/tags/v1.1.3.tar.gz" | 177 | "url": "https://github.com/Snawoot/postfix-mta-sts-resolver/archive/refs/tags/v1.1.4.tar.gz" |
178 | }, | 178 | }, |
179 | "version": "1.1.3" | 179 | "version": "1.1.4" |
180 | }, | ||
181 | "postfwd": { | ||
182 | "cargoLocks": null, | ||
183 | "extract": null, | ||
184 | "name": "postfwd", | ||
185 | "passthru": null, | ||
186 | "pinned": false, | ||
187 | "src": { | ||
188 | "name": null, | ||
189 | "sha256": "sha256-mMKXzeqg2PfXkvGL7qugOelm/I2fZnUidq6/ugXDHa0=", | ||
190 | "type": "url", | ||
191 | "url": "https://github.com/postfwd/postfwd/archive/refs/tags/v2.03.tar.gz" | ||
192 | }, | ||
193 | "version": "2.03" | ||
180 | }, | 194 | }, |
181 | "psql-versioning": { | 195 | "psql-versioning": { |
182 | "cargoLocks": null, | 196 | "cargoLocks": null, |
@@ -196,6 +210,20 @@ | |||
196 | }, | 210 | }, |
197 | "version": "3e578ff5e5aa6c7e5459dbfa842a64a1b2674b2e" | 211 | "version": "3e578ff5e5aa6c7e5459dbfa842a64a1b2674b2e" |
198 | }, | 212 | }, |
213 | "smartprom": { | ||
214 | "cargoLocks": null, | ||
215 | "extract": null, | ||
216 | "name": "smartprom", | ||
217 | "passthru": null, | ||
218 | "pinned": false, | ||
219 | "src": { | ||
220 | "name": null, | ||
221 | "sha256": "sha256-VbpFvDBygJswUfmufVjo/xXxDDmXLq/0D9ln8u+139E=", | ||
222 | "type": "url", | ||
223 | "url": "https://github.com/matusnovak/prometheus-smartctl/archive/refs/tags/v2.1.0.tar.gz" | ||
224 | }, | ||
225 | "version": "2.1.0" | ||
226 | }, | ||
199 | "uhk-agent": { | 227 | "uhk-agent": { |
200 | "cargoLocks": null, | 228 | "cargoLocks": null, |
201 | "extract": null, | 229 | "extract": null, |
@@ -223,11 +251,11 @@ | |||
223 | "name": null, | 251 | "name": null, |
224 | "owner": "umlaeute", | 252 | "owner": "umlaeute", |
225 | "repo": "v4l2loopback", | 253 | "repo": "v4l2loopback", |
226 | "rev": "4aadc417254bfa3b875bf0b69278ce400ce659b2", | 254 | "rev": "76434ab6f71d5ecbff8a218ff6bed91ea2bf73b8", |
227 | "sha256": "sha256-nHxIW5BmaZC6g7SElxboTcwtMDF4SCqi11MjYWsUZpo=", | 255 | "sha256": "sha256-c6g63jW+a+v/TxLD9NnQGn/aUgivwVkxzP+hZ65w2/o=", |
228 | "type": "github" | 256 | "type": "github" |
229 | }, | 257 | }, |
230 | "version": "4aadc417254bfa3b875bf0b69278ce400ce659b2" | 258 | "version": "76434ab6f71d5ecbff8a218ff6bed91ea2bf73b8" |
231 | }, | 259 | }, |
232 | "xcompose": { | 260 | "xcompose": { |
233 | "cargoLocks": null, | 261 | "cargoLocks": null, |
diff --git a/_sources/generated.nix b/_sources/generated.nix index 8aecd856..a77cb5d8 100644 --- a/_sources/generated.nix +++ b/_sources/generated.nix | |||
@@ -38,10 +38,10 @@ | |||
38 | }; | 38 | }; |
39 | lesspipe = { | 39 | lesspipe = { |
40 | pname = "lesspipe"; | 40 | pname = "lesspipe"; |
41 | version = "2.05"; | 41 | version = "2.06"; |
42 | src = fetchurl { | 42 | src = fetchurl { |
43 | url = "https://github.com/wofr06/lesspipe/archive/refs/tags/v2.05.tar.gz"; | 43 | url = "https://github.com/wofr06/lesspipe/archive/refs/tags/v2.06.tar.gz"; |
44 | sha256 = "sha256-eOq2PYvLB6ueIjC8Rif/p7HJKW8AkbEjf1La9/HUaC8="; | 44 | sha256 = "sha256-9Gx7Cwb5UWE1NaSA0iun6FY/TwjT0/jjcAR98SLLFjc="; |
45 | }; | 45 | }; |
46 | }; | 46 | }; |
47 | mpv-autosave = { | 47 | mpv-autosave = { |
@@ -58,13 +58,13 @@ | |||
58 | }; | 58 | }; |
59 | mpv-chapterskip = { | 59 | mpv-chapterskip = { |
60 | pname = "mpv-chapterskip"; | 60 | pname = "mpv-chapterskip"; |
61 | version = "f4c5da3e7661212eb491cc1d85beafbf951e32f0"; | 61 | version = "b26825316e3329882206ae78dc903ebc4613f039"; |
62 | src = fetchFromGitHub ({ | 62 | src = fetchFromGitHub ({ |
63 | owner = "po5"; | 63 | owner = "po5"; |
64 | repo = "chapterskip"; | 64 | repo = "chapterskip"; |
65 | rev = "f4c5da3e7661212eb491cc1d85beafbf951e32f0"; | 65 | rev = "b26825316e3329882206ae78dc903ebc4613f039"; |
66 | fetchSubmodules = false; | 66 | fetchSubmodules = false; |
67 | sha256 = "sha256-ZK64WdhXMubzfkKFVg7rX5dPc4IVHDwN0T1X9tXwsJI="; | 67 | sha256 = "sha256-OTrLQE3rYvPQamEX23D6HttNjx3vafWdTMxTiWpDy90="; |
68 | }); | 68 | }); |
69 | }; | 69 | }; |
70 | mpv-createchapter = { | 70 | mpv-createchapter = { |
@@ -80,13 +80,13 @@ | |||
80 | }; | 80 | }; |
81 | mpv-mpris = { | 81 | mpv-mpris = { |
82 | pname = "mpv-mpris"; | 82 | pname = "mpv-mpris"; |
83 | version = "0.8.1"; | 83 | version = "0.9"; |
84 | src = fetchFromGitHub ({ | 84 | src = fetchFromGitHub ({ |
85 | owner = "hoyon"; | 85 | owner = "hoyon"; |
86 | repo = "mpv-mpris"; | 86 | repo = "mpv-mpris"; |
87 | rev = "0.8.1"; | 87 | rev = "0.9"; |
88 | fetchSubmodules = false; | 88 | fetchSubmodules = false; |
89 | sha256 = "sha256-ugEiQZA1vQCVwyv3ViM84Qz8lhRvy17vcxjayYevTAs="; | 89 | sha256 = "sha256-leW7oCWTnJuprVnJJ+iXd3nuB2VXl3fw8FmPxv7d6rA="; |
90 | }); | 90 | }); |
91 | }; | 91 | }; |
92 | mpv-reload = { | 92 | mpv-reload = { |
@@ -102,10 +102,18 @@ | |||
102 | }; | 102 | }; |
103 | postfix-mta-sts-resolver = { | 103 | postfix-mta-sts-resolver = { |
104 | pname = "postfix-mta-sts-resolver"; | 104 | pname = "postfix-mta-sts-resolver"; |
105 | version = "1.1.3"; | 105 | version = "1.1.4"; |
106 | src = fetchurl { | 106 | src = fetchurl { |
107 | url = "https://github.com/Snawoot/postfix-mta-sts-resolver/archive/refs/tags/v1.1.3.tar.gz"; | 107 | url = "https://github.com/Snawoot/postfix-mta-sts-resolver/archive/refs/tags/v1.1.4.tar.gz"; |
108 | sha256 = "sha256-snvUmKZVckDNt2nnFOEa4cbGLtm825UgvA3cBpoNGLw="; | 108 | sha256 = "sha256-3vB6krsP6G25bviG27QI+9NyJN2YKOOmM5KhKUclJPc="; |
109 | }; | ||
110 | }; | ||
111 | postfwd = { | ||
112 | pname = "postfwd"; | ||
113 | version = "2.03"; | ||
114 | src = fetchurl { | ||
115 | url = "https://github.com/postfwd/postfwd/archive/refs/tags/v2.03.tar.gz"; | ||
116 | sha256 = "sha256-mMKXzeqg2PfXkvGL7qugOelm/I2fZnUidq6/ugXDHa0="; | ||
109 | }; | 117 | }; |
110 | }; | 118 | }; |
111 | psql-versioning = { | 119 | psql-versioning = { |
@@ -120,6 +128,14 @@ | |||
120 | sha256 = "sha256-j+njRssJHTdNV3FbcA3MdUmzCaJxuYBrC0qwtK3HoyY="; | 128 | sha256 = "sha256-j+njRssJHTdNV3FbcA3MdUmzCaJxuYBrC0qwtK3HoyY="; |
121 | }; | 129 | }; |
122 | }; | 130 | }; |
131 | smartprom = { | ||
132 | pname = "smartprom"; | ||
133 | version = "2.1.0"; | ||
134 | src = fetchurl { | ||
135 | url = "https://github.com/matusnovak/prometheus-smartctl/archive/refs/tags/v2.1.0.tar.gz"; | ||
136 | sha256 = "sha256-VbpFvDBygJswUfmufVjo/xXxDDmXLq/0D9ln8u+139E="; | ||
137 | }; | ||
138 | }; | ||
123 | uhk-agent = { | 139 | uhk-agent = { |
124 | pname = "uhk-agent"; | 140 | pname = "uhk-agent"; |
125 | version = "1.5.17"; | 141 | version = "1.5.17"; |
@@ -130,13 +146,13 @@ | |||
130 | }; | 146 | }; |
131 | v4l2loopback = { | 147 | v4l2loopback = { |
132 | pname = "v4l2loopback"; | 148 | pname = "v4l2loopback"; |
133 | version = "4aadc417254bfa3b875bf0b69278ce400ce659b2"; | 149 | version = "76434ab6f71d5ecbff8a218ff6bed91ea2bf73b8"; |
134 | src = fetchFromGitHub ({ | 150 | src = fetchFromGitHub ({ |
135 | owner = "umlaeute"; | 151 | owner = "umlaeute"; |
136 | repo = "v4l2loopback"; | 152 | repo = "v4l2loopback"; |
137 | rev = "4aadc417254bfa3b875bf0b69278ce400ce659b2"; | 153 | rev = "76434ab6f71d5ecbff8a218ff6bed91ea2bf73b8"; |
138 | fetchSubmodules = true; | 154 | fetchSubmodules = true; |
139 | sha256 = "sha256-nHxIW5BmaZC6g7SElxboTcwtMDF4SCqi11MjYWsUZpo="; | 155 | sha256 = "sha256-c6g63jW+a+v/TxLD9NnQGn/aUgivwVkxzP+hZ65w2/o="; |
140 | }); | 156 | }); |
141 | }; | 157 | }; |
142 | xcompose = { | 158 | xcompose = { |
diff --git a/accounts/gkleen@sif/default.nix b/accounts/gkleen@sif/default.nix index d3db91c8..2cfaa620 100644 --- a/accounts/gkleen@sif/default.nix +++ b/accounts/gkleen@sif/default.nix | |||
@@ -258,12 +258,14 @@ in { | |||
258 | screen-locker = { | 258 | screen-locker = { |
259 | enable = true; | 259 | enable = true; |
260 | lockCmd = toString (pkgs.writeShellScript "lock" '' | 260 | lockCmd = toString (pkgs.writeShellScript "lock" '' |
261 | ${pkgs.playerctl}/bin/playerctl -a status | ${pkgs.gnugrep}/bin/grep -q "Playing" && exit 0 | ||
262 | |||
261 | cleanup() { | 263 | cleanup() { |
262 | ${cfg.services.dunst.package}/bin/dunstctl set-paused false | 264 | ${cfg.services.dunst.package}/bin/dunstctl set-paused false |
263 | } | 265 | } |
264 | trap cleanup EXIT INT TERM | 266 | trap cleanup EXIT INT TERM |
265 | 267 | ||
266 | ${pkgs.playerctl}/bin/playerctl -a pause | 268 | # ${pkgs.playerctl}/bin/playerctl -a pause |
267 | ${cfg.services.dunst.package}/bin/dunstctl set-paused true | 269 | ${cfg.services.dunst.package}/bin/dunstctl set-paused true |
268 | ${pkgs.xsecurelock}/bin/xsecurelock | 270 | ${pkgs.xsecurelock}/bin/xsecurelock |
269 | ''); | 271 | ''); |
diff --git a/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__init__.py b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__init__.py | |||
diff --git a/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py new file mode 100644 index 00000000..f481090c --- /dev/null +++ b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py | |||
@@ -0,0 +1,92 @@ | |||
1 | from systemd.daemon import listen_fds | ||
2 | from sdnotify import SystemdNotifier | ||
3 | from socketserver import StreamRequestHandler, ThreadingMixIn | ||
4 | from systemd_socketserver import SystemdSocketServer | ||
5 | import sys | ||
6 | from threading import Thread | ||
7 | from psycopg_pool import ConnectionPool | ||
8 | from psycopg.rows import namedtuple_row | ||
9 | |||
10 | import logging | ||
11 | |||
12 | |||
13 | class PolicyHandler(StreamRequestHandler): | ||
14 | def handle(self): | ||
15 | logger.debug('Handling new connection...') | ||
16 | |||
17 | self.args = dict() | ||
18 | |||
19 | line = None | ||
20 | while line := self.rfile.readline().removesuffix(b'\n'): | ||
21 | if b'=' not in line: | ||
22 | break | ||
23 | |||
24 | key, val = line.split(sep=b'=', maxsplit=1) | ||
25 | self.args[key.decode()] = val.decode() | ||
26 | |||
27 | logger.info('Connection parameters: %s', self.args) | ||
28 | |||
29 | allowed = False | ||
30 | with self.server.db_pool.connection() as conn: | ||
31 | local, domain = self.args['sender'].split(sep='@', maxsplit=1) | ||
32 | extension = None | ||
33 | if '+' in local: | ||
34 | local, extension = local.split(sep='+', maxsplit=1) | ||
35 | |||
36 | logger.debug('Parsed address: %s', {'local': local, 'extension': extension, 'domain': domain}) | ||
37 | |||
38 | with conn.cursor() as cur: | ||
39 | cur.row_factory = namedtuple_row | ||
40 | cur.execute('SELECT "mailbox"."mailbox" as "user", "local", "extension", "domain" FROM "mailbox" INNER JOIN "mailbox_mapping" ON "mailbox".id = "mailbox_mapping"."mailbox" WHERE "mailbox"."mailbox" = %(user)s AND ("local" = %(local)s OR "local" IS NULL) AND ("extension" = %(extension)s OR "extension" IS NULL) AND "domain" = %(domain)s', params = {'user': self.args['ccert_subject'], 'local': local, 'extension': extension if extension is not None else '', 'domain': domain}, prepare=True) | ||
41 | for record in cur: | ||
42 | logger.debug('Received result: %s', record) | ||
43 | allowed = True | ||
44 | |||
45 | action = '550 5.7.0 Sender address not authorized for current user' | ||
46 | if allowed: | ||
47 | action = 'DUNNO' | ||
48 | |||
49 | logger.info('Reached verdict: %s', {'allowed': allowed, 'action': action}) | ||
50 | self.wfile.write(f'action={action}\n\n'.encode()) | ||
51 | |||
52 | class ThreadedSystemdSocketServer(ThreadingMixIn, SystemdSocketServer): | ||
53 | def __init__(self, fd, RequestHandlerClass): | ||
54 | super().__init__(fd, RequestHandlerClass) | ||
55 | |||
56 | self.db_pool = ConnectionPool(min_size=1) | ||
57 | self.db_pool.wait() | ||
58 | |||
59 | def main(): | ||
60 | global logger | ||
61 | logger = logging.getLogger(__name__) | ||
62 | console_handler = logging.StreamHandler() | ||
63 | console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') ) | ||
64 | if sys.stderr.isatty(): | ||
65 | console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') ) | ||
66 | logger.addHandler(console_handler) | ||
67 | logger.setLevel(logging.DEBUG) | ||
68 | |||
69 | # log uncaught exceptions | ||
70 | def log_exceptions(type, value, tb): | ||
71 | global logger | ||
72 | |||
73 | logger.error(value) | ||
74 | sys.__excepthook__(type, value, tb) # calls default excepthook | ||
75 | |||
76 | sys.excepthook = log_exceptions | ||
77 | |||
78 | fds = listen_fds() | ||
79 | servers = [ThreadedSystemdSocketServer(fd, PolicyHandler) for fd in fds] | ||
80 | |||
81 | if servers: | ||
82 | for server in servers: | ||
83 | Thread(name=f'Server for fd{server.fileno()}', target=server.serve_forever).start() | ||
84 | else: | ||
85 | return 2 | ||
86 | |||
87 | SystemdNotifier().notify('READY=1') | ||
88 | |||
89 | return 0 | ||
90 | |||
91 | if __name__ == '__main__': | ||
92 | sys.exit(main()) | ||
diff --git a/hosts/surtr/email/ccert-policy-server/setup.py b/hosts/surtr/email/ccert-policy-server/setup.py new file mode 100644 index 00000000..d8eb415a --- /dev/null +++ b/hosts/surtr/email/ccert-policy-server/setup.py | |||
@@ -0,0 +1,12 @@ | |||
1 | from setuptools import setup, find_packages | ||
2 | |||
3 | setup( | ||
4 | name = 'ccert-policy-server', | ||
5 | version = '0.0.0', | ||
6 | packages = ['ccert_policy_server'], | ||
7 | entry_points = { | ||
8 | 'console_scripts': [ | ||
9 | 'ccert-policy-server=ccert_policy_server.__main__:main' | ||
10 | ], | ||
11 | }, | ||
12 | ) | ||
diff --git a/hosts/surtr/email/default.nix b/hosts/surtr/email/default.nix index 83bf02f5..9cfba1f1 100644 --- a/hosts/surtr/email/default.nix +++ b/hosts/surtr/email/default.nix | |||
@@ -1,4 +1,4 @@ | |||
1 | { config, pkgs, lib, ... }: | 1 | { config, pkgs, lib, flakeInputs, ... }: |
2 | 2 | ||
3 | with lib; | 3 | with lib; |
4 | 4 | ||
@@ -20,6 +20,27 @@ let | |||
20 | ''; | 20 | ''; |
21 | }; | 21 | }; |
22 | 22 | ||
23 | ccert-policy-server = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage { | ||
24 | src = ./ccert-policy-server; | ||
25 | pname = "ccert-policy-server"; | ||
26 | version = "0.0.0"; | ||
27 | |||
28 | python = "python39"; | ||
29 | ignoreDataOutdated = true; | ||
30 | |||
31 | requirements = '' | ||
32 | sdnotify | ||
33 | systemd-socketserver | ||
34 | psycopg >=3.0.0 | ||
35 | psycopg-pool >=3.0.0 | ||
36 | psycopg-binary >=3.0.0 | ||
37 | ''; | ||
38 | |||
39 | overridesPre = [ | ||
40 | (self: super: { systemd-python = super.systemd.overrideAttrs (oldAttrs: { pname = "systemd-python"; }); }) | ||
41 | ]; | ||
42 | }; | ||
43 | |||
23 | spmDomains = ["bouncy.email"]; | 44 | spmDomains = ["bouncy.email"]; |
24 | in { | 45 | in { |
25 | config = { | 46 | config = { |
@@ -35,7 +56,7 @@ in { | |||
35 | }; | 56 | }; |
36 | }) | 57 | }) |
37 | ]; | 58 | ]; |
38 | 59 | ||
39 | services.postfix = { | 60 | services.postfix = { |
40 | enable = true; | 61 | enable = true; |
41 | hostname = "surtr.yggdrasil.li"; | 62 | hostname = "surtr.yggdrasil.li"; |
@@ -187,8 +208,9 @@ in { | |||
187 | "-o" "smtpd_tls_ask_ccert=yes" | 208 | "-o" "smtpd_tls_ask_ccert=yes" |
188 | "-o" "smtpd_tls_req_ccert=yes" | 209 | "-o" "smtpd_tls_req_ccert=yes" |
189 | "-o" "smtpd_client_restrictions=permit_tls_all_clientcerts,reject" | 210 | "-o" "smtpd_client_restrictions=permit_tls_all_clientcerts,reject" |
211 | "-o" "{smtpd_data_restrictions = check_policy_service unix:/run/postfwd3/postfwd3.sock}" | ||
190 | "-o" "smtpd_relay_restrictions=permit_tls_all_clientcerts,reject" | 212 | "-o" "smtpd_relay_restrictions=permit_tls_all_clientcerts,reject" |
191 | "-o" "smtpd_sender_restrictions=reject_unknown_sender_domain,reject_unverified_sender" | 213 | "-o" "{smtpd_sender_restrictions = reject_unknown_sender_domain,reject_unverified_sender,check_policy_service unix:/run/postfix-ccert-sender-policy.sock}" |
192 | "-o" "unverified_sender_reject_code=550" | 214 | "-o" "unverified_sender_reject_code=550" |
193 | "-o" "unverified_sender_reject_reason={Sender address rejected: undeliverable address}" | 215 | "-o" "unverified_sender_reject_reason={Sender address rejected: undeliverable address}" |
194 | "-o" "smtpd_recipient_restrictions=reject_unauth_pipelining,reject_non_fqdn_recipient,reject_unknown_recipient_domain,permit_tls_all_clientcerts,reject" | 216 | "-o" "smtpd_recipient_restrictions=reject_unauth_pipelining,reject_non_fqdn_recipient,reject_unknown_recipient_domain,permit_tls_all_clientcerts,reject" |
@@ -415,7 +437,7 @@ in { | |||
415 | mail_plugins = $mail_plugins quota | 437 | mail_plugins = $mail_plugins quota |
416 | mailbox_list_index = yes | 438 | mailbox_list_index = yes |
417 | postmaster_address = postmaster@yggdrasil.li | 439 | postmaster_address = postmaster@yggdrasil.li |
418 | recipient_delimiter = | 440 | recipient_delimiter = |
419 | auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-+_@ | 441 | auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-+_@ |
420 | 442 | ||
421 | service lmtp { | 443 | service lmtp { |
@@ -431,7 +453,7 @@ in { | |||
431 | namespace inbox { | 453 | namespace inbox { |
432 | separator = / | 454 | separator = / |
433 | inbox = yes | 455 | inbox = yes |
434 | prefix = | 456 | prefix = |
435 | 457 | ||
436 | mailbox Trash { | 458 | mailbox Trash { |
437 | auto = no | 459 | auto = no |
@@ -602,7 +624,7 @@ in { | |||
602 | ${pkgs.dovecot_pigeonhole}/bin/sievec $f | 624 | ${pkgs.dovecot_pigeonhole}/bin/sievec $f |
603 | done | 625 | done |
604 | ''; | 626 | ''; |
605 | 627 | ||
606 | serviceConfig = { | 628 | serviceConfig = { |
607 | LoadCredential = [ | 629 | LoadCredential = [ |
608 | "surtr.yggdrasil.li.key.pem:${config.security.acme.certs."surtr.yggdrasil.li".directory}/key.pem" | 630 | "surtr.yggdrasil.li.key.pem:${config.security.acme.certs."surtr.yggdrasil.li".directory}/key.pem" |
@@ -703,7 +725,7 @@ in { | |||
703 | }; | 725 | }; |
704 | systemd.sockets.spm = { | 726 | systemd.sockets.spm = { |
705 | wantedBy = [ "nginx.service" ]; | 727 | wantedBy = [ "nginx.service" ]; |
706 | 728 | ||
707 | socketConfig = { | 729 | socketConfig = { |
708 | ListenStream = "/run/spm/server.sock"; | 730 | ListenStream = "/run/spm/server.sock"; |
709 | SocketUser = "spm"; | 731 | SocketUser = "spm"; |
@@ -730,5 +752,57 @@ in { | |||
730 | enable = true; | 752 | enable = true; |
731 | loglevel = "debug"; | 753 | loglevel = "debug"; |
732 | }; | 754 | }; |
755 | |||
756 | systemd.sockets."postfix-ccert-sender-policy" = { | ||
757 | requiredBy = ["postfix.service"]; | ||
758 | wants = ["postfix-ccert-sender-policy.service"]; | ||
759 | socketConfig = { | ||
760 | ListenStream = "/run/postfix-ccert-sender-policy.sock"; | ||
761 | }; | ||
762 | }; | ||
763 | systemd.services."postfix-ccert-sender-policy" = { | ||
764 | serviceConfig = { | ||
765 | Type = "notify"; | ||
766 | |||
767 | ExecStart = "${ccert-policy-server}/bin/ccert-policy-server"; | ||
768 | |||
769 | Environment = [ | ||
770 | "PGDATABASE=email" | ||
771 | ]; | ||
772 | |||
773 | DynamicUser = false; | ||
774 | User = "postfix-ccert-sender-policy"; | ||
775 | Group = "postfix-ccert-sender-policy"; | ||
776 | ProtectSystem = "strict"; | ||
777 | SystemCallFilter = "@system-service"; | ||
778 | NoNewPrivileges = true; | ||
779 | ProtectKernelTunables = true; | ||
780 | ProtectKernelModules = true; | ||
781 | ProtectKernelLogs = true; | ||
782 | ProtectControlGroups = true; | ||
783 | MemoryDenyWriteExecute = true; | ||
784 | RestrictSUIDSGID = true; | ||
785 | KeyringMode = "private"; | ||
786 | ProtectClock = true; | ||
787 | RestrictRealtime = true; | ||
788 | PrivateDevices = true; | ||
789 | PrivateTmp = true; | ||
790 | ProtectHostname = true; | ||
791 | ReadWritePaths = ["/run/postgresql"]; | ||
792 | }; | ||
793 | }; | ||
794 | users.users."postfix-ccert-sender-policy" = { | ||
795 | isSystemUser = true; | ||
796 | group = "postfix-ccert-sender-policy"; | ||
797 | }; | ||
798 | users.groups."postfix-ccert-sender-policy" = {}; | ||
799 | |||
800 | services.postfwd = { | ||
801 | enable = true; | ||
802 | rules = '' | ||
803 | id=RCPT01; protocol_state=DATA; protocol_state=END-OF-MESSAGE; action=rcpt(ccert_subject/100/3600/450 4.7.1 Exceeding maximum of 100 recipients per hour [$$ratecount]) | ||
804 | id=RCPT02; protocol_state=DATA; protocol_state=END-OF-MESSAGE; action=rcpt(ccert_subject/1000/86400/450 4.7.1 Exceeding maximum of 1000 recipients per day [$$ratecount]) | ||
805 | ''; | ||
806 | }; | ||
733 | }; | 807 | }; |
734 | } | 808 | } |
diff --git a/hosts/surtr/postgresql.nix b/hosts/surtr/postgresql.nix index 66ce60eb..7013ae97 100644 --- a/hosts/surtr/postgresql.nix +++ b/hosts/surtr/postgresql.nix | |||
@@ -104,7 +104,7 @@ in { | |||
104 | ALTER TABLE mailbox_mapping ALTER local TYPE citext; | 104 | ALTER TABLE mailbox_mapping ALTER local TYPE citext; |
105 | ALTER TABLE mailbox_mapping ALTER domain TYPE citext; | 105 | ALTER TABLE mailbox_mapping ALTER domain TYPE citext; |
106 | 106 | ||
107 | CREATE VIEW mailbox_quota_rule (id, mailbox, quota_rule) AS SELECT id, mailbox, (CASE WHEN quota_bytes IS NULL THEN '*:ignore' ELSE '*:bytes=' || quota_bytes END) AS quota_rule FROM mailbox; | 107 | CREATE VIEW mailbox_quota_rule (id, mailbox, quota_rule) AS SELECT id, mailbox, (CASE WHEN quota_bytes IS NULL THEN '*:ignore' ELSE '*:bytes=' || quota_bytes END) AS quota_rule FROM mailbox; |
108 | 108 | ||
109 | CREATE VIEW virtual_mailbox_domain (domain) AS SELECT DISTINCT domain FROM mailbox_mapping; | 109 | CREATE VIEW virtual_mailbox_domain (domain) AS SELECT DISTINCT domain FROM mailbox_mapping; |
110 | CREATE VIEW virtual_mailbox_mapping (lookup) AS SELECT (CASE WHEN local IS NULL THEN ''' ELSE local END) || '@' || domain AS lookup FROM mailbox_mapping; | 110 | CREATE VIEW virtual_mailbox_mapping (lookup) AS SELECT (CASE WHEN local IS NULL THEN ''' ELSE local END) || '@' || domain AS lookup FROM mailbox_mapping; |
@@ -143,6 +143,15 @@ in { | |||
143 | 143 | ||
144 | GRANT SELECT ON ALL TABLES IN SCHEMA public TO "spm"; | 144 | GRANT SELECT ON ALL TABLES IN SCHEMA public TO "spm"; |
145 | COMMIT; | 145 | COMMIT; |
146 | |||
147 | BEGIN; | ||
148 | SELECT _v.register_patch('007-ccert-sender-policy', ARRAY['000-base'], null); | ||
149 | |||
150 | CREATE USER "postfix-ccert-sender-policy"; | ||
151 | GRANT CONNECT ON DATABASE "email" TO "postfix-ccert-sender-policy"; | ||
152 | ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO "postfix-ccert-sender-policy"; | ||
153 | GRANT SELECT ON ALL TABLES IN SCHEMA public TO "postfix-ccert-sender-policy"; | ||
154 | COMMIT; | ||
146 | ''} | 155 | ''} |
147 | ''; | 156 | ''; |
148 | }; | 157 | }; |
diff --git a/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py b/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py new file mode 100644 index 00000000..e93e6a60 --- /dev/null +++ b/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py | |||
@@ -0,0 +1,202 @@ | |||
1 | import argparse | ||
2 | import os, sys, signal | ||
3 | from pyprctl import cap_permitted, cap_inheritable, cap_effective, cap_ambient, Cap | ||
4 | from pwd import getpwnam | ||
5 | |||
6 | from datetime import datetime, timezone | ||
7 | from dateutil.parser import isoparse | ||
8 | |||
9 | from xdg import xdg_runtime_dir | ||
10 | import unshare | ||
11 | from tempfile import TemporaryDirectory | ||
12 | |||
13 | import logging | ||
14 | |||
15 | import json | ||
16 | import subprocess | ||
17 | |||
18 | import pathlib | ||
19 | from pathlib import Path | ||
20 | |||
21 | from atomicwrites import atomic_write | ||
22 | |||
23 | from traceback import format_exc | ||
24 | |||
25 | |||
26 | borg_pwd = getpwnam('borg') | ||
27 | |||
28 | def as_borg(caps=set(), cwd=None): | ||
29 | if caps: | ||
30 | cap_permitted.add(*caps) | ||
31 | cap_inheritable.add(*caps) | ||
32 | cap_effective.add(*caps) | ||
33 | cap_ambient.add(*caps) | ||
34 | |||
35 | os.setgid(borg_pwd.pw_gid) | ||
36 | os.setuid(borg_pwd.pw_uid) | ||
37 | |||
38 | if cwd is not None: | ||
39 | os.chdir(cwd) | ||
40 | |||
41 | |||
42 | def _archive_name(snapshot, target, archive_prefix): | ||
43 | _, _, ts = snapshot.rpartition('@') | ||
44 | creation_time = isoparse(ts).astimezone(timezone.utc) | ||
45 | archive_name = _archive_basename(snapshot, archive_prefix) | ||
46 | return f'{target}::{archive_name}-{creation_time.strftime("%Y-%m-%dT%H:%M:%S")}' | ||
47 | |||
48 | def _archive_basename(snapshot, archive_prefix): | ||
49 | base_name, _, _ = snapshot.rpartition('@') | ||
50 | return archive_prefix + base_name.replace('-', '--').replace('/', '-') | ||
51 | |||
52 | def check(*, snapshot, target, archive_prefix, cache_file): | ||
53 | archives = None | ||
54 | if cache_file: | ||
55 | logger.debug('Trying cache...') | ||
56 | try: | ||
57 | with open(cache_file, mode='r', encoding='utf-8') as fp: | ||
58 | archives = set(json.load(fp)) | ||
59 | logger.info('Loaded archive list from cache') | ||
60 | except FileNotFoundError: | ||
61 | pass | ||
62 | |||
63 | if not archives: | ||
64 | logger.info('Loading archive list from remote...') | ||
65 | with subprocess.Popen(['borg', 'list', '--info', '--lock-wait=600', '--json', target], stdout=subprocess.PIPE, preexec_fn=lambda: as_borg()) as proc: | ||
66 | archives = set([archive['barchive'] for archive in json.load(proc.stdout)['archives']]) | ||
67 | if cache_file: | ||
68 | logger.debug('Saving archive list to cache...') | ||
69 | with atomic_write(cache_file, mode='w', encoding='utf-8', overwrite=True) as fp: | ||
70 | json.dump(list(archives), fp) | ||
71 | |||
72 | # logger.debug(f'archives: {archives}') | ||
73 | _, _, archive_name = _archive_name(snapshot, target, archive_prefix).partition('::') | ||
74 | if archive_name in archives: | ||
75 | logger.info(f'{archive_name} found') | ||
76 | return 0 | ||
77 | else: | ||
78 | logger.info(f'{archive_name} not found') | ||
79 | return 126 | ||
80 | |||
81 | def create(*, snapshot, target, archive_prefix, dry_run): | ||
82 | basename = _archive_basename(snapshot, archive_prefix) | ||
83 | |||
84 | with TemporaryDirectory(prefix=f'borg-mount_{basename}_', dir=os.environ.get('RUNTIME_DIRECTORY')) as tmpdir: | ||
85 | child = os.fork() | ||
86 | if child == 0: | ||
87 | unshare.unshare(unshare.CLONE_NEWNS) | ||
88 | subprocess.run(['mount', '--make-rprivate', '/'], check=True) | ||
89 | chroot = pathlib.Path(tmpdir) / 'chroot' | ||
90 | upper = pathlib.Path(tmpdir) / 'upper' | ||
91 | work = pathlib.Path(tmpdir) / 'work' | ||
92 | for path in [chroot,upper,work]: | ||
93 | path.mkdir() | ||
94 | subprocess.run(['mount', '-t', 'overlay', 'overlay', '-o', f'lowerdir=/,upperdir={upper},workdir={work}', chroot], check=True) | ||
95 | bindMounts = ['nix', 'run', 'run/secrets.d', 'run/wrappers', 'proc', 'dev', 'sys', pathlib.Path(os.path.expanduser('~')).relative_to('/')] | ||
96 | if os.environ.get('BORG_BASE_DIR'): | ||
97 | bindMounts.append(pathlib.Path(os.environ['BORG_BASE_DIR']).relative_to('/')) | ||
98 | if 'SSH_AUTH_SOCK' in os.environ: | ||
99 | bindMounts.append(pathlib.Path(os.environ['SSH_AUTH_SOCK']).parent.relative_to('/')) | ||
100 | for bindMount in bindMounts: | ||
101 | (chroot / bindMount).mkdir(parents=True,exist_ok=True) | ||
102 | # print(*['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], file=stderr) | ||
103 | subprocess.run(['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], check=True) | ||
104 | os.chroot(chroot) | ||
105 | os.chdir('/') | ||
106 | dir = pathlib.Path('/borg') | ||
107 | dir.mkdir(parents=True,exist_ok=True,mode=0o0750) | ||
108 | os.chown(dir, borg_pwd.pw_uid, borg_pwd.pw_gid) | ||
109 | try: | ||
110 | subprocess.run(['mount', '-t', 'zfs', '-o', 'ro', snapshot, dir], check=True) | ||
111 | env = os.environ.copy() | ||
112 | create_args = ['borg', | ||
113 | 'create', | ||
114 | '--lock-wait=600', | ||
115 | '--one-file-system', | ||
116 | '--compression=auto,zstd,10', | ||
117 | '--chunker-params=10,23,16,4095', | ||
118 | '--files-cache=ctime,size', | ||
119 | '--show-rc', | ||
120 | # '--remote-ratelimit=20480', | ||
121 | '--progress', | ||
122 | '--list', | ||
123 | '--filter=AMEi-x?', | ||
124 | '--stats' if not dry_run else '--dry-run' | ||
125 | ] | ||
126 | _, _, ts = snapshot.rpartition('@') | ||
127 | creation_time = isoparse(ts).astimezone(timezone.utc) | ||
128 | create_args += [f'--timestamp={creation_time.strftime("%Y-%m-%dT%H:%M:%S")}'] | ||
129 | env['BORG_FILES_CACHE_SUFFIX'] = basename | ||
130 | create_args += [_archive_name(snapshot, target, archive_prefix), '.'] | ||
131 | print({'create_args': create_args, 'cwd': dir, 'env': env}, file=sys.stderr) | ||
132 | subprocess.run(create_args, stdin=subprocess.DEVNULL, env=env, preexec_fn=lambda: as_borg(caps={CAP.DAC_READ_SEARCH}, cwd=dir), check=True) | ||
133 | # subprocess.run(create_args, stdin=subprocess.DEVNULL, env=env, preexec_fn=lambda: None, cwd=dir, check=True) | ||
134 | finally: | ||
135 | subprocess.run(['umount', dir], check=True) | ||
136 | os._exit(0) | ||
137 | else: | ||
138 | while True: | ||
139 | waitpid, waitret = os.wait() | ||
140 | if waitret != 0: | ||
141 | sys.exit(waitret) | ||
142 | if waitpid == child: | ||
143 | break | ||
144 | return 0 | ||
145 | |||
146 | def sigterm(signum, frame): | ||
147 | raise SystemExit(128 + signum) | ||
148 | |||
149 | def main(): | ||
150 | signal.signal(signal.SIGTERM, sigterm) | ||
151 | |||
152 | global logger | ||
153 | logger = logging.getLogger(__name__) | ||
154 | console_handler = logging.StreamHandler() | ||
155 | console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') ) | ||
156 | if sys.stderr.isatty(): | ||
157 | console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') ) | ||
158 | logger.addHandler(console_handler) | ||
159 | |||
160 | # log uncaught exceptions | ||
161 | def log_exceptions(type, value, tb): | ||
162 | global logger | ||
163 | |||
164 | logger.error(value) | ||
165 | sys.__excepthook__(type, value, tb) # calls default excepthook | ||
166 | |||
167 | sys.excepthook = log_exceptions | ||
168 | |||
169 | parser = argparse.ArgumentParser(prog='borgsnap') | ||
170 | parser.add_argument('--verbose', '-v', action='count', default=0) | ||
171 | parser.add_argument('--target', metavar='REPO', default='yggdrasil.borgbase:repo') | ||
172 | parser.add_argument('--archive-prefix', metavar='REPO', default='yggdrasil.vidhar.') | ||
173 | subparsers = parser.add_subparsers() | ||
174 | subparsers.required = True | ||
175 | parser.set_defaults(cmd=None) | ||
176 | check_parser = subparsers.add_parser('check') | ||
177 | check_parser.add_argument('--cache-file', type=lambda p: Path(p).absolute(), default=None) | ||
178 | check_parser.add_argument('snapshot') | ||
179 | check_parser.set_defaults(cmd=check) | ||
180 | create_parser = subparsers.add_parser('create') | ||
181 | create_parser.add_argument('--dry-run', '-n', action='store_true', default=False) | ||
182 | create_parser.add_argument('snapshot') | ||
183 | create_parser.set_defaults(cmd=create) | ||
184 | args = parser.parse_args() | ||
185 | |||
186 | if args.verbose <= 0: | ||
187 | logger.setLevel(logging.WARNING) | ||
188 | elif args.verbose <= 1: | ||
189 | logger.setLevel(logging.INFO) | ||
190 | else: | ||
191 | logger.setLevel(logging.DEBUG) | ||
192 | |||
193 | cmdArgs = {} | ||
194 | for copy in {'target', 'archive_prefix', 'snapshot', 'cache_file', 'dry_run'}: | ||
195 | if copy in vars(args): | ||
196 | cmdArgs[copy] = vars(args)[copy] | ||
197 | |||
198 | return args.cmd(**cmdArgs) | ||
199 | |||
200 | |||
201 | if __name__ == '__main__': | ||
202 | sys.exit(main()) | ||
diff --git a/hosts/vidhar/borg/borgsnap/setup.py b/hosts/vidhar/borg/borgsnap/setup.py new file mode 100644 index 00000000..76356bfc --- /dev/null +++ b/hosts/vidhar/borg/borgsnap/setup.py | |||
@@ -0,0 +1,10 @@ | |||
1 | from setuptools import setup | ||
2 | |||
3 | setup(name='borgsnap', | ||
4 | packages=['borgsnap'], | ||
5 | entry_points={ | ||
6 | 'console_scripts': [ | ||
7 | 'borgsnap=borgsnap.__main__:main', | ||
8 | ], | ||
9 | } | ||
10 | ) | ||
diff --git a/hosts/vidhar/borg/copy.py b/hosts/vidhar/borg/copy.py index 4e9599b8..b9b667f2 100755 --- a/hosts/vidhar/borg/copy.py +++ b/hosts/vidhar/borg/copy.py | |||
@@ -71,7 +71,7 @@ def read_repo(path): | |||
71 | 71 | ||
72 | class ToSync: | 72 | class ToSync: |
73 | to_sync = deque() | 73 | to_sync = deque() |
74 | 74 | ||
75 | def __iter__(self): | 75 | def __iter__(self): |
76 | return self | 76 | return self |
77 | 77 | ||
@@ -267,7 +267,7 @@ def sigterm(signum, frame): | |||
267 | 267 | ||
268 | def main(): | 268 | def main(): |
269 | signal.signal(signal.SIGTERM, sigterm) | 269 | signal.signal(signal.SIGTERM, sigterm) |
270 | 270 | ||
271 | if "::" in args.source: | 271 | if "::" in args.source: |
272 | (src_repo_path, _, src_archive) = args.source.partition("::") | 272 | (src_repo_path, _, src_archive) = args.source.partition("::") |
273 | entry = None | 273 | entry = None |
diff --git a/hosts/vidhar/borg/default.nix b/hosts/vidhar/borg/default.nix index 579630a9..650c91ee 100644 --- a/hosts/vidhar/borg/default.nix +++ b/hosts/vidhar/borg/default.nix | |||
@@ -1,23 +1,28 @@ | |||
1 | { config, pkgs, lib, ... }: | 1 | { config, pkgs, lib, flakeInputs, ... }: |
2 | 2 | ||
3 | with lib; | 3 | with lib; |
4 | 4 | ||
5 | let | 5 | let |
6 | sshConfig = pkgs.writeText "config" '' | ||
7 | Include /etc/ssh/ssh_config | ||
8 | |||
9 | ControlMaster auto | ||
10 | ControlPath /var/lib/borg/.borgssh-master-%r@%n:%p | ||
11 | ControlPersist yes | ||
12 | |||
13 | Host yggdrasil.borgbase | ||
14 | HostName nx69hpl8.repo.borgbase.com | ||
15 | User nx69hpl8 | ||
16 | IdentityFile ${config.sops.secrets."append.borgbase".path} | ||
17 | IdentitiesOnly yes | ||
18 | |||
19 | BatchMode yes | ||
20 | ServerAliveInterval 10 | ||
21 | ServerAliveCountMax 30 | ||
22 | ''; | ||
23 | |||
6 | copyService = { repo, repoEscaped }: let | 24 | copyService = { repo, repoEscaped }: let |
7 | serviceName = "copy-borg@${repoEscaped}"; | 25 | serviceName = "copy-borg@${repoEscaped}"; |
8 | sshConfig = pkgs.writeText "config" '' | ||
9 | Include /etc/ssh/ssh_config | ||
10 | |||
11 | Host yggdrasil.borgbase | ||
12 | HostName nx69hpl8.repo.borgbase.com | ||
13 | User nx69hpl8 | ||
14 | IdentityFile ${config.sops.secrets."append.borgbase".path} | ||
15 | IdentitiesOnly yes | ||
16 | |||
17 | BatchMode yes | ||
18 | ServerAliveInterval 10 | ||
19 | ServerAliveCountMax 30 | ||
20 | ''; | ||
21 | in nameValuePair serviceName { | 26 | in nameValuePair serviceName { |
22 | serviceConfig = { | 27 | serviceConfig = { |
23 | Type = "oneshot"; | 28 | Type = "oneshot"; |
@@ -72,8 +77,63 @@ let | |||
72 | --prefix PATH : ${makeBinPath (with pkgs; [utillinux borgbackup])}:${config.security.wrapperDir} | 77 | --prefix PATH : ${makeBinPath (with pkgs; [utillinux borgbackup])}:${config.security.wrapperDir} |
73 | ''; | 78 | ''; |
74 | }); | 79 | }); |
80 | |||
81 | borgsnap = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage rec { | ||
82 | pname = "borgsnap"; | ||
83 | src = ./borgsnap; | ||
84 | version = "0.0.0"; | ||
85 | ignoreDataOutdated = true; | ||
86 | |||
87 | requirements = '' | ||
88 | atomicwrites | ||
89 | pyprctl | ||
90 | python-unshare | ||
91 | xdg | ||
92 | python-dateutil | ||
93 | ''; | ||
94 | postInstall = '' | ||
95 | wrapProgram $out/bin/borgsnap \ | ||
96 | --prefix PATH : ${makeBinPath (with pkgs; [utillinux borgbackup])}:${config.security.wrapperDir} | ||
97 | ''; | ||
98 | |||
99 | providers.python-unshare = "nixpkgs"; | ||
100 | overridesPre = [ | ||
101 | (self: super: { python-unshare = super.python-unshare.overrideAttrs (oldAttrs: { name = "python-unshare-0.2.1"; version = "0.2.1"; }); }) | ||
102 | ]; | ||
103 | |||
104 | _.xdg.buildInputs.add = with pkgs."python3Packages"; [ poetry ]; | ||
105 | _.tomli.buildInputs.add = with pkgs."python3Packages"; [ flit-core ]; | ||
106 | }; | ||
75 | in { | 107 | in { |
76 | config = { | 108 | config = { |
109 | services.zfssnap.config.exec = { | ||
110 | check = "${borgsnap}/bin/borgsnap -vvv --target yggdrasil.borgbase:repo --archive-prefix yggdrasil.vidhar. check --cache-file /run/zfssnap-prune/archives-cache.json"; | ||
111 | cmd = "${borgsnap}/bin/borgsnap -vvv --target yggdrasil.borgbase:repo --archive-prefix yggdrasil.vidhar. create --dry-run"; | ||
112 | |||
113 | halfweekly = "8"; | ||
114 | monthly = "-1"; | ||
115 | }; | ||
116 | |||
117 | systemd.services = { | ||
118 | "zfssnap-prune" = { | ||
119 | serviceConfig = { | ||
120 | Environment = [ | ||
121 | "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${sshConfig}\"" | ||
122 | "BORG_BASE_DIR=/var/lib/borg" | ||
123 | "BORG_CONFIG_DIR=/var/lib/borg/config" | ||
124 | "BORG_CACHE_DIR=/var/lib/borg/cache" | ||
125 | "BORG_SECURITY_DIR=/var/lib/borg/security" | ||
126 | "BORG_KEYS_DIR=/var/lib/borg/keys" | ||
127 | "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}" | ||
128 | "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes" | ||
129 | "BORG_HOSTNAME_IS_UNIQUE=yes" | ||
130 | ]; | ||
131 | RuntimeDirectory = "zfssnap-prune"; | ||
132 | }; | ||
133 | }; | ||
134 | } // listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]); | ||
135 | |||
136 | |||
77 | services.borgbackup.repos.jotnar = { | 137 | services.borgbackup.repos.jotnar = { |
78 | path = "/srv/backup/borg/jotnar"; | 138 | path = "/srv/backup/borg/jotnar"; |
79 | authorizedKeysAppendOnly = let | 139 | authorizedKeysAppendOnly = let |
@@ -111,11 +171,9 @@ in { | |||
111 | mode = "0400"; | 171 | mode = "0400"; |
112 | }; | 172 | }; |
113 | 173 | ||
114 | systemd.services = listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]); | ||
115 | |||
116 | systemd.timers."copy-borg@srv-backup-borg-jotnar" = { | 174 | systemd.timers."copy-borg@srv-backup-borg-jotnar" = { |
117 | wantedBy = ["multi-user.target"]; | 175 | wantedBy = ["multi-user.target"]; |
118 | 176 | ||
119 | timerConfig = { | 177 | timerConfig = { |
120 | OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin"; | 178 | OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin"; |
121 | }; | 179 | }; |
diff --git a/hosts/vidhar/default.nix b/hosts/vidhar/default.nix index 121cc9df..3f5d17d5 100644 --- a/hosts/vidhar/default.nix +++ b/hosts/vidhar/default.nix | |||
@@ -1,4 +1,7 @@ | |||
1 | { hostName, flake, config, pkgs, lib, ... }: | 1 | { hostName, flake, config, pkgs, lib, ... }: |
2 | |||
3 | with lib; | ||
4 | |||
2 | { | 5 | { |
3 | imports = with flake.nixosModules.systemProfiles; [ | 6 | imports = with flake.nixosModules.systemProfiles; [ |
4 | ./zfs.nix ./network ./samba.nix ./dns ./prometheus ./borg | 7 | ./zfs.nix ./network ./samba.nix ./dns ./prometheus ./borg |
@@ -39,7 +42,7 @@ | |||
39 | luks.devices = { | 42 | luks.devices = { |
40 | nvm0 = { device = "/dev/disk/by-label/${hostName}-nvm0"; bypassWorkqueues = true; }; | 43 | nvm0 = { device = "/dev/disk/by-label/${hostName}-nvm0"; bypassWorkqueues = true; }; |
41 | nvm1 = { device = "/dev/disk/by-label/${hostName}-nvm1"; bypassWorkqueues = true; }; | 44 | nvm1 = { device = "/dev/disk/by-label/${hostName}-nvm1"; bypassWorkqueues = true; }; |
42 | 45 | ||
43 | hdd0.device = "/dev/disk/by-label/${hostName}-hdd0"; | 46 | hdd0.device = "/dev/disk/by-label/${hostName}-hdd0"; |
44 | hdd1.device = "/dev/disk/by-label/${hostName}-hdd1"; | 47 | hdd1.device = "/dev/disk/by-label/${hostName}-hdd1"; |
45 | hdd2.device = "/dev/disk/by-label/${hostName}-hdd2"; | 48 | hdd2.device = "/dev/disk/by-label/${hostName}-hdd2"; |
@@ -58,7 +61,7 @@ | |||
58 | options = [ "mode=0755" ]; | 61 | options = [ "mode=0755" ]; |
59 | }; | 62 | }; |
60 | }; | 63 | }; |
61 | 64 | ||
62 | services.timesyncd.enable = false; | 65 | services.timesyncd.enable = false; |
63 | services.chrony = { | 66 | services.chrony = { |
64 | enable = true; | 67 | enable = true; |
@@ -132,6 +135,7 @@ | |||
132 | access_log syslog:server=unix:/dev/log main; | 135 | access_log syslog:server=unix:/dev/log main; |
133 | error_log syslog:server=unix:/dev/log info; | 136 | error_log syslog:server=unix:/dev/log info; |
134 | 137 | ||
138 | client_body_buffer_size 16m; | ||
135 | client_body_temp_path /run/nginx-client-bodies; | 139 | client_body_temp_path /run/nginx-client-bodies; |
136 | ''; | 140 | ''; |
137 | upstreams.grafana = { | 141 | upstreams.grafana = { |
@@ -173,12 +177,12 @@ | |||
173 | sopsFile = ./selfsigned.key; | 177 | sopsFile = ./selfsigned.key; |
174 | }; | 178 | }; |
175 | systemd.services.nginx = { | 179 | systemd.services.nginx = { |
176 | preStart = lib.mkForce config.services.nginx.preStart; | 180 | preStart = mkForce config.services.nginx.preStart; |
177 | serviceConfig = { | 181 | serviceConfig = { |
178 | ExecReload = lib.mkForce "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; | 182 | ExecReload = mkForce "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; |
179 | LoadCredential = [ "selfsigned.key:${config.sops.secrets."selfsigned.key".path}" ]; | 183 | LoadCredential = [ "selfsigned.key:${config.sops.secrets."selfsigned.key".path}" ]; |
180 | 184 | ||
181 | RuntimeDirectory = lib.mkForce [ "nginx" "nginx-client-bodies" ]; | 185 | RuntimeDirectory = mkForce [ "nginx" "nginx-client-bodies" ]; |
182 | RuntimeDirectoryMode = "0750"; | 186 | RuntimeDirectoryMode = "0750"; |
183 | }; | 187 | }; |
184 | }; | 188 | }; |
@@ -232,7 +236,7 @@ | |||
232 | }; | 236 | }; |
233 | }; | 237 | }; |
234 | systemd.services.loki.preStart = let | 238 | systemd.services.loki.preStart = let |
235 | rulesYaml = lib.generators.toYAML {} { | 239 | rulesYaml = generators.toYAML {} { |
236 | groups = [ | 240 | groups = [ |
237 | { name = "power-failures"; | 241 | { name = "power-failures"; |
238 | rules = [ | 242 | rules = [ |
@@ -311,6 +315,29 @@ | |||
311 | timers.wants = ["systemd-tmpfiles-clean.timer"]; | 315 | timers.wants = ["systemd-tmpfiles-clean.timer"]; |
312 | }; | 316 | }; |
313 | 317 | ||
318 | services.smartd = { | ||
319 | enable = true; | ||
320 | autodetect = false; | ||
321 | defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)"; | ||
322 | devices = map (dev: { device = "/dev/disk/by-path/${dev}"; }) [ | ||
323 | "pci-0000:00:1f.2-ata-1" | ||
324 | "pci-0000:00:1f.2-ata-3" | ||
325 | "pci-0000:00:1f.2-ata-4" | ||
326 | "pci-0000:00:1f.2-ata-5" | ||
327 | "pci-0000:00:1f.2-ata-6" | ||
328 | "pci-0000:02:00.0-nvme-1" | ||
329 | "pci-0000:05:00.0-sas-phy0-lun-0" | ||
330 | "pci-0000:05:00.0-sas-phy1-lun-0" | ||
331 | "pci-0000:06:00.0-nvme-1" | ||
332 | ]; | ||
333 | notifications = { | ||
334 | test = false; | ||
335 | mail.enable = false; | ||
336 | x11.enable = false; | ||
337 | wall.enable = false; | ||
338 | }; | ||
339 | }; | ||
340 | |||
314 | environment.systemPackages = with pkgs; [iotop vmtouch]; | 341 | environment.systemPackages = with pkgs; [iotop vmtouch]; |
315 | 342 | ||
316 | system.stateVersion = "21.05"; | 343 | system.stateVersion = "21.05"; |
diff --git a/hosts/vidhar/prometheus/default.nix b/hosts/vidhar/prometheus/default.nix index 4c23d8a9..7ac86c30 100644 --- a/hosts/vidhar/prometheus/default.nix +++ b/hosts/vidhar/prometheus/default.nix | |||
@@ -34,20 +34,6 @@ in { | |||
34 | enable = true; | 34 | enable = true; |
35 | enabledCollectors = []; | 35 | enabledCollectors = []; |
36 | }; | 36 | }; |
37 | smartctl = { | ||
38 | enable = true; | ||
39 | devices = map (dev: "/dev/disk/by-path/${dev}") [ | ||
40 | "pci-0000:00:1f.2-ata-1" | ||
41 | "pci-0000:00:1f.2-ata-3" | ||
42 | "pci-0000:00:1f.2-ata-4" | ||
43 | "pci-0000:00:1f.2-ata-5" | ||
44 | "pci-0000:00:1f.2-ata-6" | ||
45 | "pci-0000:02:00.0-nvme-1" | ||
46 | "pci-0000:05:00.0-sas-phy0-lun-0" | ||
47 | "pci-0000:05:00.0-sas-phy1-lun-0" | ||
48 | "pci-0000:06:00.0-nvme-1" | ||
49 | ]; | ||
50 | }; | ||
51 | snmp = { | 37 | snmp = { |
52 | enable = true; | 38 | enable = true; |
53 | configurationPath = ./snmp.yml; | 39 | configurationPath = ./snmp.yml; |
@@ -124,10 +110,10 @@ in { | |||
124 | } | 110 | } |
125 | { job_name = "smartctl"; | 111 | { job_name = "smartctl"; |
126 | static_configs = [ | 112 | static_configs = [ |
127 | { targets = ["localhost:${toString config.services.prometheus.exporters.smartctl.port}"]; } | 113 | { targets = ["localhost:9633"]; } |
128 | ]; | 114 | ]; |
129 | relabel_configs = relabelHosts; | 115 | relabel_configs = relabelHosts; |
130 | scrape_interval = "1s"; | 116 | scrape_interval = "60s"; |
131 | } | 117 | } |
132 | { job_name = "snmp"; | 118 | { job_name = "snmp"; |
133 | static_configs = [ | 119 | static_configs = [ |
@@ -376,6 +362,30 @@ in { | |||
376 | }; | 362 | }; |
377 | }; | 363 | }; |
378 | 364 | ||
365 | systemd.services."prometheus-smartctl-exporter" = { | ||
366 | wantedBy = [ "multi-user.target" ]; | ||
367 | after = [ "network.target" ]; | ||
368 | path = with pkgs; [ smartmontools ]; | ||
369 | serviceConfig = { | ||
370 | Restart = "always"; | ||
371 | |||
372 | CapabilityBoundingSet = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"]; | ||
373 | AmbientCapabilities = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"]; | ||
374 | ProtectSystem = "strict"; | ||
375 | DynamicUser = true; | ||
376 | LockPersonality = true; | ||
377 | MemoryDenyWriteExecute = true; | ||
378 | NoNewPrivileges = true; | ||
379 | PrivateDevices = false; | ||
380 | PrivateTmp = true; | ||
381 | ProcSubset = "pid"; | ||
382 | |||
383 | Type = "simple"; | ||
384 | ExecStart = "${pkgs.smartprom}/bin/smartprom"; | ||
385 | Environment = "SMARTCTL_EXPORTER_PORT=9633"; | ||
386 | }; | ||
387 | }; | ||
388 | |||
379 | systemd.services."prometheus-systemd-exporter" = let | 389 | systemd.services."prometheus-systemd-exporter" = let |
380 | cfg = config.services.prometheus.exporters.systemd; | 390 | cfg = config.services.prometheus.exporters.systemd; |
381 | in { | 391 | in { |
@@ -385,14 +395,6 @@ in { | |||
385 | ''; | 395 | ''; |
386 | }; | 396 | }; |
387 | 397 | ||
388 | systemd.services."prometheus-smartctl-exporter" = { | ||
389 | serviceConfig = { | ||
390 | DeviceAllow = lib.mkForce config.services.prometheus.exporters.smartctl.devices; | ||
391 | CapabilityBoundingSet = lib.mkForce ["CAP_SYS_ADMIN"]; | ||
392 | AmbientCapabilities = lib.mkForce ["CAP_SYS_ADMIN"]; | ||
393 | }; | ||
394 | }; | ||
395 | |||
396 | services.nginx = { | 398 | services.nginx = { |
397 | upstreams.prometheus = { | 399 | upstreams.prometheus = { |
398 | servers = { "localhost:${toString config.services.prometheus.port}" = {}; }; | 400 | servers = { "localhost:${toString config.services.prometheus.port}" = {}; }; |
diff --git a/hosts/vidhar/zfs.nix b/hosts/vidhar/zfs.nix index ef285536..52b48aca 100644 --- a/hosts/vidhar/zfs.nix +++ b/hosts/vidhar/zfs.nix | |||
@@ -130,7 +130,21 @@ | |||
130 | echo "=== ZPOOL IMPORT COMPLETE ===" | 130 | echo "=== ZPOOL IMPORT COMPLETE ===" |
131 | ''; | 131 | ''; |
132 | 132 | ||
133 | services.zfssnap.enable = true; | 133 | services.zfssnap = { |
134 | enable = true; | ||
135 | config.keep = { | ||
136 | within = "15m"; | ||
137 | "5m" = "48"; | ||
138 | "15m" = "32"; | ||
139 | hourly = "48"; | ||
140 | "4h" = "24"; | ||
141 | "12h" = "12"; | ||
142 | daily = "62"; | ||
143 | halfweekly = "32"; | ||
144 | weekly = "24"; | ||
145 | monthly = "-1"; | ||
146 | }; | ||
147 | }; | ||
134 | services.zfs.trim.enable = false; | 148 | services.zfs.trim.enable = false; |
135 | services.zfs.autoScrub = { | 149 | services.zfs.autoScrub = { |
136 | enable = true; | 150 | enable = true; |
diff --git a/modules/postfwd.nix b/modules/postfwd.nix new file mode 100644 index 00000000..4afea0a1 --- /dev/null +++ b/modules/postfwd.nix | |||
@@ -0,0 +1,65 @@ | |||
1 | { config, lib, pkgs, ... }: | ||
2 | |||
3 | with lib; | ||
4 | |||
5 | let | ||
6 | cfg = config.services.postfwd; | ||
7 | in { | ||
8 | options = { | ||
9 | services.postfwd = with types; { | ||
10 | enable = mkEnableOption "postfwd3 - postfix firewall daemon"; | ||
11 | |||
12 | rules = mkOption { | ||
13 | type = lines; | ||
14 | default = ""; | ||
15 | }; | ||
16 | }; | ||
17 | }; | ||
18 | |||
19 | config = mkIf cfg.enable { | ||
20 | systemd.services.postfwd = { | ||
21 | description = "postfwd3 - postfix firewall daemon"; | ||
22 | wantedBy = ["multi-user.target"]; | ||
23 | before = ["postfix.service"]; | ||
24 | |||
25 | serviceConfig = { | ||
26 | Type = "forking"; | ||
27 | |||
28 | ExecStart = "${pkgs.postfwd}/bin/postfwd3 ${escapeShellArgs [ | ||
29 | "-vv" | ||
30 | "--daemon" "--user" "postfwd" "--group" "postfwd" | ||
31 | "--pidfile" "/run/postfwd3/postfwd3.pid" | ||
32 | "--proto" "unix" | ||
33 | "--port" "/run/postfwd3/postfwd3.sock" | ||
34 | "--save_rates" "/var/lib/postfwd/rates" | ||
35 | "--file" (pkgs.writeText "postfwd3-rules" cfg.rules) | ||
36 | ]}"; | ||
37 | PIDFile = "/run/postfwd3/postfwd3.pid"; | ||
38 | |||
39 | Restart = "always"; | ||
40 | RestartSec = 5; | ||
41 | TimeoutSec = 10; | ||
42 | |||
43 | RuntimeDirectory = ["postfwd3"]; | ||
44 | StateDirectory = ["postfwd"]; | ||
45 | |||
46 | DynamicUser = true; | ||
47 | ProtectSystem = "strict"; | ||
48 | SystemCallFilter = "@system-service"; | ||
49 | NoNewPrivileges = true; | ||
50 | ProtectKernelTunables = true; | ||
51 | ProtectKernelModules = true; | ||
52 | ProtectKernelLogs = true; | ||
53 | ProtectControlGroups = true; | ||
54 | MemoryDenyWriteExecute = true; | ||
55 | RestrictSUIDSGID = true; | ||
56 | KeyringMode = "private"; | ||
57 | ProtectClock = true; | ||
58 | RestrictRealtime = true; | ||
59 | PrivateDevices = true; | ||
60 | PrivateTmp = true; | ||
61 | ProtectHostname = true; | ||
62 | }; | ||
63 | }; | ||
64 | }; | ||
65 | } | ||
diff --git a/modules/zfssnap/default.nix b/modules/zfssnap/default.nix index d1080e8a..f3e2f9c2 100644 --- a/modules/zfssnap/default.nix +++ b/modules/zfssnap/default.nix | |||
@@ -1,7 +1,7 @@ | |||
1 | { config, pkgs, lib, ... }: | 1 | { config, pkgs, lib, ... }: |
2 | 2 | ||
3 | with lib; | 3 | with lib; |
4 | 4 | ||
5 | let | 5 | let |
6 | zfssnap = pkgs.stdenv.mkDerivation rec { | 6 | zfssnap = pkgs.stdenv.mkDerivation rec { |
7 | name = "zfssnap"; | 7 | name = "zfssnap"; |
@@ -37,7 +37,7 @@ in { | |||
37 | options = { | 37 | options = { |
38 | services.zfssnap = { | 38 | services.zfssnap = { |
39 | enable = mkEnableOption "zfssnap service"; | 39 | enable = mkEnableOption "zfssnap service"; |
40 | 40 | ||
41 | config = mkOption { | 41 | config = mkOption { |
42 | type = with types; attrsOf (attrsOf str); | 42 | type = with types; attrsOf (attrsOf str); |
43 | default = { | 43 | default = { |
@@ -82,7 +82,7 @@ in { | |||
82 | ExecStart = let | 82 | ExecStart = let |
83 | mkSectionName = name: strings.escape [ "[" "]" ] (strings.toUpper name); | 83 | mkSectionName = name: strings.escape [ "[" "]" ] (strings.toUpper name); |
84 | zfssnapConfig = generators.toINI { inherit mkSectionName; } cfg.config; | 84 | zfssnapConfig = generators.toINI { inherit mkSectionName; } cfg.config; |
85 | in "${zfssnap}/bin/zfssnap -v prune --config=${pkgs.writeText "zfssnap.ini" zfssnapConfig}"; | 85 | in "${zfssnap}/bin/zfssnap -vv prune --config=${pkgs.writeText "zfssnap.ini" zfssnapConfig}"; |
86 | }; | 86 | }; |
87 | }; | 87 | }; |
88 | 88 | ||
diff --git a/modules/zfssnap/zfssnap.py b/modules/zfssnap/zfssnap.py index 21ed1d5b..a8dae75f 100644 --- a/modules/zfssnap/zfssnap.py +++ b/modules/zfssnap/zfssnap.py | |||
@@ -3,9 +3,9 @@ | |||
3 | import csv | 3 | import csv |
4 | import subprocess | 4 | import subprocess |
5 | import io | 5 | import io |
6 | from distutils.util import strtobool | 6 | from distutils.util import strtobool |
7 | from datetime import datetime, timezone, timedelta | 7 | from datetime import datetime, timezone, timedelta |
8 | from dateutil.tz import gettz, tzlocal | 8 | from dateutil.tz import gettz, tzutc |
9 | import pytimeparse | 9 | import pytimeparse |
10 | import argparse | 10 | import argparse |
11 | import re | 11 | import re |
@@ -27,6 +27,36 @@ from math import floor | |||
27 | 27 | ||
28 | import asyncio | 28 | import asyncio |
29 | 29 | ||
30 | from dataclasses import dataclass | ||
31 | |||
32 | |||
33 | TIME_PATTERNS = OrderedDict([ | ||
34 | ("secondly", lambda t: t.strftime('%Y-%m-%d %H:%M:%S')), | ||
35 | ("minutely", lambda t: t.strftime('%Y-%m-%d %H:%M')), | ||
36 | ("5m", lambda t: (t.strftime('%Y-%m-%d %H'), floor(t.minute / 5) * 5)), | ||
37 | ("15m", lambda t: (t.strftime('%Y-%m-%d %H'), floor(t.minute / 15) * 15)), | ||
38 | ("hourly", lambda t: t.strftime('%Y-%m-%d %H')), | ||
39 | ("4h", lambda t: (t.strftime('%Y-%m-%d'), floor(t.hour / 4) * 4)), | ||
40 | ("12h", lambda t: (t.strftime('%Y-%m-%d'), floor(t.hour / 12) * 12)), | ||
41 | ("daily", lambda t: t.strftime('%Y-%m-%d')), | ||
42 | ("halfweekly", lambda t: (t.strftime('%G-%V'), floor(int(t.strftime('%u')) / 4) * 4)), | ||
43 | ("weekly", lambda t: t.strftime('%G-%V')), | ||
44 | ("monthly", lambda t: t.strftime('%Y-%m')), | ||
45 | ("yearly", lambda t: t.strftime('%Y')), | ||
46 | ]) | ||
47 | |||
48 | @dataclass(eq=True, order=True, frozen=True) | ||
49 | class Snap: | ||
50 | name: str | ||
51 | creation: datetime | ||
52 | |||
53 | @dataclass(eq=True, order=True, frozen=True) | ||
54 | class KeptBecause: | ||
55 | rule: str | ||
56 | ix: int | ||
57 | base: str | ||
58 | period: str | ||
59 | |||
30 | 60 | ||
31 | @cache | 61 | @cache |
32 | def _now(): | 62 | def _now(): |
@@ -42,56 +72,120 @@ def _log_cmd(*args): | |||
42 | 72 | ||
43 | def _get_items(): | 73 | def _get_items(): |
44 | items = {} | 74 | items = {} |
45 | 75 | ||
46 | args = ['zfs', 'get', '-H', '-p', '-o', 'name,value', '-t', 'filesystem,volume', '-s', 'local,default,inherited,temporary,received', 'li.yggdrasil:auto-snapshot'] | 76 | args = ['zfs', 'get', '-H', '-p', '-o', 'name,value', '-t', 'filesystem,volume', '-s', 'local,default,inherited,temporary,received', 'li.yggdrasil:auto-snapshot'] |
47 | _log_cmd(*args) | 77 | _log_cmd(*args) |
48 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: | 78 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: |
49 | text_stdout = io.TextIOWrapper(proc.stdout) | 79 | text_stdout = io.TextIOWrapper(proc.stdout) |
50 | reader = csv.reader(text_stdout, delimiter='\t', quoting=csv.QUOTE_NONE) | 80 | reader = csv.DictReader(text_stdout, fieldnames=['name', 'setting'], delimiter='\t', quoting=csv.QUOTE_NONE) |
51 | Row = namedtuple('Row', ['name', 'setting']) | 81 | Row = namedtuple('Row', reader.fieldnames) |
52 | for row in map(Row._make, reader): | 82 | for row in [Row(**data) for data in reader]: |
53 | items[row.name] = bool(strtobool(row.setting)) | 83 | items[row.name] = bool(strtobool(row.setting)) |
54 | 84 | ||
55 | return items | 85 | return items |
56 | 86 | ||
57 | def prune(config, dry_run, keep_newest): | 87 | def _get_snaps(only_auto=True): |
58 | prunable_snapshots = set() | 88 | snapshots = defaultdict(list) |
59 | args = ['zfs', 'get', '-H', '-p', '-o', 'name,value', '-t', 'snapshot', '-s', 'local', 'li.yggdrasil:is-auto-snapshot'] | 89 | args = ['zfs', 'list', '-H', '-p', '-t', 'snapshot', '-o', 'name,li.yggdrasil:is-auto-snapshot,creation'] |
60 | _log_cmd(*args) | ||
61 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: | ||
62 | text_stdout = io.TextIOWrapper(proc.stdout) | ||
63 | reader = csv.reader(text_stdout, delimiter='\t', quoting=csv.QUOTE_NONE) | ||
64 | Row = namedtuple('Row', ['name', 'is_auto_snapshot']) | ||
65 | for row in map(Row._make, reader): | ||
66 | if bool(strtobool(row.is_auto_snapshot)): | ||
67 | prunable_snapshots.add(row.name) | ||
68 | |||
69 | items = defaultdict(list) | ||
70 | Snap = namedtuple('Snap', ['name', 'creation']) | ||
71 | args = ['zfs', 'get', '-H', '-p', '-o', 'name,value', '-t', 'snapshot', 'creation'] | ||
72 | _log_cmd(*args) | 90 | _log_cmd(*args) |
73 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: | 91 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: |
74 | text_stdout = io.TextIOWrapper(proc.stdout) | 92 | text_stdout = io.TextIOWrapper(proc.stdout) |
75 | reader = csv.reader(text_stdout, delimiter='\t', quoting=csv.QUOTE_NONE) | 93 | reader = csv.DictReader(text_stdout, fieldnames=['name', 'is_auto_snapshot', 'timestamp'], delimiter='\t', quoting=csv.QUOTE_NONE) |
76 | Row = namedtuple('Row', ['name', 'timestamp']) | 94 | Row = namedtuple('Row', reader.fieldnames) |
77 | for row in map(Row._make, reader): | 95 | for row in [Row(**data) for data in reader]: |
78 | if row.name not in prunable_snapshots: | 96 | if only_auto and not bool(strtobool(row.is_auto_snapshot)): |
79 | continue | 97 | continue |
80 | 98 | ||
81 | base_name, _, _ = row.name.rpartition('@') | 99 | base_name, _, _ = row.name.rpartition('@') |
82 | creation = datetime.fromtimestamp(int(row.timestamp), timezone.utc) | 100 | creation = datetime.fromtimestamp(int(row.timestamp), timezone.utc) |
83 | items[base_name].append(Snap(name=row.name, creation=creation)) | 101 | snapshots[base_name].append(Snap(name=row.name, creation=creation)) |
102 | |||
103 | return snapshots | ||
104 | |||
105 | def prune(config, dry_run, keep_newest, do_exec): | ||
106 | do_exec = do_exec and 'EXEC' in config | ||
107 | prune_timezone = config.gettimezone('KEEP', 'timezone', fallback=tzutc()) | ||
108 | logger.debug(f'prune timezone: {prune_timezone}') | ||
109 | |||
110 | items = _get_snaps() | ||
111 | |||
112 | exec_candidates = set() | ||
113 | if do_exec: | ||
114 | exec_timezone = config.gettimezone('EXEC', 'timezone', fallback=prune_timezone) | ||
115 | logger.debug(f'exec timezone: {exec_timezone}') | ||
116 | |||
117 | for rule, pattern in TIME_PATTERNS.items(): | ||
118 | desired_count = config.getint('EXEC', rule, fallback=0) | ||
119 | |||
120 | for base, snaps in items.items(): | ||
121 | periods = OrderedDict() | ||
122 | |||
123 | for snap in sorted(snaps, key=lambda snap: snap.creation): | ||
124 | period = pattern(snap.creation.astimezone(exec_timezone)) | ||
125 | if period not in periods: | ||
126 | periods[period] = deque() | ||
127 | periods[period].append(snap) | ||
128 | |||
129 | to_exec = desired_count | ||
130 | ordered_periods = periods.items() | ||
131 | for period, period_snaps in ordered_periods: | ||
132 | if to_exec == 0: | ||
133 | break | ||
134 | |||
135 | for snap in period_snaps: | ||
136 | exec_candidates.add(snap) | ||
137 | logger.debug(f'{snap.name} is exec candidate') | ||
138 | to_exec -= 1 | ||
139 | break | ||
140 | |||
141 | if to_exec > 0: | ||
142 | logger.debug(f'Missing {to_exec} to fulfill exec {rule}={desired_count} for ‘{base}’') | ||
143 | |||
144 | check_cmd = config.get('EXEC', 'check', fallback=None) | ||
145 | if check_cmd: | ||
146 | already_execed = set() | ||
147 | for snap in exec_candidates: | ||
148 | args = [] | ||
149 | args += shlex.split(check_cmd) | ||
150 | args += [snap.name] | ||
151 | _log_cmd(*args) | ||
152 | check_res = subprocess.run(args) | ||
153 | if check_res.returncode == 0: | ||
154 | already_execed.add(snap) | ||
155 | logger.debug(f'{snap.name} already execed') | ||
156 | exec_candidates -= already_execed | ||
157 | |||
158 | exec_cmd = config.get('EXEC', 'cmd', fallback=None) | ||
159 | exec_count = config.getint('EXEC', 'count', fallback=1) | ||
160 | if exec_cmd: | ||
161 | execed = set() | ||
162 | for snap in sorted(exec_candidates, key=lambda snap: snap.creation): | ||
163 | if len(execed) >= exec_count: | ||
164 | logger.debug(f'exc_count of {exec_count} reached') | ||
165 | break | ||
166 | |||
167 | args = [] | ||
168 | args += shlex.split(exec_cmd) | ||
169 | args += [snap.name] | ||
170 | _log_cmd(*args) | ||
171 | subprocess.run(args).check_returncode() | ||
172 | execed.add(snap) | ||
173 | |||
174 | exec_candidates -= execed | ||
84 | 175 | ||
85 | kept_count = defaultdict(lambda: defaultdict(lambda: 0)) | 176 | kept_count = defaultdict(lambda: defaultdict(lambda: 0)) |
86 | KeptBecause = namedtuple('KeptBecause', ['rule', 'ix', 'base', 'period']) | ||
87 | kept_because = OrderedDict() | 177 | kept_because = OrderedDict() |
88 | def keep_because(base, snap, rule, period=None): | 178 | def keep_because(base, snap, rule, period=None): |
89 | nonlocal KeptBecause, kept_count, kept_because | 179 | nonlocal kept_count, kept_because |
90 | kept_count[rule][base] += 1 | 180 | kept_count[rule][base] += 1 |
91 | if snap not in kept_because: | 181 | if snap not in kept_because: |
92 | kept_because[snap] = deque() | 182 | kept_because[snap] = deque() |
93 | kept_because[snap].append(KeptBecause(rule=rule, ix=kept_count[rule][base], base=base, period=period)) | 183 | kept_because[snap].append(KeptBecause(rule=rule, ix=kept_count[rule][base], base=base, period=period)) |
94 | 184 | ||
185 | for candidate in exec_candidates: | ||
186 | base_name, _, _ = candidate.name.rpartition('@') | ||
187 | keep_because(base_name, candidate.name, 'exec-candidate') | ||
188 | |||
95 | within = config.gettimedelta('KEEP', 'within') | 189 | within = config.gettimedelta('KEEP', 'within') |
96 | if within > timedelta(seconds=0): | 190 | if within > timedelta(seconds=0): |
97 | for base, snaps in items.items(): | 191 | for base, snaps in items.items(): |
@@ -109,31 +203,14 @@ def prune(config, dry_run, keep_newest): | |||
109 | else: | 203 | else: |
110 | logger.warn('Skipping rule ‘within’ since retention period is zero') | 204 | logger.warn('Skipping rule ‘within’ since retention period is zero') |
111 | 205 | ||
112 | prune_timezone = config.gettimezone('KEEP', 'timezone', fallback=tzlocal) | 206 | for rule, pattern in TIME_PATTERNS.items(): |
113 | |||
114 | PRUNING_PATTERNS = OrderedDict([ | ||
115 | ("secondly", lambda t: t.strftime('%Y-%m-%d %H:%M:%S')), | ||
116 | ("minutely", lambda t: t.strftime('%Y-%m-%d %H:%M')), | ||
117 | ("5m", lambda t: (t.strftime('%Y-%m-%d %H'), floor(t.minute / 5) * 5)), | ||
118 | ("15m", lambda t: (t.strftime('%Y-%m-%d %H'), floor(t.minute / 15) * 15)), | ||
119 | ("hourly", lambda t: t.strftime('%Y-%m-%d %H')), | ||
120 | ("4h", lambda t: (t.strftime('%Y-%m-%d'), floor(t.hour / 4) * 4)), | ||
121 | ("12h", lambda t: (t.strftime('%Y-%m-%d'), floor(t.hour / 12) * 12)), | ||
122 | ("daily", lambda t: t.strftime('%Y-%m-%d')), | ||
123 | ("halfweekly", lambda t: (t.strftime('%G-%V'), floor(int(t.strftime('%u')) / 4) * 4)), | ||
124 | ("weekly", lambda t: t.strftime('%G-%V')), | ||
125 | ("monthly", lambda t: t.strftime('%Y-%m')), | ||
126 | ("yearly", lambda t: t.strftime('%Y')), | ||
127 | ]) | ||
128 | |||
129 | for rule, pattern in PRUNING_PATTERNS.items(): | ||
130 | desired_count = config.getint('KEEP', rule, fallback=0) | 207 | desired_count = config.getint('KEEP', rule, fallback=0) |
131 | 208 | ||
132 | for base, snaps in items.items(): | 209 | for base, snaps in items.items(): |
133 | periods = OrderedDict() | 210 | periods = OrderedDict() |
134 | 211 | ||
135 | for snap in sorted(snaps, key=lambda snap: snap.creation, reverse=keep_newest): | 212 | for snap in sorted(snaps, key=lambda snap: snap.creation, reverse=keep_newest): |
136 | period = pattern(snap.creation) | 213 | period = pattern(snap.creation.astimezone(prune_timezone)) |
137 | if period not in periods: | 214 | if period not in periods: |
138 | periods[period] = deque() | 215 | periods[period] = deque() |
139 | periods[period].append(snap) | 216 | periods[period].append(snap) |
@@ -150,7 +227,7 @@ def prune(config, dry_run, keep_newest): | |||
150 | break | 227 | break |
151 | 228 | ||
152 | if to_keep > 0: | 229 | if to_keep > 0: |
153 | logger.debug(f'Missing {to_keep} to fulfill {rule}={desired_count} for ‘{base}’') | 230 | logger.debug(f'Missing {to_keep} to fulfill prune {rule}={desired_count} for ‘{base}’') |
154 | 231 | ||
155 | for snap, reasons in kept_because.items(): | 232 | for snap, reasons in kept_because.items(): |
156 | reasons_str = ', '.join(map(str, reasons)) | 233 | reasons_str = ', '.join(map(str, reasons)) |
@@ -171,16 +248,16 @@ def prune(config, dry_run, keep_newest): | |||
171 | logger.info(f'Would have pruned ‘{snap}’') | 248 | logger.info(f'Would have pruned ‘{snap}’') |
172 | else: | 249 | else: |
173 | logger.info(f'Pruned ‘{snap}’') | 250 | logger.info(f'Pruned ‘{snap}’') |
174 | 251 | ||
175 | def rename(snapshots, destroy=False, set_is_auto=False): | 252 | def rename(snapshots, destroy=False, set_is_auto=False): |
176 | args = ['zfs', 'get', '-H', '-p', '-o', 'name,value', 'creation', *snapshots] | 253 | args = ['zfs', 'get', '-H', '-p', '-o', 'name,value', 'creation', *snapshots] |
177 | _log_cmd(*args) | 254 | _log_cmd(*args) |
178 | renamed_to = set() | 255 | renamed_to = set() |
179 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: | 256 | with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: |
180 | text_stdout = io.TextIOWrapper(proc.stdout) | 257 | text_stdout = io.TextIOWrapper(proc.stdout) |
181 | reader = csv.reader(text_stdout, delimiter='\t', quoting=csv.QUOTE_NONE) | 258 | reader = csv.DictReader(text_stdout, fieldnames=['name', 'timestamp'], delimiter='\t', quoting=csv.QUOTE_NONE) |
182 | Row = namedtuple('Row', ['name', 'timestamp']) | 259 | Row = namedtuple('Row', reader.fieldnames) |
183 | for row in map(Row._make, reader): | 260 | for row in [Row(**data) for data in reader]: |
184 | creation = datetime.fromtimestamp(int(row.timestamp), timezone.utc) | 261 | creation = datetime.fromtimestamp(int(row.timestamp), timezone.utc) |
185 | base_name, _, _ = row.name.rpartition('@') | 262 | base_name, _, _ = row.name.rpartition('@') |
186 | new_name = _snap_name(base_name, time=creation) | 263 | new_name = _snap_name(base_name, time=creation) |
@@ -217,7 +294,7 @@ def autosnap(): | |||
217 | all_snap_names = set() | 294 | all_snap_names = set() |
218 | async def do_snapshot(*snap_items, recursive=False): | 295 | async def do_snapshot(*snap_items, recursive=False): |
219 | nonlocal items, all_snap_names | 296 | nonlocal items, all_snap_names |
220 | snap_names = {_snap_name(item) for item in snap_items} | 297 | snap_names = {_snap_name(item) for item in snap_items if items[item]} |
221 | if recursive: | 298 | if recursive: |
222 | for snap_item in snap_items: | 299 | for snap_item in snap_items: |
223 | all_snap_names |= {_snap_name(item) for item in items if item.startswith(snap_item)} | 300 | all_snap_names |= {_snap_name(item) for item in items if item.startswith(snap_item)} |
@@ -268,7 +345,7 @@ def main(): | |||
268 | sys.__excepthook__(type, value, tb) # calls default excepthook | 345 | sys.__excepthook__(type, value, tb) # calls default excepthook |
269 | 346 | ||
270 | sys.excepthook = log_exceptions | 347 | sys.excepthook = log_exceptions |
271 | 348 | ||
272 | parser = argparse.ArgumentParser(prog='zfssnap') | 349 | parser = argparse.ArgumentParser(prog='zfssnap') |
273 | parser.add_argument('--verbose', '-v', action='count', default=0) | 350 | parser.add_argument('--verbose', '-v', action='count', default=0) |
274 | subparsers = parser.add_subparsers() | 351 | subparsers = parser.add_subparsers() |
@@ -282,6 +359,7 @@ def main(): | |||
282 | prune_parser.add_argument('--config', '-c', dest='config_files', nargs='*', default=list()) | 359 | prune_parser.add_argument('--config', '-c', dest='config_files', nargs='*', default=list()) |
283 | prune_parser.add_argument('--dry-run', '-n', action='store_true', default=False) | 360 | prune_parser.add_argument('--dry-run', '-n', action='store_true', default=False) |
284 | prune_parser.add_argument('--keep-newest', action='store_true', default=False) | 361 | prune_parser.add_argument('--keep-newest', action='store_true', default=False) |
362 | prune_parser.add_argument('--no-exec', dest='do_exec', action='store_false', default=True) | ||
285 | prune_parser.set_defaults(cmd=prune) | 363 | prune_parser.set_defaults(cmd=prune) |
286 | args = parser.parse_args() | 364 | args = parser.parse_args() |
287 | 365 | ||
@@ -293,7 +371,7 @@ def main(): | |||
293 | logger.setLevel(logging.DEBUG) | 371 | logger.setLevel(logging.DEBUG) |
294 | 372 | ||
295 | cmdArgs = {} | 373 | cmdArgs = {} |
296 | for copy in {'snapshots', 'dry_run', 'destroy', 'keep_newest', 'set_is_auto'}: | 374 | for copy in {'snapshots', 'dry_run', 'destroy', 'keep_newest', 'set_is_auto', 'do_exec'}: |
297 | if copy in vars(args): | 375 | if copy in vars(args): |
298 | cmdArgs[copy] = vars(args)[copy] | 376 | cmdArgs[copy] = vars(args)[copy] |
299 | if 'config_files' in vars(args): | 377 | if 'config_files' in vars(args): |
@@ -308,7 +386,7 @@ def main(): | |||
308 | }) | 386 | }) |
309 | search_files = args.config_files if args.config_files else [*BaseDirectory.load_config_paths('zfssnap.ini')] | 387 | search_files = args.config_files if args.config_files else [*BaseDirectory.load_config_paths('zfssnap.ini')] |
310 | read_files = config.read(search_files) | 388 | read_files = config.read(search_files) |
311 | 389 | ||
312 | def format_config_files(files): | 390 | def format_config_files(files): |
313 | if not files: | 391 | if not files: |
314 | return 'no files' | 392 | return 'no files' |
@@ -323,4 +401,5 @@ def main(): | |||
323 | 401 | ||
324 | args.cmd(**cmdArgs) | 402 | args.cmd(**cmdArgs) |
325 | 403 | ||
326 | sys.exit(main()) | 404 | if __name__ == '__main__': |
405 | sys.exit(main()) | ||
diff --git a/nvfetcher.toml b/nvfetcher.toml index b05862a7..cb460076 100644 --- a/nvfetcher.toml +++ b/nvfetcher.toml | |||
@@ -58,4 +58,14 @@ fetch.url = "https://github.com/wofr06/lesspipe/archive/refs/tags/v$ver.tar.gz" | |||
58 | [postfix-mta-sts-resolver] | 58 | [postfix-mta-sts-resolver] |
59 | src.github = "Snawoot/postfix-mta-sts-resolver" | 59 | src.github = "Snawoot/postfix-mta-sts-resolver" |
60 | src.prefix = "v" | 60 | src.prefix = "v" |
61 | fetch.url = "https://github.com/Snawoot/postfix-mta-sts-resolver/archive/refs/tags/v$ver.tar.gz" \ No newline at end of file | 61 | fetch.url = "https://github.com/Snawoot/postfix-mta-sts-resolver/archive/refs/tags/v$ver.tar.gz" |
62 | |||
63 | [smartprom] | ||
64 | src.github = "matusnovak/prometheus-smartctl" | ||
65 | src.prefix = "v" | ||
66 | fetch.url = "https://github.com/matusnovak/prometheus-smartctl/archive/refs/tags/v$ver.tar.gz" | ||
67 | |||
68 | [postfwd] | ||
69 | src.github_tag = "postfwd/postfwd" | ||
70 | src.prefix = "v" | ||
71 | fetch.url = "https://github.com/postfwd/postfwd/archive/refs/tags/v$ver.tar.gz" \ No newline at end of file | ||
diff --git a/overlays/postfwd.nix b/overlays/postfwd.nix new file mode 100644 index 00000000..8a4f4bd8 --- /dev/null +++ b/overlays/postfwd.nix | |||
@@ -0,0 +1,32 @@ | |||
1 | { final, prev, sources, ... }: | ||
2 | let | ||
3 | deps = with final.perlPackages; [NetDNS NetServer IOMultiplex NetAddrIP NetCIDRLite DigestMD5 TimeHiRes Storable]; | ||
4 | in { | ||
5 | postfwd = prev.stdenv.mkDerivation rec { | ||
6 | inherit (sources.postfwd) pname version src; | ||
7 | |||
8 | nativeBuildInputs = with prev; [ makeWrapper ]; | ||
9 | propagatedBuildInputs = [final.perlPackages.perl] ++ deps; | ||
10 | |||
11 | buildPhase = '' | ||
12 | runHook preBuild | ||
13 | |||
14 | substituteInPlace sbin/postfwd3 \ | ||
15 | --replace "/usr/bin/perl -T" "/usr/bin/perl" | ||
16 | |||
17 | runHook postBuild | ||
18 | ''; | ||
19 | |||
20 | installPhase = '' | ||
21 | runHook preInstall | ||
22 | |||
23 | mkdir -p $out/bin | ||
24 | cp -t $out/bin sbin/postfwd3 | ||
25 | |||
26 | wrapProgram $out/bin/postfwd3 \ | ||
27 | --prefix PERL5LIB : ${final.perlPackages.makePerlPath deps} | ||
28 | |||
29 | runHook postInstall | ||
30 | ''; | ||
31 | }; | ||
32 | } | ||
diff --git a/overlays/smartprom/default.nix b/overlays/smartprom/default.nix new file mode 100644 index 00000000..0dd0771b --- /dev/null +++ b/overlays/smartprom/default.nix | |||
@@ -0,0 +1,19 @@ | |||
1 | { final, prev, flakeInputs, sources, ... }: | ||
2 | { | ||
3 | smartprom = flakeInputs.mach-nix.lib.${final.system}.buildPythonPackage rec { | ||
4 | inherit (sources.smartprom) src pname version; | ||
5 | ignoreDataOutdated = true; | ||
6 | |||
7 | prePatch = '' | ||
8 | mkdir smartprom | ||
9 | mv smartprom.py smartprom/__main__.py | ||
10 | echo >> smartprom/__init__.py | ||
11 | |||
12 | substituteAll ${./setup.py} ./setup.py | ||
13 | ''; | ||
14 | |||
15 | requirements = '' | ||
16 | prometheus_client | ||
17 | ''; | ||
18 | }; | ||
19 | } | ||
diff --git a/overlays/smartprom/setup.py b/overlays/smartprom/setup.py new file mode 100644 index 00000000..c30fc557 --- /dev/null +++ b/overlays/smartprom/setup.py | |||
@@ -0,0 +1,11 @@ | |||
1 | from setuptools import setup | ||
2 | |||
3 | setup(name='@pname@', | ||
4 | version='@version@', | ||
5 | packages=['@pname@'], | ||
6 | entry_points={ | ||
7 | 'console_scripts': [ | ||
8 | '@pname@=@pname@.__main__:main', | ||
9 | ], | ||
10 | } | ||
11 | ) | ||
diff --git a/overlays/worktime/worktime.py b/overlays/worktime/worktime.py index 9cfc6cd4..1fc00061 100755 --- a/overlays/worktime/worktime.py +++ b/overlays/worktime/worktime.py | |||
@@ -117,6 +117,7 @@ class Worktime(object): | |||
117 | force_day_to_work = True | 117 | force_day_to_work = True |
118 | leave_days = set() | 118 | leave_days = set() |
119 | leave_budget = dict() | 119 | leave_budget = dict() |
120 | time_per_day = None | ||
120 | 121 | ||
121 | @staticmethod | 122 | @staticmethod |
122 | def holidays(year): | 123 | def holidays(year): |
@@ -151,10 +152,10 @@ class Worktime(object): | |||
151 | def __init__(self, start_datetime=None, end_datetime=None, now=None, include_running=True, force_day_to_work=True, **kwargs): | 152 | def __init__(self, start_datetime=None, end_datetime=None, now=None, include_running=True, force_day_to_work=True, **kwargs): |
152 | self.include_running = include_running | 153 | self.include_running = include_running |
153 | self.force_day_to_work = force_day_to_work | 154 | self.force_day_to_work = force_day_to_work |
154 | 155 | ||
155 | if now: | 156 | if now: |
156 | self.now = now | 157 | self.now = now |
157 | 158 | ||
158 | config = Worktime.config() | 159 | config = Worktime.config() |
159 | config_dir = BaseDirectory.load_first_config('worktime') | 160 | config_dir = BaseDirectory.load_first_config('worktime') |
160 | api = TogglAPI(api_token=config['TOGGL']['ApiToken'], workspace_id=config['TOGGL']['Workspace']) | 161 | api = TogglAPI(api_token=config['TOGGL']['ApiToken'], workspace_id=config['TOGGL']['Workspace']) |
@@ -174,17 +175,17 @@ class Worktime(object): | |||
174 | except IOError as e: | 175 | except IOError as e: |
175 | if e.errno != 2: | 176 | if e.errno != 2: |
176 | raise e | 177 | raise e |
177 | 178 | ||
178 | 179 | ||
179 | hours_per_week = float(config.get('WORKTIME', 'HoursPerWeek', fallback=40)) | 180 | hours_per_week = float(config.get('WORKTIME', 'HoursPerWeek', fallback=40)) |
180 | workdays = set([int(d.strip()) for d in config.get('WORKTIME', 'Workdays', fallback='1,2,3,4,5').split(',')]) | 181 | workdays = set([int(d.strip()) for d in config.get('WORKTIME', 'Workdays', fallback='1,2,3,4,5').split(',')]) |
181 | time_per_day = timedelta(hours = hours_per_week) / len(workdays) | 182 | self.time_per_day = timedelta(hours = hours_per_week) / len(workdays) |
182 | 183 | ||
183 | holidays = dict() | 184 | holidays = dict() |
184 | 185 | ||
185 | leave_per_year = int(config.get('WORKTIME', 'LeavePerYear', fallback=30)) | 186 | leave_per_year = int(config.get('WORKTIME', 'LeavePerYear', fallback=30)) |
186 | for year in range(start_date.year, end_date.year + 1): | 187 | for year in range(start_date.year, end_date.year + 1): |
187 | holidays |= {k: v * time_per_day for k, v in Worktime.holidays(year).items()} | 188 | holidays |= {k: v * self.time_per_day for k, v in Worktime.holidays(year).items()} |
188 | leave_frac = 1 | 189 | leave_frac = 1 |
189 | if date(year, 1, 1) < start_date.date(): | 190 | if date(year, 1, 1) < start_date.date(): |
190 | leave_frac = (date(year + 1, 1, 1) - start_date.date()) / (date(year + 1, 1, 1) - date(year, 1, 1)) | 191 | leave_frac = (date(year + 1, 1, 1) - start_date.date()) / (date(year + 1, 1, 1) - date(year, 1, 1)) |
@@ -199,7 +200,7 @@ class Worktime(object): | |||
199 | day = datetime.strptime(datestr, date_format).replace(tzinfo=tzlocal()).date() | 200 | day = datetime.strptime(datestr, date_format).replace(tzinfo=tzlocal()).date() |
200 | if day != start_date.date(): | 201 | if day != start_date.date(): |
201 | continue | 202 | continue |
202 | 203 | ||
203 | self.leave_budget[day.year] = (self.leave_budget[day.year] if day.year in self.leave_budget else 0) + int(count) | 204 | self.leave_budget[day.year] = (self.leave_budget[day.year] if day.year in self.leave_budget else 0) + int(count) |
204 | except IOError as e: | 205 | except IOError as e: |
205 | if e.errno != 2: | 206 | if e.errno != 2: |
@@ -224,7 +225,7 @@ class Worktime(object): | |||
224 | toDay = parse_single(toDay) | 225 | toDay = parse_single(toDay) |
225 | else: | 226 | else: |
226 | fromDay = toDay = parse_single(datestr) | 227 | fromDay = toDay = parse_single(datestr) |
227 | time = time_per_day | 228 | time = self.time_per_day |
228 | if len(splitLine) == 2: | 229 | if len(splitLine) == 2: |
229 | [hours, datestr] = splitLine | 230 | [hours, datestr] = splitLine |
230 | time = timedelta(hours = float(hours)) | 231 | time = timedelta(hours = float(hours)) |
@@ -236,7 +237,7 @@ class Worktime(object): | |||
236 | if end_date.date() < day or day < start_date.date(): | 237 | if end_date.date() < day or day < start_date.date(): |
237 | continue | 238 | continue |
238 | 239 | ||
239 | if excused_kind == 'leave' and not (day in holidays and holidays[day] >= time_per_day) and day.isoweekday() in workdays: | 240 | if excused_kind == 'leave' and not (day in holidays and holidays[day] >= self.time_per_day) and day.isoweekday() in workdays: |
240 | self.leave_days.add(day) | 241 | self.leave_days.add(day) |
241 | holidays[day] = time | 242 | holidays[day] = time |
242 | except IOError as e: | 243 | except IOError as e: |
@@ -244,7 +245,7 @@ class Worktime(object): | |||
244 | raise e | 245 | raise e |
245 | 246 | ||
246 | pull_forward = dict() | 247 | pull_forward = dict() |
247 | 248 | ||
248 | start_day = start_date.date() | 249 | start_day = start_date.date() |
249 | end_day = end_date.date() | 250 | end_day = end_date.date() |
250 | 251 | ||
@@ -271,7 +272,7 @@ class Worktime(object): | |||
271 | if not d == datetime.strptime(c, date_format).replace(tzinfo=tzlocal()).date(): break | 272 | if not d == datetime.strptime(c, date_format).replace(tzinfo=tzlocal()).date(): break |
272 | else: | 273 | else: |
273 | if d >= end_date.date(): | 274 | if d >= end_date.date(): |
274 | pull_forward[d] = min(timedelta(hours = float(hours)), time_per_day - (holidays[d] if d in holidays else timedelta())) | 275 | pull_forward[d] = min(timedelta(hours = float(hours)), self.time_per_day - (holidays[d] if d in holidays else timedelta())) |
275 | except IOError as e: | 276 | except IOError as e: |
276 | if e.errno != 2: | 277 | if e.errno != 2: |
277 | raise e | 278 | raise e |
@@ -280,10 +281,10 @@ class Worktime(object): | |||
280 | 281 | ||
281 | if pull_forward: | 282 | if pull_forward: |
282 | end_day = max(end_day, max(list(pull_forward))) | 283 | end_day = max(end_day, max(list(pull_forward))) |
283 | 284 | ||
284 | for day in [start_day + timedelta(days = x) for x in range(0, (end_day - start_day).days + 1)]: | 285 | for day in [start_day + timedelta(days = x) for x in range(0, (end_day - start_day).days + 1)]: |
285 | if day.isoweekday() in workdays: | 286 | if day.isoweekday() in workdays: |
286 | time_to_work = time_per_day | 287 | time_to_work = self.time_per_day |
287 | if day in holidays.keys(): | 288 | if day in holidays.keys(): |
288 | time_to_work -= holidays[day] | 289 | time_to_work -= holidays[day] |
289 | if time_to_work > timedelta(): | 290 | if time_to_work > timedelta(): |
@@ -302,7 +303,7 @@ class Worktime(object): | |||
302 | day = datetime.strptime(datestr, date_format).replace(tzinfo=tzlocal()).date() | 303 | day = datetime.strptime(datestr, date_format).replace(tzinfo=tzlocal()).date() |
303 | extra_days_to_work[day] = timedelta(hours = float(hours)) | 304 | extra_days_to_work[day] = timedelta(hours = float(hours)) |
304 | else: | 305 | else: |
305 | extra_days_to_work[datetime.strptime(stripped_line, date_format).replace(tzinfo=tzlocal()).date()] = time_per_day | 306 | extra_days_to_work[datetime.strptime(stripped_line, date_format).replace(tzinfo=tzlocal()).date()] = self.time_per_day |
306 | except IOError as e: | 307 | except IOError as e: |
307 | if e.errno != 2: | 308 | if e.errno != 2: |
308 | raise e | 309 | raise e |
@@ -329,15 +330,15 @@ class Worktime(object): | |||
329 | 330 | ||
330 | extra_day_time_left = timedelta() | 331 | extra_day_time_left = timedelta() |
331 | for extra_day in extra_days_forward: | 332 | for extra_day in extra_days_forward: |
332 | day_time = max(timedelta(), time_per_day - extra_days_to_work[extra_day]) | 333 | day_time = max(timedelta(), self.time_per_day - extra_days_to_work[extra_day]) |
333 | extra_day_time_left += day_time | 334 | extra_day_time_left += day_time |
334 | extra_day_time = min(extra_day_time_left, pull_forward[day]) | 335 | extra_day_time = min(extra_day_time_left, pull_forward[day]) |
335 | time_forward = pull_forward[day] - extra_day_time | 336 | time_forward = pull_forward[day] - extra_day_time |
336 | if extra_day_time_left > timedelta(): | 337 | if extra_day_time_left > timedelta(): |
337 | for extra_day in extra_days_forward: | 338 | for extra_day in extra_days_forward: |
338 | day_time = max(timedelta(), time_per_day - extra_days_to_work[extra_day]) | 339 | day_time = max(timedelta(), self.time_per_day - extra_days_to_work[extra_day]) |
339 | extra_days_to_work[extra_day] += extra_day_time * (day_time / extra_day_time_left) | 340 | extra_days_to_work[extra_day] += extra_day_time * (day_time / extra_day_time_left) |
340 | 341 | ||
341 | hours_per_day_forward = time_forward / len(days_forward) if len(days_forward) > 0 else timedelta() | 342 | hours_per_day_forward = time_forward / len(days_forward) if len(days_forward) > 0 else timedelta() |
342 | days_forward.discard(end_date.date()) | 343 | days_forward.discard(end_date.date()) |
343 | 344 | ||
@@ -345,7 +346,7 @@ class Worktime(object): | |||
345 | 346 | ||
346 | if end_date.date() in extra_days_to_work: | 347 | if end_date.date() in extra_days_to_work: |
347 | self.time_pulled_forward += extra_days_to_work[end_date.date()] | 348 | self.time_pulled_forward += extra_days_to_work[end_date.date()] |
348 | 349 | ||
349 | self.time_to_work += self.time_pulled_forward | 350 | self.time_to_work += self.time_pulled_forward |
350 | 351 | ||
351 | self.time_worked += api.get_billable_hours(start_date, self.now, rounding = config.getboolean('WORKTIME', 'rounding', fallback=True)) | 352 | self.time_worked += api.get_billable_hours(start_date, self.now, rounding = config.getboolean('WORKTIME', 'rounding', fallback=True)) |
@@ -377,10 +378,10 @@ def worktime(**args): | |||
377 | 378 | ||
378 | if total_minutes_difference >= 0: | 379 | if total_minutes_difference >= 0: |
379 | difference_string = difference_string(total_minutes_difference * timedelta(minutes = 1)) | 380 | difference_string = difference_string(total_minutes_difference * timedelta(minutes = 1)) |
380 | return "{difference_string}/{clockout_time}".format(difference_string = difference_string, clockout_time = clockout_time.strftime("%H:%M")) | 381 | return f"{difference_string}/{clockout_time:%H:%M}" |
381 | else: | 382 | else: |
382 | difference_string = difference_string(abs(total_minutes_difference) * timedelta(minutes = 1)) | 383 | difference_string = difference_string(abs(total_minutes_difference) * timedelta(minutes = 1)) |
383 | return "{clockout_time}/{difference_string}".format(difference_string = difference_string, clockout_time = clockout_time.strftime("%H:%M")) | 384 | return f"{clockout_time:%H:%M}/{difference_string}" |
384 | else: | 385 | else: |
385 | if worktime.running_entry: | 386 | if worktime.running_entry: |
386 | difference_string = difference_string(abs(total_minutes_difference) * timedelta(minutes = 1)) | 387 | difference_string = difference_string(abs(total_minutes_difference) * timedelta(minutes = 1)) |
@@ -427,7 +428,20 @@ def time_worked(now, **args): | |||
427 | if hours_difference == 0 or minutes_difference != 0: | 428 | if hours_difference == 0 or minutes_difference != 0: |
428 | difference_string += f"{minutes_difference}m" | 429 | difference_string += f"{minutes_difference}m" |
429 | 430 | ||
430 | print(difference_string) | 431 | clockout_time = None |
432 | clockout_difference = None | ||
433 | if then.is_workday or now.is_workday: | ||
434 | target_time = max(then.time_per_day, now.time_per_day) if then.time_per_day and now.time_per_day else (then.time_per_day if then.time_per_day else now.time_per_day); | ||
435 | difference = target_time - worked | ||
436 | clockout_difference = 5 * ceil(difference / timedelta(minutes = 5)) | ||
437 | clockout_time = now.now + difference | ||
438 | clockout_time += (5 - clockout_time.minute % 5) * timedelta(minutes = 1) | ||
439 | clockout_time = clockout_time.replace(second = 0, microsecond = 0) | ||
440 | |||
441 | if now.running_entry and clockout_time and clockout_difference >= 0: | ||
442 | print(f"{difference_string}/{clockout_time:%H:%M}") | ||
443 | else: | ||
444 | print(difference_string) | ||
431 | else: | 445 | else: |
432 | print(worked) | 446 | print(worked) |
433 | 447 | ||
@@ -445,7 +459,7 @@ def holidays(year, **args): | |||
445 | date_format = config.get('WORKTIME', 'DateFormat', fallback='%Y-%m-%d') | 459 | date_format = config.get('WORKTIME', 'DateFormat', fallback='%Y-%m-%d') |
446 | 460 | ||
447 | table_data = [] | 461 | table_data = [] |
448 | 462 | ||
449 | holidays = Worktime.holidays(year) | 463 | holidays = Worktime.holidays(year) |
450 | for k, v in holidays.items(): | 464 | for k, v in holidays.items(): |
451 | kstr = k.strftime(date_format) | 465 | kstr = k.strftime(date_format) |
@@ -473,7 +487,7 @@ def leave(year, table, **args): | |||
473 | break | 487 | break |
474 | else: | 488 | else: |
475 | print(f'Unaccounted leave: {day}', file=stderr) | 489 | print(f'Unaccounted leave: {day}', file=stderr) |
476 | 490 | ||
477 | if table: | 491 | if table: |
478 | table_data = [] | 492 | table_data = [] |
479 | for year, days in leave_budget.items(): | 493 | for year, days in leave_budget.items(): |
diff --git a/user-profiles/utils.nix b/user-profiles/utils.nix index d0d2b2c8..c5042d41 100644 --- a/user-profiles/utils.nix +++ b/user-profiles/utils.nix | |||
@@ -24,6 +24,7 @@ | |||
24 | mosh tree vnstat file pv bc fast-cli zip nmap aspell | 24 | mosh tree vnstat file pv bc fast-cli zip nmap aspell |
25 | aspellDicts.de aspellDicts.en borgbackup man-pages rsync socat | 25 | aspellDicts.de aspellDicts.en borgbackup man-pages rsync socat |
26 | inetutils yq cached-nix-shell persistent-nix-shell rage | 26 | inetutils yq cached-nix-shell persistent-nix-shell rage |
27 | smartmontools hdparm | ||
27 | ]; | 28 | ]; |
28 | }; | 29 | }; |
29 | } | 30 | } |