summaryrefslogtreecommitdiff
path: root/hosts/vidhar/borg
diff options
context:
space:
mode:
Diffstat (limited to 'hosts/vidhar/borg')
-rw-r--r--hosts/vidhar/borg/borgsnap/borgsnap/__main__.py394
-rw-r--r--hosts/vidhar/borg/borgsnap/setup.py10
-rw-r--r--hosts/vidhar/borg/default.nix61
3 files changed, 9 insertions, 456 deletions
diff --git a/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py b/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py
deleted file mode 100644
index 15bf50c8..00000000
--- a/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py
+++ /dev/null
@@ -1,394 +0,0 @@
1import argparse
2import os, sys, signal, io
3from pyprctl import CapState, Cap, cap_ambient_raise, cap_ambient_is_set, set_keepcaps
4from pwd import getpwnam
5
6from datetime import datetime, timezone
7from dateutil.parser import isoparse
8
9import unshare
10from tempfile import TemporaryDirectory
11
12import logging
13
14import json
15import subprocess
16import csv
17from collections import namedtuple
18from distutils.util import strtobool
19
20import pathlib
21from pathlib import Path
22
23from atomicwrites import atomic_write
24
25from traceback import format_exc
26
27from multiprocessing import Process, Manager
28from contextlib import closing
29
30from enum import Enum, auto
31
32import select
33import time
34import math
35
36
37PROP_DO_BORGSNAP = 'li.yggdrasil:borgsnap'
38
39
40class DoValue(Enum):
41 BORGSNAP_DO = auto()
42 BORGSNAP_KEEP = auto()
43 BORGSNAP_DONT = auto()
44
45 @classmethod
46 def from_prop(cls, v: str):
47 if v.lower() == 'keep':
48 return cls.BORGSNAP_KEEP
49
50 return cls.BORGSNAP_DO if not v or bool(strtobool(v)) else cls.BORGSNAP_DONT
51
52 @classmethod
53 def merge(cls, v1, v2):
54 match (v1, v2):
55 case (cls.BORGSNAP_DONT, _):
56 return cls.BORGSNAP_DONT
57 case (_, cls.BORGSNAP_DONT):
58 return cls.BORGSNAP_DONT
59 case (cls.BORGSNAP_KEEP, _):
60 return cls.BORGSNAP_KEEP
61 case (_, cls.BORGSNAP_KEEP):
62 return cls.BORGSNAP_KEEP
63 case other:
64 return cls.BORGSNAP_DO
65
66 def returncode(self):
67 match self:
68 case self.__class__.BORGSNAP_DO:
69 return 126
70 case self.__class__.BORGSNAP_KEEP:
71 return 125
72 case self.__class__.BORGSNAP_DONT:
73 return 124
74
75borg_pwd = getpwnam('borg')
76
77def as_borg(caps=set()):
78 global logger
79
80 try:
81 if caps:
82 c_state = CapState.get_current()
83 c_state.permitted.add(*caps)
84 c_state.set_current()
85
86 logger.debug("before setgid/setuid: cap_permitted=%s", CapState.get_current().permitted)
87
88 set_keepcaps(True)
89
90 os.setgid(borg_pwd.pw_gid)
91 os.setuid(borg_pwd.pw_uid)
92
93 if caps:
94 logger.debug("after setgid/setuid: cap_permitted=%s", CapState.get_current().permitted)
95
96 c_state = CapState.get_current()
97 c_state.permitted = caps.copy()
98 c_state.inheritable.add(*caps)
99 c_state.set_current()
100
101 logger.debug("cap_permitted=%s", CapState.get_current().permitted)
102 logger.debug("cap_inheritable=%s", CapState.get_current().inheritable)
103
104 for cap in caps:
105 cap_ambient_raise(cap)
106 logger.debug("cap_ambient[%s]=%s", cap, cap_ambient_is_set(cap))
107 except Exception:
108 logger.error(format_exc())
109 raise
110
111
112def _archive_name(snapshot, target, archive_prefix):
113 _, _, ts = snapshot.rpartition('@')
114 creation_time = isoparse(ts).astimezone(timezone.utc)
115 archive_name = _archive_basename(snapshot, archive_prefix)
116 return f'{target}::{archive_name}-{creation_time.strftime("%Y-%m-%dT%H:%M:%S")}'
117
118def _archive_basename(snapshot, archive_prefix):
119 base_name, _, _ = snapshot.rpartition('@')
120 return archive_prefix + base_name.replace('-', '--').replace('/', '-')
121
122def check(*, snapshot, target, archive_prefix, cache_file):
123 global logger
124
125 archives = None
126 if cache_file:
127 logger.debug('Trying cache...')
128 try:
129 with open(cache_file, mode='r', encoding='utf-8') as fp:
130 archives = set(json.load(fp))
131 logger.debug('Loaded archive list from cache')
132 except FileNotFoundError:
133 pass
134
135 if not archives:
136 logger.info('Loading archive list from remote...')
137 with subprocess.Popen(['borg', 'list', '--info', '--lock-wait=600', '--json', target], stdout=subprocess.PIPE, preexec_fn=lambda: as_borg()) as proc:
138 archives = set([archive['barchive'] for archive in json.load(proc.stdout)['archives']])
139 if cache_file:
140 logger.debug('Saving archive list to cache...')
141 with atomic_write(cache_file, mode='w', encoding='utf-8', overwrite=True) as fp:
142 json.dump(list(archives), fp)
143
144 # logger.debug(f'archives: {archives}')
145 _, _, archive_name = _archive_name(snapshot, target, archive_prefix).partition('::')
146 if archive_name in archives:
147 logger.info('‘%s’ found', archive_name)
148 return 0
149 else:
150 logger.info('‘%s’ not found', archive_name)
151
152 logger.debug('Checking %s for ‘%s’...', PROP_DO_BORGSNAP, snapshot)
153 intent = DoValue.BORGSNAP_DO
154 p = subprocess.run(['zfs', 'get', '-H', '-p', '-o', 'name,value', PROP_DO_BORGSNAP, snapshot], stdout=subprocess.PIPE, text=True, check=True)
155 reader = csv.DictReader(io.StringIO(p.stdout), fieldnames=['name', 'value'], delimiter='\t', quoting=csv.QUOTE_NONE)
156 Row = namedtuple('Row', reader.fieldnames)
157 for row in [Row(**data) for data in reader]:
158 if not row.value or row.value == '-':
159 continue
160
161 logger.debug('%s=%s (parsed as %s) for ‘%s’...', PROP_DO_BORGSNAP, row.value, DoValue.from_prop(row.value), row.name)
162 intent = DoValue.merge(intent, DoValue.from_prop(row.value))
163
164 match intent:
165 case DoValue.BORGSNAP_DONT:
166 logger.warn('%s specifies to ignore, returning accordingly...', PROP_DO_BORGSNAP)
167 case DoValue.BORGSNAP_KEEP:
168 logger.info('%s specifies to ignore but keep, returning accordingly...', PROP_DO_BORGSNAP)
169 case other:
170 pass
171
172 return intent.returncode()
173
174def create(*, snapshot, target, archive_prefix, dry_run):
175 global logger
176
177 basename = _archive_basename(snapshot, archive_prefix)
178
179 def do_create(tmpdir_q):
180 global logger
181 nonlocal basename, snapshot, target, archive_prefix, dry_run
182
183 tmpdir = tmpdir_q.get()
184
185 unshare.unshare(unshare.CLONE_NEWNS)
186 subprocess.run(['mount', '--make-rprivate', '/'], check=True)
187 chroot = pathlib.Path(tmpdir) / 'chroot'
188 upper = pathlib.Path(tmpdir) / 'upper'
189 work = pathlib.Path(tmpdir) / 'work'
190 for path in [chroot,upper,work]:
191 path.mkdir()
192 subprocess.run(['mount', '-t', 'overlay', 'overlay', '-o', f'lowerdir=/,upperdir={upper},workdir={work}', chroot], check=True)
193 bindMounts = ['nix', 'run', 'run/secrets.d', 'run/wrappers', 'proc', 'dev', 'sys', pathlib.Path(os.path.expanduser('~')).relative_to('/')]
194 if borg_base_dir := os.getenv('BORG_BASE_DIR'):
195 bindMounts.append(pathlib.Path(borg_base_dir).relative_to('/'))
196 if ssh_auth_sock := os.getenv('SSH_AUTH_SOCK'):
197 bindMounts.append(pathlib.Path(ssh_auth_sock).parent.relative_to('/'))
198 for bindMount in bindMounts:
199 (chroot / bindMount).mkdir(parents=True,exist_ok=True)
200 subprocess.run(['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], check=True)
201
202 os.chroot(chroot)
203 os.chdir('/')
204 dir = pathlib.Path('/borg')
205 dir.mkdir(parents=True,exist_ok=True,mode=0o0750)
206 os.chown(dir, borg_pwd.pw_uid, borg_pwd.pw_gid)
207
208 base_name, _, _ = snapshot.rpartition('@')
209 type_val = subprocess.run(['zfs', 'get', '-H', '-p', '-o', 'value', 'type', base_name], stdout=subprocess.PIPE, text=True, check=True).stdout.strip()
210 match type_val:
211 case 'filesystem':
212 subprocess.run(['mount', '-t', 'zfs', '-o', 'ro', snapshot, dir], check=True)
213 case 'volume':
214 snapdev_val = subprocess.run(['zfs', 'get', '-H', '-p', '-o', 'value', 'snapdev', base_name], stdout=subprocess.PIPE, text=True, check=True).stdout.strip()
215 try:
216 if snapdev_val == 'hidden':
217 subprocess.run(['zfs', 'set', 'snapdev=visible', base_name], check=True)
218 subprocess.run(['mount', '-t', 'auto', '-o', 'ro', Path('/dev/zvol') / snapshot, dir], check=True)
219 finally:
220 if snapdev_val == 'hidden':
221 subprocess.run(['zfs', 'inherit', 'snapdev', base_name], check=True)
222 case other:
223 raise ValueError(f'‘{base_name}’ is of type ‘{type_val}’')
224
225 env = os.environ.copy()
226 create_args = ['borg',
227 'create',
228 '--lock-wait=600',
229 '--one-file-system',
230 '--exclude-caches',
231 '--keep-exclude-tags',
232 '--compression=auto,zstd,10',
233 '--chunker-params=10,23,16,4095',
234 '--files-cache=ctime,size',
235 '--show-rc',
236 '--upload-buffer=100',
237 '--upload-ratelimit=20480',
238 '--progress',
239 '--list',
240 '--filter=AMEi-x?',
241 '--stats' if not dry_run else '--dry-run',
242 ]
243 _, _, ts = snapshot.rpartition('@')
244 creation_time = isoparse(ts).astimezone(timezone.utc)
245 create_args += [f'--timestamp={creation_time.strftime("%Y-%m-%dT%H:%M:%S")}']
246 env['BORG_FILES_CACHE_SUFFIX'] = basename
247 archive_name = _archive_name(snapshot, target, archive_prefix)
248 target_host, _, target_path = target.rpartition(':')
249 *parents_init, _ = list(Path(target_path).parents)
250 backup_patterns = [*(map(lambda p: Path('.backup') / f'{target_host}:{p}', [Path(target_path), *parents_init])), Path('.backup') / target_host, Path('.backup')]
251 for pattern_file in backup_patterns:
252 if (dir / pattern_file).is_file():
253 logger.debug('Found backup patterns at ‘%s’', dir / pattern_file)
254 create_args += [f'--patterns-from={pattern_file}', archive_name]
255 break
256 elif (dir / pattern_file).exists():
257 logger.warn('‘%s’ exists but is no file', dir / pattern_file)
258 else:
259 logger.debug('No backup patterns exist, checked %s', list(map(lambda pattern_file: str(dir / pattern_file), backup_patterns)))
260 create_args += [archive_name, '.']
261 logger.debug('%s', {'create_args': create_args, 'cwd': dir, 'env': env})
262
263 with subprocess.Popen(create_args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, preexec_fn=lambda: as_borg(caps={Cap.DAC_READ_SEARCH}), cwd=dir, text=True) as proc:
264 proc_logger = logger.getChild('borg')
265 stdout_logger = proc_logger.getChild('stdout')
266 stderr_logger = proc_logger.getChild('stderr')
267
268 poll = select.poll()
269 poll.register(proc.stdout, select.POLLIN | select.POLLHUP)
270 poll.register(proc.stderr, select.POLLIN | select.POLLHUP)
271 pollc = 2
272 events = poll.poll()
273 while pollc > 0 and len(events) > 0:
274 for rfd, event in events:
275 if event & select.POLLIN:
276 if rfd == proc.stdout.fileno():
277 line = proc.stdout.readline()
278 if len(line) > 0:
279 stdout_logger.info(line[:-1])
280 if rfd == proc.stderr.fileno():
281 line = proc.stderr.readline()
282 if len(line) > 0:
283 stderr_logger.info(line[:-1])
284 if event & select.POLLHUP:
285 poll.unregister(rfd)
286 pollc -= 1
287
288 if pollc > 0:
289 events = poll.poll()
290
291 for handler in proc_logger.handlers:
292 handler.flush()
293
294 ret = proc.wait()
295 if ret != 0:
296 raise Exception(f'borg subprocess exited with returncode {ret}')
297
298 with Manager() as manager:
299 tmpdir_q = manager.Queue(1)
300 with closing(Process(target=do_create, args=(tmpdir_q,), name='do_create')) as p:
301 p.start()
302
303 with TemporaryDirectory(prefix=f'borg-mount_{basename}_', dir=os.getenv('RUNTIME_DIRECTORY')) as tmpdir:
304 tmpdir_q.put(tmpdir)
305 p.join()
306 if p.exitcode == 0 and dry_run:
307 return 125
308 return p.exitcode
309
310def sigterm(signum, frame):
311 raise SystemExit(128 + signum)
312
313def main():
314 signal.signal(signal.SIGTERM, sigterm)
315
316 global logger
317 logger = logging.getLogger(__name__)
318 console_handler = logging.StreamHandler()
319 console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') )
320 if sys.stderr.isatty():
321 console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') )
322
323 burst_max = 10000
324 burst = burst_max
325 last_use = None
326 inv_rate = 1e6
327 def consume_filter(record):
328 nonlocal burst, burst_max, inv_rate, last_use
329
330 delay = None
331 while True:
332 now = time.monotonic_ns()
333 burst = min(burst_max, burst + math.floor((now - last_use) / inv_rate)) if last_use else burst_max
334 last_use = now
335
336 if burst > 0:
337 burst -= 1
338 if delay:
339 delay = now - delay
340
341 return True
342
343 if delay is None:
344 delay = now
345 time.sleep(inv_rate / 1e9)
346 console_handler.addFilter(consume_filter)
347
348 logger.addHandler(console_handler)
349
350 # log uncaught exceptions
351 def log_exceptions(type, value, tb):
352 global logger
353
354 logger.error(value)
355 sys.__excepthook__(type, value, tb) # calls default excepthook
356
357 sys.excepthook = log_exceptions
358
359 parser = argparse.ArgumentParser(prog='borgsnap')
360 parser.add_argument('--verbose', '-v', dest='log_level', action='append_const', const=-1)
361 parser.add_argument('--quiet', '-q', dest='log_level', action='append_const', const=1)
362 parser.add_argument('--target', metavar='REPO', default='yggdrasil.borgbase:repo')
363 parser.add_argument('--archive-prefix', metavar='REPO', default='yggdrasil.vidhar.')
364 subparsers = parser.add_subparsers()
365 subparsers.required = True
366 parser.set_defaults(cmd=None)
367 check_parser = subparsers.add_parser('check')
368 check_parser.add_argument('--cache-file', type=lambda p: Path(p).absolute(), default=None)
369 check_parser.add_argument('snapshot')
370 check_parser.set_defaults(cmd=check)
371 create_parser = subparsers.add_parser('create')
372 create_parser.add_argument('--dry-run', '-n', action='store_true', default=False)
373 create_parser.add_argument('snapshot')
374 create_parser.set_defaults(cmd=create)
375 args = parser.parse_args()
376
377 LOG_LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]
378 DEFAULT_LOG_LEVEL = logging.ERROR
379 log_level = LOG_LEVELS.index(DEFAULT_LOG_LEVEL)
380
381 for adjustment in args.log_level or ():
382 log_level = min(len(LOG_LEVELS) - 1, max(log_level + adjustment, 0))
383 logger.setLevel(LOG_LEVELS[log_level])
384
385 cmdArgs = {}
386 for copy in {'target', 'archive_prefix', 'snapshot', 'cache_file', 'dry_run'}:
387 if copy in vars(args):
388 cmdArgs[copy] = vars(args)[copy]
389
390 return args.cmd(**cmdArgs)
391
392
393if __name__ == '__main__':
394 sys.exit(main())
diff --git a/hosts/vidhar/borg/borgsnap/setup.py b/hosts/vidhar/borg/borgsnap/setup.py
deleted file mode 100644
index 76356bfc..00000000
--- a/hosts/vidhar/borg/borgsnap/setup.py
+++ /dev/null
@@ -1,10 +0,0 @@
1from setuptools import setup
2
3setup(name='borgsnap',
4 packages=['borgsnap'],
5 entry_points={
6 'console_scripts': [
7 'borgsnap=borgsnap.__main__:main',
8 ],
9 }
10)
diff --git a/hosts/vidhar/borg/default.nix b/hosts/vidhar/borg/default.nix
index fe62c956..8d0b46ef 100644
--- a/hosts/vidhar/borg/default.nix
+++ b/hosts/vidhar/borg/default.nix
@@ -3,7 +3,7 @@
3with lib; 3with lib;
4 4
5let 5let
6 sshConfig = pkgs.writeText "config" '' 6 sshConfig = ''
7 Include /etc/ssh/ssh_config 7 Include /etc/ssh/ssh_config
8 8
9 ControlMaster auto 9 ControlMaster auto
@@ -33,7 +33,7 @@ let
33 # StateDirectory = "borg"; 33 # StateDirectory = "borg";
34 RuntimeDirectory = "copy-borg"; 34 RuntimeDirectory = "copy-borg";
35 Environment = [ 35 Environment = [
36 "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${sshConfig}\"" 36 "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${pkgs.writeText "config" sshConfig}\""
37 "BORG_BASE_DIR=/var/lib/borg" 37 "BORG_BASE_DIR=/var/lib/borg"
38 "BORG_CONFIG_DIR=/var/lib/borg/config" 38 "BORG_CONFIG_DIR=/var/lib/borg/config"
39 "BORG_CACHE_DIR=/var/lib/borg/cache" 39 "BORG_CACHE_DIR=/var/lib/borg/cache"
@@ -74,63 +74,20 @@ let
74 copy 74 copy
75 75
76 wrapProgram $out/bin/copy \ 76 wrapProgram $out/bin/copy \
77 --prefix PATH : ${makeBinPath (with pkgs; [config.boot.zfs.package util-linux borgbackup])}:${config.security.wrapperDir}
78 '';
79 });
80
81 borgsnap = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage rec {
82 pname = "borgsnap";
83 src = ./borgsnap;
84 version = "0.0.0";
85 ignoreDataOutdated = true;
86
87 requirements = ''
88 atomicwrites
89 pyprctl
90 python-unshare
91 python-dateutil
92 '';
93 postInstall = ''
94 wrapProgram $out/bin/borgsnap \
95 --prefix PATH : ${makeBinPath (with pkgs; [util-linux borgbackup])}:${config.security.wrapperDir} 77 --prefix PATH : ${makeBinPath (with pkgs; [util-linux borgbackup])}:${config.security.wrapperDir}
96 ''; 78 '';
97 79 });
98 providers.python-unshare = "nixpkgs";
99 overridesPre = [
100 (self: super: { python-unshare = super.python-unshare.overrideAttrs (oldAttrs: { name = "python-unshare-0.2.1"; version = "0.2.1"; }); })
101 ];
102
103 _.tomli.buildInputs.add = with pkgs."python3Packages"; [ flit-core ];
104 };
105in { 80in {
106 config = { 81 config = {
107 services.zfssnap.config.exec = { 82 services.borgsnap = {
108 check = "${borgsnap}/bin/borgsnap -vv --target yggdrasil.borgbase:repo --archive-prefix yggdrasil.vidhar. check --cache-file /run/zfssnap-prune/archives-cache.json"; 83 enable = true;
109 cmd = "${borgsnap}/bin/borgsnap -vv --target yggdrasil.borgbase:repo --archive-prefix yggdrasil.vidhar. create";
110 84
111 halfweekly = "8"; 85 target = "yggdrasil.borgbase:repo";
112 monthly = "-1"; 86 inherit sshConfig;
87 keyfile = config.sops.secrets."yggdrasil.borgkey".path;
113 }; 88 };
114 89
115 systemd.services = { 90 systemd.services = listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]);
116 "zfssnap-prune" = {
117 serviceConfig = {
118 Environment = [
119 "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${sshConfig}\""
120 "BORG_BASE_DIR=/var/lib/borg"
121 "BORG_CONFIG_DIR=/var/lib/borg/config"
122 "BORG_CACHE_DIR=/var/lib/borg/cache"
123 "BORG_SECURITY_DIR=/var/lib/borg/security"
124 "BORG_KEYS_DIR=/var/lib/borg/keys"
125 "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}"
126 "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes"
127 "BORG_HOSTNAME_IS_UNIQUE=yes"
128 ];
129 RuntimeDirectory = "zfssnap-prune";
130 };
131 };
132 } // listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]);
133
134 91
135 services.borgbackup.repos.jotnar = { 92 services.borgbackup.repos.jotnar = {
136 path = "/srv/backup/borg/jotnar"; 93 path = "/srv/backup/borg/jotnar";