...

15 files changed, 91 insertions, 769 deletions

diff --git a/hosts/surtr/prometheus/tls.crt b/hosts/surtr/prometheus/tls.crt
index ba958f40..d81f429f 100644
--- a/hosts/surtr/prometheus/tls.crt
+++ b/hosts/surtr/prometheus/tls.crt
@@ -1,10 +1,13 @@
 -----BEGIN CERTIFICATE-----
-MIIBXzCCARGgAwIBAgIBATAFBgMrZXAwHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55
-Z2dkcmFzaWwwIBcNMjIwNDA4MjAwMzU1WhgPMjA5MDA0MjYyMDAzNTVaMBoxGDAW
-BgNVBAMMD3N1cnRyLnlnZ2RyYXNpbDAqMAUGAytlcAMhAAJd8I32X/z9J0cO2Oz+
-4KAoIJq0igdMdbLBA+8WO+vgo3UwczAMBgNVHRMBAf8EAjAAMEQGA1UdEQQ9MDuC
-GnByb21ldGhldXMuc3VydHIueWdnZHJhc2lsgh1wcm9tZXRoZXVzLnN1cnRyLnln
-Z2RyYXNpbC5saTAdBgNVHQ4EFgQUN52tPcv5FFppzeJx2AiXk6UgPDgwBQYDK2Vw
-A0EAPN9zhaeBB2C1TursdARH0jVBz9g0dRhP7sO5ZG0K+xp24paLXiTF1rYub24p
-/yZw71p7M0BAE+hJqYBzYo5YBQ==
+MIIB5TCCAWWgAwIBAgIPQAAAAGNpYE436fsCRvVfMAUGAytlcTAfMR0wGwYDVQQD
+DBRwcm9tZXRoZXVzLnlnZ2RyYXNpbDAeFw0yMjExMDcxOTM5NDFaFw0zMjExMDcx
+OTQ0NDFaMBoxGDAWBgNVBAMMD3N1cnRyLnlnZ2RyYXNpbDAqMAUGAytlcAMhAAJd
+8I32X/z9J0cO2Oz+4KAoIJq0igdMdbLBA+8WO+vgo4G8MIG5MB8GA1UdIwQYMBaA
+FObrhCUDCZk6/JeeDMNWl8WeLr+MMB0GA1UdDgQWBBQ3na09y/kUWmnN4nHYCJeT
+pSA8ODAOBgNVHQ8BAf8EBAMCBeAwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggr
+BgEFBQcDAjBEBgNVHREEPTA7ghpwcm9tZXRoZXVzLnN1cnRyLnlnZ2RyYXNpbIId
+cHJvbWV0aGV1cy5zdXJ0ci55Z2dkcmFzaWwubGkwBQYDK2VxA3MAYHd3I/Mg/t34
+zdcxrIKOAKJ9ZVVoP0msk/viKrZ4b+Q9rKSNEnkyk0y56Z7FlLDxGLScaemqQ3uA
+5hjhdTci/xd4xYX/edLw1AWGRs2kBe3vs2WOmrdcKa849vdMH27G/P/+bgbdofCN
+fukxYHzpESYA
 -----END CERTIFICATE-----
diff --git a/hosts/vidhar/borg/copy/copy_borg/__main__.py b/hosts/vidhar/borg/copy/copy_borg/__main__.py
deleted file mode 100755
index 5b374d99..00000000
--- a/hosts/vidhar/borg/copy/copy_borg/__main__.py
+++ /dev/null
@@ -1,556 +0,0 @@
-#!@python@/bin/python
-
-import json
-import os
-import subprocess
-import re
-import sys
-import io
-from sys import stderr
-from humanize import naturalsize
-
-from tempfile import TemporaryDirectory
-
-from datetime import (datetime, timedelta)
-from dateutil.tz import (tzlocal, tzutc)
-import dateutil.parser
-import argparse
-
-from tqdm import tqdm
-
-from xdg import xdg_runtime_dir
-import pathlib
-
-import unshare
-from pyprctl import CapState, Cap, cap_ambient_raise, cap_ambient_is_set, set_keepcaps
-from pwd import getpwnam
-
-import logging
-
-import signal
-import time
-import math
-
-from halo import Halo
-
-from collections import deque
-
-import select
-import fcntl
-
-from multiprocessing import Process, Manager
-from contextlib import closing
-
-
-halo_args = {
-    'stream': stderr,
-    'enabled': stderr.isatty(),
-    'spinner': 'arc'
-}
-
-borg_pwd = getpwnam('borg')
-
-def as_borg(caps=set()):
-    global logger
-
-    try:
-        if caps:
-            c_state = CapState.get_current()
-            c_state.permitted.add(*caps)
-            c_state.set_current()
-
-            # logger.debug("before setgid/setuid: cap_permitted=%s", CapState.get_current().permitted)
-
-            set_keepcaps(True)
-
-        os.setgid(borg_pwd.pw_gid)
-        os.setuid(borg_pwd.pw_uid)
-
-        if caps:
-            # logger.debug("after setgid/setuid: cap_permitted=%s", CapState.get_current().permitted)
-
-            c_state = CapState.get_current()
-            c_state.permitted = caps.copy()
-            c_state.inheritable.add(*caps)
-            c_state.set_current()
-
-            # logger.debug("cap_permitted=%s", CapState.get_current().permitted)
-            # logger.debug("cap_inheritable=%s", CapState.get_current().inheritable)
-
-            for cap in caps:
-                cap_ambient_raise(cap)
-                # logger.debug("cap_ambient[%s]=%s", cap, cap_ambient_is_set(cap))
-    except Exception:
-        logger.error(format_exc())
-        raise
-
-def borg_json(*args, **kwargs):
-    global logger
-
-    with subprocess.Popen(*args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **kwargs) as proc:
-        stdout_buffer = io.BytesIO()
-
-        proc_logger = logger.getChild('borg')
-        stdout_logger = proc_logger.getChild('stdout')
-        stderr_logger = proc_logger.getChild('stderr')
-
-        fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
-        fcntl.fcntl(proc.stderr.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
-
-        poll = select.poll()
-        poll.register(proc.stdout, select.POLLIN | select.POLLHUP)
-        poll.register(proc.stderr, select.POLLIN | select.POLLHUP)
-        pollc = 2
-        events = poll.poll()
-        stderr_linebuf = bytearray()
-
-        while pollc > 0 and len(events) > 0:
-            for rfd, event in events:
-                if event & select.POLLIN:
-                    if rfd == proc.stdout.fileno():
-                        try:
-                            buf = os.read(proc.stdout.fileno(), 8192)
-                            # stdout_logger.debug(buf)
-                            stdout_buffer.write(buf)
-                        except BlockingIOError:
-                            pass
-                    if rfd == proc.stderr.fileno():
-                        try:
-                            stderr_linebuf.extend(os.read(proc.stderr.fileno(), 8192))
-                        except BlockingIOError:
-                            pass
-
-                        while stderr_linebuf:
-                            line, sep, stderr_linebuf = stderr_linebuf.partition(b'\n')
-                            if not sep:
-                                stderr_linebuf = line
-                                break
-
-                            stderr_logger.info(line.decode())
-                if event == select.POLLHUP:
-                    poll.unregister(rfd)
-                    pollc -= 1
-
-            if pollc > 0:
-                events = poll.poll()
-
-        for handler in proc_logger.handlers:
-            handler.flush()
-
-        ret = proc.wait()
-        if ret != 0:
-            raise Exception(f'borg subprocess exited with returncode {ret}')
-
-        stdout_buffer.seek(0)
-        return json.load(stdout_buffer)
-
-def read_repo(path):
-    global logger
-
-    with Halo(text=f'Listing {path}', **halo_args) as sp:
-        if not sp.enabled:
-            logger.debug('Listing %s...', path)
-        res = borg_json(['borg', 'list', '--info', '--lock-wait=600', '--json', path], preexec_fn=lambda: as_borg())['archives']
-        if sp.enabled:
-            sp.succeed(f'{len(res)} archives in {path}')
-        else:
-            logger.info('%d archives in ‘%s’', len(res), path)
-    return res
-
-class ToSync:
-    to_sync = deque()
-
-    def __init__(self, source, target):
-        self.source = source
-        self.target = target
-
-    def __iter__(self):
-        return self
-
-    def __next__(self):
-        global logger
-
-        if self.to_sync:
-            return self.to_sync.popleft()
-
-        while True:
-            try:
-                src = read_repo(self.source)
-                dst = read_repo(self.target)
-            except (subprocess.CalledProcessError, json.decoder.JSONDecodeError) as err:
-                logger.error(err)
-                continue
-
-            self.to_sync.extend([entry for entry in src if entry['name'] not in {dst_entry['name'] for dst_entry in dst} and not entry['name'].endswith('.checkpoint')])
-
-            if self.to_sync:
-                return self.to_sync.popleft()
-
-            raise StopIteration
-
-def copy_archive(src_repo_path, dst_repo_path, entry):
-    global logger
-
-    def do_copy(tmpdir_q):
-        global logger
-
-        nonlocal src_repo_path, dst_repo_path, entry
-
-        tmpdir = tmpdir_q.get()
-
-        cache_suffix = None
-        with Halo(text=f'Determine archive parameters', **halo_args) as sp:
-            if not sp.enabled:
-                logger.debug('Determining archive parameters...')
-            match = re.compile('^(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.(checkpoint|recreate)(\.[0-9]+)?)?').fullmatch(entry['name'])
-            if match:
-                repo_id = borg_json(['borg', 'info', '--info', '--lock-wait=600', '--json', src_repo_path], preexec_fn=lambda: as_borg())['repository']['id']
-
-                if repo_id:
-                    cache_suffix = f'{repo_id}_{match.group(1)}'
-            if sp.enabled:
-                sp.succeed(f'Will process {entry["name"]} ({dateutil.parser.isoparse(entry["start"])}, cache_suffix={cache_suffix})')
-            else:
-                logger.info('Will process ‘%s’ (%s, cache_suffix=%s)', entry['name'], dateutil.parser.isoparse(entry['start']), cache_suffix)
-
-        logger.debug('Setting up environment...')
-        unshare.unshare(unshare.CLONE_NEWNS)
-        subprocess.run(['mount', '--make-rprivate', '/'], check=True)
-        chroot = pathlib.Path(tmpdir) / 'chroot'
-        upper = pathlib.Path(tmpdir) / 'upper'
-        work = pathlib.Path(tmpdir) / 'work'
-        for path in [chroot,upper,work]:
-            path.mkdir()
-        subprocess.run(['mount', '-t', 'overlay', 'overlay', '-o', f'lowerdir=/,upperdir={upper},workdir={work}', chroot], check=True)
-        bindMounts = ['nix', 'run', 'run/secrets.d', 'run/wrappers', 'proc', 'dev', 'sys', pathlib.Path(os.path.expanduser('~')).relative_to('/')]
-        if os.environ.get('BORG_BASE_DIR'):
-            bindMounts.append(pathlib.Path(os.environ['BORG_BASE_DIR']).relative_to('/'))
-        if not ":" in src_repo_path:
-            bindMounts.append(pathlib.Path(src_repo_path).relative_to('/'))
-        if 'SSH_AUTH_SOCK' in os.environ:
-            bindMounts.append(pathlib.Path(os.environ['SSH_AUTH_SOCK']).parent.relative_to('/'))
-        for bindMount in bindMounts:
-            (chroot / bindMount).mkdir(parents=True,exist_ok=True)
-            subprocess.run(['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], check=True)
-        os.chroot(chroot)
-        os.chdir('/')
-        try:
-            os.unlink('/etc/fuse.conf')
-        except FileNotFoundError:
-            pass
-        pathlib.Path('/etc/fuse.conf').parent.mkdir(parents=True,exist_ok=True)
-        with open('/etc/fuse.conf', 'w') as fuse_conf:
-            fuse_conf.write('user_allow_other\nmount_max = 1000\n')
-        dir = pathlib.Path('/borg')
-        dir.mkdir(parents=True,exist_ok=True,mode=0o0750)
-        os.chown(dir, borg_pwd.pw_uid, borg_pwd.pw_gid)
-
-        total_size = None
-        total_files = None
-        if stderr.isatty():
-            with Halo(text=f'Determine size', **halo_args) as sp:
-                stats = borg_json(['borg', 'info', '--info', '--json', '--lock-wait=600', f'{src_repo_path}::{entry["name"]}'], preexec_fn=lambda: as_borg())['archives'][0]['stats']
-                total_size = stats['original_size']
-                total_files = stats['nfiles']
-                if sp.enabled:
-                    sp.succeed(f'{total_files} files, {naturalsize(total_size, binary=True)}')
-                else:
-                    logger.info('%d files, %s', total_files, naturalsize(total_size, binary=True))
-        with subprocess.Popen(['borg', 'mount', '-o', 'allow_other,ignore_permissions', '--foreground', '--progress', '--lock-wait=600', f'{src_repo_path}::{entry["name"]}', dir], preexec_fn=lambda: as_borg()) as mount_proc:
-            with Halo(text='Waiting for mount', **halo_args) as sp:
-                if not sp.enabled:
-                    logger.debug('Waiting for mount...')
-                wait_start = datetime.now()
-                while True:
-                    if os.path.ismount(dir):
-                        break
-                    elif datetime.now() - wait_start > timedelta(minutes=15):
-                        ret.check_returncode()
-                    time.sleep(0.1)
-                if sp.enabled:
-                    sp.succeed('Mounted')
-                else:
-                    logger.info('Mounted %s', f'{src_repo_path}::{entry["name"]}')
-
-            while True:
-                with tqdm(total=total_size, unit_scale=True, unit_divisor=1024, unit='B', smoothing=0.01, disable=None, dynamic_ncols=True, maxinterval=0.5, miniters=1) as progress:
-                    seen = 0
-                    env = os.environ.copy()
-                    create_args = ['borg',
-                                   'create',
-                                   '--lock-wait=600',
-                                   '--one-file-system',
-                                   '--compression=auto,zstd,10',
-                                   '--chunker-params=10,23,16,4095',
-                                   '--files-cache=ctime,size',
-                                   '--show-rc',
-                                   '--upload-buffer=100',
-                                   '--upload-ratelimit=20480',
-                                   '--log-json',
-                                   '--progress',
-                                   '--list',
-                                   '--filter=AMEi-x?',
-                                   '--stats'
-                                   ]
-                    archive_time = datetime.strptime(entry["time"], "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=tzlocal()).astimezone(tzutc())
-                    create_args += [f'--timestamp={archive_time.strftime("%Y-%m-%dT%H:%M:%S")}']
-                    if cache_suffix:
-                        env['BORG_FILES_CACHE_SUFFIX'] = cache_suffix
-                    else:
-                        create_args += ['--files-cache=disabled']
-                    create_args += [f'{dst_repo_path}::{entry["name"]}', '.']
-
-                    with subprocess.Popen(create_args, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, preexec_fn=lambda: as_borg(caps={Cap.DAC_READ_SEARCH}), cwd=dir) as proc:
-                        last_list = None
-                        last_list_time = time.monotonic_ns()
-                        logger.info('Creating...')
-
-                        proc_logger = logger.getChild('borg')
-                        stdout_logger = proc_logger.getChild('stdout')
-                        stderr_logger = proc_logger.getChild('stderr')
-
-                        fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
-                        fcntl.fcntl(proc.stderr.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
-
-                        poll = select.poll()
-                        poll.register(proc.stdout, select.POLLIN | select.POLLHUP)
-                        poll.register(proc.stderr, select.POLLIN | select.POLLHUP)
-                        pollc = 2
-                        events = poll.poll()
-                        stdout_linebuf = bytearray()
-                        stderr_linebuf = bytearray()
-
-                        while pollc > 0 and len(events) > 0:
-                            # logger.debug('%d events', len(events))
-                            for rfd, event in events:
-                                # logger.debug('event %s', event)
-                                if event & select.POLLIN:
-                                    if rfd == proc.stdout.fileno():
-                                        try:
-                                            # logger.debug('reading stdout...')
-                                            stdout_linebuf.extend(os.read(proc.stdout.fileno(), 8192))
-                                            # logger.debug('read stdout, len(stdout_linebuf)=%d', len(stdout_linebuf))
-                                        except BlockingIOError:
-                                            pass
-
-                                        while stdout_linebuf:
-                                            # logger.debug('stdout line...')
-                                            line, sep, stdout_linebuf = stdout_linebuf.partition(b'\n')
-                                            if not sep:
-                                                stdout_linebuf = line
-                                                break
-
-                                            stdout_logger.info(line.decode())
-                                        # logger.debug('handled stdout lines, %d leftover', len(stdout_linebuf))
-                                    if rfd == proc.stderr.fileno():
-                                        try:
-                                            # logger.debug('reading stderr...')
-                                            stderr_linebuf.extend(os.read(proc.stderr.fileno(), 8192))
-                                            # logger.debug('read stderr, len(stderr_linebuf)=%d', len(stderr_linebuf))
-                                        except BlockingIOError:
-                                            pass
-
-                                        while stderr_linebuf:
-                                            # logger.debug('stderr line...')
-                                            line, sep, stderr_linebuf = stderr_linebuf.partition(b'\n')
-                                            if not sep:
-                                                stderr_linebuf = line
-                                                break
-
-                                            try:
-                                                json_line = json.loads(line)
-                                            except json.decoder.JSONDecodeError:
-                                                if progress.disable:
-                                                    stderr_logger.error(line.decode())
-                                                else:
-                                                    tqdm.write(line.decode())
-                                                continue
-
-                                            # logger.debug('stderr line decoded: %s', json_line['type'] if 'type' in json_line else None)
-
-                                            t = ''
-                                            if 'time' in json_line and not progress.disable:
-                                                ts = datetime.fromtimestamp(json_line['time']).replace(tzinfo=tzlocal())
-                                                t = f'{ts.isoformat(timespec="minutes")} '
-                                            if json_line['type'] == 'archive_progress' and not progress.disable:
-                                                now = time.monotonic_ns()
-                                                if last_list_time is None or now - last_list_time >= 3e9:
-                                                    last_list_time = now
-                                                    if 'path' in json_line and json_line['path']:
-                                                        progress.set_description(f'… {json_line["path"]}', refresh=False)
-                                                    else:
-                                                        progress.set_description(None, refresh=False)
-                                                elif last_list is not None:
-                                                    progress.set_description(last_list, refresh=False)
-                                                nfiles=json_line["nfiles"]
-                                                if total_files is not None:
-                                                    nfiles=f'{json_line["nfiles"]}/{total_files}'
-                                                progress.set_postfix(compressed=naturalsize(json_line['compressed_size'], binary=True), deduplicated=naturalsize(json_line['deduplicated_size'], binary=True), nfiles=nfiles, refresh=False)
-                                                progress.update(json_line["original_size"] - seen)
-                                                seen = json_line["original_size"]
-                                            elif json_line['type'] == 'archive_progress':
-                                                now = time.monotonic_ns()
-                                                if last_list_time is None or now - last_list_time >= 3e9:
-                                                    last_list_time = now
-                                                    if 'path' in json_line and json_line['path']:
-                                                        stderr_logger.debug('… %s (%s)', json_line["path"], naturalsize(json_line["original_size"]))
-                                                    else:
-                                                        stderr_logger.debug('… (%s)', naturalsize(json_line["original_size"]))
-                                            elif json_line['type'] == 'file_status':
-                                                # tqdm.write(t + f'{json_line["status"]} {json_line["path"]}')
-                                                last_list = f'{json_line["status"]} {json_line["path"]}'
-                                                last_list_time = time.monotonic_ns()
-                                                progress.set_description(last_list, refresh=False)
-                                                if progress.disable:
-                                                    stderr_logger.info(last_list)
-                                            elif (json_line['type'] == 'log_message' or json_line['type'] == 'progress_message' or json_line['type'] == 'progress_percent') and ('message' in json_line or 'msgid' in json_line):
-                                                if 'message' in json_line:
-                                                    if progress.disable:
-                                                        stderr_logger.info(t + json_line['message'])
-                                                    else:
-                                                        tqdm.write(t + json_line['message'])
-                                                elif 'msgid' in json_line:
-                                                    if progress.disable:
-                                                        stderr_logger.info(t + json_line['msgid'])
-                                                    else:
-                                                        tqdm.write(t + json_line['msgid'])
-                                            else:
-                                                if progress.disable:
-                                                    stderr_logger.info(t + line.decode())
-                                                else:
-                                                    tqdm.write(t + line.decode())
-                                        # logger.debug('handled stderr lines, %d leftover', len(stderr_linebuf))
-                                if event == select.POLLHUP:
-                                    poll.unregister(rfd)
-                                    pollc -= 1
-
-                            if pollc > 0:
-                                # logger.debug('polling %d fds...', pollc)
-                                events = poll.poll()
-                                # logger.debug('done polling')
-
-                        # logger.debug('borg create closed stdout/stderr')
-                        if stdout_linebuf:
-                            logger.error('unterminated line leftover in stdout: %s', stdout_linebuf)
-                        if stderr_linebuf:
-                            logger.error('unterminated line leftover in stdout: %s', stderr_linebuf)
-                        progress.set_description(None)
-                        ret = proc.wait()
-                        # logger.debug('borg create terminated; ret=%d', ret)
-                        if ret != 0:
-                            dst = None
-                            try:
-                                dst = read_repo(dst_repo_path)
-                            except (subprocess.CalledProcessError, json.decoder.JSONDecodeError) as err:
-                                logger.error(err)
-                                continue
-                            else:
-                                if any(map(lambda other: entry['name'] == other['name'], dst)):
-                                    logger.info('destination exists, terminating')
-                                    break
-
-                            logger.warn('destination does not exist, retrying')
-                            continue
-                        else:
-                            # logger.debug('terminating')
-                            break
-            mount_proc.terminate()
-
-    with Manager() as manager:
-        tmpdir_q = manager.Queue(1)
-
-        with closing(Process(target=do_copy, args=(tmpdir_q,), name='do_copy')) as p:
-            p.start()
-
-            with TemporaryDirectory(prefix=f'borg-mount_{entry["name"]}_', dir=os.environ.get('RUNTIME_DIRECTORY')) as tmpdir:
-                tmpdir_q.put(tmpdir)
-                p.join()
-                return p.exitcode
-
-def sigterm(signum, frame):
-    raise SystemExit(128 + signum)
-
-def main():
-    signal.signal(signal.SIGTERM, sigterm)
-
-    global logger
-    logger = logging.getLogger(__name__)
-    console_handler = logging.StreamHandler()
-    console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') )
-    if sys.stderr.isatty():
-        console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') )
-
-    burst_max = 1000
-    burst = burst_max
-    last_use = None
-    inv_rate = 1e7
-    def consume_filter(record):
-        nonlocal burst, burst_max, inv_rate, last_use
-
-        delay = None
-        while True:
-            now = time.monotonic_ns()
-            burst = min(burst_max, burst + math.floor((now - last_use) / inv_rate)) if last_use else burst_max
-            last_use = now
-
-            if burst > 0:
-                burst -= 1
-                if delay:
-                    delay = now - delay
-
-                return True
-
-            if delay is None:
-                delay = now
-            time.sleep(inv_rate / 1e9)
-    console_handler.addFilter(consume_filter)
-
-    logging.getLogger().addHandler(console_handler)
-
-    # log uncaught exceptions
-    def log_exceptions(type, value, tb):
-        global logger
-
-        logger.error(value)
-        sys.__excepthook__(type, value, tb) # calls default excepthook
-
-    sys.excepthook = log_exceptions
-
-    parser = argparse.ArgumentParser(prog='copy')
-    parser.add_argument('--verbosity', dest='log_level', action='append', type=int)
-    parser.add_argument('--verbose', '-v', dest='log_level', action='append_const', const=1)
-    parser.add_argument('--quiet', '-q', dest='log_level', action='append_const', const=-1)
-    parser.add_argument('source', metavar='REPO_OR_ARCHIVE')
-    parser.add_argument('target', metavar='REPO_OR_ARCHIVE')
-    args = parser.parse_args()
-
-
-    LOG_LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]
-    DEFAULT_LOG_LEVEL = logging.ERROR
-    log_level = LOG_LEVELS.index(DEFAULT_LOG_LEVEL)
-
-    for adjustment in args.log_level or ():
-        log_level = min(len(LOG_LEVELS) - 1, max(log_level - adjustment, 0))
-    logger.setLevel(LOG_LEVELS[log_level])
-
-
-    if "::" in args.source:
-        (src_repo_path, _, src_archive) = args.source.partition("::")
-        entry = None
-        for candidate_entry in read_repo(src_repo_path):
-            if entry['name'] != src_archive:
-                continue
-            entry = candidate_entry
-            break
-
-        if entry is None:
-            logger.critical("Did not find archive ‘%s’", src_archive)
-            os.exit(1)
-
-        copy_archive(src_repo_path, args.target, entry)
-    else:
-        for entry in ToSync(args.source, args.target):
-            copy_archive(args.source, args.target, entry)
-
-if __name__ == "__main__":
-    sys.exit(main())
diff --git a/hosts/vidhar/borg/copy/setup.py b/hosts/vidhar/borg/copy/setup.py
deleted file mode 100644
index f77d9560..00000000
--- a/hosts/vidhar/borg/copy/setup.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from setuptools import setup
-
-setup(name='copy_borg',
-      packages=['copy_borg'],
-      entry_points={
-          'console_scripts': [
-              'copy_borg=copy_borg.__main__:main',
-          ],
-      }
-)
diff --git a/hosts/vidhar/borg/default.nix b/hosts/vidhar/borg/default.nix
index 7672de18..80ce9c7e 100644
--- a/hosts/vidhar/borg/default.nix
+++ b/hosts/vidhar/borg/default.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, lib, flakeInputs, ... }:
+{ config, pkgs, lib, flakeInputs, utils, ... }:
 
 with lib;
 
@@ -21,60 +21,26 @@ let
       ServerAliveCountMax 30
   '';
 
-  copyService = { repo, repoEscaped }: let
-    serviceName = "copy-borg@${repoEscaped}";
-  in nameValuePair serviceName {
+  checkBorgUnit = {
     serviceConfig = {
       Type = "oneshot";
-      ExecStart = "${copyBorg}/bin/copy_borg --verbosity 3 ${escapeShellArg repo} yggdrasil.borgbase:repo";
-      TimeoutStartSec = "8h";
-      # User = "borg";
-      # Group = "borg";
-      # StateDirectory = "borg";
-      RuntimeDirectory = "copy-borg";
+      ExecStart = "${pkgs.borgbackup}/bin/borg ${utils.escapeSystemdExecArgs [
+        "--lock-wait" "3600"
+        "--progress"
+        "check"
+        "--verify-data"
+      ]} %I";
       Environment = [
-        "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${pkgs.writeText "config" sshConfig}\""
         "BORG_BASE_DIR=/var/lib/borg"
         "BORG_CONFIG_DIR=/var/lib/borg/config"
         "BORG_CACHE_DIR=/var/lib/borg/cache"
         "BORG_SECURITY_DIR=/var/lib/borg/security"
         "BORG_KEYS_DIR=/var/lib/borg/keys"
-        "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}"
-        "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes"
         "BORG_HOSTNAME_IS_UNIQUE=yes"
+        "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${pkgs.writeText "config" sshConfig}\""
       ];
-
-      LogRateLimitIntervalSec = 0;
     };
   };
-
-  copyBorg = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage rec {
-    pname = "copy-borg";
-    src = ./copy;
-    version = "0.0.0";
-    ignoreDataOutdated = true;
-
-    requirements = ''
-      humanize
-      tqdm
-      python-dateutil
-      xdg
-      python-unshare
-      pyprctl
-      halo
-    '';
-    postInstall = ''
-      wrapProgram $out/bin/copy_borg \
-        --prefix PATH : ${makeBinPath (with pkgs; [util-linux borgbackup])}:${config.security.wrapperDir}
-    '';
-
-    providers.python-unshare = "nixpkgs";
-    overridesPre = [
-      (self: super: { python-unshare = super.python-unshare.overrideAttrs (oldAttrs: { name = "python-unshare-0.2.1"; version = "0.2.1"; }); })
-    ];
-
-    # _.tomli.buildInputs.add = with pkgs."python3Packages"; [ flit-core ];
-  };
 in {
   config = {
     services.borgsnap = {
@@ -85,7 +51,15 @@ in {
       keyfile = config.sops.secrets."yggdrasil.borgkey".path;
     };
 
-    systemd.services = listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]);
+    services.copyborg.jotnar = {
+      from = "/srv/backup/borg/jotnar";
+      to = "yggdrasil.borgbase:repo";
+      inherit sshConfig;
+      keyfile = config.sops.secrets."yggdrasil.borgkey".path;
+      timerOptions.timerConfig = {
+        OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin";
+      };
+    };
 
     services.borgbackup.repos.jotnar = {
       path = "/srv/backup/borg/jotnar";
@@ -95,6 +69,27 @@ in {
       in filter (v: v != null) (mapAttrsToList toAuthKey (builtins.readDir dir));
     };
 
+    systemd.services."check-borg@${utils.escapeSystemdPath "/srv/backup/borg/jotnar"}" = checkBorgUnit;
+    systemd.services."check-borg@${utils.escapeSystemdPath "yggdrasil.borgbase:repo"}" = recursiveUpdate checkBorgUnit {
+      serviceConfig = {
+        Environment = checkBorgUnit.serviceConfig.Environment ++ [
+          "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}"
+        ];
+      };
+    };
+    systemd.timers."check-borg@${utils.escapeSystemdPath "/srv/backup/borg/jotnar"}" = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "Sun *-*-02..08 01:30:00 Europe/Berlin";
+      };
+    };
+    systemd.timers."check-borg@${utils.escapeSystemdPath "yggdrasil.borgbase:repo"}" = {
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "Sun *-*-02..08 01:30:00 Europe/Berlin";
+      };
+    };
+
     boot.postBootCommands = mkBefore ''
       ${pkgs.findutils}/bin/find /srv/backup/borg -type d -empty -delete
     '';
@@ -123,13 +118,5 @@ in {
       group = "borg";
       mode = "0400";
     };
-
-    systemd.timers."copy-borg@srv-backup-borg-jotnar" = {
-      wantedBy = ["multi-user.target"];
-
-      timerConfig = {
-        OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin";
-      };
-    };
   };
 }
diff --git a/hosts/vidhar/borg/pyprctl-packages.nix b/hosts/vidhar/borg/pyprctl-packages.nix
deleted file mode 100644
index d3b4256a..00000000
--- a/hosts/vidhar/borg/pyprctl-packages.nix
+++ /dev/null
@@ -1,21 +0,0 @@
-# Generated by pip2nix 0.8.0.dev1
-# See https://github.com/nix-community/pip2nix
-
-{ pkgs, fetchurl, fetchgit, fetchhg }:
-
-self: super: {
-  "pyprctl" = super.buildPythonPackage rec {
-    pname = "pyprctl";
-    version = "0.1.3";
-    src = fetchurl {
-      url = "https://files.pythonhosted.org/packages/bf/5e/62765de39bbce8111fb1f4453a4a804913bf49179fa265fb713ed66c9d15/pyprctl-0.1.3-py3-none-any.whl";
-      sha256 = "1pgif990r92za5rx12mjnq5iiz72d455v0wrawzb73q79w8ya0k3";
-    };
-    format = "wheel";
-    doCheck = false;
-    buildInputs = [];
-    checkInputs = [];
-    nativeBuildInputs = [];
-    propagatedBuildInputs = [];
-  };
-}
diff --git a/hosts/vidhar/prometheus/ca/.gitignore b/hosts/vidhar/prometheus/ca/.gitignore
deleted file mode 100644
index 7c894574..00000000
--- a/hosts/vidhar/prometheus/ca/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-ca.key
-ca.cnf
-*.old
\ No newline at end of file
diff --git a/hosts/vidhar/prometheus/ca/ca.crt b/hosts/vidhar/prometheus/ca/ca.crt
index 922fed28..8cfea666 100644
--- a/hosts/vidhar/prometheus/ca/ca.crt
+++ b/hosts/vidhar/prometheus/ca/ca.crt
@@ -1,12 +1,12 @@
 -----BEGIN CERTIFICATE-----
-MIIBsjCCAWSgAwIBAgIUOzZ8XcFb8XtI2yyWp4S/WMD6QxQwBQYDK2VwMB8xHTAb
-BgNVBAMMFHByb21ldGhldXMueWdnZHJhc2lsMCAXDTIyMDQwODE5NDgwMFoYDzIw
-OTAwNDI2MTk0ODAwWjAfMR0wGwYDVQQDDBRwcm9tZXRoZXVzLnlnZ2RyYXNpbDAq
-MAUGAytlcAMhAOoxPLBH6pnCRtE7V5gejM92gg1vLNLHw3rFIXXchOJmo4GvMIGs
-MB0GA1UdDgQWBBRnwBkgZFnueEa7aV8aEAoMRzW4CTBaBgNVHSMEUzBRgBRnwBkg
-ZFnueEa7aV8aEAoMRzW4CaEjpCEwHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55Z2dk
-cmFzaWyCFDs2fF3BW/F7SNsslqeEv1jA+kMUMA8GA1UdEwEB/wQFMAMBAf8wCwYD
-VR0PBAQDAgEGMBEGCWCGSAGG+EIBAQQEAwICBDAFBgMrZXADQQD9AC2OHtzW8QSC
-HU/4rGdRWRqr3pfclKXimSWaAXMPly2M1qehPI402lhQrIAVF+D1pi/EAGJfbbzF
-aurykEMB
+MIIBrjCCAS6gAwIBAgIUYV3YPBx91CbgMpOGb5HKMZ2hzRUwBQYDK2VxMB8xHTAb
+BgNVBAMMFHByb21ldGhldXMueWdnZHJhc2lsMB4XDTIyMTEwNzE5MjgzNFoXDTMy
+MTEwNzE5MzMzNFowHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55Z2dkcmFzaWwwQzAF
+BgMrZXEDOgAVqcV3KGDhcbQt/UR3Yv6OuAGc+Kc8hrDHjAV8K9GTjahc/d49NK2v
+FAz0uK8YidIaTVJZjzHhTgCjYzBhMB8GA1UdIwQYMBaAFObrhCUDCZk6/JeeDMNW
+l8WeLr+MMB0GA1UdDgQWBBTm64QlAwmZOvyXngzDVpfFni6/jDAOBgNVHQ8BAf8E
+BAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAFBgMrZXEDcwAFAqBlI7SpHaSE+0mMzx5x
+0M6T3iJtLxP36Qz5MHx3vvcbbx1eJhZWKewuyz+9LXaCkf8Jpd5AFoC+HhoikVSz
+46yVzmTBt6TISc4bh+eiWcXEKFbxEbXkwqZd2m/oHI4Em4qnDKp96FcOfq6RQ8pR
+AwA=
 -----END CERTIFICATE-----
diff --git a/hosts/vidhar/prometheus/ca/ca.key b/hosts/vidhar/prometheus/ca/ca.key
new file mode 100644
index 00000000..32c4330a
--- /dev/null
+++ b/hosts/vidhar/prometheus/ca/ca.key
@@ -0,0 +1,21 @@
+{
+        "data": "ENC[AES256_GCM,data:yk8nI2Zz2F3XnBM9dqnA3UoWTTCGJLMZUYjpo+SW+ARmZVgYdcqHZunhoGRQP/r6qrIUvM/2Yl85Uosw43jllILCNESH17Gi6uI0gD9OE8I14oll8wCL+/GvP/IuU//1NEAeLF9cz8MBWPE0WW2wQk5DF4ikl+z3/McG+kaqeU+ka6aMmjIjUstjR2vCf+pfZN3KswylcLaeuvXP,iv:ByEIQCxQwjynCFxGZdYtg+nx9mFmwbqHL3iBhzLbKIQ=,tag:jIc+KcfbSmiZqM6Z2xIa0g==,type:str]",
+        "sops": {
+                "kms": null,
+                "gcp_kms": null,
+                "azure_kv": null,
+                "hc_vault": null,
+                "age": null,
+                "lastmodified": "2022-11-07T19:33:34Z",
+                "mac": "ENC[AES256_GCM,data:UE1+0M15ZBgsKOfEmz8DMeQsmzkRxcN5cjdpMswzc6vIgo6sRN4ArdtDKqAMcFtFhzokSZin6OIizsk6KLlsts5sgVHQHXKrqssc016OADRg4BoC9zM/MGLUXOHndrRSPGSQgRDCeVwmR9C5iE18VZ/NCcZtoztHt6DPt3xmGpo=,iv:JB6CTWUyyDpjciKfYugf78Xo+jDKCH3+tL8p9G7M5y0=,tag:n73uY6cX5EV6Rjq1/HM8kw==,type:str]",
+                "pgp": [
+                        {
+                                "created_at": "2022-11-07T19:33:34Z",
+                                "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4DXxoViZlp6dISAQdAJxBqRR1DzDPs/sQgfZNaKZTWH+mbdQo9mpGRWcWkm1ww\nOjVRJjiBDyeItfbOS9hnEOJKwKUIk1tH7F5m+U5daFLSw/Ct/xzJ7iyphcfRzNFN\n0l4BHF6sMyoPFpSGpE+0d4IRqfDPF3t9d3NL1lAGV75MoEho38ptNCbAn32kWpZ9\n7/Vk3L+oR/3xhLAwm3/7JDed01zNnKRaxFh3zpYfwZWhMtEdoUoEIkojufEJ64s2\n=KZjL\n-----END PGP MESSAGE-----\n",
+                                "fp": "30D3453B8CD02FE2A3E7C78C0FB536FB87AE8F51"
+                        }
+                ],
+                "unencrypted_suffix": "_unencrypted",
+                "version": "3.7.3"
+        }
+}
\ No newline at end of file
diff --git a/hosts/vidhar/prometheus/ca/ca.key.sops b/hosts/vidhar/prometheus/ca/ca.key.sops
deleted file mode 100644
index 5313056e..00000000
--- a/hosts/vidhar/prometheus/ca/ca.key.sops
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-        "data": "ENC[AES256_GCM,data:XW6h0psHOSV0cR03vRg479A5XRM7KfiBfVgvm4QlxCZzhkk5U1ToDJIaCxqKpxlEu8wm79wmz+/CmSLDEBcs7x05a5vBDt81mlWJ49PolOrG9bL9Qkyq5u8sB8HWXRXxCP5kg2su+n9NqdHX9AIhYCXy7VJDuGo=,iv:v661AhF2Q/O+a7JtwHtnSkSI0mL8ltu5rPny8vWCL/Q=,tag:c7b0a6o6y/MI5vG85uFuUg==,type:str]",
-        "sops": {
-                "kms": null,
-                "gcp_kms": null,
-                "azure_kv": null,
-                "hc_vault": null,
-                "age": null,
-                "lastmodified": "2022-04-08T20:12:22Z",
-                "mac": "ENC[AES256_GCM,data:W/IF6WgTscbkcMUTR3aeqM/H/UwgFgILDbKBxYJQxcFtt4kq3UqzSd/e0hk5NQ9IkagAC4X0gZDuzco2mc7caUGyzMKRdA2ekgcdDwzruQ4i+UYyr80dFhqHpV+aksdZJVR+dJzkmIRmza3Ia5e/X01XNIbIrU13JKYm9jCskd0=,iv:2g+UFcSTxcTrf+toi4BDVvAaY5ydk7yRnhpQ/rrNvVo=,tag:3X01wEqL/Q8cIiF+DEMnpg==,type:str]",
-                "pgp": [
-                        {
-                                "created_at": "2022-04-08T20:12:22Z",
-                                "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4DXxoViZlp6dISAQdADN+s7UQS8hEBc2mMRovD/zKuIoIAS3swLpP6ul9kRGMw\nDCUvOL41sxXmuodi4Pg69YB2YcL47Fod7nQWUYaK8L3CuyjWUq1cxomlYtTd03eH\n0l4BiyWTuZ+1OG4Xng8B4zdcM5jWfeTRWupDIXcnPFjwz47FetmrcCAaROKYL87e\nAjK76Y6gR/gSj0GTTAUIfKFpqsqAdBAf6oBekQcPgeqcrJcZ2ZZFWzmswGBvcGjs\n=gqhG\n-----END PGP MESSAGE-----\n",
-                                "fp": "30D3453B8CD02FE2A3E7C78C0FB536FB87AE8F51"
-                        }
-                ],
-                "unencrypted_suffix": "_unencrypted",
-                "version": "3.7.2"
-        }
-}
\ No newline at end of file
diff --git a/hosts/vidhar/prometheus/ca/certs/01.pem b/hosts/vidhar/prometheus/ca/certs/01.pem
deleted file mode 100644
index 81abe0b7..00000000
--- a/hosts/vidhar/prometheus/ca/certs/01.pem
+++ /dev/null
@@ -1,39 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 1 (0x1)
-        Signature Algorithm: ED25519
-        Issuer: CN=prometheus.yggdrasil
-        Validity
-            Not Before: Apr  8 20:03:55 2022 GMT
-            Not After : Apr 26 20:03:55 2090 GMT
-        Subject: CN=surtr.yggdrasil
-        Subject Public Key Info:
-            Public Key Algorithm: ED25519
-                ED25519 Public-Key:
-                pub:
-                    02:5d:f0:8d:f6:5f:fc:fd:27:47:0e:d8:ec:fe:e0:
-                    a0:28:20:9a:b4:8a:07:4c:75:b2:c1:03:ef:16:3b:
-                    eb:e0
-        X509v3 extensions:
-            X509v3 Basic Constraints: critical
-                CA:FALSE
-            X509v3 Subject Alternative Name: 
-                DNS:prometheus.surtr.yggdrasil, DNS:prometheus.surtr.yggdrasil.li
-            X509v3 Subject Key Identifier: 
-                37:9D:AD:3D:CB:F9:14:5A:69:CD:E2:71:D8:08:97:93:A5:20:3C:38
-    Signature Algorithm: ED25519
-         3c:df:73:85:a7:81:07:60:b5:4e:ea:ec:74:04:47:d2:35:41:
-         cf:d8:34:75:18:4f:ee:c3:b9:64:6d:0a:fb:1a:76:e2:96:8b:
-         5e:24:c5:d6:b6:2e:6f:6e:29:ff:26:70:ef:5a:7b:33:40:40:
-         13:e8:49:a9:80:73:62:8e:58:05
------BEGIN CERTIFICATE-----
-MIIBXzCCARGgAwIBAgIBATAFBgMrZXAwHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55
-Z2dkcmFzaWwwIBcNMjIwNDA4MjAwMzU1WhgPMjA5MDA0MjYyMDAzNTVaMBoxGDAW
-BgNVBAMMD3N1cnRyLnlnZ2RyYXNpbDAqMAUGAytlcAMhAAJd8I32X/z9J0cO2Oz+
-4KAoIJq0igdMdbLBA+8WO+vgo3UwczAMBgNVHRMBAf8EAjAAMEQGA1UdEQQ9MDuC
-GnByb21ldGhldXMuc3VydHIueWdnZHJhc2lsgh1wcm9tZXRoZXVzLnN1cnRyLnln
-Z2RyYXNpbC5saTAdBgNVHQ4EFgQUN52tPcv5FFppzeJx2AiXk6UgPDgwBQYDK2Vw
-A0EAPN9zhaeBB2C1TursdARH0jVBz9g0dRhP7sO5ZG0K+xp24paLXiTF1rYub24p
-/yZw71p7M0BAE+hJqYBzYo5YBQ==
------END CERTIFICATE-----
diff --git a/hosts/vidhar/prometheus/ca/certs/02.pem b/hosts/vidhar/prometheus/ca/certs/02.pem
deleted file mode 100644
index d908ca7d..00000000
--- a/hosts/vidhar/prometheus/ca/certs/02.pem
+++ /dev/null
@@ -1,38 +0,0 @@
-Certificate:
-    Data:
-        Version: 3 (0x2)
-        Serial Number: 2 (0x2)
-        Signature Algorithm: ED25519
-        Issuer: CN=prometheus.yggdrasil
-        Validity
-            Not Before: Apr  8 20:07:13 2022 GMT
-            Not After : Apr 26 20:07:13 2090 GMT
-        Subject: CN=vidhar.yggdrasil
-        Subject Public Key Info:
-            Public Key Algorithm: ED25519
-                ED25519 Public-Key:
-                pub:
-                    13:84:a6:01:07:7a:5e:8d:2b:8d:83:ee:73:1d:c6:
-                    b8:9a:ad:b9:3d:40:51:ec:2c:f3:52:7d:81:90:e7:
-                    ac:88
-        X509v3 extensions:
-            X509v3 Basic Constraints: critical
-                CA:FALSE
-            X509v3 Subject Alternative Name: 
-                DNS:prometheus.vidhar.yggdrasil
-            X509v3 Subject Key Identifier: 
-                44:AA:8E:CC:AB:C9:A7:D1:A1:D0:FA:7F:DB:87:1E:08:AA:6E:4D:59
-    Signature Algorithm: ED25519
-         47:65:87:17:50:96:77:56:20:ac:9e:f4:e4:6d:19:6d:b7:24:
-         11:af:0c:c3:f3:fd:75:19:d9:77:06:41:79:7f:a5:00:0c:18:
-         ee:82:3e:9e:09:61:34:cf:8f:f5:83:d1:5d:b2:e4:42:b6:3f:
-         9c:b6:5a:f3:40:92:e6:8f:24:0f
------BEGIN CERTIFICATE-----
-MIIBQTCB9KADAgECAgECMAUGAytlcDAfMR0wGwYDVQQDDBRwcm9tZXRoZXVzLnln
-Z2RyYXNpbDAgFw0yMjA0MDgyMDA3MTNaGA8yMDkwMDQyNjIwMDcxM1owGzEZMBcG
-A1UEAwwQdmlkaGFyLnlnZ2RyYXNpbDAqMAUGAytlcAMhABOEpgEHel6NK42D7nMd
-xriarbk9QFHsLPNSfYGQ56yIo1cwVTAMBgNVHRMBAf8EAjAAMCYGA1UdEQQfMB2C
-G3Byb21ldGhldXMudmlkaGFyLnlnZ2RyYXNpbDAdBgNVHQ4EFgQURKqOzKvJp9Gh
-0Pp/24ceCKpuTVkwBQYDK2VwA0EAR2WHF1CWd1YgrJ705G0ZbbckEa8Mw/P9dRnZ
-dwZBeX+lAAwY7oI+nglhNM+P9YPRXbLkQrY/nLZa80CS5o8kDw==
------END CERTIFICATE-----
diff --git a/hosts/vidhar/prometheus/ca/index.txt b/hosts/vidhar/prometheus/ca/index.txt
deleted file mode 100644
index 41ebb0f4..00000000
--- a/hosts/vidhar/prometheus/ca/index.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-V       20900426200355Z         01      unknown /CN=surtr.yggdrasil
-V       20900426200713Z         02      unknown /CN=vidhar.yggdrasil
diff --git a/hosts/vidhar/prometheus/ca/index.txt.attr b/hosts/vidhar/prometheus/ca/index.txt.attr
deleted file mode 100644
index 8f7e63a3..00000000
--- a/hosts/vidhar/prometheus/ca/index.txt.attr
+++ /dev/null
@@ -1 +0,0 @@
-unique_subject = yes
diff --git a/hosts/vidhar/prometheus/ca/serial b/hosts/vidhar/prometheus/ca/serial
deleted file mode 100644
index 75016ea3..00000000
--- a/hosts/vidhar/prometheus/ca/serial
+++ /dev/null
@@ -1 +0,0 @@
-03
diff --git a/hosts/vidhar/prometheus/tls.crt b/hosts/vidhar/prometheus/tls.crt
index 792ed542..6516f185 100644
--- a/hosts/vidhar/prometheus/tls.crt
+++ b/hosts/vidhar/prometheus/tls.crt
@@ -1,9 +1,12 @@
 -----BEGIN CERTIFICATE-----
-MIIBQTCB9KADAgECAgECMAUGAytlcDAfMR0wGwYDVQQDDBRwcm9tZXRoZXVzLnln
-Z2RyYXNpbDAgFw0yMjA0MDgyMDA3MTNaGA8yMDkwMDQyNjIwMDcxM1owGzEZMBcG
-A1UEAwwQdmlkaGFyLnlnZ2RyYXNpbDAqMAUGAytlcAMhABOEpgEHel6NK42D7nMd
-xriarbk9QFHsLPNSfYGQ56yIo1cwVTAMBgNVHRMBAf8EAjAAMCYGA1UdEQQfMB2C
-G3Byb21ldGhldXMudmlkaGFyLnlnZ2RyYXNpbDAdBgNVHQ4EFgQURKqOzKvJp9Gh
-0Pp/24ceCKpuTVkwBQYDK2VwA0EAR2WHF1CWd1YgrJ705G0ZbbckEa8Mw/P9dRnZ
-dwZBeX+lAAwY7oI+nglhNM+P9YPRXbLkQrY/nLZa80CS5o8kDw==
+MIIByDCCAUigAwIBAgIPQAAAAGNpXrc6y389EXtIMAUGAytlcTAfMR0wGwYDVQQD
+DBRwcm9tZXRoZXVzLnlnZ2RyYXNpbDAeFw0yMjExMDcxOTMyNTRaFw0zMjExMDcx
+OTM3NTRaMBsxGTAXBgNVBAMMEHZpZGhhci55Z2dkcmFzaWwwKjAFBgMrZXADIQAT
+hKYBB3pejSuNg+5zHca4mq25PUBR7CzzUn2BkOesiKOBnjCBmzAfBgNVHSMEGDAW
+gBTm64QlAwmZOvyXngzDVpfFni6/jDAdBgNVHQ4EFgQURKqOzKvJp9Gh0Pp/24ce
+CKpuTVkwDgYDVR0PAQH/BAQDAgXgMAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYI
+KwYBBQUHAwIwJgYDVR0RBB8wHYIbcHJvbWV0aGV1cy52aWRoYXIueWdnZHJhc2ls
+MAUGAytlcQNzAIPNcNWqVX4Ie971O/S2DL0HMFmPbR331U4snLBqPGWC1/j9NV4O
+cxJvLo8Hzb4I0BXn/nZbyk/ogCCJU69BVeK378qgLo68DIZ4TA3ka5ZPNRSt464Q
+NvbkDhtFVVxM04xUjI4dOeE9jczG9nN3jHESAA==
 -----END CERTIFICATE-----