From 0e9f1e85cd8c6f9d546ef88e971043b909017170 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Mon, 7 Nov 2022 20:51:39 +0100 Subject: ... --- hosts/surtr/prometheus/tls.crt | 19 +- hosts/vidhar/borg/copy/copy_borg/__main__.py | 556 --------------------------- hosts/vidhar/borg/copy/setup.py | 10 - hosts/vidhar/borg/default.nix | 91 ++--- hosts/vidhar/borg/pyprctl-packages.nix | 21 - hosts/vidhar/prometheus/ca/.gitignore | 3 - hosts/vidhar/prometheus/ca/ca.crt | 20 +- hosts/vidhar/prometheus/ca/ca.key | 21 + hosts/vidhar/prometheus/ca/ca.key.sops | 21 - hosts/vidhar/prometheus/ca/certs/01.pem | 39 -- hosts/vidhar/prometheus/ca/certs/02.pem | 38 -- hosts/vidhar/prometheus/ca/index.txt | 2 - hosts/vidhar/prometheus/ca/index.txt.attr | 1 - hosts/vidhar/prometheus/ca/serial | 1 - hosts/vidhar/prometheus/tls.crt | 17 +- 15 files changed, 91 insertions(+), 769 deletions(-) delete mode 100755 hosts/vidhar/borg/copy/copy_borg/__main__.py delete mode 100644 hosts/vidhar/borg/copy/setup.py delete mode 100644 hosts/vidhar/borg/pyprctl-packages.nix delete mode 100644 hosts/vidhar/prometheus/ca/.gitignore create mode 100644 hosts/vidhar/prometheus/ca/ca.key delete mode 100644 hosts/vidhar/prometheus/ca/ca.key.sops delete mode 100644 hosts/vidhar/prometheus/ca/certs/01.pem delete mode 100644 hosts/vidhar/prometheus/ca/certs/02.pem delete mode 100644 hosts/vidhar/prometheus/ca/index.txt delete mode 100644 hosts/vidhar/prometheus/ca/index.txt.attr delete mode 100644 hosts/vidhar/prometheus/ca/serial (limited to 'hosts') diff --git a/hosts/surtr/prometheus/tls.crt b/hosts/surtr/prometheus/tls.crt index ba958f40..d81f429f 100644 --- a/hosts/surtr/prometheus/tls.crt +++ b/hosts/surtr/prometheus/tls.crt @@ -1,10 +1,13 @@ -----BEGIN CERTIFICATE----- -MIIBXzCCARGgAwIBAgIBATAFBgMrZXAwHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55 -Z2dkcmFzaWwwIBcNMjIwNDA4MjAwMzU1WhgPMjA5MDA0MjYyMDAzNTVaMBoxGDAW -BgNVBAMMD3N1cnRyLnlnZ2RyYXNpbDAqMAUGAytlcAMhAAJd8I32X/z9J0cO2Oz+ -4KAoIJq0igdMdbLBA+8WO+vgo3UwczAMBgNVHRMBAf8EAjAAMEQGA1UdEQQ9MDuC -GnByb21ldGhldXMuc3VydHIueWdnZHJhc2lsgh1wcm9tZXRoZXVzLnN1cnRyLnln -Z2RyYXNpbC5saTAdBgNVHQ4EFgQUN52tPcv5FFppzeJx2AiXk6UgPDgwBQYDK2Vw -A0EAPN9zhaeBB2C1TursdARH0jVBz9g0dRhP7sO5ZG0K+xp24paLXiTF1rYub24p -/yZw71p7M0BAE+hJqYBzYo5YBQ== +MIIB5TCCAWWgAwIBAgIPQAAAAGNpYE436fsCRvVfMAUGAytlcTAfMR0wGwYDVQQD +DBRwcm9tZXRoZXVzLnlnZ2RyYXNpbDAeFw0yMjExMDcxOTM5NDFaFw0zMjExMDcx +OTQ0NDFaMBoxGDAWBgNVBAMMD3N1cnRyLnlnZ2RyYXNpbDAqMAUGAytlcAMhAAJd +8I32X/z9J0cO2Oz+4KAoIJq0igdMdbLBA+8WO+vgo4G8MIG5MB8GA1UdIwQYMBaA +FObrhCUDCZk6/JeeDMNWl8WeLr+MMB0GA1UdDgQWBBQ3na09y/kUWmnN4nHYCJeT +pSA8ODAOBgNVHQ8BAf8EBAMCBeAwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggr +BgEFBQcDAjBEBgNVHREEPTA7ghpwcm9tZXRoZXVzLnN1cnRyLnlnZ2RyYXNpbIId +cHJvbWV0aGV1cy5zdXJ0ci55Z2dkcmFzaWwubGkwBQYDK2VxA3MAYHd3I/Mg/t34 +zdcxrIKOAKJ9ZVVoP0msk/viKrZ4b+Q9rKSNEnkyk0y56Z7FlLDxGLScaemqQ3uA +5hjhdTci/xd4xYX/edLw1AWGRs2kBe3vs2WOmrdcKa849vdMH27G/P/+bgbdofCN +fukxYHzpESYA -----END CERTIFICATE----- diff --git a/hosts/vidhar/borg/copy/copy_borg/__main__.py b/hosts/vidhar/borg/copy/copy_borg/__main__.py deleted file mode 100755 index 5b374d99..00000000 --- a/hosts/vidhar/borg/copy/copy_borg/__main__.py +++ /dev/null @@ -1,556 +0,0 @@ -#!@python@/bin/python - -import json -import os -import subprocess -import re -import sys -import io -from sys import stderr -from humanize import naturalsize - -from tempfile import TemporaryDirectory - -from datetime import (datetime, timedelta) -from dateutil.tz import (tzlocal, tzutc) -import dateutil.parser -import argparse - -from tqdm import tqdm - -from xdg import xdg_runtime_dir -import pathlib - -import unshare -from pyprctl import CapState, Cap, cap_ambient_raise, cap_ambient_is_set, set_keepcaps -from pwd import getpwnam - -import logging - -import signal -import time -import math - -from halo import Halo - -from collections import deque - -import select -import fcntl - -from multiprocessing import Process, Manager -from contextlib import closing - - -halo_args = { - 'stream': stderr, - 'enabled': stderr.isatty(), - 'spinner': 'arc' -} - -borg_pwd = getpwnam('borg') - -def as_borg(caps=set()): - global logger - - try: - if caps: - c_state = CapState.get_current() - c_state.permitted.add(*caps) - c_state.set_current() - - # logger.debug("before setgid/setuid: cap_permitted=%s", CapState.get_current().permitted) - - set_keepcaps(True) - - os.setgid(borg_pwd.pw_gid) - os.setuid(borg_pwd.pw_uid) - - if caps: - # logger.debug("after setgid/setuid: cap_permitted=%s", CapState.get_current().permitted) - - c_state = CapState.get_current() - c_state.permitted = caps.copy() - c_state.inheritable.add(*caps) - c_state.set_current() - - # logger.debug("cap_permitted=%s", CapState.get_current().permitted) - # logger.debug("cap_inheritable=%s", CapState.get_current().inheritable) - - for cap in caps: - cap_ambient_raise(cap) - # logger.debug("cap_ambient[%s]=%s", cap, cap_ambient_is_set(cap)) - except Exception: - logger.error(format_exc()) - raise - -def borg_json(*args, **kwargs): - global logger - - with subprocess.Popen(*args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, **kwargs) as proc: - stdout_buffer = io.BytesIO() - - proc_logger = logger.getChild('borg') - stdout_logger = proc_logger.getChild('stdout') - stderr_logger = proc_logger.getChild('stderr') - - fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - fcntl.fcntl(proc.stderr.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - - poll = select.poll() - poll.register(proc.stdout, select.POLLIN | select.POLLHUP) - poll.register(proc.stderr, select.POLLIN | select.POLLHUP) - pollc = 2 - events = poll.poll() - stderr_linebuf = bytearray() - - while pollc > 0 and len(events) > 0: - for rfd, event in events: - if event & select.POLLIN: - if rfd == proc.stdout.fileno(): - try: - buf = os.read(proc.stdout.fileno(), 8192) - # stdout_logger.debug(buf) - stdout_buffer.write(buf) - except BlockingIOError: - pass - if rfd == proc.stderr.fileno(): - try: - stderr_linebuf.extend(os.read(proc.stderr.fileno(), 8192)) - except BlockingIOError: - pass - - while stderr_linebuf: - line, sep, stderr_linebuf = stderr_linebuf.partition(b'\n') - if not sep: - stderr_linebuf = line - break - - stderr_logger.info(line.decode()) - if event == select.POLLHUP: - poll.unregister(rfd) - pollc -= 1 - - if pollc > 0: - events = poll.poll() - - for handler in proc_logger.handlers: - handler.flush() - - ret = proc.wait() - if ret != 0: - raise Exception(f'borg subprocess exited with returncode {ret}') - - stdout_buffer.seek(0) - return json.load(stdout_buffer) - -def read_repo(path): - global logger - - with Halo(text=f'Listing {path}', **halo_args) as sp: - if not sp.enabled: - logger.debug('Listing %s...', path) - res = borg_json(['borg', 'list', '--info', '--lock-wait=600', '--json', path], preexec_fn=lambda: as_borg())['archives'] - if sp.enabled: - sp.succeed(f'{len(res)} archives in {path}') - else: - logger.info('%d archives in ‘%s’', len(res), path) - return res - -class ToSync: - to_sync = deque() - - def __init__(self, source, target): - self.source = source - self.target = target - - def __iter__(self): - return self - - def __next__(self): - global logger - - if self.to_sync: - return self.to_sync.popleft() - - while True: - try: - src = read_repo(self.source) - dst = read_repo(self.target) - except (subprocess.CalledProcessError, json.decoder.JSONDecodeError) as err: - logger.error(err) - continue - - self.to_sync.extend([entry for entry in src if entry['name'] not in {dst_entry['name'] for dst_entry in dst} and not entry['name'].endswith('.checkpoint')]) - - if self.to_sync: - return self.to_sync.popleft() - - raise StopIteration - -def copy_archive(src_repo_path, dst_repo_path, entry): - global logger - - def do_copy(tmpdir_q): - global logger - - nonlocal src_repo_path, dst_repo_path, entry - - tmpdir = tmpdir_q.get() - - cache_suffix = None - with Halo(text=f'Determine archive parameters', **halo_args) as sp: - if not sp.enabled: - logger.debug('Determining archive parameters...') - match = re.compile('^(.*)-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.(checkpoint|recreate)(\.[0-9]+)?)?').fullmatch(entry['name']) - if match: - repo_id = borg_json(['borg', 'info', '--info', '--lock-wait=600', '--json', src_repo_path], preexec_fn=lambda: as_borg())['repository']['id'] - - if repo_id: - cache_suffix = f'{repo_id}_{match.group(1)}' - if sp.enabled: - sp.succeed(f'Will process {entry["name"]} ({dateutil.parser.isoparse(entry["start"])}, cache_suffix={cache_suffix})') - else: - logger.info('Will process ‘%s’ (%s, cache_suffix=%s)', entry['name'], dateutil.parser.isoparse(entry['start']), cache_suffix) - - logger.debug('Setting up environment...') - unshare.unshare(unshare.CLONE_NEWNS) - subprocess.run(['mount', '--make-rprivate', '/'], check=True) - chroot = pathlib.Path(tmpdir) / 'chroot' - upper = pathlib.Path(tmpdir) / 'upper' - work = pathlib.Path(tmpdir) / 'work' - for path in [chroot,upper,work]: - path.mkdir() - subprocess.run(['mount', '-t', 'overlay', 'overlay', '-o', f'lowerdir=/,upperdir={upper},workdir={work}', chroot], check=True) - bindMounts = ['nix', 'run', 'run/secrets.d', 'run/wrappers', 'proc', 'dev', 'sys', pathlib.Path(os.path.expanduser('~')).relative_to('/')] - if os.environ.get('BORG_BASE_DIR'): - bindMounts.append(pathlib.Path(os.environ['BORG_BASE_DIR']).relative_to('/')) - if not ":" in src_repo_path: - bindMounts.append(pathlib.Path(src_repo_path).relative_to('/')) - if 'SSH_AUTH_SOCK' in os.environ: - bindMounts.append(pathlib.Path(os.environ['SSH_AUTH_SOCK']).parent.relative_to('/')) - for bindMount in bindMounts: - (chroot / bindMount).mkdir(parents=True,exist_ok=True) - subprocess.run(['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], check=True) - os.chroot(chroot) - os.chdir('/') - try: - os.unlink('/etc/fuse.conf') - except FileNotFoundError: - pass - pathlib.Path('/etc/fuse.conf').parent.mkdir(parents=True,exist_ok=True) - with open('/etc/fuse.conf', 'w') as fuse_conf: - fuse_conf.write('user_allow_other\nmount_max = 1000\n') - dir = pathlib.Path('/borg') - dir.mkdir(parents=True,exist_ok=True,mode=0o0750) - os.chown(dir, borg_pwd.pw_uid, borg_pwd.pw_gid) - - total_size = None - total_files = None - if stderr.isatty(): - with Halo(text=f'Determine size', **halo_args) as sp: - stats = borg_json(['borg', 'info', '--info', '--json', '--lock-wait=600', f'{src_repo_path}::{entry["name"]}'], preexec_fn=lambda: as_borg())['archives'][0]['stats'] - total_size = stats['original_size'] - total_files = stats['nfiles'] - if sp.enabled: - sp.succeed(f'{total_files} files, {naturalsize(total_size, binary=True)}') - else: - logger.info('%d files, %s', total_files, naturalsize(total_size, binary=True)) - with subprocess.Popen(['borg', 'mount', '-o', 'allow_other,ignore_permissions', '--foreground', '--progress', '--lock-wait=600', f'{src_repo_path}::{entry["name"]}', dir], preexec_fn=lambda: as_borg()) as mount_proc: - with Halo(text='Waiting for mount', **halo_args) as sp: - if not sp.enabled: - logger.debug('Waiting for mount...') - wait_start = datetime.now() - while True: - if os.path.ismount(dir): - break - elif datetime.now() - wait_start > timedelta(minutes=15): - ret.check_returncode() - time.sleep(0.1) - if sp.enabled: - sp.succeed('Mounted') - else: - logger.info('Mounted %s', f'{src_repo_path}::{entry["name"]}') - - while True: - with tqdm(total=total_size, unit_scale=True, unit_divisor=1024, unit='B', smoothing=0.01, disable=None, dynamic_ncols=True, maxinterval=0.5, miniters=1) as progress: - seen = 0 - env = os.environ.copy() - create_args = ['borg', - 'create', - '--lock-wait=600', - '--one-file-system', - '--compression=auto,zstd,10', - '--chunker-params=10,23,16,4095', - '--files-cache=ctime,size', - '--show-rc', - '--upload-buffer=100', - '--upload-ratelimit=20480', - '--log-json', - '--progress', - '--list', - '--filter=AMEi-x?', - '--stats' - ] - archive_time = datetime.strptime(entry["time"], "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=tzlocal()).astimezone(tzutc()) - create_args += [f'--timestamp={archive_time.strftime("%Y-%m-%dT%H:%M:%S")}'] - if cache_suffix: - env['BORG_FILES_CACHE_SUFFIX'] = cache_suffix - else: - create_args += ['--files-cache=disabled'] - create_args += [f'{dst_repo_path}::{entry["name"]}', '.'] - - with subprocess.Popen(create_args, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, preexec_fn=lambda: as_borg(caps={Cap.DAC_READ_SEARCH}), cwd=dir) as proc: - last_list = None - last_list_time = time.monotonic_ns() - logger.info('Creating...') - - proc_logger = logger.getChild('borg') - stdout_logger = proc_logger.getChild('stdout') - stderr_logger = proc_logger.getChild('stderr') - - fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - fcntl.fcntl(proc.stderr.fileno(), fcntl.F_SETFL, fcntl.fcntl(proc.stderr.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) - - poll = select.poll() - poll.register(proc.stdout, select.POLLIN | select.POLLHUP) - poll.register(proc.stderr, select.POLLIN | select.POLLHUP) - pollc = 2 - events = poll.poll() - stdout_linebuf = bytearray() - stderr_linebuf = bytearray() - - while pollc > 0 and len(events) > 0: - # logger.debug('%d events', len(events)) - for rfd, event in events: - # logger.debug('event %s', event) - if event & select.POLLIN: - if rfd == proc.stdout.fileno(): - try: - # logger.debug('reading stdout...') - stdout_linebuf.extend(os.read(proc.stdout.fileno(), 8192)) - # logger.debug('read stdout, len(stdout_linebuf)=%d', len(stdout_linebuf)) - except BlockingIOError: - pass - - while stdout_linebuf: - # logger.debug('stdout line...') - line, sep, stdout_linebuf = stdout_linebuf.partition(b'\n') - if not sep: - stdout_linebuf = line - break - - stdout_logger.info(line.decode()) - # logger.debug('handled stdout lines, %d leftover', len(stdout_linebuf)) - if rfd == proc.stderr.fileno(): - try: - # logger.debug('reading stderr...') - stderr_linebuf.extend(os.read(proc.stderr.fileno(), 8192)) - # logger.debug('read stderr, len(stderr_linebuf)=%d', len(stderr_linebuf)) - except BlockingIOError: - pass - - while stderr_linebuf: - # logger.debug('stderr line...') - line, sep, stderr_linebuf = stderr_linebuf.partition(b'\n') - if not sep: - stderr_linebuf = line - break - - try: - json_line = json.loads(line) - except json.decoder.JSONDecodeError: - if progress.disable: - stderr_logger.error(line.decode()) - else: - tqdm.write(line.decode()) - continue - - # logger.debug('stderr line decoded: %s', json_line['type'] if 'type' in json_line else None) - - t = '' - if 'time' in json_line and not progress.disable: - ts = datetime.fromtimestamp(json_line['time']).replace(tzinfo=tzlocal()) - t = f'{ts.isoformat(timespec="minutes")} ' - if json_line['type'] == 'archive_progress' and not progress.disable: - now = time.monotonic_ns() - if last_list_time is None or now - last_list_time >= 3e9: - last_list_time = now - if 'path' in json_line and json_line['path']: - progress.set_description(f'… {json_line["path"]}', refresh=False) - else: - progress.set_description(None, refresh=False) - elif last_list is not None: - progress.set_description(last_list, refresh=False) - nfiles=json_line["nfiles"] - if total_files is not None: - nfiles=f'{json_line["nfiles"]}/{total_files}' - progress.set_postfix(compressed=naturalsize(json_line['compressed_size'], binary=True), deduplicated=naturalsize(json_line['deduplicated_size'], binary=True), nfiles=nfiles, refresh=False) - progress.update(json_line["original_size"] - seen) - seen = json_line["original_size"] - elif json_line['type'] == 'archive_progress': - now = time.monotonic_ns() - if last_list_time is None or now - last_list_time >= 3e9: - last_list_time = now - if 'path' in json_line and json_line['path']: - stderr_logger.debug('… %s (%s)', json_line["path"], naturalsize(json_line["original_size"])) - else: - stderr_logger.debug('… (%s)', naturalsize(json_line["original_size"])) - elif json_line['type'] == 'file_status': - # tqdm.write(t + f'{json_line["status"]} {json_line["path"]}') - last_list = f'{json_line["status"]} {json_line["path"]}' - last_list_time = time.monotonic_ns() - progress.set_description(last_list, refresh=False) - if progress.disable: - stderr_logger.info(last_list) - elif (json_line['type'] == 'log_message' or json_line['type'] == 'progress_message' or json_line['type'] == 'progress_percent') and ('message' in json_line or 'msgid' in json_line): - if 'message' in json_line: - if progress.disable: - stderr_logger.info(t + json_line['message']) - else: - tqdm.write(t + json_line['message']) - elif 'msgid' in json_line: - if progress.disable: - stderr_logger.info(t + json_line['msgid']) - else: - tqdm.write(t + json_line['msgid']) - else: - if progress.disable: - stderr_logger.info(t + line.decode()) - else: - tqdm.write(t + line.decode()) - # logger.debug('handled stderr lines, %d leftover', len(stderr_linebuf)) - if event == select.POLLHUP: - poll.unregister(rfd) - pollc -= 1 - - if pollc > 0: - # logger.debug('polling %d fds...', pollc) - events = poll.poll() - # logger.debug('done polling') - - # logger.debug('borg create closed stdout/stderr') - if stdout_linebuf: - logger.error('unterminated line leftover in stdout: %s', stdout_linebuf) - if stderr_linebuf: - logger.error('unterminated line leftover in stdout: %s', stderr_linebuf) - progress.set_description(None) - ret = proc.wait() - # logger.debug('borg create terminated; ret=%d', ret) - if ret != 0: - dst = None - try: - dst = read_repo(dst_repo_path) - except (subprocess.CalledProcessError, json.decoder.JSONDecodeError) as err: - logger.error(err) - continue - else: - if any(map(lambda other: entry['name'] == other['name'], dst)): - logger.info('destination exists, terminating') - break - - logger.warn('destination does not exist, retrying') - continue - else: - # logger.debug('terminating') - break - mount_proc.terminate() - - with Manager() as manager: - tmpdir_q = manager.Queue(1) - - with closing(Process(target=do_copy, args=(tmpdir_q,), name='do_copy')) as p: - p.start() - - with TemporaryDirectory(prefix=f'borg-mount_{entry["name"]}_', dir=os.environ.get('RUNTIME_DIRECTORY')) as tmpdir: - tmpdir_q.put(tmpdir) - p.join() - return p.exitcode - -def sigterm(signum, frame): - raise SystemExit(128 + signum) - -def main(): - signal.signal(signal.SIGTERM, sigterm) - - global logger - logger = logging.getLogger(__name__) - console_handler = logging.StreamHandler() - console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') ) - if sys.stderr.isatty(): - console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') ) - - burst_max = 1000 - burst = burst_max - last_use = None - inv_rate = 1e7 - def consume_filter(record): - nonlocal burst, burst_max, inv_rate, last_use - - delay = None - while True: - now = time.monotonic_ns() - burst = min(burst_max, burst + math.floor((now - last_use) / inv_rate)) if last_use else burst_max - last_use = now - - if burst > 0: - burst -= 1 - if delay: - delay = now - delay - - return True - - if delay is None: - delay = now - time.sleep(inv_rate / 1e9) - console_handler.addFilter(consume_filter) - - logging.getLogger().addHandler(console_handler) - - # log uncaught exceptions - def log_exceptions(type, value, tb): - global logger - - logger.error(value) - sys.__excepthook__(type, value, tb) # calls default excepthook - - sys.excepthook = log_exceptions - - parser = argparse.ArgumentParser(prog='copy') - parser.add_argument('--verbosity', dest='log_level', action='append', type=int) - parser.add_argument('--verbose', '-v', dest='log_level', action='append_const', const=1) - parser.add_argument('--quiet', '-q', dest='log_level', action='append_const', const=-1) - parser.add_argument('source', metavar='REPO_OR_ARCHIVE') - parser.add_argument('target', metavar='REPO_OR_ARCHIVE') - args = parser.parse_args() - - - LOG_LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL] - DEFAULT_LOG_LEVEL = logging.ERROR - log_level = LOG_LEVELS.index(DEFAULT_LOG_LEVEL) - - for adjustment in args.log_level or (): - log_level = min(len(LOG_LEVELS) - 1, max(log_level - adjustment, 0)) - logger.setLevel(LOG_LEVELS[log_level]) - - - if "::" in args.source: - (src_repo_path, _, src_archive) = args.source.partition("::") - entry = None - for candidate_entry in read_repo(src_repo_path): - if entry['name'] != src_archive: - continue - entry = candidate_entry - break - - if entry is None: - logger.critical("Did not find archive ‘%s’", src_archive) - os.exit(1) - - copy_archive(src_repo_path, args.target, entry) - else: - for entry in ToSync(args.source, args.target): - copy_archive(args.source, args.target, entry) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/hosts/vidhar/borg/copy/setup.py b/hosts/vidhar/borg/copy/setup.py deleted file mode 100644 index f77d9560..00000000 --- a/hosts/vidhar/borg/copy/setup.py +++ /dev/null @@ -1,10 +0,0 @@ -from setuptools import setup - -setup(name='copy_borg', - packages=['copy_borg'], - entry_points={ - 'console_scripts': [ - 'copy_borg=copy_borg.__main__:main', - ], - } -) diff --git a/hosts/vidhar/borg/default.nix b/hosts/vidhar/borg/default.nix index 7672de18..80ce9c7e 100644 --- a/hosts/vidhar/borg/default.nix +++ b/hosts/vidhar/borg/default.nix @@ -1,4 +1,4 @@ -{ config, pkgs, lib, flakeInputs, ... }: +{ config, pkgs, lib, flakeInputs, utils, ... }: with lib; @@ -21,60 +21,26 @@ let ServerAliveCountMax 30 ''; - copyService = { repo, repoEscaped }: let - serviceName = "copy-borg@${repoEscaped}"; - in nameValuePair serviceName { + checkBorgUnit = { serviceConfig = { Type = "oneshot"; - ExecStart = "${copyBorg}/bin/copy_borg --verbosity 3 ${escapeShellArg repo} yggdrasil.borgbase:repo"; - TimeoutStartSec = "8h"; - # User = "borg"; - # Group = "borg"; - # StateDirectory = "borg"; - RuntimeDirectory = "copy-borg"; + ExecStart = "${pkgs.borgbackup}/bin/borg ${utils.escapeSystemdExecArgs [ + "--lock-wait" "3600" + "--progress" + "check" + "--verify-data" + ]} %I"; Environment = [ - "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${pkgs.writeText "config" sshConfig}\"" "BORG_BASE_DIR=/var/lib/borg" "BORG_CONFIG_DIR=/var/lib/borg/config" "BORG_CACHE_DIR=/var/lib/borg/cache" "BORG_SECURITY_DIR=/var/lib/borg/security" "BORG_KEYS_DIR=/var/lib/borg/keys" - "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}" - "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes" "BORG_HOSTNAME_IS_UNIQUE=yes" + "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${pkgs.writeText "config" sshConfig}\"" ]; - - LogRateLimitIntervalSec = 0; }; }; - - copyBorg = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage rec { - pname = "copy-borg"; - src = ./copy; - version = "0.0.0"; - ignoreDataOutdated = true; - - requirements = '' - humanize - tqdm - python-dateutil - xdg - python-unshare - pyprctl - halo - ''; - postInstall = '' - wrapProgram $out/bin/copy_borg \ - --prefix PATH : ${makeBinPath (with pkgs; [util-linux borgbackup])}:${config.security.wrapperDir} - ''; - - providers.python-unshare = "nixpkgs"; - overridesPre = [ - (self: super: { python-unshare = super.python-unshare.overrideAttrs (oldAttrs: { name = "python-unshare-0.2.1"; version = "0.2.1"; }); }) - ]; - - # _.tomli.buildInputs.add = with pkgs."python3Packages"; [ flit-core ]; - }; in { config = { services.borgsnap = { @@ -85,7 +51,15 @@ in { keyfile = config.sops.secrets."yggdrasil.borgkey".path; }; - systemd.services = listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]); + services.copyborg.jotnar = { + from = "/srv/backup/borg/jotnar"; + to = "yggdrasil.borgbase:repo"; + inherit sshConfig; + keyfile = config.sops.secrets."yggdrasil.borgkey".path; + timerOptions.timerConfig = { + OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin"; + }; + }; services.borgbackup.repos.jotnar = { path = "/srv/backup/borg/jotnar"; @@ -95,6 +69,27 @@ in { in filter (v: v != null) (mapAttrsToList toAuthKey (builtins.readDir dir)); }; + systemd.services."check-borg@${utils.escapeSystemdPath "/srv/backup/borg/jotnar"}" = checkBorgUnit; + systemd.services."check-borg@${utils.escapeSystemdPath "yggdrasil.borgbase:repo"}" = recursiveUpdate checkBorgUnit { + serviceConfig = { + Environment = checkBorgUnit.serviceConfig.Environment ++ [ + "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}" + ]; + }; + }; + systemd.timers."check-borg@${utils.escapeSystemdPath "/srv/backup/borg/jotnar"}" = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "Sun *-*-02..08 01:30:00 Europe/Berlin"; + }; + }; + systemd.timers."check-borg@${utils.escapeSystemdPath "yggdrasil.borgbase:repo"}" = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "Sun *-*-02..08 01:30:00 Europe/Berlin"; + }; + }; + boot.postBootCommands = mkBefore '' ${pkgs.findutils}/bin/find /srv/backup/borg -type d -empty -delete ''; @@ -123,13 +118,5 @@ in { group = "borg"; mode = "0400"; }; - - systemd.timers."copy-borg@srv-backup-borg-jotnar" = { - wantedBy = ["multi-user.target"]; - - timerConfig = { - OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin"; - }; - }; }; } diff --git a/hosts/vidhar/borg/pyprctl-packages.nix b/hosts/vidhar/borg/pyprctl-packages.nix deleted file mode 100644 index d3b4256a..00000000 --- a/hosts/vidhar/borg/pyprctl-packages.nix +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by pip2nix 0.8.0.dev1 -# See https://github.com/nix-community/pip2nix - -{ pkgs, fetchurl, fetchgit, fetchhg }: - -self: super: { - "pyprctl" = super.buildPythonPackage rec { - pname = "pyprctl"; - version = "0.1.3"; - src = fetchurl { - url = "https://files.pythonhosted.org/packages/bf/5e/62765de39bbce8111fb1f4453a4a804913bf49179fa265fb713ed66c9d15/pyprctl-0.1.3-py3-none-any.whl"; - sha256 = "1pgif990r92za5rx12mjnq5iiz72d455v0wrawzb73q79w8ya0k3"; - }; - format = "wheel"; - doCheck = false; - buildInputs = []; - checkInputs = []; - nativeBuildInputs = []; - propagatedBuildInputs = []; - }; -} diff --git a/hosts/vidhar/prometheus/ca/.gitignore b/hosts/vidhar/prometheus/ca/.gitignore deleted file mode 100644 index 7c894574..00000000 --- a/hosts/vidhar/prometheus/ca/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -ca.key -ca.cnf -*.old \ No newline at end of file diff --git a/hosts/vidhar/prometheus/ca/ca.crt b/hosts/vidhar/prometheus/ca/ca.crt index 922fed28..8cfea666 100644 --- a/hosts/vidhar/prometheus/ca/ca.crt +++ b/hosts/vidhar/prometheus/ca/ca.crt @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- -MIIBsjCCAWSgAwIBAgIUOzZ8XcFb8XtI2yyWp4S/WMD6QxQwBQYDK2VwMB8xHTAb -BgNVBAMMFHByb21ldGhldXMueWdnZHJhc2lsMCAXDTIyMDQwODE5NDgwMFoYDzIw -OTAwNDI2MTk0ODAwWjAfMR0wGwYDVQQDDBRwcm9tZXRoZXVzLnlnZ2RyYXNpbDAq -MAUGAytlcAMhAOoxPLBH6pnCRtE7V5gejM92gg1vLNLHw3rFIXXchOJmo4GvMIGs -MB0GA1UdDgQWBBRnwBkgZFnueEa7aV8aEAoMRzW4CTBaBgNVHSMEUzBRgBRnwBkg -ZFnueEa7aV8aEAoMRzW4CaEjpCEwHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55Z2dk -cmFzaWyCFDs2fF3BW/F7SNsslqeEv1jA+kMUMA8GA1UdEwEB/wQFMAMBAf8wCwYD -VR0PBAQDAgEGMBEGCWCGSAGG+EIBAQQEAwICBDAFBgMrZXADQQD9AC2OHtzW8QSC -HU/4rGdRWRqr3pfclKXimSWaAXMPly2M1qehPI402lhQrIAVF+D1pi/EAGJfbbzF -aurykEMB +MIIBrjCCAS6gAwIBAgIUYV3YPBx91CbgMpOGb5HKMZ2hzRUwBQYDK2VxMB8xHTAb +BgNVBAMMFHByb21ldGhldXMueWdnZHJhc2lsMB4XDTIyMTEwNzE5MjgzNFoXDTMy +MTEwNzE5MzMzNFowHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55Z2dkcmFzaWwwQzAF +BgMrZXEDOgAVqcV3KGDhcbQt/UR3Yv6OuAGc+Kc8hrDHjAV8K9GTjahc/d49NK2v +FAz0uK8YidIaTVJZjzHhTgCjYzBhMB8GA1UdIwQYMBaAFObrhCUDCZk6/JeeDMNW +l8WeLr+MMB0GA1UdDgQWBBTm64QlAwmZOvyXngzDVpfFni6/jDAOBgNVHQ8BAf8E +BAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAFBgMrZXEDcwAFAqBlI7SpHaSE+0mMzx5x +0M6T3iJtLxP36Qz5MHx3vvcbbx1eJhZWKewuyz+9LXaCkf8Jpd5AFoC+HhoikVSz +46yVzmTBt6TISc4bh+eiWcXEKFbxEbXkwqZd2m/oHI4Em4qnDKp96FcOfq6RQ8pR +AwA= -----END CERTIFICATE----- diff --git a/hosts/vidhar/prometheus/ca/ca.key b/hosts/vidhar/prometheus/ca/ca.key new file mode 100644 index 00000000..32c4330a --- /dev/null +++ b/hosts/vidhar/prometheus/ca/ca.key @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:yk8nI2Zz2F3XnBM9dqnA3UoWTTCGJLMZUYjpo+SW+ARmZVgYdcqHZunhoGRQP/r6qrIUvM/2Yl85Uosw43jllILCNESH17Gi6uI0gD9OE8I14oll8wCL+/GvP/IuU//1NEAeLF9cz8MBWPE0WW2wQk5DF4ikl+z3/McG+kaqeU+ka6aMmjIjUstjR2vCf+pfZN3KswylcLaeuvXP,iv:ByEIQCxQwjynCFxGZdYtg+nx9mFmwbqHL3iBhzLbKIQ=,tag:jIc+KcfbSmiZqM6Z2xIa0g==,type:str]", + "sops": { + "kms": null, + "gcp_kms": null, + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2022-11-07T19:33:34Z", + "mac": "ENC[AES256_GCM,data:UE1+0M15ZBgsKOfEmz8DMeQsmzkRxcN5cjdpMswzc6vIgo6sRN4ArdtDKqAMcFtFhzokSZin6OIizsk6KLlsts5sgVHQHXKrqssc016OADRg4BoC9zM/MGLUXOHndrRSPGSQgRDCeVwmR9C5iE18VZ/NCcZtoztHt6DPt3xmGpo=,iv:JB6CTWUyyDpjciKfYugf78Xo+jDKCH3+tL8p9G7M5y0=,tag:n73uY6cX5EV6Rjq1/HM8kw==,type:str]", + "pgp": [ + { + "created_at": "2022-11-07T19:33:34Z", + "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4DXxoViZlp6dISAQdAJxBqRR1DzDPs/sQgfZNaKZTWH+mbdQo9mpGRWcWkm1ww\nOjVRJjiBDyeItfbOS9hnEOJKwKUIk1tH7F5m+U5daFLSw/Ct/xzJ7iyphcfRzNFN\n0l4BHF6sMyoPFpSGpE+0d4IRqfDPF3t9d3NL1lAGV75MoEho38ptNCbAn32kWpZ9\n7/Vk3L+oR/3xhLAwm3/7JDed01zNnKRaxFh3zpYfwZWhMtEdoUoEIkojufEJ64s2\n=KZjL\n-----END PGP MESSAGE-----\n", + "fp": "30D3453B8CD02FE2A3E7C78C0FB536FB87AE8F51" + } + ], + "unencrypted_suffix": "_unencrypted", + "version": "3.7.3" + } +} \ No newline at end of file diff --git a/hosts/vidhar/prometheus/ca/ca.key.sops b/hosts/vidhar/prometheus/ca/ca.key.sops deleted file mode 100644 index 5313056e..00000000 --- a/hosts/vidhar/prometheus/ca/ca.key.sops +++ /dev/null @@ -1,21 +0,0 @@ -{ - "data": "ENC[AES256_GCM,data:XW6h0psHOSV0cR03vRg479A5XRM7KfiBfVgvm4QlxCZzhkk5U1ToDJIaCxqKpxlEu8wm79wmz+/CmSLDEBcs7x05a5vBDt81mlWJ49PolOrG9bL9Qkyq5u8sB8HWXRXxCP5kg2su+n9NqdHX9AIhYCXy7VJDuGo=,iv:v661AhF2Q/O+a7JtwHtnSkSI0mL8ltu5rPny8vWCL/Q=,tag:c7b0a6o6y/MI5vG85uFuUg==,type:str]", - "sops": { - "kms": null, - "gcp_kms": null, - "azure_kv": null, - "hc_vault": null, - "age": null, - "lastmodified": "2022-04-08T20:12:22Z", - "mac": "ENC[AES256_GCM,data:W/IF6WgTscbkcMUTR3aeqM/H/UwgFgILDbKBxYJQxcFtt4kq3UqzSd/e0hk5NQ9IkagAC4X0gZDuzco2mc7caUGyzMKRdA2ekgcdDwzruQ4i+UYyr80dFhqHpV+aksdZJVR+dJzkmIRmza3Ia5e/X01XNIbIrU13JKYm9jCskd0=,iv:2g+UFcSTxcTrf+toi4BDVvAaY5ydk7yRnhpQ/rrNvVo=,tag:3X01wEqL/Q8cIiF+DEMnpg==,type:str]", - "pgp": [ - { - "created_at": "2022-04-08T20:12:22Z", - "enc": "-----BEGIN PGP MESSAGE-----\n\nhF4DXxoViZlp6dISAQdADN+s7UQS8hEBc2mMRovD/zKuIoIAS3swLpP6ul9kRGMw\nDCUvOL41sxXmuodi4Pg69YB2YcL47Fod7nQWUYaK8L3CuyjWUq1cxomlYtTd03eH\n0l4BiyWTuZ+1OG4Xng8B4zdcM5jWfeTRWupDIXcnPFjwz47FetmrcCAaROKYL87e\nAjK76Y6gR/gSj0GTTAUIfKFpqsqAdBAf6oBekQcPgeqcrJcZ2ZZFWzmswGBvcGjs\n=gqhG\n-----END PGP MESSAGE-----\n", - "fp": "30D3453B8CD02FE2A3E7C78C0FB536FB87AE8F51" - } - ], - "unencrypted_suffix": "_unencrypted", - "version": "3.7.2" - } -} \ No newline at end of file diff --git a/hosts/vidhar/prometheus/ca/certs/01.pem b/hosts/vidhar/prometheus/ca/certs/01.pem deleted file mode 100644 index 81abe0b7..00000000 --- a/hosts/vidhar/prometheus/ca/certs/01.pem +++ /dev/null @@ -1,39 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 1 (0x1) - Signature Algorithm: ED25519 - Issuer: CN=prometheus.yggdrasil - Validity - Not Before: Apr 8 20:03:55 2022 GMT - Not After : Apr 26 20:03:55 2090 GMT - Subject: CN=surtr.yggdrasil - Subject Public Key Info: - Public Key Algorithm: ED25519 - ED25519 Public-Key: - pub: - 02:5d:f0:8d:f6:5f:fc:fd:27:47:0e:d8:ec:fe:e0: - a0:28:20:9a:b4:8a:07:4c:75:b2:c1:03:ef:16:3b: - eb:e0 - X509v3 extensions: - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Alternative Name: - DNS:prometheus.surtr.yggdrasil, DNS:prometheus.surtr.yggdrasil.li - X509v3 Subject Key Identifier: - 37:9D:AD:3D:CB:F9:14:5A:69:CD:E2:71:D8:08:97:93:A5:20:3C:38 - Signature Algorithm: ED25519 - 3c:df:73:85:a7:81:07:60:b5:4e:ea:ec:74:04:47:d2:35:41: - cf:d8:34:75:18:4f:ee:c3:b9:64:6d:0a:fb:1a:76:e2:96:8b: - 5e:24:c5:d6:b6:2e:6f:6e:29:ff:26:70:ef:5a:7b:33:40:40: - 13:e8:49:a9:80:73:62:8e:58:05 ------BEGIN CERTIFICATE----- -MIIBXzCCARGgAwIBAgIBATAFBgMrZXAwHzEdMBsGA1UEAwwUcHJvbWV0aGV1cy55 -Z2dkcmFzaWwwIBcNMjIwNDA4MjAwMzU1WhgPMjA5MDA0MjYyMDAzNTVaMBoxGDAW -BgNVBAMMD3N1cnRyLnlnZ2RyYXNpbDAqMAUGAytlcAMhAAJd8I32X/z9J0cO2Oz+ -4KAoIJq0igdMdbLBA+8WO+vgo3UwczAMBgNVHRMBAf8EAjAAMEQGA1UdEQQ9MDuC -GnByb21ldGhldXMuc3VydHIueWdnZHJhc2lsgh1wcm9tZXRoZXVzLnN1cnRyLnln -Z2RyYXNpbC5saTAdBgNVHQ4EFgQUN52tPcv5FFppzeJx2AiXk6UgPDgwBQYDK2Vw -A0EAPN9zhaeBB2C1TursdARH0jVBz9g0dRhP7sO5ZG0K+xp24paLXiTF1rYub24p -/yZw71p7M0BAE+hJqYBzYo5YBQ== ------END CERTIFICATE----- diff --git a/hosts/vidhar/prometheus/ca/certs/02.pem b/hosts/vidhar/prometheus/ca/certs/02.pem deleted file mode 100644 index d908ca7d..00000000 --- a/hosts/vidhar/prometheus/ca/certs/02.pem +++ /dev/null @@ -1,38 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: 2 (0x2) - Signature Algorithm: ED25519 - Issuer: CN=prometheus.yggdrasil - Validity - Not Before: Apr 8 20:07:13 2022 GMT - Not After : Apr 26 20:07:13 2090 GMT - Subject: CN=vidhar.yggdrasil - Subject Public Key Info: - Public Key Algorithm: ED25519 - ED25519 Public-Key: - pub: - 13:84:a6:01:07:7a:5e:8d:2b:8d:83:ee:73:1d:c6: - b8:9a:ad:b9:3d:40:51:ec:2c:f3:52:7d:81:90:e7: - ac:88 - X509v3 extensions: - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Alternative Name: - DNS:prometheus.vidhar.yggdrasil - X509v3 Subject Key Identifier: - 44:AA:8E:CC:AB:C9:A7:D1:A1:D0:FA:7F:DB:87:1E:08:AA:6E:4D:59 - Signature Algorithm: ED25519 - 47:65:87:17:50:96:77:56:20:ac:9e:f4:e4:6d:19:6d:b7:24: - 11:af:0c:c3:f3:fd:75:19:d9:77:06:41:79:7f:a5:00:0c:18: - ee:82:3e:9e:09:61:34:cf:8f:f5:83:d1:5d:b2:e4:42:b6:3f: - 9c:b6:5a:f3:40:92:e6:8f:24:0f ------BEGIN CERTIFICATE----- -MIIBQTCB9KADAgECAgECMAUGAytlcDAfMR0wGwYDVQQDDBRwcm9tZXRoZXVzLnln -Z2RyYXNpbDAgFw0yMjA0MDgyMDA3MTNaGA8yMDkwMDQyNjIwMDcxM1owGzEZMBcG -A1UEAwwQdmlkaGFyLnlnZ2RyYXNpbDAqMAUGAytlcAMhABOEpgEHel6NK42D7nMd -xriarbk9QFHsLPNSfYGQ56yIo1cwVTAMBgNVHRMBAf8EAjAAMCYGA1UdEQQfMB2C -G3Byb21ldGhldXMudmlkaGFyLnlnZ2RyYXNpbDAdBgNVHQ4EFgQURKqOzKvJp9Gh -0Pp/24ceCKpuTVkwBQYDK2VwA0EAR2WHF1CWd1YgrJ705G0ZbbckEa8Mw/P9dRnZ -dwZBeX+lAAwY7oI+nglhNM+P9YPRXbLkQrY/nLZa80CS5o8kDw== ------END CERTIFICATE----- diff --git a/hosts/vidhar/prometheus/ca/index.txt b/hosts/vidhar/prometheus/ca/index.txt deleted file mode 100644 index 41ebb0f4..00000000 --- a/hosts/vidhar/prometheus/ca/index.txt +++ /dev/null @@ -1,2 +0,0 @@ -V 20900426200355Z 01 unknown /CN=surtr.yggdrasil -V 20900426200713Z 02 unknown /CN=vidhar.yggdrasil diff --git a/hosts/vidhar/prometheus/ca/index.txt.attr b/hosts/vidhar/prometheus/ca/index.txt.attr deleted file mode 100644 index 8f7e63a3..00000000 --- a/hosts/vidhar/prometheus/ca/index.txt.attr +++ /dev/null @@ -1 +0,0 @@ -unique_subject = yes diff --git a/hosts/vidhar/prometheus/ca/serial b/hosts/vidhar/prometheus/ca/serial deleted file mode 100644 index 75016ea3..00000000 --- a/hosts/vidhar/prometheus/ca/serial +++ /dev/null @@ -1 +0,0 @@ -03 diff --git a/hosts/vidhar/prometheus/tls.crt b/hosts/vidhar/prometheus/tls.crt index 792ed542..6516f185 100644 --- a/hosts/vidhar/prometheus/tls.crt +++ b/hosts/vidhar/prometheus/tls.crt @@ -1,9 +1,12 @@ -----BEGIN CERTIFICATE----- -MIIBQTCB9KADAgECAgECMAUGAytlcDAfMR0wGwYDVQQDDBRwcm9tZXRoZXVzLnln -Z2RyYXNpbDAgFw0yMjA0MDgyMDA3MTNaGA8yMDkwMDQyNjIwMDcxM1owGzEZMBcG -A1UEAwwQdmlkaGFyLnlnZ2RyYXNpbDAqMAUGAytlcAMhABOEpgEHel6NK42D7nMd -xriarbk9QFHsLPNSfYGQ56yIo1cwVTAMBgNVHRMBAf8EAjAAMCYGA1UdEQQfMB2C -G3Byb21ldGhldXMudmlkaGFyLnlnZ2RyYXNpbDAdBgNVHQ4EFgQURKqOzKvJp9Gh -0Pp/24ceCKpuTVkwBQYDK2VwA0EAR2WHF1CWd1YgrJ705G0ZbbckEa8Mw/P9dRnZ -dwZBeX+lAAwY7oI+nglhNM+P9YPRXbLkQrY/nLZa80CS5o8kDw== +MIIByDCCAUigAwIBAgIPQAAAAGNpXrc6y389EXtIMAUGAytlcTAfMR0wGwYDVQQD +DBRwcm9tZXRoZXVzLnlnZ2RyYXNpbDAeFw0yMjExMDcxOTMyNTRaFw0zMjExMDcx +OTM3NTRaMBsxGTAXBgNVBAMMEHZpZGhhci55Z2dkcmFzaWwwKjAFBgMrZXADIQAT +hKYBB3pejSuNg+5zHca4mq25PUBR7CzzUn2BkOesiKOBnjCBmzAfBgNVHSMEGDAW +gBTm64QlAwmZOvyXngzDVpfFni6/jDAdBgNVHQ4EFgQURKqOzKvJp9Gh0Pp/24ce +CKpuTVkwDgYDVR0PAQH/BAQDAgXgMAwGA1UdEwEB/wQCMAAwEwYDVR0lBAwwCgYI +KwYBBQUHAwIwJgYDVR0RBB8wHYIbcHJvbWV0aGV1cy52aWRoYXIueWdnZHJhc2ls +MAUGAytlcQNzAIPNcNWqVX4Ie971O/S2DL0HMFmPbR331U4snLBqPGWC1/j9NV4O +cxJvLo8Hzb4I0BXn/nZbyk/ogCCJU69BVeK378qgLo68DIZ4TA3ka5ZPNRSt464Q +NvbkDhtFVVxM04xUjI4dOeE9jczG9nN3jHESAA== -----END CERTIFICATE----- -- cgit v1.2.3