12 files changed, 558 insertions, 58 deletions

diff --git a/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__init__.py b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__init__.py
diff --git a/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py
new file mode 100644
index 00000000..f481090c
--- /dev/null
+++ b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py
@@ -0,0 +1,92 @@
+from systemd.daemon import listen_fds
+from sdnotify import SystemdNotifier
+from socketserver import StreamRequestHandler, ThreadingMixIn
+from systemd_socketserver import SystemdSocketServer
+import sys
+from threading import Thread
+from psycopg_pool import ConnectionPool
+from psycopg.rows import namedtuple_row
+
+import logging
+
+
+class PolicyHandler(StreamRequestHandler):
+    def handle(self):
+        logger.debug('Handling new connection...')
+
+        self.args = dict()
+
+        line = None
+        while line := self.rfile.readline().removesuffix(b'\n'):
+            if b'=' not in line:
+                break
+
+            key, val = line.split(sep=b'=', maxsplit=1)
+            self.args[key.decode()] = val.decode()
+
+        logger.info('Connection parameters: %s', self.args)
+
+        allowed = False
+        with self.server.db_pool.connection() as conn:
+            local, domain = self.args['sender'].split(sep='@', maxsplit=1)
+            extension = None
+            if '+' in local:
+                local, extension = local.split(sep='+', maxsplit=1)
+
+            logger.debug('Parsed address: %s', {'local': local, 'extension': extension, 'domain': domain})
+
+            with conn.cursor() as cur:
+                cur.row_factory = namedtuple_row
+                cur.execute('SELECT "mailbox"."mailbox" as "user", "local", "extension", "domain" FROM "mailbox" INNER JOIN "mailbox_mapping" ON "mailbox".id = "mailbox_mapping"."mailbox" WHERE "mailbox"."mailbox" = %(user)s AND ("local" = %(local)s OR "local" IS NULL) AND ("extension" = %(extension)s OR "extension" IS NULL) AND "domain" = %(domain)s', params = {'user': self.args['ccert_subject'], 'local': local, 'extension': extension if extension is not None else '', 'domain': domain}, prepare=True)
+                for record in cur:
+                    logger.debug('Received result: %s', record)
+                    allowed = True
+
+        action = '550 5.7.0 Sender address not authorized for current user'
+        if allowed:
+            action = 'DUNNO'
+
+        logger.info('Reached verdict: %s', {'allowed': allowed, 'action': action})
+        self.wfile.write(f'action={action}\n\n'.encode())
+
+class ThreadedSystemdSocketServer(ThreadingMixIn, SystemdSocketServer):
+    def __init__(self, fd, RequestHandlerClass):
+        super().__init__(fd, RequestHandlerClass)
+
+        self.db_pool = ConnectionPool(min_size=1)
+        self.db_pool.wait()
+
+def main():
+    global logger
+    logger = logging.getLogger(__name__)
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') )
+    if sys.stderr.isatty():
+        console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') )
+    logger.addHandler(console_handler)
+    logger.setLevel(logging.DEBUG)
+
+    # log uncaught exceptions
+    def log_exceptions(type, value, tb):
+        global logger
+
+        logger.error(value)
+        sys.__excepthook__(type, value, tb) # calls default excepthook
+
+    sys.excepthook = log_exceptions
+
+    fds = listen_fds()
+    servers = [ThreadedSystemdSocketServer(fd, PolicyHandler) for fd in fds]
+
+    if servers:
+        for server in servers:
+            Thread(name=f'Server for fd{server.fileno()}', target=server.serve_forever).start()
+    else:
+        return 2
+
+    SystemdNotifier().notify('READY=1')
+
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/hosts/surtr/email/ccert-policy-server/setup.py b/hosts/surtr/email/ccert-policy-server/setup.py
new file mode 100644
index 00000000..d8eb415a
--- /dev/null
+++ b/hosts/surtr/email/ccert-policy-server/setup.py
@@ -0,0 +1,12 @@
+from setuptools import setup, find_packages
+
+setup(
+    name = 'ccert-policy-server',
+    version = '0.0.0',
+    packages = ['ccert_policy_server'],
+    entry_points = {
+        'console_scripts': [
+            'ccert-policy-server=ccert_policy_server.__main__:main'
+        ],
+    },
+)
diff --git a/hosts/surtr/email/default.nix b/hosts/surtr/email/default.nix
index 83bf02f5..9cfba1f1 100644
--- a/hosts/surtr/email/default.nix
+++ b/hosts/surtr/email/default.nix
@@ -1,4 +1,4 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, lib, flakeInputs, ... }:
 
 with lib;
 
@@ -20,6 +20,27 @@ let
     '';
   };
 
+  ccert-policy-server = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage {
+    src = ./ccert-policy-server;
+    pname = "ccert-policy-server";
+    version = "0.0.0";
+
+    python = "python39";
+    ignoreDataOutdated = true;
+
+    requirements = ''
+      sdnotify
+      systemd-socketserver
+      psycopg >=3.0.0
+      psycopg-pool >=3.0.0
+      psycopg-binary >=3.0.0
+    '';
+
+    overridesPre = [
+      (self: super: { systemd-python = super.systemd.overrideAttrs (oldAttrs: { pname = "systemd-python"; }); })
+    ];
+  };
+
   spmDomains = ["bouncy.email"];
 in {
   config = {
@@ -35,7 +56,7 @@ in {
         };
       })
     ];
-    
+
     services.postfix = {
       enable = true;
       hostname = "surtr.yggdrasil.li";
@@ -187,8 +208,9 @@ in {
             "-o" "smtpd_tls_ask_ccert=yes"
             "-o" "smtpd_tls_req_ccert=yes"
             "-o" "smtpd_client_restrictions=permit_tls_all_clientcerts,reject"
+            "-o" "{smtpd_data_restrictions = check_policy_service unix:/run/postfwd3/postfwd3.sock}"
             "-o" "smtpd_relay_restrictions=permit_tls_all_clientcerts,reject"
-            "-o" "smtpd_sender_restrictions=reject_unknown_sender_domain,reject_unverified_sender"
+            "-o" "{smtpd_sender_restrictions = reject_unknown_sender_domain,reject_unverified_sender,check_policy_service unix:/run/postfix-ccert-sender-policy.sock}"
             "-o" "unverified_sender_reject_code=550"
             "-o" "unverified_sender_reject_reason={Sender address rejected: undeliverable address}"
             "-o" "smtpd_recipient_restrictions=reject_unauth_pipelining,reject_non_fqdn_recipient,reject_unknown_recipient_domain,permit_tls_all_clientcerts,reject"
@@ -415,7 +437,7 @@ in {
         mail_plugins = $mail_plugins quota
         mailbox_list_index = yes
         postmaster_address = postmaster@yggdrasil.li
-        recipient_delimiter = 
+        recipient_delimiter =
         auth_username_chars = abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-+_@
 
         service lmtp {
@@ -431,7 +453,7 @@ in {
         namespace inbox {
           separator = /
           inbox = yes
-          prefix = 
+          prefix =
 
           mailbox Trash {
             auto = no
@@ -602,7 +624,7 @@ in {
           ${pkgs.dovecot_pigeonhole}/bin/sievec $f
         done
       '';
-      
+
       serviceConfig = {
         LoadCredential = [
           "surtr.yggdrasil.li.key.pem:${config.security.acme.certs."surtr.yggdrasil.li".directory}/key.pem"
@@ -703,7 +725,7 @@ in {
     };
     systemd.sockets.spm = {
       wantedBy = [ "nginx.service" ];
-      
+
       socketConfig = {
         ListenStream = "/run/spm/server.sock";
         SocketUser = "spm";
@@ -730,5 +752,57 @@ in {
       enable = true;
       loglevel = "debug";
     };
+
+    systemd.sockets."postfix-ccert-sender-policy" = {
+      requiredBy = ["postfix.service"];
+      wants = ["postfix-ccert-sender-policy.service"];
+      socketConfig = {
+        ListenStream = "/run/postfix-ccert-sender-policy.sock";
+      };
+    };
+    systemd.services."postfix-ccert-sender-policy" = {
+      serviceConfig = {
+        Type = "notify";
+
+        ExecStart = "${ccert-policy-server}/bin/ccert-policy-server";
+
+        Environment = [
+          "PGDATABASE=email"
+        ];
+
+        DynamicUser = false;
+        User = "postfix-ccert-sender-policy";
+        Group = "postfix-ccert-sender-policy";
+        ProtectSystem = "strict";
+        SystemCallFilter = "@system-service";
+        NoNewPrivileges = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        MemoryDenyWriteExecute = true;
+        RestrictSUIDSGID = true;
+        KeyringMode = "private";
+        ProtectClock = true;
+        RestrictRealtime = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        ProtectHostname = true;
+        ReadWritePaths = ["/run/postgresql"];
+      };
+    };
+    users.users."postfix-ccert-sender-policy" = {
+      isSystemUser = true;
+      group = "postfix-ccert-sender-policy";
+    };
+    users.groups."postfix-ccert-sender-policy" = {};
+
+    services.postfwd = {
+      enable = true;
+      rules = ''
+        id=RCPT01; protocol_state=DATA; protocol_state=END-OF-MESSAGE; action=rcpt(ccert_subject/100/3600/450 4.7.1 Exceeding maximum of 100 recipients per hour [$$ratecount])
+        id=RCPT02; protocol_state=DATA; protocol_state=END-OF-MESSAGE; action=rcpt(ccert_subject/1000/86400/450 4.7.1 Exceeding maximum of 1000 recipients per day [$$ratecount])
+      '';
+    };
   };
 }
diff --git a/hosts/surtr/postgresql.nix b/hosts/surtr/postgresql.nix
index 66ce60eb..7013ae97 100644
--- a/hosts/surtr/postgresql.nix
+++ b/hosts/surtr/postgresql.nix
@@ -104,7 +104,7 @@ in {
           ALTER TABLE mailbox_mapping ALTER local TYPE citext;
           ALTER TABLE mailbox_mapping ALTER domain TYPE citext;
 
-          CREATE VIEW mailbox_quota_rule (id, mailbox, quota_rule) AS SELECT id, mailbox, (CASE WHEN quota_bytes IS NULL THEN '*:ignore' ELSE '*:bytes=' || quota_bytes END) AS quota_rule FROM mailbox; 
+          CREATE VIEW mailbox_quota_rule (id, mailbox, quota_rule) AS SELECT id, mailbox, (CASE WHEN quota_bytes IS NULL THEN '*:ignore' ELSE '*:bytes=' || quota_bytes END) AS quota_rule FROM mailbox;
 
           CREATE VIEW virtual_mailbox_domain (domain) AS SELECT DISTINCT domain FROM mailbox_mapping;
           CREATE VIEW virtual_mailbox_mapping (lookup) AS SELECT (CASE WHEN local IS NULL THEN ''' ELSE local END) || '@' || domain AS lookup FROM mailbox_mapping;
@@ -143,6 +143,15 @@ in {
 
           GRANT SELECT ON ALL TABLES IN SCHEMA public TO "spm";
           COMMIT;
+
+          BEGIN;
+          SELECT _v.register_patch('007-ccert-sender-policy', ARRAY['000-base'], null);
+
+          CREATE USER "postfix-ccert-sender-policy";
+          GRANT CONNECT ON DATABASE "email" TO "postfix-ccert-sender-policy";
+          ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO "postfix-ccert-sender-policy";
+          GRANT SELECT ON ALL TABLES IN SCHEMA public TO "postfix-ccert-sender-policy";
+          COMMIT;
         ''}
       '';
     };
diff --git a/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py b/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py
new file mode 100644
index 00000000..e93e6a60
--- /dev/null
+++ b/hosts/vidhar/borg/borgsnap/borgsnap/__main__.py
@@ -0,0 +1,202 @@
+import argparse
+import os, sys, signal
+from pyprctl import cap_permitted, cap_inheritable, cap_effective, cap_ambient, Cap
+from pwd import getpwnam
+
+from datetime import datetime, timezone
+from dateutil.parser import isoparse
+
+from xdg import xdg_runtime_dir
+import unshare
+from tempfile import TemporaryDirectory
+
+import logging
+
+import json
+import subprocess
+
+import pathlib
+from pathlib import Path
+
+from atomicwrites import atomic_write
+
+from traceback import format_exc
+
+
+borg_pwd = getpwnam('borg')
+
+def as_borg(caps=set(), cwd=None):
+    if caps:
+        cap_permitted.add(*caps)
+        cap_inheritable.add(*caps)
+        cap_effective.add(*caps)
+        cap_ambient.add(*caps)
+
+    os.setgid(borg_pwd.pw_gid)
+    os.setuid(borg_pwd.pw_uid)
+
+    if cwd is not None:
+        os.chdir(cwd)
+
+
+def _archive_name(snapshot, target, archive_prefix):
+    _, _, ts = snapshot.rpartition('@')
+    creation_time = isoparse(ts).astimezone(timezone.utc)
+    archive_name = _archive_basename(snapshot, archive_prefix)
+    return f'{target}::{archive_name}-{creation_time.strftime("%Y-%m-%dT%H:%M:%S")}'
+
+def _archive_basename(snapshot, archive_prefix):
+    base_name, _, _ = snapshot.rpartition('@')
+    return archive_prefix + base_name.replace('-', '--').replace('/', '-')
+
+def check(*, snapshot, target, archive_prefix, cache_file):
+    archives = None
+    if cache_file:
+        logger.debug('Trying cache...')
+        try:
+            with open(cache_file, mode='r', encoding='utf-8') as fp:
+                archives = set(json.load(fp))
+            logger.info('Loaded archive list from cache')
+        except FileNotFoundError:
+            pass
+
+    if not archives:
+        logger.info('Loading archive list from remote...')
+        with subprocess.Popen(['borg', 'list', '--info', '--lock-wait=600', '--json', target], stdout=subprocess.PIPE, preexec_fn=lambda: as_borg()) as proc:
+            archives = set([archive['barchive'] for archive in json.load(proc.stdout)['archives']])
+        if cache_file:
+            logger.debug('Saving archive list to cache...')
+            with atomic_write(cache_file, mode='w', encoding='utf-8', overwrite=True) as fp:
+                json.dump(list(archives), fp)
+
+    # logger.debug(f'archives: {archives}')
+    _, _, archive_name = _archive_name(snapshot, target, archive_prefix).partition('::')
+    if archive_name in archives:
+        logger.info(f'{archive_name} found')
+        return 0
+    else:
+        logger.info(f'{archive_name} not found')
+        return 126
+
+def create(*, snapshot, target, archive_prefix, dry_run):
+    basename = _archive_basename(snapshot, archive_prefix)
+
+    with TemporaryDirectory(prefix=f'borg-mount_{basename}_', dir=os.environ.get('RUNTIME_DIRECTORY')) as tmpdir:
+        child = os.fork()
+        if child == 0:
+            unshare.unshare(unshare.CLONE_NEWNS)
+            subprocess.run(['mount', '--make-rprivate', '/'], check=True)
+            chroot = pathlib.Path(tmpdir) / 'chroot'
+            upper = pathlib.Path(tmpdir) / 'upper'
+            work = pathlib.Path(tmpdir) / 'work'
+            for path in [chroot,upper,work]:
+                path.mkdir()
+            subprocess.run(['mount', '-t', 'overlay', 'overlay', '-o', f'lowerdir=/,upperdir={upper},workdir={work}', chroot], check=True)
+            bindMounts = ['nix', 'run', 'run/secrets.d', 'run/wrappers', 'proc', 'dev', 'sys', pathlib.Path(os.path.expanduser('~')).relative_to('/')]
+            if os.environ.get('BORG_BASE_DIR'):
+                bindMounts.append(pathlib.Path(os.environ['BORG_BASE_DIR']).relative_to('/'))
+            if 'SSH_AUTH_SOCK' in os.environ:
+                bindMounts.append(pathlib.Path(os.environ['SSH_AUTH_SOCK']).parent.relative_to('/'))
+            for bindMount in bindMounts:
+                (chroot / bindMount).mkdir(parents=True,exist_ok=True)
+                # print(*['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], file=stderr)
+                subprocess.run(['mount', '--bind', pathlib.Path('/') / bindMount, chroot / bindMount], check=True)
+            os.chroot(chroot)
+            os.chdir('/')
+            dir = pathlib.Path('/borg')
+            dir.mkdir(parents=True,exist_ok=True,mode=0o0750)
+            os.chown(dir, borg_pwd.pw_uid, borg_pwd.pw_gid)
+            try:
+                subprocess.run(['mount', '-t', 'zfs', '-o', 'ro', snapshot, dir], check=True)
+                env = os.environ.copy()
+                create_args = ['borg',
+                               'create',
+                               '--lock-wait=600',
+                               '--one-file-system',
+                               '--compression=auto,zstd,10',
+                               '--chunker-params=10,23,16,4095',
+                               '--files-cache=ctime,size',
+                               '--show-rc',
+                               # '--remote-ratelimit=20480',
+                               '--progress',
+                               '--list',
+                               '--filter=AMEi-x?',
+                               '--stats' if not dry_run else '--dry-run'
+                               ]
+                _, _, ts = snapshot.rpartition('@')
+                creation_time = isoparse(ts).astimezone(timezone.utc)
+                create_args += [f'--timestamp={creation_time.strftime("%Y-%m-%dT%H:%M:%S")}']
+                env['BORG_FILES_CACHE_SUFFIX'] = basename
+                create_args += [_archive_name(snapshot, target, archive_prefix), '.']
+                print({'create_args': create_args, 'cwd': dir, 'env': env}, file=sys.stderr)
+                subprocess.run(create_args, stdin=subprocess.DEVNULL, env=env, preexec_fn=lambda: as_borg(caps={CAP.DAC_READ_SEARCH}, cwd=dir), check=True)
+                #  subprocess.run(create_args, stdin=subprocess.DEVNULL, env=env, preexec_fn=lambda: None, cwd=dir, check=True)
+            finally:
+                subprocess.run(['umount', dir], check=True)
+            os._exit(0)
+        else:
+            while True:
+                waitpid, waitret = os.wait()
+                if waitret != 0:
+                    sys.exit(waitret)
+                if waitpid == child:
+                    break
+    return 0
+
+def sigterm(signum, frame):
+    raise SystemExit(128 + signum)
+
+def main():
+    signal.signal(signal.SIGTERM, sigterm)
+
+    global logger
+    logger = logging.getLogger(__name__)
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter( logging.Formatter('[%(levelname)s](%(name)s): %(message)s') )
+    if sys.stderr.isatty():
+        console_handler.setFormatter( logging.Formatter('%(asctime)s [%(levelname)s](%(name)s): %(message)s') )
+    logger.addHandler(console_handler)
+
+    # log uncaught exceptions
+    def log_exceptions(type, value, tb):
+        global logger
+
+        logger.error(value)
+        sys.__excepthook__(type, value, tb) # calls default excepthook
+
+    sys.excepthook = log_exceptions
+
+    parser = argparse.ArgumentParser(prog='borgsnap')
+    parser.add_argument('--verbose', '-v', action='count', default=0)
+    parser.add_argument('--target', metavar='REPO', default='yggdrasil.borgbase:repo')
+    parser.add_argument('--archive-prefix', metavar='REPO', default='yggdrasil.vidhar.')
+    subparsers = parser.add_subparsers()
+    subparsers.required = True
+    parser.set_defaults(cmd=None)
+    check_parser = subparsers.add_parser('check')
+    check_parser.add_argument('--cache-file', type=lambda p: Path(p).absolute(), default=None)
+    check_parser.add_argument('snapshot')
+    check_parser.set_defaults(cmd=check)
+    create_parser = subparsers.add_parser('create')
+    create_parser.add_argument('--dry-run', '-n', action='store_true', default=False)
+    create_parser.add_argument('snapshot')
+    create_parser.set_defaults(cmd=create)
+    args = parser.parse_args()
+
+    if args.verbose <= 0:
+        logger.setLevel(logging.WARNING)
+    elif args.verbose <= 1:
+        logger.setLevel(logging.INFO)
+    else:
+        logger.setLevel(logging.DEBUG)
+
+    cmdArgs = {}
+    for copy in {'target', 'archive_prefix', 'snapshot', 'cache_file', 'dry_run'}:
+        if copy in vars(args):
+            cmdArgs[copy] = vars(args)[copy]
+
+    return args.cmd(**cmdArgs)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/hosts/vidhar/borg/borgsnap/setup.py b/hosts/vidhar/borg/borgsnap/setup.py
new file mode 100644
index 00000000..76356bfc
--- /dev/null
+++ b/hosts/vidhar/borg/borgsnap/setup.py
@@ -0,0 +1,10 @@
+from setuptools import setup
+
+setup(name='borgsnap',
+      packages=['borgsnap'],
+      entry_points={
+          'console_scripts': [
+              'borgsnap=borgsnap.__main__:main',
+          ],
+      }
+)
diff --git a/hosts/vidhar/borg/copy.py b/hosts/vidhar/borg/copy.py
index 4e9599b8..b9b667f2 100755
--- a/hosts/vidhar/borg/copy.py
+++ b/hosts/vidhar/borg/copy.py
@@ -71,7 +71,7 @@ def read_repo(path):
 
 class ToSync:
     to_sync = deque()
-    
+
     def __iter__(self):
         return self
 
@@ -267,7 +267,7 @@ def sigterm(signum, frame):
 
 def main():
     signal.signal(signal.SIGTERM, sigterm)
-    
+
     if "::" in args.source:
         (src_repo_path, _, src_archive) = args.source.partition("::")
         entry = None
diff --git a/hosts/vidhar/borg/default.nix b/hosts/vidhar/borg/default.nix
index 579630a9..650c91ee 100644
--- a/hosts/vidhar/borg/default.nix
+++ b/hosts/vidhar/borg/default.nix
@@ -1,23 +1,28 @@
-{ config, pkgs, lib, ... }:
+{ config, pkgs, lib, flakeInputs, ... }:
 
 with lib;
 
 let
+  sshConfig = pkgs.writeText "config" ''
+    Include /etc/ssh/ssh_config
+
+    ControlMaster auto
+    ControlPath /var/lib/borg/.borgssh-master-%r@%n:%p
+    ControlPersist yes
+
+    Host yggdrasil.borgbase
+      HostName nx69hpl8.repo.borgbase.com
+      User nx69hpl8
+      IdentityFile ${config.sops.secrets."append.borgbase".path}
+      IdentitiesOnly yes
+
+      BatchMode yes
+      ServerAliveInterval 10
+      ServerAliveCountMax 30
+  '';
+
   copyService = { repo, repoEscaped }: let
     serviceName = "copy-borg@${repoEscaped}";
-    sshConfig = pkgs.writeText "config" ''
-      Include /etc/ssh/ssh_config
-
-      Host yggdrasil.borgbase
-        HostName nx69hpl8.repo.borgbase.com
-        User nx69hpl8
-        IdentityFile ${config.sops.secrets."append.borgbase".path}
-        IdentitiesOnly yes
-
-        BatchMode yes
-        ServerAliveInterval 10
-        ServerAliveCountMax 30
-    '';
   in nameValuePair serviceName {
     serviceConfig = {
       Type = "oneshot";
@@ -72,8 +77,63 @@ let
         --prefix PATH : ${makeBinPath (with pkgs; [utillinux borgbackup])}:${config.security.wrapperDir}
     '';
   });
+
+  borgsnap = flakeInputs.mach-nix.lib.${config.nixpkgs.system}.buildPythonPackage rec {
+    pname = "borgsnap";
+    src = ./borgsnap;
+    version = "0.0.0";
+    ignoreDataOutdated = true;
+
+    requirements = ''
+      atomicwrites
+      pyprctl
+      python-unshare
+      xdg
+      python-dateutil
+    '';
+    postInstall = ''
+      wrapProgram $out/bin/borgsnap \
+        --prefix PATH : ${makeBinPath (with pkgs; [utillinux borgbackup])}:${config.security.wrapperDir}
+    '';
+
+    providers.python-unshare = "nixpkgs";
+    overridesPre = [
+      (self: super: { python-unshare = super.python-unshare.overrideAttrs (oldAttrs: { name = "python-unshare-0.2.1"; version = "0.2.1"; }); })
+    ];
+
+    _.xdg.buildInputs.add = with pkgs."python3Packages"; [ poetry ];
+    _.tomli.buildInputs.add = with pkgs."python3Packages"; [ flit-core ];
+  };
 in {
   config = {
+    services.zfssnap.config.exec = {
+      check = "${borgsnap}/bin/borgsnap -vvv --target yggdrasil.borgbase:repo --archive-prefix yggdrasil.vidhar. check --cache-file /run/zfssnap-prune/archives-cache.json";
+      cmd = "${borgsnap}/bin/borgsnap -vvv --target yggdrasil.borgbase:repo --archive-prefix yggdrasil.vidhar. create --dry-run";
+
+      halfweekly = "8";
+      monthly = "-1";
+    };
+
+    systemd.services = {
+      "zfssnap-prune" = {
+        serviceConfig = {
+          Environment = [
+            "BORG_RSH=\"${pkgs.openssh}/bin/ssh -F ${sshConfig}\""
+            "BORG_BASE_DIR=/var/lib/borg"
+            "BORG_CONFIG_DIR=/var/lib/borg/config"
+            "BORG_CACHE_DIR=/var/lib/borg/cache"
+            "BORG_SECURITY_DIR=/var/lib/borg/security"
+            "BORG_KEYS_DIR=/var/lib/borg/keys"
+            "BORG_KEY_FILE=${config.sops.secrets."yggdrasil.borgkey".path}"
+            "BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=yes"
+            "BORG_HOSTNAME_IS_UNIQUE=yes"
+          ];
+          RuntimeDirectory = "zfssnap-prune";
+        };
+      };
+    } // listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]);
+
+
     services.borgbackup.repos.jotnar = {
       path = "/srv/backup/borg/jotnar";
       authorizedKeysAppendOnly = let
@@ -111,11 +171,9 @@ in {
       mode = "0400";
     };
 
-    systemd.services = listToAttrs (map copyService [{ repo = "/srv/backup/borg/jotnar"; repoEscaped = "srv-backup-borg-jotnar"; }]);
-
     systemd.timers."copy-borg@srv-backup-borg-jotnar" = {
       wantedBy = ["multi-user.target"];
-      
+
       timerConfig = {
         OnCalendar = "*-*-* 00/4:00:00 Europe/Berlin";
       };
diff --git a/hosts/vidhar/default.nix b/hosts/vidhar/default.nix
index 121cc9df..3f5d17d5 100644
--- a/hosts/vidhar/default.nix
+++ b/hosts/vidhar/default.nix
@@ -1,4 +1,7 @@
 { hostName, flake, config, pkgs, lib, ... }:
+
+with lib;
+
 {
   imports = with flake.nixosModules.systemProfiles; [
     ./zfs.nix ./network ./samba.nix ./dns ./prometheus ./borg
@@ -39,7 +42,7 @@
         luks.devices = {
           nvm0 = { device = "/dev/disk/by-label/${hostName}-nvm0"; bypassWorkqueues = true; };
           nvm1 = { device = "/dev/disk/by-label/${hostName}-nvm1"; bypassWorkqueues = true; };
-          
+
           hdd0.device = "/dev/disk/by-label/${hostName}-hdd0";
           hdd1.device = "/dev/disk/by-label/${hostName}-hdd1";
           hdd2.device = "/dev/disk/by-label/${hostName}-hdd2";
@@ -58,7 +61,7 @@
         options = [ "mode=0755" ];
       };
     };
-    
+
     services.timesyncd.enable = false;
     services.chrony = {
       enable = true;
@@ -132,6 +135,7 @@
         access_log syslog:server=unix:/dev/log main;
         error_log syslog:server=unix:/dev/log info;
 
+        client_body_buffer_size 16m;
         client_body_temp_path /run/nginx-client-bodies;
       '';
       upstreams.grafana = {
@@ -173,12 +177,12 @@
       sopsFile = ./selfsigned.key;
     };
     systemd.services.nginx = {
-      preStart = lib.mkForce config.services.nginx.preStart;
+      preStart = mkForce config.services.nginx.preStart;
       serviceConfig = {
-        ExecReload = lib.mkForce "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        ExecReload = mkForce "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
         LoadCredential = [ "selfsigned.key:${config.sops.secrets."selfsigned.key".path}" ];
 
-        RuntimeDirectory = lib.mkForce [ "nginx" "nginx-client-bodies" ];
+        RuntimeDirectory = mkForce [ "nginx" "nginx-client-bodies" ];
         RuntimeDirectoryMode = "0750";
       };
     };
@@ -232,7 +236,7 @@
       };
     };
     systemd.services.loki.preStart = let
-      rulesYaml = lib.generators.toYAML {} {
+      rulesYaml = generators.toYAML {} {
         groups = [
           { name = "power-failures";
             rules = [
@@ -311,6 +315,29 @@
       timers.wants = ["systemd-tmpfiles-clean.timer"];
     };
 
+    services.smartd = {
+      enable = true;
+      autodetect = false;
+      defaults.monitored = "-a -o on -s (S/../.././02|L/../../7/04)";
+      devices = map (dev: { device = "/dev/disk/by-path/${dev}"; }) [
+        "pci-0000:00:1f.2-ata-1"
+        "pci-0000:00:1f.2-ata-3"
+        "pci-0000:00:1f.2-ata-4"
+        "pci-0000:00:1f.2-ata-5"
+        "pci-0000:00:1f.2-ata-6"
+        "pci-0000:02:00.0-nvme-1"
+        "pci-0000:05:00.0-sas-phy0-lun-0"
+        "pci-0000:05:00.0-sas-phy1-lun-0"
+        "pci-0000:06:00.0-nvme-1"
+      ];
+      notifications = {
+        test = false;
+        mail.enable = false;
+        x11.enable = false;
+        wall.enable = false;
+      };
+    };
+
     environment.systemPackages = with pkgs; [iotop vmtouch];
 
     system.stateVersion = "21.05";
diff --git a/hosts/vidhar/prometheus/default.nix b/hosts/vidhar/prometheus/default.nix
index 4c23d8a9..7ac86c30 100644
--- a/hosts/vidhar/prometheus/default.nix
+++ b/hosts/vidhar/prometheus/default.nix
@@ -34,20 +34,6 @@ in {
           enable = true;
           enabledCollectors = [];
         };
-        smartctl = {
-          enable = true;
-          devices = map (dev: "/dev/disk/by-path/${dev}") [
-            "pci-0000:00:1f.2-ata-1"
-            "pci-0000:00:1f.2-ata-3"
-            "pci-0000:00:1f.2-ata-4"
-            "pci-0000:00:1f.2-ata-5"
-            "pci-0000:00:1f.2-ata-6"
-            "pci-0000:02:00.0-nvme-1"
-            "pci-0000:05:00.0-sas-phy0-lun-0"
-            "pci-0000:05:00.0-sas-phy1-lun-0"
-            "pci-0000:06:00.0-nvme-1"
-          ];
-        };
         snmp = {
           enable = true;
           configurationPath = ./snmp.yml;
@@ -124,10 +110,10 @@ in {
         }
         { job_name = "smartctl";
           static_configs = [
-            { targets = ["localhost:${toString config.services.prometheus.exporters.smartctl.port}"]; }
+            { targets = ["localhost:9633"]; }
           ];
           relabel_configs = relabelHosts;
-          scrape_interval = "1s";
+          scrape_interval = "60s";
         }
         { job_name = "snmp";
           static_configs = [
@@ -376,6 +362,30 @@ in {
       };
     };
 
+    systemd.services."prometheus-smartctl-exporter" = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = with pkgs; [ smartmontools ];
+      serviceConfig = {
+        Restart = "always";
+
+        CapabilityBoundingSet = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"];
+        AmbientCapabilities = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"];
+        ProtectSystem = "strict";
+        DynamicUser = true;
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = false;
+        PrivateTmp = true;
+        ProcSubset = "pid";
+
+        Type = "simple";
+        ExecStart = "${pkgs.smartprom}/bin/smartprom";
+        Environment = "SMARTCTL_EXPORTER_PORT=9633";
+      };
+    };
+
     systemd.services."prometheus-systemd-exporter" = let
       cfg = config.services.prometheus.exporters.systemd;
     in {
@@ -385,14 +395,6 @@ in {
       '';
     };
 
-    systemd.services."prometheus-smartctl-exporter" = {
-      serviceConfig = {
-        DeviceAllow = lib.mkForce config.services.prometheus.exporters.smartctl.devices;
-        CapabilityBoundingSet = lib.mkForce ["CAP_SYS_ADMIN"];
-        AmbientCapabilities = lib.mkForce ["CAP_SYS_ADMIN"];
-      };
-    };
-
     services.nginx = {
       upstreams.prometheus = {
         servers = { "localhost:${toString config.services.prometheus.port}" = {}; };
diff --git a/hosts/vidhar/zfs.nix b/hosts/vidhar/zfs.nix
index ef285536..52b48aca 100644
--- a/hosts/vidhar/zfs.nix
+++ b/hosts/vidhar/zfs.nix
@@ -130,7 +130,21 @@
       echo "=== ZPOOL IMPORT COMPLETE ==="
     '';
 
-    services.zfssnap.enable = true;
+    services.zfssnap = {
+      enable = true;
+      config.keep = {
+        within = "15m";
+        "5m" = "48";
+        "15m" = "32";
+        hourly = "48";
+        "4h" = "24";
+        "12h" = "12";
+        daily = "62";
+        halfweekly = "32";
+        weekly = "24";
+        monthly = "-1";
+      };
+    };
     services.zfs.trim.enable = false;
     services.zfs.autoScrub = {
       enable = true;