2 files changed, 113 insertions, 88 deletions

diff --git a/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py
index 7117eb63..45619fb0 100644
--- a/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py
+++ b/hosts/surtr/email/ccert-policy-server/ccert_policy_server/__main__.py
@@ -28,12 +28,14 @@ class PolicyHandler(StreamRequestHandler):
 
         allowed = False
         user = None
+        relay_eligible = False
         if self.args['sasl_username']:
             user = self.args['sasl_username']
         if self.args['ccert_subject']:
             user = self.args['ccert_subject']
+            relay_eligible = True
 
-        if user:
+        if user and '@' in self.args['sender']:
             with self.server.db_pool.connection() as conn:
                 local, domain = self.args['sender'].split(sep='@', maxsplit=1)
                 extension = None
@@ -44,9 +46,16 @@ class PolicyHandler(StreamRequestHandler):
 
                 with conn.cursor() as cur:
                     cur.row_factory = namedtuple_row
-                    cur.execute('SELECT EXISTS(SELECT true FROM "mailbox" INNER JOIN "mailbox_mapping" ON "mailbox".id = "mailbox_mapping"."mailbox" WHERE "mailbox"."mailbox" = %(user)s AND ("local" = %(local)s OR "local" IS NULL) AND ("extension" = %(extension)s OR "extension" IS NULL) AND "domain" = %(domain)s) as "exists"', params = {'user': user, 'local': local, 'extension': extension if extension is not None else '', 'domain': domain}, prepare=True)
-                    if (row := cur.fetchone()) is not None:
-                        allowed = row.exists
+
+                    if relay_eligible:
+                        cur.execute('SELECT EXISTS(SELECT true FROM "mailbox" INNER JOIN "relay_access" ON "mailbox".id = "relay_access"."mailbox" WHERE "mailbox"."mailbox" = %(user)s AND ("domain" = %(domain)s OR %(domain)s ilike CONCAT(\'%%_.\', "domain"))) as "exists"', params = {'user': user, 'domain': domain})
+                        if (row := cur.fetchone()) is not None:
+                            allowed = row.exists
+
+                    if not allowed:
+                        cur.execute('SELECT EXISTS(SELECT true FROM "mailbox" INNER JOIN "mailbox_mapping" ON "mailbox".id = "mailbox_mapping"."mailbox" WHERE "mailbox"."mailbox" = %(user)s AND ("local" = %(local)s OR "local" IS NULL) AND ("extension" = %(extension)s OR "extension" IS NULL) AND "domain" = %(domain)s) as "exists"', params = {'user': user, 'local': local, 'extension': extension if extension is not None else '', 'domain': domain}, prepare=True)
+                        if (row := cur.fetchone()) is not None:
+                            allowed = row.exists
 
         action = '550 5.7.0 Sender address not authorized for current user'
         if allowed:
diff --git a/hosts/surtr/email/default.nix b/hosts/surtr/email/default.nix
index c6253e4c..b0e95a0e 100644
--- a/hosts/surtr/email/default.nix
+++ b/hosts/surtr/email/default.nix
@@ -124,19 +124,20 @@ in {
     services.postfix = {
       enable = true;
       enableSmtp = false;
-      hostname = "surtr.yggdrasil.li";
-      recipientDelimiter = "";
       setSendmail = true;
       postmasterAlias = ""; rootAlias = ""; extraAliases = "";
-      destination = [];
-      sslCert = "/run/credentials/postfix.service/surtr.yggdrasil.li.pem";
-      sslKey = "/run/credentials/postfix.service/surtr.yggdrasil.li.key.pem";
-      networks = [];
-      config = let
-        relay_ccert = "texthash:${pkgs.writeText "relay_ccert" ""}";
-      in {
+      settings.main = {
+        recpipient_delimiter = "";
+        mydestination = [];
+        mynetworks = [];
+        myhostname = "surtr.yggdrasil.li";
+
         smtpd_tls_security_level = "may";
 
+        smtpd_tls_chain_files = [
+          "/run/credentials/postfix.service/surtr.yggdrasil.li.full.pem"
+        ];
+
         #the dh params
         smtpd_tls_dh1024_param_file = toString config.security.dhparams.params."postfix-1024".path;
         smtpd_tls_dh512_param_file = toString config.security.dhparams.params."postfix-512".path;
@@ -171,21 +172,14 @@ in {
 
         smtp_tls_connection_reuse = true;
 
-        tls_server_sni_maps = ''texthash:${pkgs.writeText "sni" (
-          concatMapStringsSep "\n\n" (domain:
-            concatMapStringsSep "\n" (subdomain: "${subdomain} /run/credentials/postfix.service/${removePrefix "." subdomain}.full.pem")
-              [domain "mailin.${domain}" "mailsub.${domain}" ".${domain}"]
-          ) emailDomains
-        )}'';
+        tls_server_sni_maps = "inline:{${concatMapStringsSep ", " (domain: "{ ${domain} = /run/credentials/postfix.service/${removePrefix "." domain}.full.pem }") (concatMap (domain: [domain "mailin.${domain}" "mailsub.${domain}" ".${domain}"]) emailDomains)}}";
 
         smtp_tls_policy_maps = "socketmap:unix:${config.services.postfix-mta-sts-resolver.settings.path}:postfix";
 
         local_recipient_maps = "";
 
-        # 10 GiB
-        message_size_limit = "10737418240";
-        # 10 GiB
-        mailbox_size_limit = "10737418240";
+        message_size_limit = 10 * 1024 * 1024 * 1024;
+        mailbox_size_limit = 10 * 1024 * 1024 * 1024;
 
         smtpd_delay_reject = true;
         smtpd_helo_required = true;
@@ -200,7 +194,6 @@ in {
             dbname = email
             query = SELECT action FROM virtual_mailbox_access WHERE lookup = '%s'
           ''}"
-          "check_ccert_access ${relay_ccert}"
           "reject_non_fqdn_helo_hostname"
           "reject_invalid_helo_hostname"
           "reject_unauth_destination"
@@ -221,7 +214,6 @@ in {
         address_verify_sender_ttl = "30045s";
 
         smtpd_relay_restrictions = [
-          "check_ccert_access ${relay_ccert}"
           "reject_unauth_destination"
         ];
 
@@ -244,6 +236,37 @@ in {
         bounce_queue_lifetime = "20m";
         delay_warning_time = "10m";
 
+        failure_template_file = toString (pkgs.writeText "failure.cf" ''
+          Charset: us-ascii
+          From: Mail Delivery System <MAILER-DAEMON>
+          Subject: Undelivered Mail Returned to Sender
+          Postmaster-Subject: Postmaster Copy: Undelivered Mail
+
+          This is the mail system at host $myhostname.
+
+          I'm sorry to have to inform you that your message could not
+          be delivered to one or more recipients. It's attached below.
+
+                             The mail system
+        '');
+        delay_template_file = toString (pkgs.writeText "delay.cf" ''
+          Charset: us-ascii
+          From: Mail Delivery System <MAILER-DAEMON>
+          Subject: Delayed Mail (still being retried)
+          Postmaster-Subject: Postmaster Warning: Delayed Mail
+
+          This is the mail system at host $myhostname.
+
+          ####################################################################
+          # THIS IS A WARNING ONLY.  YOU DO NOT NEED TO RESEND YOUR MESSAGE. #
+          ####################################################################
+
+          Your message could not be delivered for more than $delay_warning_time_minutes minute(s).
+          It will be retried until it is $maximal_queue_lifetime_minutes minute(s) old.
+
+                             The mail system
+        '');
+
         smtpd_discard_ehlo_keyword_address_maps = "cidr:${pkgs.writeText "esmtp_access" ''
           # Allow DSN requests from local subnet only
           192.168.0.0/16      silent-discard
@@ -268,13 +291,26 @@ in {
         virtual_transport = "dvlmtp:unix:/run/dovecot-lmtp";
         smtputf8_enable = false;
 
-        authorized_submit_users = "inline:{ root= postfwd= dovecot2= }";
+        authorized_submit_users = "inline:{ root= postfwd= ${config.services.dovecot2.user}= }";
+        authorized_flush_users = "inline:{ root= }";
+        authorized_mailq_users = "inline:{ root= }";
 
         postscreen_access_list = "";
         postscreen_denylist_action = "drop";
         postscreen_greet_action = "enforce";
+
+        sender_bcc_maps = ''pgsql:${pkgs.writeText "sender_bcc_maps.cf" ''
+          hosts = postgresql:///email
+          dbname = email
+          query = SELECT value FROM sender_bcc_maps WHERE key = '%s'
+        ''}'';
+        recipient_bcc_maps = ''pgsql:${pkgs.writeText "recipient_bcc_maps.cf" ''
+          hosts = postgresql:///email
+          dbname = email
+          query = SELECT value FROM recipient_bcc_maps WHERE key = '%s'
+        ''}'';
       };
-      masterConfig = {
+      settings.master = {
         "465" = {
           type = "inet";
           private = false;
@@ -342,7 +378,10 @@ in {
           maxproc = 0;
           args = [
             "-o" "header_checks=pcre:${pkgs.writeText "header_checks_submission" ''
+              if /^Received: /
+              !/by surtr\.yggdrasil\.li/ STRIP
               /^Received: from [^ ]+ \([^ ]+ [^ ]+\)\s+(.*)$/ REPLACE Received: $1
+              endif
             ''}"
           ];
         };
@@ -390,7 +429,7 @@ in {
       enable = true;
       user = "postfix"; group = "postfix";
       socket = "local:/run/opendkim/opendkim.sock";
-      domains = ''csl:${concatStringsSep "," (["surtr.yggdrasil.li"] ++ emailDomains)}'';
+      domains = ''csl:${concatStringsSep "," (["surtr.yggdrasil.li" "yggdrasil.li" "141.li" "kleen.li" "synapse.li" "praseodym.org"] ++ emailDomains)}'';
       selector = "surtr";
       configFile = builtins.toFile "opendkim.conf" ''
         Syslog true
@@ -494,49 +533,49 @@ in {
       };
     };
 
-    users.groups.${config.services.rspamd.group}.members = [ config.services.postfix.user "dovecot2" ];
+    users.groups.${config.services.rspamd.group}.members = [ config.services.postfix.user config.services.dovecot2.user ];
 
     services.redis.servers.rspamd.enable = true;
 
     users.groups.${config.services.redis.servers.rspamd.user}.members = [ config.services.rspamd.user ];
 
-    environment.systemPackages = with pkgs; [ dovecot_pigeonhole dovecot_fts_xapian ];
+    environment.systemPackages = with pkgs; [ dovecot_pigeonhole dovecot-fts-flatcurve ];
     services.dovecot2 = {
       enable = true;
       enablePAM = false;
-      sslServerCert = "/run/credentials/dovecot2.service/surtr.yggdrasil.li.pem";
-      sslServerKey = "/run/credentials/dovecot2.service/surtr.yggdrasil.li.key.pem";
+      sslServerCert = "/run/credentials/dovecot.service/surtr.yggdrasil.li.pem";
+      sslServerKey = "/run/credentials/dovecot.service/surtr.yggdrasil.li.key.pem";
       sslCACert = toString ./ca/ca.crt;
       mailLocation = "maildir:/var/lib/mail/%u/maildir:UTF-8:INDEX=/var/lib/dovecot/indices/%u";
-      mailPlugins.globally.enable = [ "fts" "fts_xapian" ];
+      mailPlugins.globally.enable = [ "fts" "fts_flatcurve" ];
       protocols = [ "lmtp" "sieve" ];
       sieve = {
-        extensions = ["copy" "imapsieve" "variables" "imap4flags" "vacation"];
-        globalExtensions = ["copy" "imapsieve" "variables" "imap4flags" "vacation"];
+        extensions = ["copy" "imapsieve" "variables" "imap4flags" "vacation" "vacation-seconds" "vnd.dovecot.debug"];
+        globalExtensions = ["copy" "imapsieve" "variables" "imap4flags" "vacation" "vacation-seconds" "vnd.dovecot.debug"];
       };
       extraConfig = let
         dovecotSqlConf = pkgs.writeText "dovecot-sql.conf" ''
           driver = pgsql
           connect = dbname=email
-          password_query = SELECT (CASE WHEN '%k' = 'valid' AND '%m' = 'EXTERNAL' THEN NULL ELSE "password" END) as password, (CASE WHEN '%k' = 'valid' AND '%m' = 'EXTERNAL' THEN true WHEN password IS NULL THEN true ELSE NULL END) as nopassword, "user", quota_rule, 'dovecot2' as uid, 'dovecot2' as gid FROM imap_user WHERE "user" = '%n'
-          user_query = SELECT "user", quota_rule, 'dovecot2' as uid, 'dovecot2' as gid FROM imap_user WHERE "user" = '%n'
+          password_query = SELECT (CASE WHEN '%k' = 'valid' AND '%m' = 'EXTERNAL' THEN NULL ELSE "password" END) as password, (CASE WHEN '%k' = 'valid' AND '%m' = 'EXTERNAL' THEN true WHEN password IS NULL THEN true ELSE NULL END) as nopassword, "user", quota_rule, '${config.services.dovecot2.user}' as uid, '${config.services.dovecot2.group}' as gid FROM imap_user WHERE "user" = '%n'
+          user_query = SELECT "user", quota_rule, '${config.services.dovecot2.user}' as uid, 'dovecot2' as gid FROM imap_user WHERE "user" = '%n'
           iterate_query = SELECT "user" FROM imap_user
         '';
       in ''
         mail_home = /var/lib/mail/%u
 
-        mail_plugins = $mail_plugins quota
+        mail_plugins = $mail_plugins quota fts fts_flatcurve
 
-        first_valid_uid = ${toString config.users.users.dovecot2.uid}
-        last_valid_uid = ${toString config.users.users.dovecot2.uid}
-        first_valid_gid = ${toString config.users.groups.dovecot2.gid}
-        last_valid_gid = ${toString config.users.groups.dovecot2.gid}
+        first_valid_uid = ${toString config.users.users.${config.services.dovecot2.user}.uid}
+        last_valid_uid = ${toString config.users.users.${config.services.dovecot2.user}.uid}
+        first_valid_gid = ${toString config.users.groups.${config.services.dovecot2.group}.gid}
+        last_valid_gid = ${toString config.users.groups.${config.services.dovecot2.group}.gid}
 
         ${concatMapStringsSep "\n\n" (domain:
           concatMapStringsSep "\n" (subdomain: ''
             local_name ${subdomain} {
-              ssl_cert = </run/credentials/dovecot2.service/${subdomain}.pem
-              ssl_key = </run/credentials/dovecot2.service/${subdomain}.key.pem
+              ssl_cert = </run/credentials/dovecot.service/${subdomain}.pem
+              ssl_key = </run/credentials/dovecot.service/${subdomain}.key.pem
             }
           '') ["imap.${domain}" domain]
         ) emailDomains}
@@ -557,10 +596,10 @@ in {
         auth_debug = yes
 
         service auth {
-          user = dovecot2
+          user = ${config.services.dovecot2.user}
         }
         service auth-worker {
-          user = dovecot2
+          user = ${config.services.dovecot2.user}
         }
 
         userdb {
@@ -581,7 +620,7 @@ in {
             args = ${pkgs.writeText "dovecot-sql.conf" ''
               driver = pgsql
               connect = dbname=email
-              user_query = SELECT DISTINCT ON (extension IS NULL, local IS NULL) "user", quota_rule, 'dovecot2' as uid, 'dovecot2' as gid FROM lmtp_mapping WHERE CASE WHEN extension IS NOT NULL AND local IS NOT NULL THEN ('%n' :: citext) = local || '+' || extension AND domain = ('%d' :: citext) WHEN local IS NOT NULL THEN (local = ('%n' :: citext) OR ('%n' :: citext) ILIKE local || '+%%') AND domain = ('%d' :: citext) WHEN extension IS NOT NULL THEN ('%n' :: citext) ILIKE '%%+' || extension AND domain = ('%d' :: citext) ELSE domain = ('%d' :: citext) END ORDER BY (extension IS NULL) ASC, (local IS NULL) ASC
+              user_query = SELECT DISTINCT ON (extension IS NULL, local IS NULL) "user", quota_rule, '${config.services.dovecot2.user}' as uid, '${config.services.dovecot2.group}' as gid FROM lmtp_mapping WHERE CASE WHEN extension IS NOT NULL AND local IS NOT NULL THEN ('%n' :: citext) = local || '+' || extension AND domain = ('%d' :: citext) WHEN local IS NOT NULL THEN (local = ('%n' :: citext) OR ('%n' :: citext) ILIKE local || '+%%') AND domain = ('%d' :: citext) WHEN extension IS NOT NULL THEN ('%n' :: citext) ILIKE '%%+' || extension AND domain = ('%d' :: citext) ELSE domain = ('%d' :: citext) END ORDER BY (extension IS NULL) ASC, (local IS NULL) ASC
             ''}
 
             skip = never
@@ -651,7 +690,7 @@ in {
           quota_status_success = DUNNO
           quota_status_nouser = DUNNO
           quota_grace = 10%%
-          quota_max_mail_size = ${config.services.postfix.config.message_size_limit}
+          quota_max_mail_size = ${toString config.services.postfix.settings.main.message_size_limit}
           quota_vsizes = yes
         }
 
@@ -687,13 +726,16 @@ in {
         }
 
         plugin {
-          plugin = fts fts_xapian
-          fts = xapian
-          fts_xapian = partial=3 full=20 attachments=1 verbose=1
+          fts = flatcurve
 
-          fts_autoindex = yes
+          fts_languages = en de
+          fts_tokenizers = generic email-address
 
-          fts_enforced = no
+          fts_tokenizer_email_address = maxlen=100
+          fts_tokenizer_generic = algorithm=simple maxlen=30
+
+          fts_filters = normalizer-icu snowball stopwords
+          fts_filters_en = lowercase snowball stopwords
         }
 
         service indexer-worker {
@@ -702,30 +744,6 @@ in {
       '';
     };
 
-    systemd.services.dovecot-fts-xapian-optimize = {
-      description = "Optimize dovecot indices for fts_xapian";
-      requisite = [ "dovecot2.service" ];
-      after = [ "dovecot2.service" ];
-      startAt = "*-*-* 22:00:00 Europe/Berlin";
-      serviceConfig = {
-        Type = "oneshot";
-        ExecStart = "${getExe' pkgs.dovecot "doveadm"} fts optimize -A";
-        PrivateDevices = true;
-        PrivateNetwork = true;
-        ProtectKernelTunables = true;
-        ProtectKernelModules = true;
-        ProtectControlGroups = true;
-        ProtectHome = true;
-        ProtectSystem = true;
-        PrivateTmp = true;
-      };
-    };
-    systemd.timers.dovecot-fts-xapian-optimize = {
-      timerConfig = {
-        RandomizedDelaySec = 4 * 3600;
-      };
-    };
-
     environment.etc = {
       "dovecot/sieve_before.d/tag-junk.sieve".text = ''
         require ["imap4flags"];
@@ -770,28 +788,26 @@ in {
 
     security.acme.rfc2136Domains = {
       "surtr.yggdrasil.li" = {
-        restartUnits = [ "postfix.service" "dovecot2.service" ];
+        restartUnits = [ "postfix.service" "dovecot.service" ];
       };
     } // listToAttrs (map (domain: nameValuePair "spm.${domain}" { restartUnits = ["nginx.service"]; }) spmDomains)
     // listToAttrs (concatMap (domain: [
-      (nameValuePair domain { restartUnits = ["postfix.service" "dovecot2.service"]; })
+      (nameValuePair domain { restartUnits = ["postfix.service" "dovecot.service"]; })
       (nameValuePair "mailin.${domain}" { restartUnits = ["postfix.service"]; })
       (nameValuePair "mailsub.${domain}" { restartUnits = ["postfix.service"]; })
-      (nameValuePair "imap.${domain}" { restartUnits = ["dovecot2.service"]; })
+      (nameValuePair "imap.${domain}" { restartUnits = ["dovecot.service"]; })
       (nameValuePair "mta-sts.${domain}" { restartUnits = ["nginx.service"]; })
     ]) emailDomains);
 
     systemd.services.postfix = {
-      serviceConfig.LoadCredential = [
-        "surtr.yggdrasil.li.key.pem:${config.security.acme.certs."surtr.yggdrasil.li".directory}/key.pem"
-        "surtr.yggdrasil.li.pem:${config.security.acme.certs."surtr.yggdrasil.li".directory}/fullchain.pem"
-      ] ++ concatMap (domain:
-        map (subdomain: "${subdomain}.full.pem:${config.security.acme.certs.${subdomain}.directory}/full.pem")
-          [domain "mailin.${domain}" "mailsub.${domain}"]
-      ) emailDomains;
+      serviceConfig.LoadCredential = let
+        tlsCredential = domain: "${domain}.full.pem:${config.security.acme.certs.${domain}.directory}/full.pem";
+      in [
+        (tlsCredential "surtr.yggdrasil.li")
+      ] ++ concatMap (domain: map tlsCredential [domain "mailin.${domain}" "mailsub.${domain}"]) emailDomains;
     };
 
-    systemd.services.dovecot2 = {
+    systemd.services.dovecot = {
       preStart = ''
         for f in /etc/dovecot/sieve_flag.d/*.sieve /etc/dovecot/sieve_before.d/*.sieve; do
           ${getExe' pkgs.dovecot_pigeonhole "sievec"} $f