{ config, lib, pkgs, ... }: with lib; let relabelHosts = [ { source_labels = ["__address__"]; target_label = "instance"; regex = "(localhost|127\.[0-9]+\.[0-9]+\.[0-9]+)(:[0-9]+)?"; replacement = "surtr"; } ]; in { config = { services.prometheus = { enable = true; exporters = { node = { enable = true; enabledCollectors = []; }; unbound = { enable = true; controlInterface = "/run/unbound/unbound.ctl"; group = config.services.unbound.group; }; wireguard = { enable = true; wireguardConfig = let keys = { "sif" = ["yioRagUtRvalJLrTtLp8NPiym6a3RpIcqgVfNL1iyRA=" "zIgyMw5wSernKPmMfDZ+fqaYUjbIQUhsXe+7hIZgJho="]; "surtr" = ["YP/sWEUWw51czlGxvgrgyEZ+ssx/3C9siufgd0a8d3g=" "6V2EjwvZ07Pebc9g9TNqIlQu57MvqyUsCeIOzky4Txw="]; "vidhar" = ["IOuHpNQ2ff09HCPKtKY95lDXoRhd8FIBsbB8kaMeUUA=" "jdaF4sx+dhdkTNGxQI6g6JV4XwXgD9QQJQ4f0NYy1gY=" "moESFbO3qUTuoOv6lbzSLrNYSjHkM5hyvAs5XZtQzRA="]; }; in pkgs.writeText "wireguard-config" (concatMapStringsSep "\n" ({ name, value }: '' [Peer] # friendly_name = ${name} PublicKey = ${value} AllowedIPs = ::/0 '') (concatLists (mapAttrsToList (host: hostKeys: map (nameValuePair host) hostKeys) keys))); }; blackbox = { enable = true; configFile = pkgs.writeText "blackbox-config.yaml" (builtins.toJSON { modules = { "dns_soa" = { prober = "dns"; dns = { query_name = "."; query_type = "SOA"; }; }; }; }); }; }; globalConfig = { evaluation_interval = "1s"; }; remoteWrite = [ { url = "https://prometheus.vidhar.yggdrasil/api/v1/write"; name = "vidhar"; tls_config = { ca_file = toString ../../vidhar/prometheus/ca/ca.crt; cert_file = toString ./tls.crt; key_file = "/run/credentials/prometheus.service/tls.key"; }; } ]; scrapeConfigs = [ { job_name = "prometheus"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "node"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "unbound"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.unbound.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "wireguard"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.wireguard.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "nftables"; static_configs = [ { targets = ["localhost:9901"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "blackbox"; metrics_path = "/probe"; params = { module = ["dns_soa"]; }; static_configs = [ { targets = ["127.0.0.53:53" "127.0.0.1:5353"]; } ]; relabel_configs = [ { source_labels = ["__address__"]; target_label = "__param_target"; } ] ++ relabelHosts ++ [ { source_labels = ["__param_target"]; target_label = "job"; regex = "127\.0\.0\.53:53"; replacement = "systemd-resolved.dns_soa"; } { source_labels = ["__param_target"]; target_label = "job"; regex = "127\.0\.0\.1:5353"; replacement = "unbound.dns_soa"; } { replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}"; target_label = "__address__"; } ]; scrape_interval = "5s"; } { job_name = "synapse"; metrics_path = "/_synapse/metrics"; static_configs = [ { targets = ["localhost:9092"]; } ]; relabel_configs = relabelHosts; scrape_interval = "5s"; } ]; rules = [ (generators.toYAML {} { groups = [ ]; }) ]; ruleFiles = [ ./synapse-v2.rules ]; }; users.users.${config.services.prometheus.exporters.unbound.user} = { description = "Prometheus unbound exporter service user"; isSystemUser = true; group = config.services.unbound.group; }; systemd.services."prometheus-unbound-exporter".serviceConfig = { DynamicUser = false; }; systemd.services."prometheus-nftables-exporter" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; path = with pkgs; [ nftables ]; serviceConfig = { Restart = "always"; PrivateTmp = true; WorkingDirectory = "/tmp"; CapabilityBoundingSet = ["CAP_NET_ADMIN"]; DynamicUser = true; DeviceAllow = [""]; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = true; ProtectClock = true; ProtectControlGroups = true; ProtectHome = true; ProtectHostname = true; ProtectKernelLogs = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RemoveIPC = true; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; SystemCallArchitectures = "native"; UMask = "0077"; AmbientCapabilities = [ "CAP_NET_ADMIN" ]; Type = "simple"; ExecStart = "${pkgs.nftables-prometheus-exporter}/bin/nftables-prometheus-exporter"; Environment = "NFT_HOSTNAME=localhost NFT_PORT=9901"; }; }; systemd.services.prometheus = { serviceConfig = { SystemCallFilter = mkForce [ "@system-service" "~@privileged" ]; }; }; sops.secrets."prometheus.key" = { format = "binary"; sopsFile = ./tls.key; }; systemd.services.prometheus.serviceConfig.LoadCredential = [ "tls.key:${config.sops.secrets."prometheus.key".path}" ]; }; }