{ config, lib, pkgs, utils, ... }: with lib; let relabelHosts = [ { source_labels = ["__address__"]; target_label = "instance"; regex = "(localhost|127\.[0-9]+\.[0-9]+\.[0-9]+)(:[0-9]+)?"; replacement = "vidhar"; } { source_labels = ["__address__"]; target_label = "instance"; regex = "10.141.1.2(:[0-9]+)?"; replacement = "switch01"; } { source_labels = ["__address__"]; target_label = "instance"; regex = "10.141.1.4(:[0-9]+)?"; replacement = "ap01"; } ]; in { config = { services.prometheus = { enable = true; extraFlags = [ "--enable-feature=remote-write-receiver" ]; exporters = { node = { enable = true; enabledCollectors = []; }; snmp = { enable = true; configurationPath = ./snmp.yml; }; unbound = { enable = true; controlInterface = "/run/unbound/unbound.ctl"; group = config.services.unbound.group; }; wireguard = { enable = true; wireguardConfig = let keys = { "sif" = ["yioRagUtRvalJLrTtLp8NPiym6a3RpIcqgVfNL1iyRA=" "zIgyMw5wSernKPmMfDZ+fqaYUjbIQUhsXe+7hIZgJho="]; "surtr" = ["YP/sWEUWw51czlGxvgrgyEZ+ssx/3C9siufgd0a8d3g=" "6V2EjwvZ07Pebc9g9TNqIlQu57MvqyUsCeIOzky4Txw=" "moESFbO3qUTuoOv6lbzSLrNYSjHkM5hyvAs5XZtQzRA="]; "vidhar" = ["IOuHpNQ2ff09HCPKtKY95lDXoRhd8FIBsbB8kaMeUUA=" "jdaF4sx+dhdkTNGxQI6g6JV4XwXgD9QQJQ4f0NYy1gY="]; }; in pkgs.writeText "wireguard-config" (concatMapStringsSep "\n" ({ name, value }: '' [Peer] # friendly_name = ${name} PublicKey = ${value} AllowedIPs = ::/0 '') (concatLists (mapAttrsToList (host: hostKeys: map (nameValuePair host) hostKeys) keys))); }; apcupsd.enable = true; systemd = { enable = true; extraFlags = [ "--systemd.collector.unit-include=(dhcpcd-dsl|pppd-telekom|corerad|ndppd)\.service" "--systemd.collector.enable-restart-count" "--systemd.collector.enable-ip-accounting" ]; }; lvm.enable = true; blackbox = { enable = true; configFile = pkgs.writeText "blackbox-config.yaml" (builtins.toJSON { modules = { "dns_soa" = { prober = "dns"; dns = { query_name = "."; query_type = "SOA"; }; }; }; }); }; }; globalConfig = { evaluation_interval = "1s"; }; scrapeConfigs = [ { job_name = "prometheus"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "node"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "lvm"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.lvm.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "5s"; } { job_name = "systemd"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.systemd.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "smartctl"; static_configs = [ { targets = ["localhost:9633"]; } ]; relabel_configs = relabelHosts; scrape_interval = "60s"; } { job_name = "snmp"; static_configs = [ { targets = ["10.141.1.2"]; } ]; metrics_path = "/snmp"; params = { module = ["if_mib"]; }; relabel_configs = [ { source_labels = ["__address__"]; target_label = "__param_target"; } ] ++ relabelHosts ++ [ { replacement = "localhost:${toString config.services.prometheus.exporters.snmp.port}"; target_label = "__address__"; } ]; scrape_interval = "15s"; } { job_name = "zte"; static_configs = [ { targets = ["localhost:9900"]; } ]; relabel_configs = [ { replacement = "dsl01"; target_label = "instance"; } ]; scrape_interval = "15s"; } { job_name = "unbound"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.unbound.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "wireguard"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.wireguard.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "corerad"; static_configs = [ { targets = ["localhost:9430"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "nftables"; static_configs = [ { targets = ["localhost:9901"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "cake"; static_configs = [ { targets = ["localhost:9902"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "loki"; static_configs = [ { targets = ["localhost:9094"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "promtail"; static_configs = [ { targets = ["localhost:9080"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "apcupsd"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.apcupsd.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "openwrt"; static_configs = [ { targets = ["10.141.1.4:9100"]; } ]; relabel_configs = relabelHosts; scrape_interval = "5s"; } { job_name = "blackbox"; metrics_path = "/probe"; params = { module = ["dns_soa"]; }; static_configs = [ { targets = ["127.0.0.1:53"]; } ]; relabel_configs = [ { source_labels = ["__address__"]; target_label = "__param_target"; } ] ++ relabelHosts ++ [ { source_labels = ["__param_target"]; target_label = "job"; regex = "127\.0\.0\.1:53"; replacement = "unbound.dns_soa"; } { replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}"; target_label = "__address__"; } ]; scrape_interval = "5s"; } { job_name = "pushgateway"; honor_labels = true; static_configs = [ { targets = ["localhost:9091"];} ]; relabel_configs = relabelHosts; scrape_interval = "5s"; } ]; rules = [ (generators.toYAML {} { groups = [ { name = "lvm-with-info"; rules = [ { record = "lvm_lv_sync_percent_info"; expr = "lvm_lv_sync_percent * on (lv_uuid) group_left(instance,lv_full_name) lvm_lv_info"; } ]; } { name = "systemd-starts"; rules = [ { record = "systemd_unit_runtime_seconds"; expr = "timestamp(systemd_unit_start_time_seconds) - (systemd_unit_start_time_seconds > 0)"; } { record = "systemd_unit_runtime_seconds:resets_per_hour"; expr = "resets(systemd_unit_runtime_seconds[1h])"; } { record = "systemd_unit_runtime_seconds:resets_per_day"; expr = "resets(systemd_unit_runtime_seconds[1d])"; } { record = "systemd_unit_runtime_seconds:resets_per_week"; expr = "resets(systemd_unit_runtime_seconds[1w])"; } ]; } { name = "apcupsd-transfers"; rules = [ { record = "apcupsd_last_transfer_on_battery_counter_seconds"; expr = "timestamp(apcupsd_last_transfer_on_battery_time_seconds) - (apcupsd_last_transfer_on_battery_time_seconds > 0)"; } { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_hour"; expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1h])"; } { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_day"; expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1d])"; } { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_week"; expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1w])"; } ]; } ]; }) ]; pushgateway = { enable = true; }; }; users.users.${config.services.prometheus.exporters.unbound.user} = { description = "Prometheus unbound exporter service user"; isSystemUser = true; group = config.services.unbound.group; }; systemd.services."prometheus-unbound-exporter" = { wantedBy = [ "unbound.service" ]; serviceConfig = { DynamicUser = false; Restart = lib.mkForce "always"; RestartSec = "5"; }; }; systemd.services."prometheus-zte-exporter@dsl01.mgmt.yggdrasil" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; serviceConfig = { Restart = "always"; PrivateTmp = true; WorkingDirectory = "/tmp"; DynamicUser = true; CapabilityBoundingSet = [""]; DeviceAllow = [""]; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = true; ProtectClock = true; ProtectControlGroups = true; ProtectHome = true; ProtectHostname = true; ProtectKernelLogs = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RemoveIPC = true; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; SystemCallArchitectures = "native"; UMask = "0077"; Type = "simple"; ExecStart = "${pkgs.zte-prometheus-exporter}/bin/zte-prometheus-exporter"; Environment = "ZTE_BASEURL=http://%I ZTE_HOSTNAME=localhost ZTE_PORT=9900"; EnvironmentFile = config.sops.secrets."zte_dsl01.mgmt.yggdrasil".path; }; }; sops.secrets."zte_dsl01.mgmt.yggdrasil" = { format = "binary"; sopsFile = ./zte_dsl01.mgmt.yggdrasil; }; systemd.services."prometheus-nftables-exporter" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; path = with pkgs; [ nftables ]; serviceConfig = { Restart = "always"; PrivateTmp = true; WorkingDirectory = "/tmp"; CapabilityBoundingSet = ["CAP_NET_ADMIN"]; DynamicUser = true; DeviceAllow = [""]; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = true; ProtectClock = true; ProtectControlGroups = true; ProtectHome = true; ProtectHostname = true; ProtectKernelLogs = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RemoveIPC = true; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; SystemCallArchitectures = "native"; UMask = "0077"; AmbientCapabilities = [ "CAP_NET_ADMIN" ]; Type = "simple"; ExecStart = "${pkgs.nftables-prometheus-exporter}/bin/nftables-prometheus-exporter"; Environment = "NFT_HOSTNAME=localhost NFT_PORT=9901"; }; }; systemd.services."prometheus-smartctl-exporter" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; path = with pkgs; [ smartmontools ]; serviceConfig = { Restart = "always"; CapabilityBoundingSet = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"]; AmbientCapabilities = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"]; ProtectSystem = "strict"; DynamicUser = true; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = false; PrivateTmp = true; ProcSubset = "pid"; Type = "simple"; ExecStart = "${pkgs.smartprom}/bin/smartprom"; Environment = "SMARTCTL_EXPORTER_PORT=9633"; }; }; systemd.services."prometheus-systemd-exporter" = let cfg = config.services.prometheus.exporters.systemd; in { serviceConfig = { ExecStart = mkForce "${pkgs.prometheus-systemd-exporter}/bin/systemd_exporter ${utils.escapeSystemdExecArgs ([ "--web.listen-address" "${cfg.listenAddress}:${toString cfg.port}" ] ++ cfg.extraFlags)}"; ReadWritePaths = ["/run/dbus/system_bus_socket"]; BindReadOnlyPaths = ["/proc" "/sys"]; CapabilityBoundingSet = ["CAP_DAC_OVERRIDE"]; AmbientCapabilities = ["CAP_DAC_OVERRIDE"]; }; }; systemd.services."prometheus-cake-exporter" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; path = with pkgs; [ iproute2 ]; serviceConfig = { Restart = "always"; PrivateTmp = true; WorkingDirectory = "/tmp"; CapabilityBoundingSet = ["CAP_NET_ADMIN"]; DynamicUser = true; DeviceAllow = [""]; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = true; ProtectClock = true; ProtectControlGroups = true; ProtectHome = true; ProtectHostname = true; ProtectKernelLogs = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RemoveIPC = true; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; SystemCallArchitectures = "native"; UMask = "0077"; AmbientCapabilities = [ "CAP_NET_ADMIN" ]; Type = "simple"; ExecStart = "${pkgs.cake-prometheus-exporter}/bin/cake-prometheus-exporter"; Environment = "CAKE_HOSTNAME=localhost CAKE_PORT=9902"; }; }; services.nginx = { upstreams.prometheus = { servers = { "localhost:${toString config.services.prometheus.port}" = {}; }; }; virtualHosts."prometheus.vidhar.yggdrasil" = { forceSSL = true; sslCertificate = ./tls.crt; sslCertificateKey = "/run/credentials/nginx.service/prometheus.key"; extraConfig = '' ssl_client_certificate ${./ca/ca.crt}; ssl_trusted_certificate ${./ca/ca.crt}; ssl_verify_client on; client_max_body_size 0; ''; locations."/" = { proxyPass = "http://prometheus/"; proxyWebsockets = true; }; }; }; sops.secrets."prometheus.key" = { format = "binary"; sopsFile = ./tls.key; }; systemd.services.nginx.serviceConfig.LoadCredential = [ "prometheus.key:${config.sops.secrets."prometheus.key".path}" ]; }; }