{ config, lib, pkgs, ... }: with lib; let relabelHosts = [ { source_labels = ["__address__"]; target_label = "instance"; regex = "(localhost|127\.[0-9]+\.[0-9]+\.[0-9]+)(:[0-9]+)?"; replacement = "vidhar"; } { source_labels = ["__address__"]; target_label = "instance"; regex = "10.141.1.2(:[0-9]+)?"; replacement = "switch01"; } { source_labels = ["__address__"]; target_label = "instance"; regex = "10.141.1.4(:[0-9]+)?"; replacement = "ap01"; } ]; in { config = { services.prometheus = { enable = true; extraFlags = [ "--enable-feature=remote-write-receiver" ]; exporters = { node = { enable = true; enabledCollectors = []; }; smartctl = { enable = true; devices = map (dev: "/dev/disk/by-path/${dev}") [ "pci-0000:00:1f.2-ata-1" "pci-0000:00:1f.2-ata-3" "pci-0000:00:1f.2-ata-4" "pci-0000:00:1f.2-ata-5" "pci-0000:00:1f.2-ata-6" "pci-0000:02:00.0-nvme-1" "pci-0000:05:00.0-sas-phy0-lun-0" "pci-0000:05:00.0-sas-phy1-lun-0" "pci-0000:06:00.0-nvme-1" ]; }; snmp = { enable = true; configurationPath = ./snmp.yml; }; unbound = { enable = true; controlInterface = "/run/unbound/unbound.ctl"; group = config.services.unbound.group; }; wireguard = { enable = true; wireguardConfig = let keys = { "sif" = ["yioRagUtRvalJLrTtLp8NPiym6a3RpIcqgVfNL1iyRA=" "zIgyMw5wSernKPmMfDZ+fqaYUjbIQUhsXe+7hIZgJho="]; "surtr" = ["YP/sWEUWw51czlGxvgrgyEZ+ssx/3C9siufgd0a8d3g=" "6V2EjwvZ07Pebc9g9TNqIlQu57MvqyUsCeIOzky4Txw="]; "vidhar" = ["IOuHpNQ2ff09HCPKtKY95lDXoRhd8FIBsbB8kaMeUUA=" "jdaF4sx+dhdkTNGxQI6g6JV4XwXgD9QQJQ4f0NYy1gY="]; }; in pkgs.writeText "wireguard-config" (concatMapStringsSep "\n" ({ name, value }: '' [Peer] # friendly_name = ${name} PublicKey = ${value} AllowedIPs = ::/0 '') (concatLists (mapAttrsToList (host: hostKeys: map (nameValuePair host) hostKeys) keys))); }; apcupsd.enable = true; systemd = { enable = true; extraFlags = [ "--collector.unit-whitelist=(dhcpcd-dsl|pppd-telekom|corerad|ndppd)\.service" ]; }; blackbox = { enable = true; configFile = pkgs.writeText "blackbox-config.yaml" (builtins.toJSON { modules = { "dns_soa" = { prober = "dns"; dns = { query_name = "."; query_type = "SOA"; }; }; }; }); }; }; globalConfig = { evaluation_interval = "1s"; }; scrapeConfigs = [ { job_name = "prometheus"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "node"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "systemd"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.systemd.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "smartctl"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.smartctl.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "snmp"; static_configs = [ { targets = ["10.141.1.2"]; } ]; metrics_path = "/snmp"; params = { module = ["if_mib"]; }; relabel_configs = [ { source_labels = ["__address__"]; target_label = "__param_target"; } ] ++ relabelHosts ++ [ { replacement = "localhost:${toString config.services.prometheus.exporters.snmp.port}"; target_label = "__address__"; } ]; scrape_interval = "15s"; } { job_name = "zte"; static_configs = [ { targets = ["localhost:9900"]; } ]; relabel_configs = [ { replacement = "telekom"; target_label = "instance"; } ]; scrape_interval = "15s"; } { job_name = "unbound"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.unbound.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "wireguard"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.wireguard.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "corerad"; static_configs = [ { targets = ["localhost:9430"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "nftables"; static_configs = [ { targets = ["localhost:9901"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "loki"; static_configs = [ { targets = ["localhost:9094"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "promtail"; static_configs = [ { targets = ["localhost:9080"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "apcupsd"; static_configs = [ { targets = ["localhost:${toString config.services.prometheus.exporters.apcupsd.port}"]; } ]; relabel_configs = relabelHosts; scrape_interval = "1s"; } { job_name = "openwrt"; static_configs = [ { targets = ["10.141.1.4:9100"]; } ]; relabel_configs = relabelHosts; scrape_interval = "5s"; } { job_name = "blackbox"; metrics_path = "/probe"; params = { module = ["dns_soa"]; }; static_configs = [ { targets = ["127.0.0.53:53" "127.0.0.1:53"]; } ]; relabel_configs = [ { source_labels = ["__address__"]; target_label = "__param_target"; } ] ++ relabelHosts ++ [ { source_labels = ["__param_target"]; target_label = "instance"; regex = "127\.0\.0\.53:53"; replacement = "systemd-resolved.dns_soa"; } { source_labels = ["__param_target"]; target_label = "instance"; regex = "127\.0\.0\.1:53"; replacement = "unbound.dns_soa"; } { replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}"; target_label = "__address__"; } ]; scrape_interval = "5s"; } ]; rules = [ (generators.toYAML {} { groups = [ { name = "systemd-starts"; rules = [ { record = "systemd_unit_runtime_seconds"; expr = "timestamp(systemd_unit_start_time_seconds) - (systemd_unit_start_time_seconds > 0)"; } { record = "systemd_unit_runtime_seconds:resets_per_hour"; expr = "resets(systemd_unit_runtime_seconds[1h])"; } { record = "systemd_unit_runtime_seconds:resets_per_day"; expr = "resets(systemd_unit_runtime_seconds[1d])"; } { record = "systemd_unit_runtime_seconds:resets_per_week"; expr = "resets(systemd_unit_runtime_seconds[1w])"; } ]; } { name = "apcupsd-transfers"; rules = [ { record = "apcupsd_last_transfer_on_battery_counter_seconds"; expr = "timestamp(apcupsd_last_transfer_on_battery_time_seconds) - (apcupsd_last_transfer_on_battery_time_seconds > 0)"; } { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_hour"; expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1h])"; } { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_day"; expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1d])"; } { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_week"; expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1w])"; } ]; } ]; }) ]; }; users.users.${config.services.prometheus.exporters.unbound.user} = { description = "Prometheus unbound exporter service user"; isSystemUser = true; group = config.services.unbound.group; }; systemd.services."prometheus-unbound-exporter".serviceConfig = { DynamicUser = false; }; systemd.services."prometheus-zte-exporter@10.141.1.3" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; serviceConfig = { Restart = "always"; PrivateTmp = true; WorkingDirectory = "/tmp"; DynamicUser = true; CapabilityBoundingSet = [""]; DeviceAllow = [""]; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = true; ProtectClock = true; ProtectControlGroups = true; ProtectHome = true; ProtectHostname = true; ProtectKernelLogs = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RemoveIPC = true; RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; SystemCallArchitectures = "native"; UMask = "0077"; Type = "simple"; ExecStart = "${pkgs.zte-prometheus-exporter}/bin/zte-prometheus-exporter"; Environment = "ZTE_BASEURL=http://%I ZTE_HOSTNAME=localhost ZTE_PORT=9900"; EnvironmentFile = config.sops.secrets."zte_10.141.1.3".path; }; }; sops.secrets."zte_10.141.1.3" = { format = "binary"; sopsFile = ./zte_10.141.1.3; }; systemd.services."prometheus-nftables-exporter" = { wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; path = with pkgs; [ nftables ]; serviceConfig = { Restart = "always"; PrivateTmp = true; WorkingDirectory = "/tmp"; CapabilityBoundingSet = ["CAP_NET_ADMIN"]; DynamicUser = true; DeviceAllow = [""]; LockPersonality = true; MemoryDenyWriteExecute = true; NoNewPrivileges = true; PrivateDevices = true; ProtectClock = true; ProtectControlGroups = true; ProtectHome = true; ProtectHostname = true; ProtectKernelLogs = true; ProtectKernelModules = true; ProtectKernelTunables = true; ProtectSystem = "strict"; RemoveIPC = true; RestrictNamespaces = true; RestrictRealtime = true; RestrictSUIDSGID = true; SystemCallArchitectures = "native"; UMask = "0077"; AmbientCapabilities = [ "CAP_NET_ADMIN" ]; Type = "simple"; ExecStart = "${pkgs.nftables-prometheus-exporter}/bin/nftables-prometheus-exporter"; Environment = "NFT_HOSTNAME=localhost NFT_PORT=9901"; }; }; systemd.services."prometheus-systemd-exporter" = let cfg = config.services.prometheus.exporters.systemd; in { serviceConfig.ExecStart = mkForce '' ${pkgs.prometheus-systemd-exporter}/bin/systemd_exporter \ --web.listen-address ${cfg.listenAddress}:${toString cfg.port} ${concatStringsSep " " cfg.extraFlags} ''; }; systemd.services."prometheus-smartctl-exporter" = { serviceConfig = { DeviceAllow = lib.mkForce config.services.prometheus.exporters.smartctl.devices; CapabilityBoundingSet = lib.mkForce ["CAP_SYS_ADMIN"]; AmbientCapabilities = lib.mkForce ["CAP_SYS_ADMIN"]; }; }; }; }