{ config, lib, pkgs, utils, ... }:

with lib;

let
  relabelHosts = [
    { source_labels = ["__address__"];
      target_label = "instance";
      regex = "(localhost|127\.[0-9]+\.[0-9]+\.[0-9]+)(:[0-9]+)?";
      replacement = "vidhar";
    }
    { source_labels = ["__address__"];
      target_label = "instance";
      regex = "10.141.1.2(:[0-9]+)?";
      replacement = "switch01";
    }
    { source_labels = ["__address__"];
      target_label = "instance";
      regex = "10.141.1.4(:[0-9]+)?";
      replacement = "ap01";
    }
  ];
in {
  config = {
    services.prometheus = {
      enable = true;

      extraFlags = [
        "--enable-feature=remote-write-receiver"
      ];

      exporters = {
        node = {
          enable = true;
          enabledCollectors = [];
        };
        snmp = {
          enable = true;
          configurationPath = ./snmp.yml;
        };
        unbound = {
          enable = true;
          unbound.host = "unix:///run/unbound/unbound.ctl";
          group = config.services.unbound.group;
        };
        wireguard = {
          enable = true;
          wireguardConfig =
            let
              keys = {
                "sif" = ["yioRagUtRvalJLrTtLp8NPiym6a3RpIcqgVfNL1iyRA=" "zIgyMw5wSernKPmMfDZ+fqaYUjbIQUhsXe+7hIZgJho="];
                "surtr" = ["YP/sWEUWw51czlGxvgrgyEZ+ssx/3C9siufgd0a8d3g=" "6V2EjwvZ07Pebc9g9TNqIlQu57MvqyUsCeIOzky4Txw=" "moESFbO3qUTuoOv6lbzSLrNYSjHkM5hyvAs5XZtQzRA="];
                "vidhar" = ["IOuHpNQ2ff09HCPKtKY95lDXoRhd8FIBsbB8kaMeUUA=" "jdaF4sx+dhdkTNGxQI6g6JV4XwXgD9QQJQ4f0NYy1gY="];
              };
            in pkgs.writeText "wireguard-config" (concatMapStringsSep "\n" ({ name, value }: ''
              [Peer]
              # friendly_name = ${name}
              PublicKey = ${value}
              AllowedIPs = ::/0
            '') (concatLists (mapAttrsToList (host: hostKeys: map (nameValuePair host) hostKeys) keys)));
        };
        apcupsd.enable = true;
        systemd = {
          enable = true;
          extraFlags = [
            "--systemd.collector.unit-include=(dhcpcd-.*|pppd-telekom|corerad|ndppd)\.service"
            "--systemd.collector.enable-restart-count"
            "--systemd.collector.enable-ip-accounting"
          ];
        };
        lvm.enable = true;
        blackbox = {
          enable = true;
          configFile = pkgs.writeText "blackbox-config.yaml" (builtins.toJSON {
            modules = {
              "dns_soa" = {
                prober = "dns";
                dns = {
                  query_name = ".";
                  query_type = "SOA";
                };
              };
            };
          });
        };
      };

      globalConfig = {
        evaluation_interval = "1s";
      };

      scrapeConfigs = [
        { job_name = "prometheus";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "node";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "lvm";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.lvm.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "5s";
        }
        { job_name = "systemd";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.systemd.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "smartctl";
          static_configs = [
            { targets = ["localhost:9633"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "60s";
        }
        { job_name = "snmp";
          static_configs = [
            { targets = ["10.141.1.2"]; }
          ];
          metrics_path = "/snmp";
          params = {
            module = ["if_mib"];
          };
          relabel_configs = [
            { source_labels = ["__address__"];
              target_label = "__param_target";
            }
          ] ++ relabelHosts ++
          [ { replacement = "localhost:${toString config.services.prometheus.exporters.snmp.port}";
              target_label = "__address__";
            }
          ];
          scrape_interval = "15s";
        }
        { job_name = "unbound";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.unbound.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "wireguard";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.wireguard.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "corerad";
          static_configs = [
            { targets = ["localhost:9430"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "nftables";
          static_configs = [
            { targets = ["localhost:9901"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "cake";
          static_configs = [
            { targets = ["localhost:9902"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "loki";
          static_configs = [
            { targets = ["localhost:9094"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "promtail";
          static_configs = [
            { targets = ["localhost:9080"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "apcupsd";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.apcupsd.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "openwrt";
          static_configs = [
            { targets = ["10.141.1.4:9100"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "5s";
        }
        { job_name = "blackbox";
          metrics_path = "/probe";
          params = { module = ["dns_soa"]; };
          static_configs = [
            { targets = ["127.0.0.1:53"]; }
          ];
          relabel_configs = [
            { source_labels = ["__address__"];
              target_label = "__param_target";
            }
          ] ++ relabelHosts ++
          [ { source_labels = ["__param_target"];
              target_label = "job";
              regex = "127\.0\.0\.1:53";
              replacement = "unbound.dns_soa";
            }
            { replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
              target_label = "__address__";
            }
          ];
          scrape_interval = "5s";
        }
        { job_name = "pushgateway";
          honor_labels = true;
          static_configs = [
            { targets = ["localhost:9091"];}
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "5s";
        }
      ];

      rules = [
        (generators.toYAML {} {
          groups = [
            { name = "lvm-with-info";
              rules = [
                { record = "lvm_lv_sync_percent_info";
                  expr = "lvm_lv_sync_percent * on (lv_uuid) group_left(instance,lv_full_name) lvm_lv_info";
                }
              ];
            }
            { name = "systemd-starts";
              rules = [
                { record = "systemd_unit_runtime_seconds";
                  expr = "timestamp(systemd_unit_start_time_seconds) - (systemd_unit_start_time_seconds > 0)";
                }
                { record = "systemd_unit_runtime_seconds:resets_per_hour";
                  expr = "resets(systemd_unit_runtime_seconds[1h])";
                }
                { record = "systemd_unit_runtime_seconds:resets_per_day";
                  expr = "resets(systemd_unit_runtime_seconds[1d])";
                }
                { record = "systemd_unit_runtime_seconds:resets_per_week";
                  expr = "resets(systemd_unit_runtime_seconds[1w])";
                }
                { record = "systemd_unit_runtime_seconds:avg_resets_per_day";
                  expr = "avg_over_time(systemd_unit_runtime_seconds:resets_per_day[1w])";
                }
              ];
            }
            { name = "apcupsd-transfers";
              rules = [
                { record = "apcupsd_last_transfer_on_battery_counter_seconds";
                  expr = "timestamp(apcupsd_last_transfer_on_battery_time_seconds) - (apcupsd_last_transfer_on_battery_time_seconds > 0)";
                }
                { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_hour";
                  expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1h])";
                }
                { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_day";
                  expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1d])";
                }
                { record = "apcupsd_last_transfer_on_battery_counter_seconds:resets_per_week";
                  expr = "resets(apcupsd_last_transfer_on_battery_counter_seconds[1w])";
                }
                { record = "apcupsd_last_transfer_on_battery_counter_seconds:avg_resets_per_day";
                  expr = "avg_over_time(apcupsd_last_transfer_on_battery_counter_seconds:resets_per_day[1w])";
                }
              ];
            }
          ];
        })
      ];

      pushgateway = {
        enable = true;
      };
    };

    users.users.${config.services.prometheus.exporters.unbound.user} = {
      description = "Prometheus unbound exporter service user";
      isSystemUser = true;
      group = config.services.unbound.group;
    };
    systemd.services."prometheus-unbound-exporter" = {
      wantedBy = [ "unbound.service" ];
      serviceConfig = {
        DynamicUser = false;
        Restart = lib.mkForce "always";
        RestartSec = "5";
      };
    };

    systemd.services."prometheus-nftables-exporter" = {
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];
      path = with pkgs; [ nftables ];
      serviceConfig = {
        Restart = "always";

        PrivateTmp = true;
        WorkingDirectory = "/tmp";
        CapabilityBoundingSet = ["CAP_NET_ADMIN"];
        DynamicUser = true;
        DeviceAllow = [""];
        LockPersonality = true;
        MemoryDenyWriteExecute = true;
        NoNewPrivileges = true;
        PrivateDevices = true;
        ProtectClock = true;
        ProtectControlGroups = true;
        ProtectHome = true;
        ProtectHostname = true;
        ProtectKernelLogs = true;
        ProtectKernelModules = true;
        ProtectKernelTunables = true;
        ProtectSystem = "strict";
        RemoveIPC = true;
        RestrictNamespaces = true;
        RestrictRealtime = true;
        RestrictSUIDSGID = true;
        SystemCallArchitectures = "native";
        UMask = "0077";
        AmbientCapabilities = [ "CAP_NET_ADMIN" ];

        Type = "simple";
        ExecStart = "${pkgs.nftables-prometheus-exporter}/bin/nftables-prometheus-exporter";
        Environment = "NFT_HOSTNAME=localhost NFT_PORT=9901";
      };
    };

    systemd.services."prometheus-smartctl-exporter" = {
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];
      path = with pkgs; [ smartmontools ];
      serviceConfig = {
        Restart = "always";

        CapabilityBoundingSet = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"];
        AmbientCapabilities = ["CAP_DAC_OVERRIDE" "CAP_SYS_RAWIO" "CAP_SYS_ADMIN"];
        ProtectSystem = "strict";
        DynamicUser = true;
        LockPersonality = true;
        MemoryDenyWriteExecute = true;
        NoNewPrivileges = true;
        PrivateDevices = false;
        PrivateTmp = true;
        ProcSubset = "pid";

        Type = "simple";
        ExecStart = "${pkgs.smartprom}/bin/smartprom";
        Environment = "SMARTCTL_EXPORTER_PORT=9633";
      };
    };

    systemd.services."prometheus-systemd-exporter" = let
      cfg = config.services.prometheus.exporters.systemd;
    in {
      serviceConfig = {
        ExecStart = mkForce "${pkgs.prometheus-systemd-exporter}/bin/systemd_exporter ${utils.escapeSystemdExecArgs ([
          "--web.listen-address" "${cfg.listenAddress}:${toString cfg.port}"
        ] ++ cfg.extraFlags)}";
        ReadWritePaths = ["/run/dbus/system_bus_socket"];
        BindReadOnlyPaths = ["/proc" "/sys"];
        CapabilityBoundingSet = ["CAP_DAC_OVERRIDE"];
        AmbientCapabilities = ["CAP_DAC_OVERRIDE"];
      };
    };

    systemd.services."prometheus-cake-exporter" = {
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];
      path = with pkgs; [ iproute2 ];
      serviceConfig = {
        Restart = "always";

        PrivateTmp = true;
        WorkingDirectory = "/tmp";
        CapabilityBoundingSet = ["CAP_NET_ADMIN"];
        DynamicUser = true;
        DeviceAllow = [""];
        LockPersonality = true;
        MemoryDenyWriteExecute = true;
        NoNewPrivileges = true;
        PrivateDevices = true;
        ProtectClock = true;
        ProtectControlGroups = true;
        ProtectHome = true;
        ProtectHostname = true;
        ProtectKernelLogs = true;
        ProtectKernelModules = true;
        ProtectKernelTunables = true;
        ProtectSystem = "strict";
        RemoveIPC = true;
        RestrictNamespaces = true;
        RestrictRealtime = true;
        RestrictSUIDSGID = true;
        SystemCallArchitectures = "native";
        UMask = "0077";
        AmbientCapabilities = [ "CAP_NET_ADMIN" ];

        Type = "simple";
        ExecStart = "${pkgs.cake-prometheus-exporter}/bin/cake-prometheus-exporter";
        Environment = "CAKE_HOSTNAME=localhost CAKE_PORT=9902";
      };
    };

    services.nginx = {
      upstreams.prometheus = {
        servers = { "localhost:${toString config.services.prometheus.port}" = {}; };
      };
      virtualHosts."prometheus.vidhar.yggdrasil" = {
        forceSSL = true;
        sslCertificate = ./tls.crt;
        sslCertificateKey = "/run/credentials/nginx.service/prometheus.key";
        extraConfig = ''
          ssl_client_certificate ${./ca/ca.crt};
          ssl_trusted_certificate ${./ca/ca.crt};
          ssl_verify_client on;

          client_max_body_size    0;
        '';
        locations."/" = {
          proxyPass = "http://prometheus/";
          proxyWebsockets = true;
        };
      };
    };

    sops.secrets."prometheus.key" = {
      format = "binary";
      sopsFile = ./tls.key;
    };
    systemd.services.nginx.serviceConfig.LoadCredential = [
      "prometheus.key:${config.sops.secrets."prometheus.key".path}"
    ];
  };
}