summaryrefslogtreecommitdiff
path: root/hosts/surtr/prometheus/default.nix
blob: 685d117ba39114e555df01c72cd2dae8fd7fddd9 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
{ config, lib, pkgs, ... }:

with lib;

let
  relabelHosts = [
    { source_labels = ["__address__"];
      target_label = "instance";
      regex = "(localhost|127\.[0-9]+\.[0-9]+\.[0-9]+)(:[0-9]+)?";
      replacement = "surtr";
    }
  ];
in {
  config = {
    services.prometheus = {
      enable = true;

      exporters = {
        node = {
          enable = true;
          enabledCollectors = [];
        };
        unbound = {
          enable = true;
          controlInterface = "/run/unbound/unbound.ctl";
          group = config.services.unbound.group;
        };
        wireguard = {
          enable = true;
          wireguardConfig =
            let
              keys = {
                "sif" = ["yioRagUtRvalJLrTtLp8NPiym6a3RpIcqgVfNL1iyRA=" "zIgyMw5wSernKPmMfDZ+fqaYUjbIQUhsXe+7hIZgJho="];
                "surtr" = ["YP/sWEUWw51czlGxvgrgyEZ+ssx/3C9siufgd0a8d3g=" "6V2EjwvZ07Pebc9g9TNqIlQu57MvqyUsCeIOzky4Txw="];
                "vidhar" = ["IOuHpNQ2ff09HCPKtKY95lDXoRhd8FIBsbB8kaMeUUA=" "jdaF4sx+dhdkTNGxQI6g6JV4XwXgD9QQJQ4f0NYy1gY=" "moESFbO3qUTuoOv6lbzSLrNYSjHkM5hyvAs5XZtQzRA="];
              };
            in pkgs.writeText "wireguard-config" (concatMapStringsSep "\n" ({ name, value }: ''
              [Peer]
              # friendly_name = ${name}
              PublicKey = ${value}
              AllowedIPs = ::/0
            '') (concatLists (mapAttrsToList (host: hostKeys: map (nameValuePair host) hostKeys) keys)));
        };
        blackbox = {
          enable = true;
          configFile = pkgs.writeText "blackbox-config.yaml" (builtins.toJSON {
            modules = {
              "dns_soa" = {
                prober = "dns";
                dns = {
                  query_name = ".";
                  query_type = "SOA";
                };
              };
            };
          });
        };
      };

      globalConfig = {
        evaluation_interval = "1s";
      };

      remoteWrite = [
        {
          url = "https://prometheus.vidhar.yggdrasil/api/v1/write";
          name = "vidhar";
          tls_config = {
            ca_file = toString ../../vidhar/prometheus/ca/ca.crt;
            cert_file = toString ./tls.crt;
            key_file = "/run/credentials/prometheus.service/tls.key";
          };
        }
      ];

      scrapeConfigs = [
        { job_name = "prometheus";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "node";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "unbound";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.unbound.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "wireguard";
          static_configs = [
            { targets = ["localhost:${toString config.services.prometheus.exporters.wireguard.port}"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "nftables";
          static_configs = [
            { targets = ["localhost:9901"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "1s";
        }
        { job_name = "blackbox";
          metrics_path = "/probe";
          params = { module = ["dns_soa"]; };
          static_configs = [
            { targets = ["127.0.0.53:53" "127.0.0.1:5353"]; }
          ];
          relabel_configs = [
            { source_labels = ["__address__"];
              target_label = "__param_target";
            }
          ] ++ relabelHosts ++
          [ { source_labels = ["__param_target"];
              target_label = "job";
              regex = "127\.0\.0\.53:53";
              replacement = "systemd-resolved.dns_soa";
            }
            { source_labels = ["__param_target"];
              target_label = "job";
              regex = "127\.0\.0\.1:5353";
              replacement = "unbound.dns_soa";
            }
            { replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}";
              target_label = "__address__";
            }
          ];
          scrape_interval = "5s";
        }
        { job_name = "synapse";
          metrics_path = "/_synapse/metrics";
          static_configs = [
            { targets = ["localhost:9092"]; }
          ];
          relabel_configs = relabelHosts;
          scrape_interval = "5s";
        }
      ];

      rules = [
        (generators.toYAML {} {
          groups = [
          ];
        })
      ];
      ruleFiles = [
        ./synapse-v2.rules
      ];
    };
    users.users.${config.services.prometheus.exporters.unbound.user} = {
      description = "Prometheus unbound exporter service user";
      isSystemUser = true;
      group = config.services.unbound.group;
    };
    systemd.services."prometheus-unbound-exporter".serviceConfig = {
      DynamicUser = false;
    };

    systemd.services."prometheus-nftables-exporter" = {
      wantedBy = [ "multi-user.target" ];
      after = [ "network.target" ];
      path = with pkgs; [ nftables ];
      serviceConfig = {
        Restart = "always";

        PrivateTmp = true;
        WorkingDirectory = "/tmp";
        CapabilityBoundingSet = ["CAP_NET_ADMIN"];
        DynamicUser = true;
        DeviceAllow = [""];
        LockPersonality = true;
        MemoryDenyWriteExecute = true;
        NoNewPrivileges = true;
        PrivateDevices = true;
        ProtectClock = true;
        ProtectControlGroups = true;
        ProtectHome = true;
        ProtectHostname = true;
        ProtectKernelLogs = true;
        ProtectKernelModules = true;
        ProtectKernelTunables = true;
        ProtectSystem = "strict";
        RemoveIPC = true;
        RestrictNamespaces = true;
        RestrictRealtime = true;
        RestrictSUIDSGID = true;
        SystemCallArchitectures = "native";
        UMask = "0077";
        AmbientCapabilities = [ "CAP_NET_ADMIN" ];

        Type = "simple";
        ExecStart = "${pkgs.nftables-prometheus-exporter}/bin/nftables-prometheus-exporter";
        Environment = "NFT_HOSTNAME=localhost NFT_PORT=9901";
      };
    };

    systemd.services.prometheus = {
      serviceConfig = {
        SystemCallFilter = mkForce [ "@system-service" "~@privileged" ];
      };
    };

    sops.secrets."prometheus.key" = {
      format = "binary";
      sopsFile = ./tls.key;
    };

    systemd.services.prometheus.serviceConfig.LoadCredential = [
      "tls.key:${config.sops.secrets."prometheus.key".path}"
    ];
  };
}