{ildkule/prometheus,base}: send stats over HTTPS through nginx

base/default.nix

@@ -14,7 +14,6 @@
 
     ./mitigations.nix
 
-    ./flake-input-exporter.nix
     ./hardening.nix
     ./networking.nix
     ./nix.nix
@@ -34,6 +33,7 @@
     ./services/openssh.nix
     ./services/polkit.nix
     ./services/postfix.nix
+    ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix
     ./services/roowho2.nix

base/flake-input-exporter.nix

@@ -1,55 +0,0 @@
-{
-  config,
-  inputs,
-  lib,
-  pkgs,
-  values,
-  ...
-}:
-let
-  data = lib.flip lib.mapAttrs inputs (
-    name: input: {
-      inherit (input)
-        lastModified
-        ;
-    }
-  );
-  folder = pkgs.writeTextDir "share/flake-inputs" (
-    lib.concatMapStringsSep "\n" (
-      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
-    ) (lib.attrsToList data)
-  );
-  port = 9102;
-in
-{
-  services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = {
-    serverName = config.networking.fqdn;
-    serverAliases = [
-      "${config.networking.hostName}.pvv.org"
-    ];
-    locations."/metrics" = {
-      root = "${folder}/share";
-      tryFiles = "/flake-inputs =404";
-      extraConfig = ''
-        default_type text/plain;
-      '';
-    };
-    listen = [
-      {
-        inherit port;
-        addr = "0.0.0.0";
-      }
-    ];
-    extraConfig = ''
-      allow ${values.hosts.ildkule.ipv4}/32;
-      allow ${values.hosts.ildkule.ipv6}/128;
-      allow 127.0.0.1/32;
-      allow ::1/128;
-      allow ${values.ipv4-space};
-      allow ${values.ipv6-space};
-      deny all;
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [ port ];
-}

base/services/nginx.nix

@@ -1,18 +1,5 @@
 { config, lib, ... }:
 {
-  # nginx return 444 for all nonexistent virtualhosts
-
-  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
-
-  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
-    "/etc/certs/nginx" = {
-      owner = "nginx";
-      group = "nginx";
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-
   services.nginx = {
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
@@ -60,17 +47,8 @@
           ];
         }
       ];
-      sslCertificate = "/etc/certs/nginx.crt";
-      sslCertificateKey = "/etc/certs/nginx.key";
-      addSSL = true;
-      extraConfig = "return 444;";
-    };
-
-    ${config.networking.fqdn} = {
-      sslCertificate = lib.mkDefault "/etc/certs/nginx.crt";
-      sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key";
-      addSSL = lib.mkDefault true;
-      extraConfig = lib.mkDefault "return 444;";
     };
   };
+
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
 }

base/services/prometheus-flake-input-exporter.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  values,
+  ...
+}:
+let
+  data = lib.flip lib.mapAttrs inputs (
+    name: input: {
+      inherit (input)
+        lastModified
+        ;
+    }
+  );
+  folder = pkgs.writeTextDir "share/flake-inputs" (
+    lib.concatMapStringsSep "\n" (
+      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
+    ) (lib.attrsToList data)
+  );
+in
+{
+  services.nginx = {
+    enable = lib.mkDefault true;
+
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-nixos-flake-input-exporter/metrics" = {
+        root = "${folder}/share";
+        tryFiles = "/flake-inputs =404";
+        extraConfig = ''
+          default_type text/plain;
+
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

base/services/prometheus-node-exporter.nix

@@ -5,19 +5,30 @@ in
 {
   services.prometheus.exporters.node = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9100;
     enabledCollectors = [ "systemd" ];
   };
 
-  systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable {
+  services.nginx = {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
-      "127.0.0.1"
-      "::1"
-      values.hosts.ildkule.ipv4
-      values.hosts.ildkule.ipv6
-    ];
-  };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-node-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -5,6 +5,7 @@ in
 {
   services.prometheus.exporters.systemd = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9101;
     extraFlags = [
       "--systemd.collector.enable-restart-count"
@@ -12,15 +13,25 @@ in
     ];
   };
 
-  systemd.services.prometheus-systemd-exporter.serviceConfig = {
+  services.nginx = {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
-      "127.0.0.1"
-      "::1"
-      values.hosts.ildkule.ipv4
-      values.hosts.ildkule.ipv6
-    ];
-  };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-systemd-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
 }

hosts/ildkule/services/monitoring/prometheus/machines.nix

@@ -6,32 +6,63 @@
     targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports;
   };
 
+  nixosMachines = [
+    "ildkule"
+    "bekkalokk"
+    "bicep"
+    "brzeczyszczykiewicz"
+    "georg"
+    "gluttony"
+    "kommode"
+    "lupine-1"
+    "lupine-2"
+    "lupine-3"
+    "lupine-4"
+    "lupine-5"
+    # TODO: export prometheus stats via apache on temmie
+    # "temmie"
+    "wenche"
+  ];
+
   defaultNodeExporterPort = 9100;
-  defaultSystemdExporterPort = 9101;
-  defaultNixosExporterPort = 9102;
 in {
-  services.prometheus.scrapeConfigs = [{
+  services.prometheus.scrapeConfigs = [
-    job_name = "base_info";
+    {
-    static_configs = [
+      job_name = "nixos-node";
-      (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ])
+      scheme = "https";
-
+      metrics_path = "/prometheus-node-exporter/metrics";
-      (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      static_configs = map (name: {
-      (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+        labels.hostname = name;
-      (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+        targets = [ "${name}.pvv.ntnu.no:443" ];
-      (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      }) nixosMachines;
-      (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+    }
-      (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+    {
-      (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      job_name = "nixos-systemd";
-      (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      scheme = "https";
-      (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      metrics_path = "/prometheus-systemd-exporter/metrics";
-      (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      static_configs = map (name: {
-      (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+        labels.hostname = name;
-      (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+        targets = [ "${name}.pvv.ntnu.no:443" ];
-      (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      }) nixosMachines;
-
+    }
-      (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
+    {
-      (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
+      job_name = "nixos-flake-input";
-      (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
+      scheme = "https";
-    ];
+      metrics_path = "/prometheus-nixos-flake-input-exporter/metrics";
-  }];
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "non-nixos-node";
+      scheme = "http";
+      metrics_path = "/metrics";
+      static_configs = [
+        (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
+        (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
+        (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
+      ];
+    }
+  ];
 }