diff --git a/base/default.nix b/base/default.nix index 0fc64da..c99cd20 100644 --- a/base/default.nix +++ b/base/default.nix @@ -14,7 +14,6 @@ ./mitigations.nix - ./flake-input-exporter.nix ./hardening.nix ./networking.nix ./nix.nix @@ -34,6 +33,7 @@ ./services/openssh.nix ./services/polkit.nix ./services/postfix.nix + ./services/prometheus-flake-input-exporter.nix ./services/prometheus-node-exporter.nix ./services/prometheus-systemd-exporter.nix ./services/roowho2.nix diff --git a/base/flake-input-exporter.nix b/base/flake-input-exporter.nix deleted file mode 100644 index dcb0016..0000000 --- a/base/flake-input-exporter.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ - config, - inputs, - lib, - pkgs, - values, - ... -}: -let - data = lib.flip lib.mapAttrs inputs ( - name: input: { - inherit (input) - lastModified - ; - } - ); - folder = pkgs.writeTextDir "share/flake-inputs" ( - lib.concatMapStringsSep "\n" ( - { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}'' - ) (lib.attrsToList data) - ); - port = 9102; -in -{ - services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = { - serverName = config.networking.fqdn; - serverAliases = [ - "${config.networking.hostName}.pvv.org" - ]; - locations."/metrics" = { - root = "${folder}/share"; - tryFiles = "/flake-inputs =404"; - extraConfig = '' - default_type text/plain; - ''; - }; - listen = [ - { - inherit port; - addr = "0.0.0.0"; - } - ]; - extraConfig = '' - allow ${values.hosts.ildkule.ipv4}/32; - allow ${values.hosts.ildkule.ipv6}/128; - allow 127.0.0.1/32; - allow ::1/128; - allow ${values.ipv4-space}; - allow ${values.ipv6-space}; - deny all; - ''; - }; - - networking.firewall.allowedTCPPorts = [ port ]; -} diff --git a/base/services/nginx.nix b/base/services/nginx.nix index 9053c09..b31e7c6 100644 --- a/base/services/nginx.nix +++ b/base/services/nginx.nix @@ -1,18 +1,5 @@ { config, lib, ... }: { - # nginx return 444 for all nonexistent virtualhosts - - systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ]; - - environment.snakeoil-certs = lib.mkIf config.services.nginx.enable { - "/etc/certs/nginx" = { - owner = "nginx"; - group = "nginx"; - }; - }; - - networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ]; - services.nginx = { recommendedTlsSettings = true; recommendedProxySettings = true; @@ -60,17 +47,8 @@ ]; } ]; - sslCertificate = "/etc/certs/nginx.crt"; - sslCertificateKey = "/etc/certs/nginx.key"; - addSSL = true; - extraConfig = "return 444;"; - }; - - ${config.networking.fqdn} = { - sslCertificate = lib.mkDefault "/etc/certs/nginx.crt"; - sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key"; - addSSL = lib.mkDefault true; - extraConfig = lib.mkDefault "return 444;"; }; }; + + networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ]; } diff --git a/base/services/prometheus-flake-input-exporter.nix b/base/services/prometheus-flake-input-exporter.nix new file mode 100644 index 0000000..56d7655 --- /dev/null +++ b/base/services/prometheus-flake-input-exporter.nix @@ -0,0 +1,47 @@ +{ + config, + inputs, + lib, + pkgs, + values, + ... +}: +let + data = lib.flip lib.mapAttrs inputs ( + name: input: { + inherit (input) + lastModified + ; + } + ); + folder = pkgs.writeTextDir "share/flake-inputs" ( + lib.concatMapStringsSep "\n" ( + { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}'' + ) (lib.attrsToList data) + ); +in +{ + services.nginx = { + enable = lib.mkDefault true; + + virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable { + forceSSL = true; + enableACME = true; + kTLS = true; + + locations."/prometheus-nixos-flake-input-exporter/metrics" = { + root = "${folder}/share"; + tryFiles = "/flake-inputs =404"; + extraConfig = '' + default_type text/plain; + + allow 127.0.0.1; + allow ::1; + allow ${values.hosts.ildkule.ipv4}; + allow ${values.hosts.ildkule.ipv6}; + deny all; + ''; + }; + }; + }; +} diff --git a/base/services/prometheus-node-exporter.nix b/base/services/prometheus-node-exporter.nix index bdacdb1..24d5843 100644 --- a/base/services/prometheus-node-exporter.nix +++ b/base/services/prometheus-node-exporter.nix @@ -5,19 +5,30 @@ in { services.prometheus.exporters.node = { enable = lib.mkDefault true; + listenAddress = "127.0.0.1"; port = 9100; enabledCollectors = [ "systemd" ]; }; - systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable { - IPAddressDeny = "any"; - IPAddressAllow = [ - "127.0.0.1" - "::1" - values.hosts.ildkule.ipv4 - values.hosts.ildkule.ipv6 - ]; - }; + services.nginx = { + enable = lib.mkDefault true; - networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ]; + virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable { + forceSSL = true; + enableACME = true; + kTLS = true; + + locations."/prometheus-node-exporter/metrics" = { + proxyPass = "http://localhost:${toString cfg.port}/metrics"; + + extraConfig = '' + allow 127.0.0.1; + allow ::1; + allow ${values.hosts.ildkule.ipv4}; + allow ${values.hosts.ildkule.ipv6}; + deny all; + ''; + }; + }; + }; } diff --git a/base/services/prometheus-systemd-exporter.nix b/base/services/prometheus-systemd-exporter.nix index 0599c04..986bb14 100644 --- a/base/services/prometheus-systemd-exporter.nix +++ b/base/services/prometheus-systemd-exporter.nix @@ -5,6 +5,7 @@ in { services.prometheus.exporters.systemd = { enable = lib.mkDefault true; + listenAddress = "127.0.0.1"; port = 9101; extraFlags = [ "--systemd.collector.enable-restart-count" @@ -12,15 +13,25 @@ in ]; }; - systemd.services.prometheus-systemd-exporter.serviceConfig = { - IPAddressDeny = "any"; - IPAddressAllow = [ - "127.0.0.1" - "::1" - values.hosts.ildkule.ipv4 - values.hosts.ildkule.ipv6 - ]; - }; + services.nginx = { + enable = lib.mkDefault true; - networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ]; + virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable { + forceSSL = true; + enableACME = true; + kTLS = true; + + locations."/prometheus-systemd-exporter/metrics" = { + proxyPass = "http://localhost:${toString cfg.port}/metrics"; + + extraConfig = '' + allow 127.0.0.1; + allow ::1; + allow ${values.hosts.ildkule.ipv4}; + allow ${values.hosts.ildkule.ipv6}; + deny all; + ''; + }; + }; + }; } diff --git a/hosts/ildkule/services/monitoring/prometheus/machines.nix b/hosts/ildkule/services/monitoring/prometheus/machines.nix index 0f22fa5..1b1bfa5 100644 --- a/hosts/ildkule/services/monitoring/prometheus/machines.nix +++ b/hosts/ildkule/services/monitoring/prometheus/machines.nix @@ -6,32 +6,63 @@ targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports; }; + nixosMachines = [ + "ildkule" + "bekkalokk" + "bicep" + "brzeczyszczykiewicz" + "georg" + "gluttony" + "kommode" + "lupine-1" + "lupine-2" + "lupine-3" + "lupine-4" + "lupine-5" + # TODO: export prometheus stats via apache on temmie + # "temmie" + "wenche" + ]; + defaultNodeExporterPort = 9100; - defaultSystemdExporterPort = 9101; - defaultNixosExporterPort = 9102; in { - services.prometheus.scrapeConfigs = [{ - job_name = "base_info"; - static_configs = [ - (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ]) - - (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ]) - - (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ]) - (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ]) - (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ]) - ]; - }]; + services.prometheus.scrapeConfigs = [ + { + job_name = "nixos-node"; + scheme = "https"; + metrics_path = "/prometheus-node-exporter/metrics"; + static_configs = map (name: { + labels.hostname = name; + targets = [ "${name}.pvv.ntnu.no:443" ]; + }) nixosMachines; + } + { + job_name = "nixos-systemd"; + scheme = "https"; + metrics_path = "/prometheus-systemd-exporter/metrics"; + static_configs = map (name: { + labels.hostname = name; + targets = [ "${name}.pvv.ntnu.no:443" ]; + }) nixosMachines; + } + { + job_name = "nixos-flake-input"; + scheme = "https"; + metrics_path = "/prometheus-nixos-flake-input-exporter/metrics"; + static_configs = map (name: { + labels.hostname = name; + targets = [ "${name}.pvv.ntnu.no:443" ]; + }) nixosMachines; + } + { + job_name = "non-nixos-node"; + scheme = "http"; + metrics_path = "/metrics"; + static_configs = [ + (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ]) + (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ]) + (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ]) + ]; + } + ]; }