ildkule/loki: restrict incoming connections to pvv + ntnu

Co-authored-by: Øystein Kristoffer Tveit <oysteikt@pvv.ntnu.no>

base/default.nix

@@ -95,10 +95,6 @@
     AllowHibernation = lib.mkDefault false;
   };
 
-  systemd.slices."system-monitoring" = {
-    description = "Monitoring related services";
-  };
-
   # users.mutableUsers = lib.mkDefault false;
 
   users.groups."drift".name = "drift";

base/services/fluentbit.nix

@@ -62,8 +62,10 @@ in
           name = "loki";
           match = "*";
 
-          host = "ildkule.pvv.ntnu.no";
-          port = 3100;
+          host = "loki.pvv.ntnu.no";
+          port = 443;
+          tls = "on";
+          "tls.verify" = "on";
           uri = "/loki/api/v1/push";
           compress = "gzip";
 
@@ -88,7 +90,6 @@ in
 
   systemd.services.fluent-bit = lib.mkIf cfg.enable {
     serviceConfig = {
-      Slice = "system-monitoring.slice";
       StateDirectory = "fluent-bit";
 
       # NOTE: This hardening might be way too strong for general purpose use, don't upstream this.

base/services/journald-upload.nix

@@ -14,7 +14,6 @@ in
   };
 
   systemd.services."systemd-journal-upload".serviceConfig = lib.mkIf cfg.enable {
-    Slice = "system-monitoring.slice";
     IPAddressDeny = "any";
     IPAddressAllow = [
       values.hosts.ildkule.ipv4

base/services/prometheus-node-exporter.nix

@@ -10,7 +10,7 @@ in
     enabledCollectors = [ "systemd" ];
   };
 
-  services.nginx = lib.mkIf cfg.enable {
+  services.nginx = {
     enable = lib.mkDefault true;
 
     virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
@@ -31,8 +31,4 @@ in
       };
     };
   };
-
-  systemd.services = lib.mkIf cfg.enable {
-    "prometheus-node-exporter".serviceConfig.Slice = "system-monitoring.slice";
-  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -13,7 +13,7 @@ in
     ];
   };
 
-  services.nginx = lib.mkIf cfg.enable {
+  services.nginx = {
     enable = lib.mkDefault true;
 
     virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
@@ -34,8 +34,4 @@ in
       };
     };
   };
-
-  systemd.services = lib.mkIf cfg.enable {
-    "prometheus-systemd-exporter".serviceConfig.Slice = "system-monitoring.slice";
-  };
 }

base/services/rsyslogd.nix

@@ -1,20 +1,13 @@
-{ config, lib, ... }:
-let
-  cfg = config.services.rsyslogd;
-in
+{ ... }:
 {
   services.rsyslogd = {
-    enable = lib.mkDefault true;
+    enable = true;
     defaultConfig = ''
       *.* @loghost.pvv.ntnu.no
     '';
   };
 
-  services.journald.extraConfig = lib.mkIf cfg.enable ''
+  services.journald.extraConfig = ''
     ForwardToSyslog=yes
   '';
-
-  systemd.services = lib.mkIf cfg.enable {
-    "syslog".serviceConfig.Slice = "system-monitoring.slice";
-  };
 }

base/services/uptimed.nix

@@ -23,7 +23,7 @@ in
       };
     };
 
-    systemd.services.uptimed = lib.mkIf cfg.enable {
+    systemd.services.uptimed = lib.mkIf (cfg.enable) {
       serviceConfig = let
         uptimed = pkgs.uptimed.overrideAttrs (prev: {
           postPatch = ''
@@ -35,8 +35,6 @@ in
         });
 
       in {
-        Slice = "system-monitoring.slice";
-
         Type = "notify";
 
         ExecStart = lib.mkForce "${uptimed}/sbin/uptimed -f";

hosts/ildkule/services/monitoring/loki.nix

@@ -1,4 +1,4 @@
-{ config, pkgs, ... }:
+{ config, pkgs, values, ... }:
 
 let
   cfg = config.services.loki;
@@ -9,8 +9,8 @@ in {
     configuration = {
       auth_enabled = false;
       server = {
-        http_listen_port = 3100;
-        http_listen_address = "0.0.0.0";
+        http_listen_port = 31832;
+        http_listen_address = "127.0.0.1";
         grpc_listen_port = 9096;
       };
 
@@ -81,5 +81,25 @@ in {
     };
   };
 
-  networking.firewall.allowedTCPPorts = [ cfg.configuration.server.http_listen_port ];
+  services.nginx.virtualHosts."loki.pvv.ntnu.no" = {
+    forceSSL = true;
+    enableACME = true;
+    kTLS = true;
+
+    locations = {
+      "/".return = "403";
+      "/loki/api/v1/push" = {
+        proxyPass = "http://${cfg.configuration.server.http_listen_address}:${toString cfg.configuration.server.http_listen_port}/loki/api/v1/push";
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.ipv4-space};
+          allow ${values.ipv6-space};
+          allow ${values.ntnu.ipv4-space};
+          allow ${values.ntnu.ipv6-space};
+          deny all;
+        '';
+      };
+    };
+  };
 }

hosts/temmie/services/userweb/default.nix

@@ -14,6 +14,8 @@ let
     upload_max_filesize = "40M";
   });
 
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+
   # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
   phpEnv = pkgs.php.buildEnv {
     extensions = { all, ... }: with all; [
@@ -159,9 +161,13 @@ in
 {
   imports = [
     ./mail.nix
-    ./log-processor.nix
   ];
 
+  sops.secrets = {
+    "httpd/passwd-ssh-key" = { };
+    "httpd/ssh-known-hosts" = { };
+  };
+
   services.httpd = {
     enable = true;
     adminAddr = "drift@pvv.ntnu.no";
@@ -231,8 +237,8 @@ in
 
       extraConfig = ''
         CustomLog "${cfg.logDir}/access.log" combined
-        CustomLog "/run/httpd-log-processor-access.fifo" combined
-        ErrorLog "/run/httpd-log-processor-error.fifo"
+        CustomLog "|${lib.getExe apache-log-processor} access" combined
+        ErrorLog "|${lib.getExe apache-log-processor} error"
         ScriptLog "${cfg.logDir}/cgi.log"
 
         UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
@@ -292,26 +298,9 @@ in
   users.users."wwwrun".uid = lib.mkForce 33;
   users.groups."wwwrun".gid = lib.mkForce 33;
 
-  systemd.targets.userweb = {
-    description = "PVV HTTPD UserWeb";
-  };
-
-  systemd.slices.system-userweb = {
-    description = "PVV HTTPD UserWeb";
-  };
-
   systemd.services.httpd = {
-    after = [
-      "pvv-homedirs.target"
-      "httpd-log-processor@access.socket"
-      "httpd-log-processor@error.socket"
-    ];
-    requires = [
-      "pvv-homedirs.target"
-      "httpd-log-processor@access.socket"
-      "httpd-log-processor@error.socket"
-    ];
-    requiredBy = [ "userweb.target" ];
+    after = [ "pvv-homedirs.target" ];
+    requires = [ "pvv-homedirs.target" ];
 
     environment = {
       PATH = lib.mkForce "/usr/bin";
@@ -319,18 +308,64 @@ in
 
     serviceConfig = {
       Type = lib.mkForce "notify";
+
+      ExecStartPre = let
+        rsyncCommand = ''${lib.getExe pkgs.rsync} -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey" -avz'';
+      in lib.mkForce [
+        "${lib.getExe (pkgs.writeShellApplication {
+          name = "http-exec-start-pre-remove-old-semaphores";
+          text = ''
+            # Get rid of old semaphores.  These tend to accumulate across
+            # server restarts, eventually preventing it from restarting
+            # successfully.
+            for i in $(${pkgs.util-linux}/bin/ipcs -s | grep ' ${cfg.user} ' | cut -f2 -d ' '); do
+                ${pkgs.util-linux}/bin/ipcrm -s "$i"
+            done
+          '';
+        })}"
+
+        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
+        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
+
+        (let
+          args = lib.cli.toCommandLineShellGNU { } {
+            passwd-file = "/run/httpd/pamunix-in/passwd";
+            group-file = "/run/httpd/pamunix-in/group";
+            output-dir = "/run/httpd/pamunix-out";
+            shadow-file = pkgs.emptyFile;
+
+            output-passwd = true;
+
+            ignore-user-file = toString ./ignore_user_file.txt;
+            ignore-group-file = toString ./ignore_group_file.txt;
+          };
+        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
+        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
+        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
+      ];
       ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
       ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
       ExecStop = lib.mkForce "";
       KillMode = "mixed";
-      Slice = "system-userweb.slice";
+
+      LoadCredential=[
+        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
+        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
+      ];
 
       ConfigurationDirectory = [ "httpd" ];
       LogsDirectory = [ "httpd" ];
       LogsDirectoryMode = "0700";
 
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
       LockPersonality = !debug;
       PrivateDevices = true;
       PrivateTmp = true;
@@ -358,19 +393,48 @@ in
         "tcp:443"
       ];
       SystemCallArchitectures = "native";
-      SystemCallFilter = lib.mkIf (!debug) [ "@system-service" ];
+      SystemCallFilter = lib.mkIf (!debug) [
+         "@system-service"
+         "@setuid"
+      ];
       UMask = "0077";
 
       RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [ "httpd/root-mnt" ];
+      RuntimeDirectory = [
+        "httpd/root-mnt"
+        "httpd/pamunix-in"
+        "httpd/pamunix-out"
+      ];
       RootDirectory = "/run/httpd/root-mnt";
       MountAPIVFS = true;
       BindReadOnlyPaths = [
         builtins.storeDir
         "/etc"
         "/dev/null"
+        "/etc/resolv.conf"
         "/var/lib/acme"
-        "/var/run/nscd"
+
+        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd/pamunix-out/group:/etc/group"
+
+        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
+          passwd:    files
+          group:     files
+          shadow:    files
+          sudoers:   files
+
+          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+
+          subuid:    files
+          subgid:    files
+        ''}:/etc/nsswitch.conf"
+
         "${fhsEnv}/bin:/bin"
         "${fhsEnv}/sbin:/sbin"
         "${fhsEnv}/lib:/lib"
@@ -395,16 +459,13 @@ in
           "/share"
         ];
       });
-      BindPaths = (lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
+      BindPaths = lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
         directoryFn = [
           (_: "/home/pvv/")
           (l: "/amd/homepvv${l}/")
         ];
         letter = homeLetters;
-      }) ++ [
-        "/run/httpd-log-processor-access.fifo"
-        "/run/httpd-log-processor-error.fifo"
-      ];
+      };
     };
   };
 

hosts/temmie/services/userweb/log-processor.nix

@@ -1,167 +0,0 @@
-{ config, lib, pkgs, values, ... }:
-let
-  homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
-
-  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
-in
-{
-  sops.secrets = {
-    "httpd/passwd-ssh-key" = { };
-    "httpd/ssh-known-hosts" = { };
-  };
-
-  systemd.targets.sockets.wants = [
-    "httpd-log-processor@access.socket"
-    "httpd-log-processor@error.socket"
-  ];
-
-  systemd.sockets."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
-    requiredBy = [ "userweb.target" ];
-    socketConfig = {
-      ListenFIFO = "/run/httpd-log-processor-%i.fifo";
-      RemoveOnStop = true;
-
-      SocketUser = "wwwrun";
-      SocketGroup = "wwwrun";
-      SocketMode = "0600";
-    };
-  };
-
-  systemd.services."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
-    requiredBy = [ "userweb.target" ];
-    serviceConfig = {
-      User = "wwwrun";
-      Group = "wwwrun";
-      Slice = "system-userweb.slice";
-      Restart = "on-failure";
-
-      StandardInput = "socket";
-      StandardOutput = "journal";
-      StandardError = "journal";
-
-      LoadCredential = [
-        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
-        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
-      ];
-      ExecStartPre = let
-        rsyncArgs = lib.cli.toCommandLineShellGNU { } {
-          archive = true;
-          verbose = true;
-          compress = true;
-          rsh = "${lib.getExe' pkgs.openssh "ssh"} -o BatchMode=yes -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey";
-        };
-        inputDir = "/run/httpd-log-processor-%i/pamunix-in";
-        outputDir = "/run/httpd-log-processor-%i/pamunix-out";
-      in lib.mkForce [
-        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/passwd ${inputDir}/"
-        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/group ${inputDir}/"
-
-        (let
-          args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "${inputDir}/passwd";
-            group-file = "${inputDir}/group";
-            output-dir = outputDir;
-            shadow-file = pkgs.emptyFile;
-
-            output-passwd = true;
-
-            ignore-user-file = toString ./ignore_user_file.txt;
-            ignore-group-file = toString ./ignore_group_file.txt;
-          };
-        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u ${inputDir}/passwd ${inputDir}/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' ${outputDir}/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' ${outputDir}/group"
-        "+${lib.getExe' pkgs.coreutils "chown"} root:root ${outputDir}/passwd ${outputDir}/group"
-        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 ${outputDir}/passwd ${outputDir}/group"
-        "+${lib.getExe pkgs.mount} --bind ${outputDir}/passwd /etc/passwd"
-        "+${lib.getExe pkgs.mount} --bind ${outputDir}/group  /etc/group"
-      ];
-
-      ExecStart = "${lib.getExe apache-log-processor} %i";
-
-      AmbientCapabilities = [ "CAP_SETUID" "CAP_SETGID" ];
-      CapabilityBoundingSet = [ "CAP_SETUID" "CAP_SETGID" ];
-      DeviceAllow = [ "" ];
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      PrivateDevices = true;
-      PrivateIPC = true;
-      PrivateTmp = true;
-      # PrivateUsers = true;
-      ProcSubset = "pid";
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = "tmpfs";
-      ProtectHostname = true;
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectProc = "invisible";
-      ProtectSystem = "strict";
-      ProtectKernelTunables = true;
-      RemoveIPC = true;
-      RestrictAddressFamilies = [
-        "AF_INET"
-        "AF_INET6"
-      ];
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SocketBindDeny = "any";
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-         "@system-service"
-         "@setuid"
-      ];
-      UMask = "0077";
-
-      IPAddressAllow = [
-        "127.0.0.53" # systemd-resolved
-        values.hosts.microbel.ipv4
-        values.hosts.microbel.ipv6
-      ];
-      IPAddressDeny = "any";
-
-      RootDirectory = "/run/httpd-log-processor-%i/root-mnt";
-      MountAPIVFS = true;
-
-      RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [
-        "httpd-log-processor-%i/root-mnt"
-        "httpd-log-processor-%i/pamunix-in"
-        "httpd-log-processor-%i/pamunix-out"
-      ];
-      BindReadOnlyPaths = [
-        builtins.storeDir
-        "/etc"
-        "/etc/resolv.conf"
-
-        "-/run/httpd-log-processor-%i/pamunix-out/passwd:/etc/passwd"
-        "-/run/httpd-log-processor-%i/pamunix-out/group:/etc/group"
-
-        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    files
-          group:     files
-          shadow:    files
-          sudoers:   files
-
-          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
-          networks:  files
-
-          ethers:    files
-          services:  files
-          protocols: files
-          rpc:       files
-
-          subuid:    files
-          subgid:    files
-        ''}:/etc/nsswitch.conf"
-      ];
-      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters ++ [
-        "/var/log/httpd"
-      ];
-    };
-  };
-}