temmie/userweb: run log processors as separate systemd units

This lets us divide up some of the logic making httpd itself less
brittle, and also reduces the amount of privileges for httpd.

base/default.nix

@@ -14,7 +14,6 @@
 
     ./mitigations.nix
 
-    ./flake-input-exporter.nix
     ./hardening.nix
     ./networking.nix
     ./nix.nix
@@ -34,12 +33,15 @@
     ./services/openssh.nix
     ./services/polkit.nix
     ./services/postfix.nix
+    ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix
     ./services/roowho2.nix
+    ./services/rsyslogd.nix
     ./services/scrutiny-collector.nix
     ./services/smartd.nix
     ./services/thermald.nix
+    ./services/timesyncd.nix
     ./services/uptimed.nix
     ./services/userborn.nix
     ./services/userdbd.nix
@@ -93,6 +95,10 @@
     AllowHibernation = lib.mkDefault false;
   };
 
+  systemd.slices."system-monitoring" = {
+    description = "Monitoring related services";
+  };
+
   # users.mutableUsers = lib.mkDefault false;
 
   users.groups."drift".name = "drift";

base/flake-input-exporter.nix

@@ -1,55 +0,0 @@
-{
-  config,
-  inputs,
-  lib,
-  pkgs,
-  values,
-  ...
-}:
-let
-  data = lib.flip lib.mapAttrs inputs (
-    name: input: {
-      inherit (input)
-        lastModified
-        ;
-    }
-  );
-  folder = pkgs.writeTextDir "share/flake-inputs" (
-    lib.concatMapStringsSep "\n" (
-      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
-    ) (lib.attrsToList data)
-  );
-  port = 9102;
-in
-{
-  services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = {
-    serverName = config.networking.fqdn;
-    serverAliases = [
-      "${config.networking.hostName}.pvv.org"
-    ];
-    locations."/metrics" = {
-      root = "${folder}/share";
-      tryFiles = "/flake-inputs =404";
-      extraConfig = ''
-        default_type text/plain;
-      '';
-    };
-    listen = [
-      {
-        inherit port;
-        addr = "0.0.0.0";
-      }
-    ];
-    extraConfig = ''
-      allow ${values.hosts.ildkule.ipv4}/32;
-      allow ${values.hosts.ildkule.ipv6}/128;
-      allow 127.0.0.1/32;
-      allow ::1/128;
-      allow ${values.ipv4-space};
-      allow ${values.ipv6-space};
-      deny all;
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [ port ];
-}

base/services/fluentbit.nix

@@ -88,6 +88,7 @@ in
 
   systemd.services.fluent-bit = lib.mkIf cfg.enable {
     serviceConfig = {
+      Slice = "system-monitoring.slice";
       StateDirectory = "fluent-bit";
 
       # NOTE: This hardening might be way too strong for general purpose use, don't upstream this.

base/services/journald-upload.nix

@@ -14,6 +14,7 @@ in
   };
 
   systemd.services."systemd-journal-upload".serviceConfig = lib.mkIf cfg.enable {
+    Slice = "system-monitoring.slice";
     IPAddressDeny = "any";
     IPAddressAllow = [
       values.hosts.ildkule.ipv4

base/services/nginx.nix

@@ -1,18 +1,5 @@
 { config, lib, ... }:
 {
-  # nginx return 444 for all nonexistent virtualhosts
-
-  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
-
-  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
-    "/etc/certs/nginx" = {
-      owner = "nginx";
-      group = "nginx";
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-
   services.nginx = {
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
@@ -60,17 +47,8 @@
           ];
         }
       ];
-      sslCertificate = "/etc/certs/nginx.crt";
+    };
-      sslCertificateKey = "/etc/certs/nginx.key";
-      addSSL = true;
-      extraConfig = "return 444;";
   };
 
-    ${config.networking.fqdn} = {
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-      sslCertificate = lib.mkDefault "/etc/certs/nginx.crt";
-      sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key";
-      addSSL = lib.mkDefault true;
-      extraConfig = lib.mkDefault "return 444;";
-    };
-  };
 }

base/services/prometheus-flake-input-exporter.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  values,
+  ...
+}:
+let
+  data = lib.flip lib.mapAttrs inputs (
+    name: input: {
+      inherit (input)
+        lastModified
+        ;
+    }
+  );
+  folder = pkgs.writeTextDir "share/flake-inputs" (
+    lib.concatMapStringsSep "\n" (
+      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
+    ) (lib.attrsToList data)
+  );
+in
+{
+  services.nginx = {
+    enable = lib.mkDefault true;
+
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-nixos-flake-input-exporter/metrics" = {
+        root = "${folder}/share";
+        tryFiles = "/flake-inputs =404";
+        extraConfig = ''
+          default_type text/plain;
+
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

base/services/prometheus-node-exporter.nix

@@ -5,19 +5,34 @@ in
 {
   services.prometheus.exporters.node = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9100;
     enabledCollectors = [ "systemd" ];
   };
 
-  systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable {
+  services.nginx = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
+
-      "127.0.0.1"
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      "::1"
+      forceSSL = true;
-      values.hosts.ildkule.ipv4
+      enableACME = true;
-      values.hosts.ildkule.ipv6
+      kTLS = true;
-    ];
+
+      locations."/prometheus-node-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
   };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-node-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -5,6 +5,7 @@ in
 {
   services.prometheus.exporters.systemd = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9101;
     extraFlags = [
       "--systemd.collector.enable-restart-count"
@@ -12,15 +13,29 @@ in
     ];
   };
 
-  systemd.services.prometheus-systemd-exporter.serviceConfig = {
+  services.nginx = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
+
-      "127.0.0.1"
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      "::1"
+      forceSSL = true;
-      values.hosts.ildkule.ipv4
+      enableACME = true;
-      values.hosts.ildkule.ipv6
+      kTLS = true;
-    ];
+
+      locations."/prometheus-systemd-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
   };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-systemd-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/rsyslogd.nix

@@ -0,0 +1,20 @@
+{ config, lib, ... }:
+let
+  cfg = config.services.rsyslogd;
+in
+{
+  services.rsyslogd = {
+    enable = lib.mkDefault true;
+    defaultConfig = ''
+      *.* @loghost.pvv.ntnu.no
+    '';
+  };
+
+  services.journald.extraConfig = lib.mkIf cfg.enable ''
+    ForwardToSyslog=yes
+  '';
+
+  systemd.services = lib.mkIf cfg.enable {
+    "syslog".serviceConfig.Slice = "system-monitoring.slice";
+  };
+}

base/services/timesyncd.nix

@@ -0,0 +1,12 @@
+{ ... }:
+{
+  services.timesyncd = {
+    servers = [ "ntp.ntnu.no" ];
+    fallbackServers = [
+      "0.pool.ntp.org"
+      "1.pool.ntp.org"
+      "0.no.pool.ntp.org"
+    ];
+  };
+}
+

base/services/uptimed.nix

@@ -23,7 +23,7 @@ in
       };
     };
 
-    systemd.services.uptimed = lib.mkIf (cfg.enable) {
+    systemd.services.uptimed = lib.mkIf cfg.enable {
       serviceConfig = let
         uptimed = pkgs.uptimed.overrideAttrs (prev: {
           postPatch = ''
@@ -35,6 +35,8 @@ in
         });
 
       in {
+        Slice = "system-monitoring.slice";
+
         Type = "notify";
 
         ExecStart = lib.mkForce "${uptimed}/sbin/uptimed -f";

hosts/ildkule/services/monitoring/prometheus/machines.nix

@@ -6,32 +6,63 @@
     targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports;
   };
 
+  nixosMachines = [
+    "ildkule"
+    "bekkalokk"
+    "bicep"
+    "brzeczyszczykiewicz"
+    "georg"
+    "gluttony"
+    "kommode"
+    "lupine-1"
+    "lupine-2"
+    "lupine-3"
+    "lupine-4"
+    "lupine-5"
+    # TODO: export prometheus stats via apache on temmie
+    # "temmie"
+    "wenche"
+  ];
+
   defaultNodeExporterPort = 9100;
-  defaultSystemdExporterPort = 9101;
-  defaultNixosExporterPort = 9102;
 in {
-  services.prometheus.scrapeConfigs = [{
+  services.prometheus.scrapeConfigs = [
-    job_name = "base_info";
+    {
+      job_name = "nixos-node";
+      scheme = "https";
+      metrics_path = "/prometheus-node-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-systemd";
+      scheme = "https";
+      metrics_path = "/prometheus-systemd-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-flake-input";
+      scheme = "https";
+      metrics_path = "/prometheus-nixos-flake-input-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "non-nixos-node";
+      scheme = "http";
+      metrics_path = "/metrics";
       static_configs = [
-      (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ])
-
-      (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-
         (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
         (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
         (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
       ];
-  }];
+    }
+  ];
 }

hosts/temmie/services/userweb/default.nix

@@ -14,8 +14,6 @@ let
     upload_max_filesize = "40M";
   });
 
-  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
-
   # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
   phpEnv = pkgs.php.buildEnv {
     extensions = { all, ... }: with all; [
@@ -161,13 +159,9 @@ in
 {
   imports = [
     ./mail.nix
+    ./log-processor.nix
   ];
 
-  sops.secrets = {
-    "httpd/passwd-ssh-key" = { };
-    "httpd/ssh-known-hosts" = { };
-  };
-
   services.httpd = {
     enable = true;
     adminAddr = "drift@pvv.ntnu.no";
@@ -212,6 +206,19 @@ in
     extraConfig = ''
       TraceEnable on
       LogLevel warn rewrite:trace3
+
+      <FilesMatch ".+\.ph(p[3457]?|t|tml)$">
+          SetHandler application/x-httpd-php
+      </FilesMatch>
+      <FilesMatch ".+\.phps$">
+          SetHandler application/x-httpd-php-source
+          Require all denied
+      </FilesMatch>
+      <FilesMatch "\.pl$">
+          SetHandler modperl
+          PerlResponseHandler ModPerl::Registry
+          Options +ExecCGI
+      </FilesMatch>
     '';
 
     virtualHosts."temmie.pvv.ntnu.no" = {
@@ -224,8 +231,8 @@ in
 
       extraConfig = ''
         CustomLog "${cfg.logDir}/access.log" combined
-        CustomLog "|${lib.getExe apache-log-processor} access" combined
+        CustomLog "/run/httpd-log-processor-access.fifo" combined
-        ErrorLog "|${lib.getExe apache-log-processor} error"
+        ErrorLog "/run/httpd-log-processor-error.fifo"
         ScriptLog "${cfg.logDir}/cgi.log"
 
         UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
@@ -285,9 +292,26 @@ in
   users.users."wwwrun".uid = lib.mkForce 33;
   users.groups."wwwrun".gid = lib.mkForce 33;
 
+  systemd.targets.userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
+  systemd.slices.system-userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
   systemd.services.httpd = {
-    after = [ "pvv-homedirs.target" ];
+    after = [
-    requires = [ "pvv-homedirs.target" ];
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requires = [
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requiredBy = [ "userweb.target" ];
 
     environment = {
       PATH = lib.mkForce "/usr/bin";
@@ -295,65 +319,18 @@ in
 
     serviceConfig = {
       Type = lib.mkForce "notify";
-
-      ExecStartPre = let
-        rsyncCommand = ''${lib.getExe pkgs.rsync} -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey" -avz'';
-      in lib.mkForce [
-        "${lib.getExe (pkgs.writeShellApplication {
-          name = "http-exec-start-pre-remove-old-semaphores";
-          text = ''
-            # Get rid of old semaphores.  These tend to accumulate across
-            # server restarts, eventually preventing it from restarting
-            # successfully.
-            for i in $(${pkgs.util-linux}/bin/ipcs -s | grep ' ${cfg.user} ' | cut -f2 -d ' '); do
-                ${pkgs.util-linux}/bin/ipcrm -s "$i"
-            done
-          '';
-        })}"
-
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
-
-        (let
-          args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "/run/httpd/pamunix-in/passwd";
-            group-file = "/run/httpd/pamunix-in/group";
-            output-dir = "/run/httpd/pamunix-out";
-            shadow-file = pkgs.emptyFile;
-
-            output-passwd = true;
-
-            ignore-user-file = toString ./ignore_user_file.txt;
-            ignore-group-file = toString ./ignore_group_file.txt;
-          };
-        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
-        "${lib.getExe' pkgs.coreutils "cat"} /run/httpd/pamunix-out/passwd"
-        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
-      ];
       ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
       ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
       ExecStop = lib.mkForce "";
       KillMode = "mixed";
-
+      Slice = "system-userweb.slice";
-      LoadCredential=[
-        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
-        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
-      ];
 
       ConfigurationDirectory = [ "httpd" ];
       LogsDirectory = [ "httpd" ];
       LogsDirectoryMode = "0700";
 
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
       LockPersonality = !debug;
       PrivateDevices = true;
       PrivateTmp = true;
@@ -381,48 +358,19 @@ in
         "tcp:443"
       ];
       SystemCallArchitectures = "native";
-      SystemCallFilter = lib.mkIf (!debug) [
+      SystemCallFilter = lib.mkIf (!debug) [ "@system-service" ];
-         "@system-service"
-         "@setuid"
-      ];
       UMask = "0077";
 
       RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [
+      RuntimeDirectory = [ "httpd/root-mnt" ];
-        "httpd/root-mnt"
-        "httpd/pamunix-in"
-        "httpd/pamunix-out"
-      ];
       RootDirectory = "/run/httpd/root-mnt";
       MountAPIVFS = true;
       BindReadOnlyPaths = [
         builtins.storeDir
         "/etc"
         "/dev/null"
-        "/etc/resolv.conf"
         "/var/lib/acme"
-
+        "/var/run/nscd"
-        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
-        "-/run/httpd/pamunix-out/group:/etc/group"
-
-        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    files
-          group:     files
-          shadow:    files
-          sudoers:   files
-
-          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
-          networks:  files
-
-          ethers:    files
-          services:  files
-          protocols: files
-          rpc:       files
-
-          subuid:    files
-          subgid:    files
-        ''}:/etc/nsswitch.conf"
-
         "${fhsEnv}/bin:/bin"
         "${fhsEnv}/sbin:/sbin"
         "${fhsEnv}/lib:/lib"
@@ -447,7 +395,16 @@ in
           "/share"
         ];
       });
-      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters;
+      BindPaths = (lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
+        directoryFn = [
+          (_: "/home/pvv/")
+          (l: "/amd/homepvv${l}/")
+        ];
+        letter = homeLetters;
+      }) ++ [
+        "/run/httpd-log-processor-access.fifo"
+        "/run/httpd-log-processor-error.fifo"
+      ];
     };
   };
 

hosts/temmie/services/userweb/log-processor.nix

@@ -0,0 +1,167 @@
+{ config, lib, pkgs, values, ... }:
+let
+  homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
+
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+in
+{
+  sops.secrets = {
+    "httpd/passwd-ssh-key" = { };
+    "httpd/ssh-known-hosts" = { };
+  };
+
+  systemd.targets.sockets.wants = [
+    "httpd-log-processor@access.socket"
+    "httpd-log-processor@error.socket"
+  ];
+
+  systemd.sockets."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    socketConfig = {
+      ListenFIFO = "/run/httpd-log-processor-%i.fifo";
+      RemoveOnStop = true;
+
+      SocketUser = "wwwrun";
+      SocketGroup = "wwwrun";
+      SocketMode = "0600";
+    };
+  };
+
+  systemd.services."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    serviceConfig = {
+      User = "wwwrun";
+      Group = "wwwrun";
+      Slice = "system-userweb.slice";
+      Restart = "on-failure";
+
+      StandardInput = "socket";
+      StandardOutput = "journal";
+      StandardError = "journal";
+
+      LoadCredential = [
+        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
+        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
+      ];
+      ExecStartPre = let
+        rsyncArgs = lib.cli.toCommandLineShellGNU { } {
+          archive = true;
+          verbose = true;
+          compress = true;
+          rsh = "${lib.getExe' pkgs.openssh "ssh"} -o BatchMode=yes -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey";
+        };
+        inputDir = "/run/httpd-log-processor-%i/pamunix-in";
+        outputDir = "/run/httpd-log-processor-%i/pamunix-out";
+      in lib.mkForce [
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/passwd ${inputDir}/"
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/group ${inputDir}/"
+
+        (let
+          args = lib.cli.toCommandLineShellGNU { } {
+            passwd-file = "${inputDir}/passwd";
+            group-file = "${inputDir}/group";
+            output-dir = outputDir;
+            shadow-file = pkgs.emptyFile;
+
+            output-passwd = true;
+
+            ignore-user-file = toString ./ignore_user_file.txt;
+            ignore-group-file = toString ./ignore_group_file.txt;
+          };
+        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
+        "${lib.getExe' pkgs.coreutils "shred"} -u ${inputDir}/passwd ${inputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' ${outputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/group  /etc/group"
+      ];
+
+      ExecStart = "${lib.getExe apache-log-processor} %i";
+
+      AmbientCapabilities = [ "CAP_SETUID" "CAP_SETGID" ];
+      CapabilityBoundingSet = [ "CAP_SETUID" "CAP_SETGID" ];
+      DeviceAllow = [ "" ];
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      PrivateDevices = true;
+      PrivateIPC = true;
+      PrivateTmp = true;
+      # PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = "tmpfs";
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      ProtectKernelTunables = true;
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SocketBindDeny = "any";
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+         "@system-service"
+         "@setuid"
+      ];
+      UMask = "0077";
+
+      IPAddressAllow = [
+        "127.0.0.53" # systemd-resolved
+        values.hosts.microbel.ipv4
+        values.hosts.microbel.ipv6
+      ];
+      IPAddressDeny = "any";
+
+      RootDirectory = "/run/httpd-log-processor-%i/root-mnt";
+      MountAPIVFS = true;
+
+      RuntimeDirectoryMode = "0750";
+      RuntimeDirectory = [
+        "httpd-log-processor-%i/root-mnt"
+        "httpd-log-processor-%i/pamunix-in"
+        "httpd-log-processor-%i/pamunix-out"
+      ];
+      BindReadOnlyPaths = [
+        builtins.storeDir
+        "/etc"
+        "/etc/resolv.conf"
+
+        "-/run/httpd-log-processor-%i/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd-log-processor-%i/pamunix-out/group:/etc/group"
+
+        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
+          passwd:    files
+          group:     files
+          shadow:    files
+          sudoers:   files
+
+          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+
+          subuid:    files
+          subgid:    files
+        ''}:/etc/nsswitch.conf"
+      ];
+      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters ++ [
+        "/var/log/httpd"
+      ];
+    };
+  };
+}

secrets/skrot/skrot.yaml

@@ -1,13 +1,12 @@
 dibbler:
     postgresql:
-        password: ENC[AES256_GCM,data:3X9A3jOpFVRuBg0gRiCEsZVKfLI=,iv:XC7LBNUhALk9IEhItV8fO5p/m7VKL0REBY1W2IZt7G4=,tag:l18R7EhbOlucZHFQiEvpHw==,type:str]
+        password: ENC[AES256_GCM,data:ZeNKipcCB+z8QVGeg1iV3MUXqALjotVz,iv:xCtgOoe6Pkr6Cq3vL+T4L+GW1KAcgP/xUz3YbHs5bCc=,tag:/X5phRYDAws8Aam1j+UaTw==,type:str]
 worblehat:
     postgresql:
         password: ENC[AES256_GCM,data:WpJR6MumY+7WUYdVVgAqv1af+NmqecTMO9aP5lidSpE=,iv:7aoN8mjXckd81LxasMSG3R2vqj0SvzSl7wrEQ1LwToo=,tag:zeeNcEpkYnqyd8be0ZS+kQ==,type:str]
 sops:
     age:
-        - recipient: age1hzkvnktkr8t5gvtq0ccw69e44z5z6wf00n3xhk3hj24emf07je5s6q2evr
+        - enc: |
-          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvTk5YU3Z2Yy9HS1R4ME5I
             UU1PRWVncHJYcXY5RlFpOWVQUWZsdy93ZDFBCnlxWkpaL1g5WmNSckNYd202WE40
@@ -15,8 +14,8 @@ sops:
             ZnllQzJiK1ZkRmFndmtYdW9IclFWY1EK82f1iGt3nt8dJnEQlMujNqConf6Qq6GX
             hqoqPoc2EM4kun28Bbpq4pAY7eEPRrWFqOkjYVvgIRoS88D7xT3LWg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
+          recipient: age1hzkvnktkr8t5gvtq0ccw69e44z5z6wf00n3xhk3hj24emf07je5s6q2evr
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5WTJIOUcxRlBuNmRrNUZo
             MXFxeVJBTEhDK00yTUw1U2dHckNFYWZKWkhNCnYxYmtrUEVvd1RaYUI5WTRTRW16
@@ -24,8 +23,8 @@ sops:
             eTB4WldMNW9GNUwwaEUzRThsemxRVzQKGpa0J2PBzDRdHijm0e3nFAaxQCHUjz+L
             KataXJEMCijJ6k+7vpb5QMxe2jB1J2PMxNGFp0bWAy2Al3p/Ez2Kww==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+          recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZaW1ZSXhVeFVTQW9WYzVh
             WkVUM2JkOU5VNU9oQXE2Y2pvcFlOWTdvbnpJClduS0RHL2xja291a2doQ0wzbzhQ
@@ -33,8 +32,8 @@ sops:
             ZUdnS2RvOXI1dGNYQTl6ZHE1cUdMWHMK4ycAJQLyKCgJIzjQ02bPjz4Ct9eO6ivw
             kfWhyMaoWwM9PhFcwSak0cLpX0C/IOzSzO78pf3WhG16pV7aXapdog==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
+          recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqaml0OVlhcUJSU1hSY3lP
             bkM0cUV4Z2ZLeERHZ3BUNExuYS9KSU5CekQ4CmQ3SE1vdDBtdFJ6czZYR3U5Tk1X
@@ -42,8 +41,8 @@ sops:
             Sy9XbjhwOFR6SFpaNHZLd3ZxdmxOVUEKBBbGmdVVlKHxO+/iODznLP3+dJGppybW
             +1k9uenVHzie+pDKcrQpSyX2WDnmgg7hUAUiXPuz1eEWmwbRJnU/5w==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
+          recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXK01vOVV5YlhsZ2ljYS91
             OUVEaEpTbXFKOHVNVDVoMTlrS05wRmsyM2dvCjZHOXlCUGowd0J4UlQzSzM5dWJ0
@@ -51,8 +50,8 @@ sops:
             RUR6Yi9SUDFCUkZmRk5hYTVFeGloZXcKY/XtaSoW8Pu2wS4oistLSc0T5JvMnt+w
             s3yfe/zx9/1K6OtbeljF9FZVOB/dOamvk+Qlfl0T5qush7/WgGzErA==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
+          recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOM0pFb2tRTURtWmp6elRN
             M0xtajlzMTNPMnppcGhJMVlsNHdwWmNGbFVFCnlxM1JQTkR2elAvdytKUEJ3djBS
@@ -60,8 +59,8 @@ sops:
             eWlyWGhaS1JCNitUSVVScFk2WGEvOG8K2rpYPGx5jhyyRK4UkeJR96wDFr4Frzsr
             QWz7fYZRWKWf0H0qn+bm9IfVJiBAlS5i16D1FnipZVmdWefFaZSEPg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
+          recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJVFV0WVZrK0wzbnhkcmcz
             c2lIdVlKcFpoYjZIWlNPN0M5N2g2WG9YdlRJCjg5YlNoSzQ5YW5yRUVSeTEzRThY
@@ -69,8 +68,9 @@ sops:
             MmxPMWNPYzJiOFRqY2VYczhvRm5IR3cKpUVV+zsMolsHI2YK9YqC6ecNT6QXv0TV
             d1SpXRAexZBeWCCHBjSdvQBl8AT4EwrAIP2M2o++6i5DaGoGiEIWZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-03-18T14:56:22Z"
+          recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
-    mac: ENC[AES256_GCM,data:nBKtFmFKx/Mt9TIFnKuuznsPAXCQpc3+WIspNu5TN9TpIqw75nzYXpxIb2hxRfRu0nbjHXpBy4bkzeMi41BGkvkvV57CZyq11J5i/iIKwuvllaB1IWrdDT2u+6RH3jIspp3KoyxFWdRqcGfNma9dSmtI+1Dd5z7XaxVaoVK2QMI=,iv:6joviyJ2cXmGh/9HH7VEcoK3+4GK5I6i2N/1d65PAN0=,tag:0BFVPWL3BByJH8HbrBTKOw==,type:str]
+    lastmodified: "2026-06-07T08:58:13Z"
+    mac: ENC[AES256_GCM,data:rchs8pGkw7dthGOQNDB5p/kgQdfdystaC+jRr0bZnA4Q41+PVMu+vBSMIZ+9zZek6oENgchmV5rRS0CEeb9UQMMiPXCk2Q2jMaDfiNCmOmjf0YYeFWRM6g2lA+IZ3RgKjwhXa6i5JOeNrNewMjtx7MFcHTn3EBlg2mztyn1xbT0=,iv:0yvoFPDxpugaBmTtXSmhNz9XusJHrU3E02tBm1hVsZo=,tag:hZ1BzqfmlT1OhPSsn+CCTg==,type:str]
     pgp:
         - created_at: "2026-02-10T20:01:32Z"
           enc: |-
@@ -93,4 +93,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.12.1
+    version: 3.13.0

users/vegardbm.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, config, ... }:
 {
   users.users.vegardbm = {
     isNormalUser = true;
@@ -8,12 +8,14 @@
       "drift"
       "nix-builder-users"
     ];
+    shell = if config.programs.zsh.enable then pkgs.zsh else pkgs.bash;
     packages = with pkgs; [
       btop
       eza
     ];
     openssh.authorizedKeys.keys = [
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVA3HqEx3je6L1AC+bP8sTxu3ZTKvTCR0npCyOVAYK5 vbm@arch-xeon"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICrYATHNvBNAHr9G+VwZIaAQPe02iRgAjqtZkW4x/dje vbm@talos"
     ];
   };
 }