temmie/userweb: run log processors as separate systemd units

This lets us divide up some of the logic making httpd itself less
brittle, and also reduces the amount of privileges for httpd.

base/default.nix

@@ -14,7 +14,6 @@
 
     ./mitigations.nix
 
-    ./flake-input-exporter.nix
     ./hardening.nix
     ./networking.nix
     ./nix.nix
@@ -34,9 +33,11 @@
     ./services/openssh.nix
     ./services/polkit.nix
     ./services/postfix.nix
+    ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix
     ./services/roowho2.nix
+    ./services/rsyslogd.nix
     ./services/scrutiny-collector.nix
     ./services/smartd.nix
     ./services/thermald.nix
@@ -94,6 +95,10 @@
     AllowHibernation = lib.mkDefault false;
   };
 
+  systemd.slices."system-monitoring" = {
+    description = "Monitoring related services";
+  };
+
   # users.mutableUsers = lib.mkDefault false;
 
   users.groups."drift".name = "drift";

base/flake-input-exporter.nix

@@ -1,55 +0,0 @@
-{
-  config,
-  inputs,
-  lib,
-  pkgs,
-  values,
-  ...
-}:
-let
-  data = lib.flip lib.mapAttrs inputs (
-    name: input: {
-      inherit (input)
-        lastModified
-        ;
-    }
-  );
-  folder = pkgs.writeTextDir "share/flake-inputs" (
-    lib.concatMapStringsSep "\n" (
-      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
-    ) (lib.attrsToList data)
-  );
-  port = 9102;
-in
-{
-  services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = {
-    serverName = config.networking.fqdn;
-    serverAliases = [
-      "${config.networking.hostName}.pvv.org"
-    ];
-    locations."/metrics" = {
-      root = "${folder}/share";
-      tryFiles = "/flake-inputs =404";
-      extraConfig = ''
-        default_type text/plain;
-      '';
-    };
-    listen = [
-      {
-        inherit port;
-        addr = "0.0.0.0";
-      }
-    ];
-    extraConfig = ''
-      allow ${values.hosts.ildkule.ipv4}/32;
-      allow ${values.hosts.ildkule.ipv6}/128;
-      allow 127.0.0.1/32;
-      allow ::1/128;
-      allow ${values.ipv4-space};
-      allow ${values.ipv6-space};
-      deny all;
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [ port ];
-}

base/services/fluentbit.nix

@@ -88,6 +88,7 @@ in
 
   systemd.services.fluent-bit = lib.mkIf cfg.enable {
     serviceConfig = {
+      Slice = "system-monitoring.slice";
       StateDirectory = "fluent-bit";
 
       # NOTE: This hardening might be way too strong for general purpose use, don't upstream this.

base/services/journald-upload.nix

@@ -14,6 +14,7 @@ in
   };
 
   systemd.services."systemd-journal-upload".serviceConfig = lib.mkIf cfg.enable {
+    Slice = "system-monitoring.slice";
     IPAddressDeny = "any";
     IPAddressAllow = [
       values.hosts.ildkule.ipv4

base/services/nginx.nix

@@ -1,18 +1,5 @@
 { config, lib, ... }:
 {
-  # nginx return 444 for all nonexistent virtualhosts
-
-  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
-
-  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
-    "/etc/certs/nginx" = {
-      owner = "nginx";
-      group = "nginx";
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-
   services.nginx = {
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
@@ -60,17 +47,8 @@
           ];
         }
       ];
-      sslCertificate = "/etc/certs/nginx.crt";
+    };
-      sslCertificateKey = "/etc/certs/nginx.key";
-      addSSL = true;
-      extraConfig = "return 444;";
   };
 
-    ${config.networking.fqdn} = {
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-      sslCertificate = lib.mkDefault "/etc/certs/nginx.crt";
-      sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key";
-      addSSL = lib.mkDefault true;
-      extraConfig = lib.mkDefault "return 444;";
-    };
-  };
 }

base/services/prometheus-flake-input-exporter.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  values,
+  ...
+}:
+let
+  data = lib.flip lib.mapAttrs inputs (
+    name: input: {
+      inherit (input)
+        lastModified
+        ;
+    }
+  );
+  folder = pkgs.writeTextDir "share/flake-inputs" (
+    lib.concatMapStringsSep "\n" (
+      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
+    ) (lib.attrsToList data)
+  );
+in
+{
+  services.nginx = {
+    enable = lib.mkDefault true;
+
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-nixos-flake-input-exporter/metrics" = {
+        root = "${folder}/share";
+        tryFiles = "/flake-inputs =404";
+        extraConfig = ''
+          default_type text/plain;
+
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

base/services/prometheus-node-exporter.nix

@@ -5,19 +5,34 @@ in
 {
   services.prometheus.exporters.node = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9100;
     enabledCollectors = [ "systemd" ];
   };
 
-  systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable {
+  services.nginx = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
+
-      "127.0.0.1"
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      "::1"
+      forceSSL = true;
-      values.hosts.ildkule.ipv4
+      enableACME = true;
-      values.hosts.ildkule.ipv6
+      kTLS = true;
-    ];
+
+      locations."/prometheus-node-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
   };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-node-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -5,6 +5,7 @@ in
 {
   services.prometheus.exporters.systemd = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9101;
     extraFlags = [
       "--systemd.collector.enable-restart-count"
@@ -12,15 +13,29 @@ in
     ];
   };
 
-  systemd.services.prometheus-systemd-exporter.serviceConfig = {
+  services.nginx = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
+
-      "127.0.0.1"
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      "::1"
+      forceSSL = true;
-      values.hosts.ildkule.ipv4
+      enableACME = true;
-      values.hosts.ildkule.ipv6
+      kTLS = true;
-    ];
+
+      locations."/prometheus-systemd-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
   };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-systemd-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/rsyslogd.nix

@@ -0,0 +1,20 @@
+{ config, lib, ... }:
+let
+  cfg = config.services.rsyslogd;
+in
+{
+  services.rsyslogd = {
+    enable = lib.mkDefault true;
+    defaultConfig = ''
+      *.* @loghost.pvv.ntnu.no
+    '';
+  };
+
+  services.journald.extraConfig = lib.mkIf cfg.enable ''
+    ForwardToSyslog=yes
+  '';
+
+  systemd.services = lib.mkIf cfg.enable {
+    "syslog".serviceConfig.Slice = "system-monitoring.slice";
+  };
+}

base/services/uptimed.nix

@@ -23,7 +23,7 @@ in
       };
     };
 
-    systemd.services.uptimed = lib.mkIf (cfg.enable) {
+    systemd.services.uptimed = lib.mkIf cfg.enable {
       serviceConfig = let
         uptimed = pkgs.uptimed.overrideAttrs (prev: {
           postPatch = ''
@@ -35,6 +35,8 @@ in
         });
 
       in {
+        Slice = "system-monitoring.slice";
+
         Type = "notify";
 
         ExecStart = lib.mkForce "${uptimed}/sbin/uptimed -f";

hosts/ildkule/services/monitoring/prometheus/machines.nix

@@ -6,32 +6,63 @@
     targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports;
   };
 
+  nixosMachines = [
+    "ildkule"
+    "bekkalokk"
+    "bicep"
+    "brzeczyszczykiewicz"
+    "georg"
+    "gluttony"
+    "kommode"
+    "lupine-1"
+    "lupine-2"
+    "lupine-3"
+    "lupine-4"
+    "lupine-5"
+    # TODO: export prometheus stats via apache on temmie
+    # "temmie"
+    "wenche"
+  ];
+
   defaultNodeExporterPort = 9100;
-  defaultSystemdExporterPort = 9101;
-  defaultNixosExporterPort = 9102;
 in {
-  services.prometheus.scrapeConfigs = [{
+  services.prometheus.scrapeConfigs = [
-    job_name = "base_info";
+    {
+      job_name = "nixos-node";
+      scheme = "https";
+      metrics_path = "/prometheus-node-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-systemd";
+      scheme = "https";
+      metrics_path = "/prometheus-systemd-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-flake-input";
+      scheme = "https";
+      metrics_path = "/prometheus-nixos-flake-input-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "non-nixos-node";
+      scheme = "http";
+      metrics_path = "/metrics";
       static_configs = [
-      (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ])
-
-      (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-
         (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
         (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
         (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
       ];
-  }];
+    }
+  ];
 }

hosts/temmie/services/userweb/default.nix

@@ -14,8 +14,6 @@ let
     upload_max_filesize = "40M";
   });
 
-  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
-
   # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
   phpEnv = pkgs.php.buildEnv {
     extensions = { all, ... }: with all; [
@@ -161,13 +159,9 @@ in
 {
   imports = [
     ./mail.nix
+    ./log-processor.nix
   ];
 
-  sops.secrets = {
-    "httpd/passwd-ssh-key" = { };
-    "httpd/ssh-known-hosts" = { };
-  };
-
   services.httpd = {
     enable = true;
     adminAddr = "drift@pvv.ntnu.no";
@@ -212,6 +206,19 @@ in
     extraConfig = ''
       TraceEnable on
       LogLevel warn rewrite:trace3
+
+      <FilesMatch ".+\.ph(p[3457]?|t|tml)$">
+          SetHandler application/x-httpd-php
+      </FilesMatch>
+      <FilesMatch ".+\.phps$">
+          SetHandler application/x-httpd-php-source
+          Require all denied
+      </FilesMatch>
+      <FilesMatch "\.pl$">
+          SetHandler modperl
+          PerlResponseHandler ModPerl::Registry
+          Options +ExecCGI
+      </FilesMatch>
     '';
 
     virtualHosts."temmie.pvv.ntnu.no" = {
@@ -224,8 +231,8 @@ in
 
       extraConfig = ''
         CustomLog "${cfg.logDir}/access.log" combined
-        CustomLog "|${lib.getExe apache-log-processor} access" combined
+        CustomLog "/run/httpd-log-processor-access.fifo" combined
-        ErrorLog "|${lib.getExe apache-log-processor} error"
+        ErrorLog "/run/httpd-log-processor-error.fifo"
         ScriptLog "${cfg.logDir}/cgi.log"
 
         UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
@@ -285,9 +292,26 @@ in
   users.users."wwwrun".uid = lib.mkForce 33;
   users.groups."wwwrun".gid = lib.mkForce 33;
 
+  systemd.targets.userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
+  systemd.slices.system-userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
   systemd.services.httpd = {
-    after = [ "pvv-homedirs.target" ];
+    after = [
-    requires = [ "pvv-homedirs.target" ];
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requires = [
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requiredBy = [ "userweb.target" ];
 
     environment = {
       PATH = lib.mkForce "/usr/bin";
@@ -295,65 +319,18 @@ in
 
     serviceConfig = {
       Type = lib.mkForce "notify";
-
-      ExecStartPre = let
-        rsyncCommand = ''${lib.getExe pkgs.rsync} -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey" -avz'';
-      in lib.mkForce [
-        "${lib.getExe (pkgs.writeShellApplication {
-          name = "http-exec-start-pre-remove-old-semaphores";
-          text = ''
-            # Get rid of old semaphores.  These tend to accumulate across
-            # server restarts, eventually preventing it from restarting
-            # successfully.
-            for i in $(${pkgs.util-linux}/bin/ipcs -s | grep ' ${cfg.user} ' | cut -f2 -d ' '); do
-                ${pkgs.util-linux}/bin/ipcrm -s "$i"
-            done
-          '';
-        })}"
-
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
-
-        (let
-          args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "/run/httpd/pamunix-in/passwd";
-            group-file = "/run/httpd/pamunix-in/group";
-            output-dir = "/run/httpd/pamunix-out";
-            shadow-file = pkgs.emptyFile;
-
-            output-passwd = true;
-
-            ignore-user-file = toString ./ignore_user_file.txt;
-            ignore-group-file = toString ./ignore_group_file.txt;
-          };
-        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
-        "${lib.getExe' pkgs.coreutils "cat"} /run/httpd/pamunix-out/passwd"
-        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
-      ];
       ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
       ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
       ExecStop = lib.mkForce "";
       KillMode = "mixed";
-
+      Slice = "system-userweb.slice";
-      LoadCredential=[
-        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
-        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
-      ];
 
       ConfigurationDirectory = [ "httpd" ];
       LogsDirectory = [ "httpd" ];
       LogsDirectoryMode = "0700";
 
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
       LockPersonality = !debug;
       PrivateDevices = true;
       PrivateTmp = true;
@@ -381,48 +358,19 @@ in
         "tcp:443"
       ];
       SystemCallArchitectures = "native";
-      SystemCallFilter = lib.mkIf (!debug) [
+      SystemCallFilter = lib.mkIf (!debug) [ "@system-service" ];
-         "@system-service"
-         "@setuid"
-      ];
       UMask = "0077";
 
       RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [
+      RuntimeDirectory = [ "httpd/root-mnt" ];
-        "httpd/root-mnt"
-        "httpd/pamunix-in"
-        "httpd/pamunix-out"
-      ];
       RootDirectory = "/run/httpd/root-mnt";
       MountAPIVFS = true;
       BindReadOnlyPaths = [
         builtins.storeDir
         "/etc"
         "/dev/null"
-        "/etc/resolv.conf"
         "/var/lib/acme"
-
+        "/var/run/nscd"
-        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
-        "-/run/httpd/pamunix-out/group:/etc/group"
-
-        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    files
-          group:     files
-          shadow:    files
-          sudoers:   files
-
-          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
-          networks:  files
-
-          ethers:    files
-          services:  files
-          protocols: files
-          rpc:       files
-
-          subuid:    files
-          subgid:    files
-        ''}:/etc/nsswitch.conf"
-
         "${fhsEnv}/bin:/bin"
         "${fhsEnv}/sbin:/sbin"
         "${fhsEnv}/lib:/lib"
@@ -447,7 +395,16 @@ in
           "/share"
         ];
       });
-      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters;
+      BindPaths = (lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
+        directoryFn = [
+          (_: "/home/pvv/")
+          (l: "/amd/homepvv${l}/")
+        ];
+        letter = homeLetters;
+      }) ++ [
+        "/run/httpd-log-processor-access.fifo"
+        "/run/httpd-log-processor-error.fifo"
+      ];
     };
   };
 

hosts/temmie/services/userweb/log-processor.nix

@@ -0,0 +1,167 @@
+{ config, lib, pkgs, values, ... }:
+let
+  homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
+
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+in
+{
+  sops.secrets = {
+    "httpd/passwd-ssh-key" = { };
+    "httpd/ssh-known-hosts" = { };
+  };
+
+  systemd.targets.sockets.wants = [
+    "httpd-log-processor@access.socket"
+    "httpd-log-processor@error.socket"
+  ];
+
+  systemd.sockets."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    socketConfig = {
+      ListenFIFO = "/run/httpd-log-processor-%i.fifo";
+      RemoveOnStop = true;
+
+      SocketUser = "wwwrun";
+      SocketGroup = "wwwrun";
+      SocketMode = "0600";
+    };
+  };
+
+  systemd.services."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    serviceConfig = {
+      User = "wwwrun";
+      Group = "wwwrun";
+      Slice = "system-userweb.slice";
+      Restart = "on-failure";
+
+      StandardInput = "socket";
+      StandardOutput = "journal";
+      StandardError = "journal";
+
+      LoadCredential = [
+        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
+        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
+      ];
+      ExecStartPre = let
+        rsyncArgs = lib.cli.toCommandLineShellGNU { } {
+          archive = true;
+          verbose = true;
+          compress = true;
+          rsh = "${lib.getExe' pkgs.openssh "ssh"} -o BatchMode=yes -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey";
+        };
+        inputDir = "/run/httpd-log-processor-%i/pamunix-in";
+        outputDir = "/run/httpd-log-processor-%i/pamunix-out";
+      in lib.mkForce [
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/passwd ${inputDir}/"
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/group ${inputDir}/"
+
+        (let
+          args = lib.cli.toCommandLineShellGNU { } {
+            passwd-file = "${inputDir}/passwd";
+            group-file = "${inputDir}/group";
+            output-dir = outputDir;
+            shadow-file = pkgs.emptyFile;
+
+            output-passwd = true;
+
+            ignore-user-file = toString ./ignore_user_file.txt;
+            ignore-group-file = toString ./ignore_group_file.txt;
+          };
+        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
+        "${lib.getExe' pkgs.coreutils "shred"} -u ${inputDir}/passwd ${inputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' ${outputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/group  /etc/group"
+      ];
+
+      ExecStart = "${lib.getExe apache-log-processor} %i";
+
+      AmbientCapabilities = [ "CAP_SETUID" "CAP_SETGID" ];
+      CapabilityBoundingSet = [ "CAP_SETUID" "CAP_SETGID" ];
+      DeviceAllow = [ "" ];
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      PrivateDevices = true;
+      PrivateIPC = true;
+      PrivateTmp = true;
+      # PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = "tmpfs";
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      ProtectKernelTunables = true;
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SocketBindDeny = "any";
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+         "@system-service"
+         "@setuid"
+      ];
+      UMask = "0077";
+
+      IPAddressAllow = [
+        "127.0.0.53" # systemd-resolved
+        values.hosts.microbel.ipv4
+        values.hosts.microbel.ipv6
+      ];
+      IPAddressDeny = "any";
+
+      RootDirectory = "/run/httpd-log-processor-%i/root-mnt";
+      MountAPIVFS = true;
+
+      RuntimeDirectoryMode = "0750";
+      RuntimeDirectory = [
+        "httpd-log-processor-%i/root-mnt"
+        "httpd-log-processor-%i/pamunix-in"
+        "httpd-log-processor-%i/pamunix-out"
+      ];
+      BindReadOnlyPaths = [
+        builtins.storeDir
+        "/etc"
+        "/etc/resolv.conf"
+
+        "-/run/httpd-log-processor-%i/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd-log-processor-%i/pamunix-out/group:/etc/group"
+
+        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
+          passwd:    files
+          group:     files
+          shadow:    files
+          sudoers:   files
+
+          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+
+          subuid:    files
+          subgid:    files
+        ''}:/etc/nsswitch.conf"
+      ];
+      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters ++ [
+        "/var/log/httpd"
+      ];
+    };
+  };
+}

users/vegardbm.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, config, ... }:
 {
   users.users.vegardbm = {
     isNormalUser = true;
@@ -8,12 +8,14 @@
       "drift"
       "nix-builder-users"
     ];
+    shell = if config.programs.zsh.enable then pkgs.zsh else pkgs.bash;
     packages = with pkgs; [
       btop
       eza
     ];
     openssh.authorizedKeys.keys = [
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVA3HqEx3je6L1AC+bP8sTxu3ZTKvTCR0npCyOVAYK5 vbm@arch-xeon"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICrYATHNvBNAHr9G+VwZIaAQPe02iRgAjqtZkW4x/dje vbm@talos"
     ];
   };
 }