temmie/userweb: run log processors as separate systemd units

This lets us divide up some of the logic making httpd itself less
brittle, and also reduces the amount of privileges for httpd.

base/default.nix

@@ -14,7 +14,6 @@
 
     ./mitigations.nix
 
-    ./flake-input-exporter.nix
     ./hardening.nix
     ./networking.nix
     ./nix.nix
@@ -34,9 +33,11 @@
     ./services/openssh.nix
     ./services/polkit.nix
     ./services/postfix.nix
+    ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix
     ./services/roowho2.nix
+    ./services/rsyslogd.nix
     ./services/scrutiny-collector.nix
     ./services/smartd.nix
     ./services/thermald.nix
@@ -94,6 +95,10 @@
     AllowHibernation = lib.mkDefault false;
   };
 
+  systemd.slices."system-monitoring" = {
+    description = "Monitoring related services";
+  };
+
   # users.mutableUsers = lib.mkDefault false;
 
   users.groups."drift".name = "drift";

base/flake-input-exporter.nix

@@ -1,55 +0,0 @@
-{
-  config,
-  inputs,
-  lib,
-  pkgs,
-  values,
-  ...
-}:
-let
-  data = lib.flip lib.mapAttrs inputs (
-    name: input: {
-      inherit (input)
-        lastModified
-        ;
-    }
-  );
-  folder = pkgs.writeTextDir "share/flake-inputs" (
-    lib.concatMapStringsSep "\n" (
-      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
-    ) (lib.attrsToList data)
-  );
-  port = 9102;
-in
-{
-  services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = {
-    serverName = config.networking.fqdn;
-    serverAliases = [
-      "${config.networking.hostName}.pvv.org"
-    ];
-    locations."/metrics" = {
-      root = "${folder}/share";
-      tryFiles = "/flake-inputs =404";
-      extraConfig = ''
-        default_type text/plain;
-      '';
-    };
-    listen = [
-      {
-        inherit port;
-        addr = "0.0.0.0";
-      }
-    ];
-    extraConfig = ''
-      allow ${values.hosts.ildkule.ipv4}/32;
-      allow ${values.hosts.ildkule.ipv6}/128;
-      allow 127.0.0.1/32;
-      allow ::1/128;
-      allow ${values.ipv4-space};
-      allow ${values.ipv6-space};
-      deny all;
-    '';
-  };
-
-  networking.firewall.allowedTCPPorts = [ port ];
-}

base/services/fluentbit.nix

@@ -88,6 +88,7 @@ in
 
   systemd.services.fluent-bit = lib.mkIf cfg.enable {
     serviceConfig = {
+      Slice = "system-monitoring.slice";
       StateDirectory = "fluent-bit";
 
       # NOTE: This hardening might be way too strong for general purpose use, don't upstream this.

base/services/journald-upload.nix

@@ -14,6 +14,7 @@ in
   };
 
   systemd.services."systemd-journal-upload".serviceConfig = lib.mkIf cfg.enable {
+    Slice = "system-monitoring.slice";
     IPAddressDeny = "any";
     IPAddressAllow = [
       values.hosts.ildkule.ipv4

base/services/nginx.nix

@@ -1,18 +1,5 @@
 { config, lib, ... }:
 {
-  # nginx return 444 for all nonexistent virtualhosts
-
-  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
-
-  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
-    "/etc/certs/nginx" = {
-      owner = "nginx";
-      group = "nginx";
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-
   services.nginx = {
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
@@ -60,17 +47,8 @@
           ];
         }
       ];
-      sslCertificate = "/etc/certs/nginx.crt";
+    };
-      sslCertificateKey = "/etc/certs/nginx.key";
-      addSSL = true;
-      extraConfig = "return 444;";
   };
 
-    ${config.networking.fqdn} = {
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-      sslCertificate = lib.mkDefault "/etc/certs/nginx.crt";
-      sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key";
-      addSSL = lib.mkDefault true;
-      extraConfig = lib.mkDefault "return 444;";
-    };
-  };
 }

base/services/prometheus-flake-input-exporter.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  values,
+  ...
+}:
+let
+  data = lib.flip lib.mapAttrs inputs (
+    name: input: {
+      inherit (input)
+        lastModified
+        ;
+    }
+  );
+  folder = pkgs.writeTextDir "share/flake-inputs" (
+    lib.concatMapStringsSep "\n" (
+      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
+    ) (lib.attrsToList data)
+  );
+in
+{
+  services.nginx = {
+    enable = lib.mkDefault true;
+
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      forceSSL = true;
+      enableACME = true;
+      kTLS = true;
+
+      locations."/prometheus-nixos-flake-input-exporter/metrics" = {
+        root = "${folder}/share";
+        tryFiles = "/flake-inputs =404";
+        extraConfig = ''
+          default_type text/plain;
+
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
+  };
+}

base/services/prometheus-node-exporter.nix

@@ -5,19 +5,34 @@ in
 {
   services.prometheus.exporters.node = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9100;
     enabledCollectors = [ "systemd" ];
   };
 
-  systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable {
+  services.nginx = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
+
-      "127.0.0.1"
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      "::1"
+      forceSSL = true;
-      values.hosts.ildkule.ipv4
+      enableACME = true;
-      values.hosts.ildkule.ipv6
+      kTLS = true;
-    ];
+
+      locations."/prometheus-node-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
   };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-node-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -5,6 +5,7 @@ in
 {
   services.prometheus.exporters.systemd = {
     enable = lib.mkDefault true;
+    listenAddress = "127.0.0.1";
     port = 9101;
     extraFlags = [
       "--systemd.collector.enable-restart-count"
@@ -12,15 +13,29 @@ in
     ];
   };
 
-  systemd.services.prometheus-systemd-exporter.serviceConfig = {
+  services.nginx = lib.mkIf cfg.enable {
-    IPAddressDeny = "any";
+    enable = lib.mkDefault true;
-    IPAddressAllow = [
+
-      "127.0.0.1"
+    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      "::1"
+      forceSSL = true;
-      values.hosts.ildkule.ipv4
+      enableACME = true;
-      values.hosts.ildkule.ipv6
+      kTLS = true;
-    ];
+
+      locations."/prometheus-systemd-exporter/metrics" = {
+        proxyPass = "http://localhost:${toString cfg.port}/metrics";
+
+        extraConfig = ''
+          allow 127.0.0.1;
+          allow ::1;
+          allow ${values.hosts.ildkule.ipv4};
+          allow ${values.hosts.ildkule.ipv6};
+          deny all;
+        '';
+      };
+    };
   };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
+  systemd.services = lib.mkIf cfg.enable {
+    "prometheus-systemd-exporter".serviceConfig.Slice = "system-monitoring.slice";
+  };
 }

base/services/rsyslogd.nix

@@ -0,0 +1,20 @@
+{ config, lib, ... }:
+let
+  cfg = config.services.rsyslogd;
+in
+{
+  services.rsyslogd = {
+    enable = lib.mkDefault true;
+    defaultConfig = ''
+      *.* @loghost.pvv.ntnu.no
+    '';
+  };
+
+  services.journald.extraConfig = lib.mkIf cfg.enable ''
+    ForwardToSyslog=yes
+  '';
+
+  systemd.services = lib.mkIf cfg.enable {
+    "syslog".serviceConfig.Slice = "system-monitoring.slice";
+  };
+}

base/services/uptimed.nix

@@ -23,7 +23,7 @@ in
       };
     };
 
-    systemd.services.uptimed = lib.mkIf (cfg.enable) {
+    systemd.services.uptimed = lib.mkIf cfg.enable {
       serviceConfig = let
         uptimed = pkgs.uptimed.overrideAttrs (prev: {
           postPatch = ''
@@ -35,6 +35,8 @@ in
         });
 
       in {
+        Slice = "system-monitoring.slice";
+
         Type = "notify";
 
         ExecStart = lib.mkForce "${uptimed}/sbin/uptimed -f";

hosts/ildkule/services/monitoring/prometheus/default.nix

@@ -8,7 +8,6 @@ in {
     ./matrix-synapse.nix
     ./mysqld.nix
     ./postgres.nix
-    ./dibbler.nix
   ];
 
   services.prometheus = {

hosts/ildkule/services/monitoring/prometheus/dibbler.nix

@@ -1,96 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  ...
-}:
-let
-  cfg = config.services.prometheus.exporters.sql;
-  configFile =
-    if cfg.configFile != null then
-      cfg.configFile
-    else
-      let
-        nameInline = lib.mapAttrsToList (k: v: v // { name = k; });
-        renameStartupSql = j: removeAttrs (j // { startup_sql = j.startupSql; }) [ "startupSql" ];
-        configuration = {
-          jobs = map renameStartupSql (
-            nameInline (lib.mapAttrs (k: v: (v // { queries = nameInline v.queries; })) cfg.configuration.jobs)
-          );
-        };
-      in
-      builtins.toFile "config.yaml" (builtins.toJSON configuration);
-in
-{
-  sops.secrets."config/postgresql_dibbler_password" = { };
-
-  services.prometheus.scrapeConfigs = [
-    {
-      job_name = "sql_exporter";
-      scrape_interval = "1m";
-      scheme = "http";
-
-      static_configs = [
-        {
-          targets = [ "localhost:9237" ];
-        }
-      ];
-    }
-  ];
-
-  services.prometheus.exporters.sql = {
-    enable = true;
-    configuration = {
-      jobs.dibbler = {
-        interval = "1m";
-
-        queries."daily_purchase_sum" = {
-          help = "Sum of purchases for the current day.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1";
-        };
-
-        queries."total_purchase_sum" = {
-          help = "Sum of all purchases.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(price) FROM purchases";
-        };
-
-        queries."total_stock_value" = {
-          help = "The value of all stock in dibbler.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(price * stock) FROM products";
-        };
-
-        queries."user_credit_sum" = {
-          help = "The sum of all user credit.";
-          labels = [ "thing" ];
-          values = [ "sum" ];
-          query = "SELECT SUM(credit) FROM users";
-        };
-      };
-    };
-  };
-
-  systemd.services."prometheus-sql-exporter".serviceConfig = {
-    RuntimeDirectory = "prometheus-sql-exporter";
-    LoadCredential = "postgresql_dibbler_password:${
-      config.sops.secrets."config/postgresql_dibbler_password".path
-    }";
-    ExecStartPre = ''
-      |${lib.getExe pkgs.jq} \
-      --null-input \
-      --compact-output \
-      --slurpfile config '${configFile}' \
-      --rawfile pw '%d/postgresql_dibbler_password' \
-      --from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" ''
-        ("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri
-        | $config[0]
-        | .jobs[0].connections[0] = $pg_uri
-      ''}' > /run/prometheus-sql-exporter/config.yaml
-    '';
-  };
-}

hosts/ildkule/services/monitoring/prometheus/machines.nix

@@ -6,32 +6,63 @@
     targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports;
   };
 
+  nixosMachines = [
+    "ildkule"
+    "bekkalokk"
+    "bicep"
+    "brzeczyszczykiewicz"
+    "georg"
+    "gluttony"
+    "kommode"
+    "lupine-1"
+    "lupine-2"
+    "lupine-3"
+    "lupine-4"
+    "lupine-5"
+    # TODO: export prometheus stats via apache on temmie
+    # "temmie"
+    "wenche"
+  ];
+
   defaultNodeExporterPort = 9100;
-  defaultSystemdExporterPort = 9101;
-  defaultNixosExporterPort = 9102;
 in {
-  services.prometheus.scrapeConfigs = [{
+  services.prometheus.scrapeConfigs = [
-    job_name = "base_info";
+    {
+      job_name = "nixos-node";
+      scheme = "https";
+      metrics_path = "/prometheus-node-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-systemd";
+      scheme = "https";
+      metrics_path = "/prometheus-systemd-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "nixos-flake-input";
+      scheme = "https";
+      metrics_path = "/prometheus-nixos-flake-input-exporter/metrics";
+      static_configs = map (name: {
+        labels.hostname = name;
+        targets = [ "${name}.pvv.ntnu.no:443" ];
+      }) nixosMachines;
+    }
+    {
+      job_name = "non-nixos-node";
+      scheme = "http";
+      metrics_path = "/metrics";
       static_configs = [
-      (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ])
-
-      (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-      (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
-
         (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
         (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
         (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
       ];
-  }];
+    }
+  ];
 }

hosts/temmie/services/userweb/default.nix

@@ -14,8 +14,6 @@ let
     upload_max_filesize = "40M";
   });
 
-  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
-
   # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
   phpEnv = pkgs.php.buildEnv {
     extensions = { all, ... }: with all; [
@@ -161,13 +159,9 @@ in
 {
   imports = [
     ./mail.nix
+    ./log-processor.nix
   ];
 
-  sops.secrets = {
-    "httpd/passwd-ssh-key" = { };
-    "httpd/ssh-known-hosts" = { };
-  };
-
   services.httpd = {
     enable = true;
     adminAddr = "drift@pvv.ntnu.no";
@@ -212,6 +206,19 @@ in
     extraConfig = ''
       TraceEnable on
       LogLevel warn rewrite:trace3
+
+      <FilesMatch ".+\.ph(p[3457]?|t|tml)$">
+          SetHandler application/x-httpd-php
+      </FilesMatch>
+      <FilesMatch ".+\.phps$">
+          SetHandler application/x-httpd-php-source
+          Require all denied
+      </FilesMatch>
+      <FilesMatch "\.pl$">
+          SetHandler modperl
+          PerlResponseHandler ModPerl::Registry
+          Options +ExecCGI
+      </FilesMatch>
     '';
 
     virtualHosts."temmie.pvv.ntnu.no" = {
@@ -224,8 +231,8 @@ in
 
       extraConfig = ''
         CustomLog "${cfg.logDir}/access.log" combined
-        CustomLog "|${lib.getExe apache-log-processor} access" combined
+        CustomLog "/run/httpd-log-processor-access.fifo" combined
-        ErrorLog "|${lib.getExe apache-log-processor} error"
+        ErrorLog "/run/httpd-log-processor-error.fifo"
         ScriptLog "${cfg.logDir}/cgi.log"
 
         UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
@@ -285,9 +292,26 @@ in
   users.users."wwwrun".uid = lib.mkForce 33;
   users.groups."wwwrun".gid = lib.mkForce 33;
 
+  systemd.targets.userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
+  systemd.slices.system-userweb = {
+    description = "PVV HTTPD UserWeb";
+  };
+
   systemd.services.httpd = {
-    after = [ "pvv-homedirs.target" ];
+    after = [
-    requires = [ "pvv-homedirs.target" ];
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requires = [
+      "pvv-homedirs.target"
+      "httpd-log-processor@access.socket"
+      "httpd-log-processor@error.socket"
+    ];
+    requiredBy = [ "userweb.target" ];
 
     environment = {
       PATH = lib.mkForce "/usr/bin";
@@ -295,65 +319,18 @@ in
 
     serviceConfig = {
       Type = lib.mkForce "notify";
-
-      ExecStartPre = let
-        rsyncCommand = ''${lib.getExe pkgs.rsync} -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey" -avz'';
-      in lib.mkForce [
-        "${lib.getExe (pkgs.writeShellApplication {
-          name = "http-exec-start-pre-remove-old-semaphores";
-          text = ''
-            # Get rid of old semaphores.  These tend to accumulate across
-            # server restarts, eventually preventing it from restarting
-            # successfully.
-            for i in $(${pkgs.util-linux}/bin/ipcs -s | grep ' ${cfg.user} ' | cut -f2 -d ' '); do
-                ${pkgs.util-linux}/bin/ipcrm -s "$i"
-            done
-          '';
-        })}"
-
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
-
-        (let
-          args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "/run/httpd/pamunix-in/passwd";
-            group-file = "/run/httpd/pamunix-in/group";
-            output-dir = "/run/httpd/pamunix-out";
-            shadow-file = pkgs.emptyFile;
-
-            output-passwd = true;
-
-            ignore-user-file = toString ./ignore_user_file.txt;
-            ignore-group-file = toString ./ignore_group_file.txt;
-          };
-        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
-        "${lib.getExe' pkgs.coreutils "cat"} /run/httpd/pamunix-out/passwd"
-        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
-        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
-      ];
       ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
       ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
       ExecStop = lib.mkForce "";
       KillMode = "mixed";
-
+      Slice = "system-userweb.slice";
-      LoadCredential=[
-        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
-        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
-      ];
 
       ConfigurationDirectory = [ "httpd" ];
       LogsDirectory = [ "httpd" ];
       LogsDirectoryMode = "0700";
 
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
       LockPersonality = !debug;
       PrivateDevices = true;
       PrivateTmp = true;
@@ -381,48 +358,19 @@ in
         "tcp:443"
       ];
       SystemCallArchitectures = "native";
-      SystemCallFilter = lib.mkIf (!debug) [
+      SystemCallFilter = lib.mkIf (!debug) [ "@system-service" ];
-         "@system-service"
-         "@setuid"
-      ];
       UMask = "0077";
 
       RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [
+      RuntimeDirectory = [ "httpd/root-mnt" ];
-        "httpd/root-mnt"
-        "httpd/pamunix-in"
-        "httpd/pamunix-out"
-      ];
       RootDirectory = "/run/httpd/root-mnt";
       MountAPIVFS = true;
       BindReadOnlyPaths = [
         builtins.storeDir
         "/etc"
         "/dev/null"
-        "/etc/resolv.conf"
         "/var/lib/acme"
-
+        "/var/run/nscd"
-        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
-        "-/run/httpd/pamunix-out/group:/etc/group"
-
-        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    files
-          group:     files
-          shadow:    files
-          sudoers:   files
-
-          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
-          networks:  files
-
-          ethers:    files
-          services:  files
-          protocols: files
-          rpc:       files
-
-          subuid:    files
-          subgid:    files
-        ''}:/etc/nsswitch.conf"
-
         "${fhsEnv}/bin:/bin"
         "${fhsEnv}/sbin:/sbin"
         "${fhsEnv}/lib:/lib"
@@ -447,7 +395,16 @@ in
           "/share"
         ];
       });
-      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters;
+      BindPaths = (lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
+        directoryFn = [
+          (_: "/home/pvv/")
+          (l: "/amd/homepvv${l}/")
+        ];
+        letter = homeLetters;
+      }) ++ [
+        "/run/httpd-log-processor-access.fifo"
+        "/run/httpd-log-processor-error.fifo"
+      ];
     };
   };
 

hosts/temmie/services/userweb/log-processor.nix

@@ -0,0 +1,167 @@
+{ config, lib, pkgs, values, ... }:
+let
+  homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
+
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+in
+{
+  sops.secrets = {
+    "httpd/passwd-ssh-key" = { };
+    "httpd/ssh-known-hosts" = { };
+  };
+
+  systemd.targets.sockets.wants = [
+    "httpd-log-processor@access.socket"
+    "httpd-log-processor@error.socket"
+  ];
+
+  systemd.sockets."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    socketConfig = {
+      ListenFIFO = "/run/httpd-log-processor-%i.fifo";
+      RemoveOnStop = true;
+
+      SocketUser = "wwwrun";
+      SocketGroup = "wwwrun";
+      SocketMode = "0600";
+    };
+  };
+
+  systemd.services."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
+    requiredBy = [ "userweb.target" ];
+    serviceConfig = {
+      User = "wwwrun";
+      Group = "wwwrun";
+      Slice = "system-userweb.slice";
+      Restart = "on-failure";
+
+      StandardInput = "socket";
+      StandardOutput = "journal";
+      StandardError = "journal";
+
+      LoadCredential = [
+        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
+        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
+      ];
+      ExecStartPre = let
+        rsyncArgs = lib.cli.toCommandLineShellGNU { } {
+          archive = true;
+          verbose = true;
+          compress = true;
+          rsh = "${lib.getExe' pkgs.openssh "ssh"} -o BatchMode=yes -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey";
+        };
+        inputDir = "/run/httpd-log-processor-%i/pamunix-in";
+        outputDir = "/run/httpd-log-processor-%i/pamunix-out";
+      in lib.mkForce [
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/passwd ${inputDir}/"
+        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/group ${inputDir}/"
+
+        (let
+          args = lib.cli.toCommandLineShellGNU { } {
+            passwd-file = "${inputDir}/passwd";
+            group-file = "${inputDir}/group";
+            output-dir = outputDir;
+            shadow-file = pkgs.emptyFile;
+
+            output-passwd = true;
+
+            ignore-user-file = toString ./ignore_user_file.txt;
+            ignore-group-file = toString ./ignore_group_file.txt;
+          };
+        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
+        "${lib.getExe' pkgs.coreutils "shred"} -u ${inputDir}/passwd ${inputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' ${outputDir}/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 ${outputDir}/passwd ${outputDir}/group"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind ${outputDir}/group  /etc/group"
+      ];
+
+      ExecStart = "${lib.getExe apache-log-processor} %i";
+
+      AmbientCapabilities = [ "CAP_SETUID" "CAP_SETGID" ];
+      CapabilityBoundingSet = [ "CAP_SETUID" "CAP_SETGID" ];
+      DeviceAllow = [ "" ];
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      PrivateDevices = true;
+      PrivateIPC = true;
+      PrivateTmp = true;
+      # PrivateUsers = true;
+      ProcSubset = "pid";
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = "tmpfs";
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      ProtectKernelTunables = true;
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SocketBindDeny = "any";
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+         "@system-service"
+         "@setuid"
+      ];
+      UMask = "0077";
+
+      IPAddressAllow = [
+        "127.0.0.53" # systemd-resolved
+        values.hosts.microbel.ipv4
+        values.hosts.microbel.ipv6
+      ];
+      IPAddressDeny = "any";
+
+      RootDirectory = "/run/httpd-log-processor-%i/root-mnt";
+      MountAPIVFS = true;
+
+      RuntimeDirectoryMode = "0750";
+      RuntimeDirectory = [
+        "httpd-log-processor-%i/root-mnt"
+        "httpd-log-processor-%i/pamunix-in"
+        "httpd-log-processor-%i/pamunix-out"
+      ];
+      BindReadOnlyPaths = [
+        builtins.storeDir
+        "/etc"
+        "/etc/resolv.conf"
+
+        "-/run/httpd-log-processor-%i/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd-log-processor-%i/pamunix-out/group:/etc/group"
+
+        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
+          passwd:    files
+          group:     files
+          shadow:    files
+          sudoers:   files
+
+          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+
+          subuid:    files
+          subgid:    files
+        ''}:/etc/nsswitch.conf"
+      ];
+      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters ++ [
+        "/var/log/httpd"
+      ];
+    };
+  };
+}

secrets/ildkule/ildkule.yaml

@@ -1,6 +1,5 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
-    postgresql_dibbler_password: ENC[AES256_GCM,data:wP4CVz9qRE3CJrblWWYqOIkcH3LM5H81,iv:j8zr1TRNlPPqIppYlWhDoKlL7m2Ph2wQlU6bvFj8R9A=,tag:Cfp8zbYkoLqI7DTDpOBlJw==,type:str]
 keys:
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
@@ -73,8 +72,8 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-06-07T12:51:04Z"
+    lastmodified: "2025-03-16T20:08:18Z"
-    mac: ENC[AES256_GCM,data:otGwzc3Bme1PGRU4zWRRf3kmAf5EDjT8sPkDK/zDrO0ve+d3x99Qls4DBZV8G6FsFtUXbeKVo70I5VpqUgaIef+nyMl6zzWkW1wpbKIBYjh5fkaP0xxhLnw+shrB088PAXT0wkP7hJBLO0ZJgTrpprJKhmeqVj2HEXBSJ1wSjk8=,iv:cjqtdbWBVLUfpQdykb3vKDKa/VC/kGBybwYtG/eStqc=,tag:remOj4bX/H/LlAPgcXHAdA==,type:str]
+    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -97,4 +96,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.12.2
+    version: 3.9.4

users/vegardbm.nix

@@ -1,4 +1,4 @@
-{ pkgs, ... }:
+{ pkgs, config, ... }:
 {
   users.users.vegardbm = {
     isNormalUser = true;
@@ -8,12 +8,14 @@
       "drift"
       "nix-builder-users"
     ];
+    shell = if config.programs.zsh.enable then pkgs.zsh else pkgs.bash;
     packages = with pkgs; [
       btop
       eza
     ];
     openssh.authorizedKeys.keys = [
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVA3HqEx3je6L1AC+bP8sTxu3ZTKvTCR0npCyOVAYK5 vbm@arch-xeon"
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICrYATHNvBNAHr9G+VwZIaAQPe02iRgAjqtZkW4x/dje vbm@talos"
     ];
   };
 }