prometheus for dibbler

base/default.nix

@@ -14,6 +14,7 @@
 
     ./mitigations.nix
 
+    ./flake-input-exporter.nix
     ./hardening.nix
     ./networking.nix
     ./nix.nix
@@ -33,11 +34,9 @@
     ./services/openssh.nix
     ./services/polkit.nix
     ./services/postfix.nix
-    ./services/prometheus-flake-input-exporter.nix
     ./services/prometheus-node-exporter.nix
     ./services/prometheus-systemd-exporter.nix
     ./services/roowho2.nix
-    ./services/rsyslogd.nix
     ./services/scrutiny-collector.nix
     ./services/smartd.nix
     ./services/thermald.nix
@@ -95,10 +94,6 @@
     AllowHibernation = lib.mkDefault false;
   };
 
-  systemd.slices."system-monitoring" = {
-    description = "Monitoring related services";
-  };
-
   # users.mutableUsers = lib.mkDefault false;
 
   users.groups."drift".name = "drift";

base/flake-input-exporter.nix

@@ -0,0 +1,55 @@
+{
+  config,
+  inputs,
+  lib,
+  pkgs,
+  values,
+  ...
+}:
+let
+  data = lib.flip lib.mapAttrs inputs (
+    name: input: {
+      inherit (input)
+        lastModified
+        ;
+    }
+  );
+  folder = pkgs.writeTextDir "share/flake-inputs" (
+    lib.concatMapStringsSep "\n" (
+      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
+    ) (lib.attrsToList data)
+  );
+  port = 9102;
+in
+{
+  services.nginx.virtualHosts."${config.networking.fqdn}-nixos-metrics" = {
+    serverName = config.networking.fqdn;
+    serverAliases = [
+      "${config.networking.hostName}.pvv.org"
+    ];
+    locations."/metrics" = {
+      root = "${folder}/share";
+      tryFiles = "/flake-inputs =404";
+      extraConfig = ''
+        default_type text/plain;
+      '';
+    };
+    listen = [
+      {
+        inherit port;
+        addr = "0.0.0.0";
+      }
+    ];
+    extraConfig = ''
+      allow ${values.hosts.ildkule.ipv4}/32;
+      allow ${values.hosts.ildkule.ipv6}/128;
+      allow 127.0.0.1/32;
+      allow ::1/128;
+      allow ${values.ipv4-space};
+      allow ${values.ipv6-space};
+      deny all;
+    '';
+  };
+
+  networking.firewall.allowedTCPPorts = [ port ];
+}

base/services/fluentbit.nix

@@ -88,7 +88,6 @@ in
 
   systemd.services.fluent-bit = lib.mkIf cfg.enable {
     serviceConfig = {
-      Slice = "system-monitoring.slice";
       StateDirectory = "fluent-bit";
 
       # NOTE: This hardening might be way too strong for general purpose use, don't upstream this.

base/services/journald-upload.nix

@@ -14,7 +14,6 @@ in
   };
 
   systemd.services."systemd-journal-upload".serviceConfig = lib.mkIf cfg.enable {
-    Slice = "system-monitoring.slice";
     IPAddressDeny = "any";
     IPAddressAllow = [
       values.hosts.ildkule.ipv4

base/services/nginx.nix

@@ -1,5 +1,18 @@
 { config, lib, ... }:
 {
+  # nginx return 444 for all nonexistent virtualhosts
+
+  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
+
+  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
+    "/etc/certs/nginx" = {
+      owner = "nginx";
+      group = "nginx";
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
+
   services.nginx = {
     recommendedTlsSettings = true;
     recommendedProxySettings = true;
@@ -47,8 +60,17 @@
           ];
         }
       ];
-    };
+      sslCertificate = "/etc/certs/nginx.crt";
+      sslCertificateKey = "/etc/certs/nginx.key";
+      addSSL = true;
+      extraConfig = "return 444;";
     };
 
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
+    ${config.networking.fqdn} = {
+      sslCertificate = lib.mkDefault "/etc/certs/nginx.crt";
+      sslCertificateKey = lib.mkDefault "/etc/certs/nginx.key";
+      addSSL = lib.mkDefault true;
+      extraConfig = lib.mkDefault "return 444;";
+    };
+  };
 }

base/services/prometheus-flake-input-exporter.nix

@@ -1,47 +0,0 @@
-{
-  config,
-  inputs,
-  lib,
-  pkgs,
-  values,
-  ...
-}:
-let
-  data = lib.flip lib.mapAttrs inputs (
-    name: input: {
-      inherit (input)
-        lastModified
-        ;
-    }
-  );
-  folder = pkgs.writeTextDir "share/flake-inputs" (
-    lib.concatMapStringsSep "\n" (
-      { name, value }: ''nixos_last_modified_input{flake="${name}"} ${toString value.lastModified}''
-    ) (lib.attrsToList data)
-  );
-in
-{
-  services.nginx = {
-    enable = lib.mkDefault true;
-
-    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
-      forceSSL = true;
-      enableACME = true;
-      kTLS = true;
-
-      locations."/prometheus-nixos-flake-input-exporter/metrics" = {
-        root = "${folder}/share";
-        tryFiles = "/flake-inputs =404";
-        extraConfig = ''
-          default_type text/plain;
-
-          allow 127.0.0.1;
-          allow ::1;
-          allow ${values.hosts.ildkule.ipv4};
-          allow ${values.hosts.ildkule.ipv6};
-          deny all;
-        '';
-      };
-    };
-  };
-}

base/services/prometheus-node-exporter.nix

@@ -5,34 +5,19 @@ in
 {
   services.prometheus.exporters.node = {
     enable = lib.mkDefault true;
-    listenAddress = "127.0.0.1";
     port = 9100;
     enabledCollectors = [ "systemd" ];
   };
 
-  services.nginx = lib.mkIf cfg.enable {
+  systemd.services.prometheus-node-exporter.serviceConfig = lib.mkIf cfg.enable {
-    enable = lib.mkDefault true;
+    IPAddressDeny = "any";
-
+    IPAddressAllow = [
-    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      "127.0.0.1"
-      forceSSL = true;
+      "::1"
-      enableACME = true;
+      values.hosts.ildkule.ipv4
-      kTLS = true;
+      values.hosts.ildkule.ipv6
-
+    ];
-      locations."/prometheus-node-exporter/metrics" = {
-        proxyPass = "http://localhost:${toString cfg.port}/metrics";
-
-        extraConfig = ''
-          allow 127.0.0.1;
-          allow ::1;
-          allow ${values.hosts.ildkule.ipv4};
-          allow ${values.hosts.ildkule.ipv6};
-          deny all;
-        '';
-      };
-    };
   };
 
-  systemd.services = lib.mkIf cfg.enable {
+  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
-    "prometheus-node-exporter".serviceConfig.Slice = "system-monitoring.slice";
-  };
 }

base/services/prometheus-systemd-exporter.nix

@@ -5,7 +5,6 @@ in
 {
   services.prometheus.exporters.systemd = {
     enable = lib.mkDefault true;
-    listenAddress = "127.0.0.1";
     port = 9101;
     extraFlags = [
       "--systemd.collector.enable-restart-count"
@@ -13,29 +12,15 @@ in
     ];
   };
 
-  services.nginx = lib.mkIf cfg.enable {
+  systemd.services.prometheus-systemd-exporter.serviceConfig = {
-    enable = lib.mkDefault true;
+    IPAddressDeny = "any";
-
+    IPAddressAllow = [
-    virtualHosts.${config.networking.fqdn} = lib.mkIf config.services.nginx.enable {
+      "127.0.0.1"
-      forceSSL = true;
+      "::1"
-      enableACME = true;
+      values.hosts.ildkule.ipv4
-      kTLS = true;
+      values.hosts.ildkule.ipv6
-
+    ];
-      locations."/prometheus-systemd-exporter/metrics" = {
-        proxyPass = "http://localhost:${toString cfg.port}/metrics";
-
-        extraConfig = ''
-          allow 127.0.0.1;
-          allow ::1;
-          allow ${values.hosts.ildkule.ipv4};
-          allow ${values.hosts.ildkule.ipv6};
-          deny all;
-        '';
-      };
-    };
   };
 
-  systemd.services = lib.mkIf cfg.enable {
+  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ cfg.port ];
-    "prometheus-systemd-exporter".serviceConfig.Slice = "system-monitoring.slice";
-  };
 }

base/services/rsyslogd.nix

@@ -1,20 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.services.rsyslogd;
-in
-{
-  services.rsyslogd = {
-    enable = lib.mkDefault true;
-    defaultConfig = ''
-      *.* @loghost.pvv.ntnu.no
-    '';
-  };
-
-  services.journald.extraConfig = lib.mkIf cfg.enable ''
-    ForwardToSyslog=yes
-  '';
-
-  systemd.services = lib.mkIf cfg.enable {
-    "syslog".serviceConfig.Slice = "system-monitoring.slice";
-  };
-}

base/services/uptimed.nix

@@ -23,7 +23,7 @@ in
       };
     };
 
-    systemd.services.uptimed = lib.mkIf cfg.enable {
+    systemd.services.uptimed = lib.mkIf (cfg.enable) {
       serviceConfig = let
         uptimed = pkgs.uptimed.overrideAttrs (prev: {
           postPatch = ''
@@ -35,8 +35,6 @@ in
         });
 
       in {
-        Slice = "system-monitoring.slice";
-
         Type = "notify";
 
         ExecStart = lib.mkForce "${uptimed}/sbin/uptimed -f";

flake.lock

@@ -379,11 +379,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781362709,
+        "lastModified": 1779903528,
-        "narHash": "sha256-zTzLvvtRdOKKcQMWydoZuduj9fwwkYRfB1RBzKegwHM=",
+        "narHash": "sha256-4rajaHeBeQ4PjbNSpslE9G3A5mZM1J/64ls+VoufWZo=",
         "ref": "main",
-        "rev": "5fb08a462263e5a8d742ecd93e559e534d5c3af2",
+        "rev": "bba7413a1c611d4918fbef4d3aa55e465ca3f3fb",
-        "revCount": 586,
+        "revCount": 585,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/nettsiden.git"
       },

hosts/ildkule/services/monitoring/prometheus/default.nix

@@ -8,6 +8,7 @@ in {
     ./matrix-synapse.nix
     ./mysqld.nix
     ./postgres.nix
+    ./dibbler.nix
   ];
 
   services.prometheus = {

hosts/ildkule/services/monitoring/prometheus/dibbler.nix

@@ -0,0 +1,96 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.exporters.sql;
+  configFile =
+    if cfg.configFile != null then
+      cfg.configFile
+    else
+      let
+        nameInline = lib.mapAttrsToList (k: v: v // { name = k; });
+        renameStartupSql = j: removeAttrs (j // { startup_sql = j.startupSql; }) [ "startupSql" ];
+        configuration = {
+          jobs = map renameStartupSql (
+            nameInline (lib.mapAttrs (k: v: (v // { queries = nameInline v.queries; })) cfg.configuration.jobs)
+          );
+        };
+      in
+      builtins.toFile "config.yaml" (builtins.toJSON configuration);
+in
+{
+  sops.secrets."config/postgresql_dibbler_password" = { };
+
+  services.prometheus.scrapeConfigs = [
+    {
+      job_name = "sql_exporter";
+      scrape_interval = "1m";
+      scheme = "http";
+
+      static_configs = [
+        {
+          targets = [ "localhost:9237" ];
+        }
+      ];
+    }
+  ];
+
+  services.prometheus.exporters.sql = {
+    enable = true;
+    configuration = {
+      jobs.dibbler = {
+        interval = "1m";
+
+        queries."daily_purchase_sum" = {
+          help = "Sum of purchases for the current day.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1";
+        };
+
+        queries."total_purchase_sum" = {
+          help = "Sum of all purchases.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(price) FROM purchases";
+        };
+
+        queries."total_stock_value" = {
+          help = "The value of all stock in dibbler.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(price * stock) FROM products";
+        };
+
+        queries."user_credit_sum" = {
+          help = "The sum of all user credit.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(credit) FROM users";
+        };
+      };
+    };
+  };
+
+  systemd.services."prometheus-sql-exporter".serviceConfig = {
+    RuntimeDirectory = "prometheus-sql-exporter";
+    LoadCredential = "postgresql_dibbler_password:${
+      config.sops.secrets."config/postgresql_dibbler_password".path
+    }";
+    ExecStartPre = ''
+      |${lib.getExe pkgs.jq} \
+      --null-input \
+      --compact-output \
+      --slurpfile config '${configFile}' \
+      --rawfile pw '%d/postgresql_dibbler_password' \
+      --from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" ''
+        ("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri
+        | $config[0]
+        | .jobs[0].connections[0] = $pg_uri
+      ''}' > /run/prometheus-sql-exporter/config.yaml
+    '';
+  };
+}

hosts/ildkule/services/monitoring/prometheus/machines.nix

@@ -6,63 +6,32 @@
     targets = map (port: "${name}.pvv.ntnu.no:${toString port}") ports;
   };
 
-  nixosMachines = [
-    "ildkule"
-    "bekkalokk"
-    "bicep"
-    "brzeczyszczykiewicz"
-    "georg"
-    "gluttony"
-    "kommode"
-    "lupine-1"
-    "lupine-2"
-    "lupine-3"
-    "lupine-4"
-    "lupine-5"
-    # TODO: export prometheus stats via apache on temmie
-    # "temmie"
-    "wenche"
-  ];
-
   defaultNodeExporterPort = 9100;
+  defaultSystemdExporterPort = 9101;
+  defaultNixosExporterPort = 9102;
 in {
-  services.prometheus.scrapeConfigs = [
+  services.prometheus.scrapeConfigs = [{
-    {
+    job_name = "base_info";
-      job_name = "nixos-node";
-      scheme = "https";
-      metrics_path = "/prometheus-node-exporter/metrics";
-      static_configs = map (name: {
-        labels.hostname = name;
-        targets = [ "${name}.pvv.ntnu.no:443" ];
-      }) nixosMachines;
-    }
-    {
-      job_name = "nixos-systemd";
-      scheme = "https";
-      metrics_path = "/prometheus-systemd-exporter/metrics";
-      static_configs = map (name: {
-        labels.hostname = name;
-        targets = [ "${name}.pvv.ntnu.no:443" ];
-      }) nixosMachines;
-    }
-    {
-      job_name = "nixos-flake-input";
-      scheme = "https";
-      metrics_path = "/prometheus-nixos-flake-input-exporter/metrics";
-      static_configs = map (name: {
-        labels.hostname = name;
-        targets = [ "${name}.pvv.ntnu.no:443" ];
-      }) nixosMachines;
-    }
-    {
-      job_name = "non-nixos-node";
-      scheme = "http";
-      metrics_path = "/metrics";
     static_configs = [
+      (mkHostScrapeConfig "ildkule" [ cfg.exporters.node.port cfg.exporters.systemd.port defaultNixosExporterPort ])
+
+      (mkHostScrapeConfig "bekkalokk" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "bicep" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "brzeczyszczykiewicz" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "georg" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "gluttony" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "kommode" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "lupine-1" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "lupine-2" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "lupine-3" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "lupine-4" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "lupine-5" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "temmie" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+      (mkHostScrapeConfig "wenche" [ defaultNodeExporterPort defaultSystemdExporterPort defaultNixosExporterPort ])
+
       (mkHostScrapeConfig "hildring" [ defaultNodeExporterPort ])
       (mkHostScrapeConfig "isvegg" [ defaultNodeExporterPort ])
       (mkHostScrapeConfig "microbel" [ defaultNodeExporterPort ])
     ];
-    }
+  }];
-  ];
 }

hosts/temmie/services/userweb/default.nix

@@ -1,9 +1,455 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.httpd;
+
+  # NOTE Enable this if you want to strace stuff in the sandbox...
+  debug = false;
+
+  homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
+
+  phpOptions = lib.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "${k} = ${v}"){
+    display_errors = "Off";
+    display_startup_errors = "Off";
+    post_max_size = "40M";
+    upload_max_filesize = "40M";
+  });
+
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+
+  # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
+  phpEnv = pkgs.php.buildEnv {
+    extensions = { all, ... }: with all; [
+      bz2
+      curl
+      decimal
+      gd
+      imagick
+      mysqli
+      mysqlnd
+      pgsql
+      posix
+      protobuf sqlite3
+      uuid
+      xml
+      xsl
+      zlib
+      zstd
+
+      pdo
+      pdo_mysql
+      pdo_pgsql
+      pdo_sqlite
+    ];
+
+    extraConfig = phpOptions;
+  };
+
+  perlEnv = (pkgs.perl.withPackages (ps: with ps; [
+    pkgs.exiftool
+    pkgs.ikiwiki
+    pkgs.irssi
+    pkgs.nix.libs.nix-perl-bindings
+
+    CGI
+    DBDPg
+    DBDSQLite
+    DBDmysql
+    DBI
+    Git
+    ImageMagick
+    JSON
+    TemplateToolkit
+  ])).overrideAttrs (prev: {
+    # NOTE: `pkgs.perl.propagatedBuildInputs` don't actually propagate through the
+    #       wrapper derivation created by `withPackages`. This should compensate
+    #       for that.
+    postBuild = prev.postBuild + ''
+      cp -r '${pkgs.perl}/nix-support' "$out"/nix-support
+    '';
+  });
+
+  # https://nixos.org/manual/nixpkgs/stable/#python.buildenv-function
+  pythonEnv = pkgs.python3.buildEnv.override {
+    extraLibs = with pkgs.python3Packages; [
+      legacy-cgi
+
+      matplotlib
+      requests
+    ];
+    ignoreCollisions = true;
+  };
+
+  # https://nixos.org/manual/nixpkgs/stable/#sec-building-environment
+  fhsEnv = pkgs.buildEnv {
+    name = "userweb-env";
+    ignoreCollisions = true;
+    paths = with pkgs; [
+      bash
+
+      config.services.bro.instances.userweb-sendmail.client.package
+
+      perlEnv
+      pythonEnv
+      phpEnv
+    ]
+    ++ (with phpEnv.packages; [
+      # composer
+    ])
+    ++ [
+      # Useful packages for homepages
+      exiftool
+      gnuplot
+      ikiwiki-full
+      imagemagick
+      jhead
+      ruby
+      sbcl
+      sourceHighlight
+
+      # Missing packages from tom
+      # blosxom
+      # pyblosxom
+      # mediawiki (TODO: do people host their own mediawikis in userweb?)
+      # nanoblogger
+
+      # Version control
+      cvs
+      rcs
+      git
+
+      # Compression/Archival
+      bzip2
+      gnutar
+      gzip
+      lz4
+      unzip
+      xz
+      zip
+      zstd
+
+      # Other tools you might expect to find on a normal system
+      acl
+      coreutils-full
+      curl
+      diffutils
+      file
+      findutils
+      gawk
+      gnugrep
+      gnumake
+      gnupg
+      gnused
+      less
+      man
+      util-linux
+      vim
+      wget
+      which
+      xdg-utils
+    ] ++ lib.optionals debug [
+      glibc.getent
+      strace
+      systemd
+    ];
+
+    extraOutputsToInstall = [
+      "man"
+      "doc"
+    ];
+  };
+in
 {
   imports = [
-    ./httpd.nix
-    ./log-processor.nix
     ./mail.nix
-    ./module.nix
-    ./packages.nix
   ];
+
+  sops.secrets = {
+    "httpd/passwd-ssh-key" = { };
+    "httpd/ssh-known-hosts" = { };
+  };
+
+  services.httpd = {
+    enable = true;
+    adminAddr = "drift@pvv.ntnu.no";
+
+    # TODO: consider upstreaming systemd support
+    # TODO: mod_log_journald in v2.5
+    package = pkgs.apacheHttpd.overrideAttrs (prev: {
+      nativeBuildInputs = prev.nativeBuildInputs ++ [ pkgs.pkg-config ];
+      buildInputs = prev.buildInputs ++ [ pkgs.systemdLibs ];
+      configureFlags = prev.configureFlags ++ [ "--enable-systemd" ];
+    });
+
+    enablePHP = true;
+    phpPackage = phpEnv;
+    inherit phpOptions;
+
+    enablePerl = true;
+
+    # TODO: mod_log_journald in v2.5
+    extraModules = [
+      "systemd"
+      "userdir"
+      {
+        name = "perl";
+        path = let
+          mod_perl = pkgs.symlinkJoin {
+            name = "userweb_modperl_with_custom_perl_env";
+            ignoreCollisions = true;
+            paths = [
+              (pkgs.apacheHttpdPackages.mod_perl.override {
+                apacheHttpd = cfg.package.out;
+              })
+              perlEnv
+            ];
+          };
+        in "${mod_perl}/modules/mod_perl.so";
+      }
+    ];
+
+    logPerVirtualHost = false;
+
+    extraConfig = ''
+      TraceEnable on
+      LogLevel warn rewrite:trace3
+    '';
+
+    virtualHosts."temmie.pvv.ntnu.no" = {
+      forceSSL = true;
+      enableACME = true;
+
+      serverAliases = [
+        "www2.pvv.ntnu.no"
+      ];
+
+      extraConfig = ''
+        CustomLog "${cfg.logDir}/access.log" combined
+        CustomLog "|${lib.getExe apache-log-processor} access" combined
+        ErrorLog "|${lib.getExe apache-log-processor} error"
+        ScriptLog "${cfg.logDir}/cgi.log"
+
+        UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
+        UserDir disabled root
+        AddHandler cgi-script .cgi
+        DirectoryIndex index.html index.html.var index.php index.php3 index.cgi index.phtml index.shtml meg.html
+        SetEnvIf Request_URI "^/~([^/]+)" USERDIR_USER=$1
+
+        <Directory "/home/pvv/?/*/web-docs">
+          Options MultiViews Indexes SymLinksIfOwnerMatch ExecCGI IncludesNoExec
+          AllowOverride All
+          Require all granted
+        </Directory>
+
+        <DirectoryMatch "^/home/pvv/.*/web-docs/(${lib.concatStringsSep "|" [
+          "\\.git"
+          "\\.hg"
+          "\\.svn"
+          "\\.ssh"
+          "\\.env"
+          "\\.envrc"
+          "\\.bzr"
+          "\\.venv"
+          "CVS"
+          "RCS"
+          ".*\\.swp"
+          ".*\\.bak"
+          ".*~"
+        ]})(/|$)">
+            AllowOverride All
+            Require all denied
+        </DirectoryMatch>
+      '';
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
+  # socket activation comes in v2.5
+  # systemd.sockets.httpd = {
+  #   wantedBy = [ "sockets.target" ];
+  #   description = "HTTPD socket";
+  #   listenStreams = [
+  #     "0.0.0.0:80"
+  #     "0.0.0.0:443"
+  #   ];
+  # };
+
+  # NOTE: 54 -> 33, this is the UID/GID we used for www-data on tom in the past.
+  #       Any files accessed by or created by httpd will do so over NFS with this
+  #       UID/GID pair as its credentials.
+  #       This overlaps with the hardcoded `disnix` uid in nixpkgs, but we *probably*
+  #       won't be using that for the foreseeable future.
+  users.users."wwwrun".uid = lib.mkForce 33;
+  users.groups."wwwrun".gid = lib.mkForce 33;
+
+  systemd.services.httpd = {
+    after = [ "pvv-homedirs.target" ];
+    requires = [ "pvv-homedirs.target" ];
+
+    environment = {
+      PATH = lib.mkForce "/usr/bin";
+    };
+
+    serviceConfig = {
+      Type = lib.mkForce "notify";
+
+      ExecStartPre = let
+        rsyncCommand = ''${lib.getExe pkgs.rsync} -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey" -avz'';
+      in lib.mkForce [
+        "${lib.getExe (pkgs.writeShellApplication {
+          name = "http-exec-start-pre-remove-old-semaphores";
+          text = ''
+            # Get rid of old semaphores.  These tend to accumulate across
+            # server restarts, eventually preventing it from restarting
+            # successfully.
+            for i in $(${pkgs.util-linux}/bin/ipcs -s | grep ' ${cfg.user} ' | cut -f2 -d ' '); do
+                ${pkgs.util-linux}/bin/ipcrm -s "$i"
+            done
+          '';
+        })}"
+
+        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
+        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
+
+        (let
+          args = lib.cli.toCommandLineShellGNU { } {
+            passwd-file = "/run/httpd/pamunix-in/passwd";
+            group-file = "/run/httpd/pamunix-in/group";
+            output-dir = "/run/httpd/pamunix-out";
+            shadow-file = pkgs.emptyFile;
+
+            output-passwd = true;
+
+            ignore-user-file = toString ./ignore_user_file.txt;
+            ignore-group-file = toString ./ignore_group_file.txt;
+          };
+        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
+        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
+        "${lib.getExe' pkgs.coreutils "cat"} /run/httpd/pamunix-out/passwd"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
+        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
+      ];
+      ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
+      ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
+      ExecStop = lib.mkForce "";
+      KillMode = "mixed";
+
+      LoadCredential=[
+        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
+        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
+      ];
+
+      ConfigurationDirectory = [ "httpd" ];
+      LogsDirectory = [ "httpd" ];
+      LogsDirectoryMode = "0700";
+
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
+      LockPersonality = !debug;
+      PrivateDevices = true;
+      PrivateTmp = true;
+      # NOTE: this removes CAP_NET_BIND_SERVICE...
+      # PrivateUsers = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = "tmpfs";
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectSystem = true;
+      RemoveIPC = true;
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+        "AF_UNIX"
+        "AF_NETLINK"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      SocketBindDeny = "any";
+      SocketBindAllow = [
+        "tcp:80"
+        "tcp:443"
+      ];
+      SystemCallArchitectures = "native";
+      SystemCallFilter = lib.mkIf (!debug) [
+         "@system-service"
+         "@setuid"
+      ];
+      UMask = "0077";
+
+      RuntimeDirectoryMode = "0750";
+      RuntimeDirectory = [
+        "httpd/root-mnt"
+        "httpd/pamunix-in"
+        "httpd/pamunix-out"
+      ];
+      RootDirectory = "/run/httpd/root-mnt";
+      MountAPIVFS = true;
+      BindReadOnlyPaths = [
+        builtins.storeDir
+        "/etc"
+        "/dev/null"
+        "/etc/resolv.conf"
+        "/var/lib/acme"
+
+        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd/pamunix-out/group:/etc/group"
+
+        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
+          passwd:    files
+          group:     files
+          shadow:    files
+          sudoers:   files
+
+          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
+          networks:  files
+
+          ethers:    files
+          services:  files
+          protocols: files
+          rpc:       files
+
+          subuid:    files
+          subgid:    files
+        ''}:/etc/nsswitch.conf"
+
+        "${fhsEnv}/bin:/bin"
+        "${fhsEnv}/sbin:/sbin"
+        "${fhsEnv}/lib:/lib"
+        "${fhsEnv}/share:/share"
+      ] ++ (lib.mapCartesianProduct ({ parent, child }: "${fhsEnv}${child}:${parent}${child}") {
+        parent = [
+          "/local"
+          "/opt"
+          "/opt/local"
+          "/store"
+          "/store/gnu"
+          "/usr"
+          "/usr/local"
+          "/run/current-system/sw"
+        ];
+        child = [
+          "/bin"
+          "/sbin"
+          "/lib"
+          "/libexec"
+          "/include"
+          "/share"
+        ];
+      });
+      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") homeLetters;
+    };
+  };
+
+  # TODO: create phpfpm pools with php environments that contain packages similar to those present on tom
 }

hosts/temmie/services/userweb/httpd.nix

@@ -1,291 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.httpd;
-  mcfg = config.services.pvv-userweb;
-in
-{
-  services.httpd = {
-    enable = true;
-    adminAddr = "drift@pvv.ntnu.no";
-
-    # TODO: consider upstreaming systemd support
-    # TODO: mod_log_journald in v2.5
-    package = pkgs.apacheHttpd.overrideAttrs (prev: {
-      nativeBuildInputs = prev.nativeBuildInputs ++ [ pkgs.pkg-config ];
-      buildInputs = prev.buildInputs ++ [ pkgs.systemdLibs ];
-      configureFlags = prev.configureFlags ++ [ "--enable-systemd" ];
-    });
-
-    enablePHP = true;
-    phpPackage = mcfg.php.env;
-    phpOptions = mcfg.php.options;
-
-    enablePerl = true;
-
-    # TODO: mod_log_journald in v2.5
-    extraModules = [
-      "systemd"
-      "userdir"
-      {
-        name = "perl";
-        path = let
-          mod_perl = pkgs.symlinkJoin {
-            name = "userweb_modperl_with_custom_perl_env";
-            ignoreCollisions = true;
-            paths = [
-              (pkgs.apacheHttpdPackages.mod_perl.override {
-                apacheHttpd = cfg.package.out;
-              })
-              mcfg.perl.env
-            ];
-          };
-        in "${mod_perl}/modules/mod_perl.so";
-      }
-    ];
-
-    logPerVirtualHost = false;
-
-    extraConfig = ''
-      TraceEnable on
-      LogLevel warn rewrite:trace3
-
-      <FilesMatch ".+\.ph(p[3457]?|t|tml)$">
-          SetHandler application/x-httpd-php
-      </FilesMatch>
-      <FilesMatch ".+\.phps$">
-          SetHandler application/x-httpd-php-source
-          Require all denied
-      </FilesMatch>
-      <FilesMatch "\.pl$">
-          SetHandler modperl
-          PerlResponseHandler ModPerl::Registry
-          Options +ExecCGI
-      </FilesMatch>
-    '';
-
-    virtualHosts."temmie.pvv.ntnu.no" = {
-      forceSSL = true;
-      enableACME = true;
-
-      serverAliases = [
-        "www2.pvv.ntnu.no"
-      ];
-
-      extraConfig = ''
-        CustomLog "${cfg.logDir}/access.log" combined
-        CustomLog "/run/httpd-log-processor-access.fifo" combined
-        ErrorLog "/run/httpd-log-processor-error.fifo"
-        ScriptLog "${cfg.logDir}/cgi.log"
-
-        UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") mcfg.homeLetters}
-
-        UserDir disabled root
-        UserDir disabled pvv
-
-        AddHandler cgi-script .cgi
-
-        DirectoryIndex ${lib.concatStringsSep " " [
-          "index.htm"
-          "index.html"
-          "index.html.var"
-
-          "index.shtml"
-          "index.xhtml"
-
-          "index.php"
-          "index.php3"
-          "index.php4"
-          "index.pht"
-          "index.phtml"
-
-          "index.cgi"
-          "index.txt"
-
-          "meg.html"
-        ]}
-
-        SetEnvIf Request_URI "^/~([^/]+)" USERDIR_USER=$1
-
-        <Directory "/home/pvv/?/*/web-docs">
-          Options MultiViews Indexes SymLinksIfOwnerMatch ExecCGI IncludesNoExec
-          AllowOverride All
-          Require all granted
-        </Directory>
-
-        <DirectoryMatch "^/home/pvv/.*/web-docs/(${lib.concatStringsSep "|" [
-          "\\.git"
-          "\\.hg"
-          "\\.svn"
-          "\\.ssh"
-          "\\.env"
-          "\\.env\\..*"
-          "\\.envs"
-          "\\.envs\\..*"
-          "\\.envrc"
-          "\\.bzr"
-          "\\.venv"
-          "CVS"
-          "RCS"
-
-          ".*\\.swp"
-          ".*~"
-
-          ".*\\.bak"
-          ".*\\.bak.*"
-          ".*\\.bkp"
-          ".*\\.bkp.*"
-          ".*\\.backup"
-          ".*\\.backup.*"
-
-          ".*\\.lck"
-          ".*\\.lock"
-          "LCK\\.\\..*"
-        ]})(/|$)">
-            AllowOverride All
-            Require all denied
-        </DirectoryMatch>
-      '';
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = [
-    80
-    443
-  ];
-
-  # socket activation comes in v2.5
-  # systemd.sockets.httpd = {
-  #   wantedBy = [ "sockets.target" ];
-  #   description = "HTTPD socket";
-  #   listenStreams = [
-  #     "0.0.0.0:80"
-  #     "0.0.0.0:443"
-  #   ];
-  # };
-
-  # NOTE: 54 -> 33, this is the UID/GID we used for www-data on tom in the past.
-  #       Any files accessed by or created by httpd will do so over NFS with this
-  #       UID/GID pair as its credentials.
-  #       This overlaps with the hardcoded `disnix` uid in nixpkgs, but we *probably*
-  #       won't be using that for the foreseeable future.
-  users.users."wwwrun".uid = lib.mkForce 33;
-  users.groups."wwwrun".gid = lib.mkForce 33;
-
-  systemd.targets.userweb = {
-    description = "PVV HTTPD UserWeb";
-  };
-
-  systemd.slices.system-userweb = {
-    description = "PVV HTTPD UserWeb";
-  };
-
-  systemd.services.httpd = {
-    after = [
-      "pvv-homedirs.target"
-      "httpd-log-processor@access.socket"
-      "httpd-log-processor@error.socket"
-    ];
-    requires = [
-      "pvv-homedirs.target"
-      "httpd-log-processor@access.socket"
-      "httpd-log-processor@error.socket"
-    ];
-    requiredBy = [ "userweb.target" ];
-
-    environment = {
-      PATH = lib.mkForce "/usr/bin";
-    };
-
-    serviceConfig = {
-      Type = lib.mkForce "notify";
-      ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
-      ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
-      ExecStop = lib.mkForce "";
-      KillMode = "mixed";
-      Slice = "system-userweb.slice";
-
-      ConfigurationDirectory = [ "httpd" ];
-      LogsDirectory = [ "httpd" ];
-      LogsDirectoryMode = "0700";
-
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals mcfg.debugMode [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ] ++ lib.optionals mcfg.debugMode [ "CAP_SYS_PTRACE" ];
-      LockPersonality = !mcfg.debugMode;
-      PrivateDevices = true;
-      PrivateTmp = true;
-      # NOTE: this removes CAP_NET_BIND_SERVICE...
-      # PrivateUsers = true;
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = "tmpfs";
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectSystem = true;
-      RemoveIPC = true;
-      RestrictAddressFamilies = [
-        "AF_INET"
-        "AF_INET6"
-        "AF_UNIX"
-        "AF_NETLINK"
-      ];
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SocketBindDeny = "any";
-      SocketBindAllow = [
-        "tcp:80"
-        "tcp:443"
-      ];
-      SystemCallArchitectures = "native";
-      SystemCallFilter = lib.mkIf (!mcfg.debugMode) [ "@system-service" ];
-      UMask = "0077";
-
-      RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [ "httpd/root-mnt" ];
-      RootDirectory = "/run/httpd/root-mnt";
-      MountAPIVFS = true;
-      BindReadOnlyPaths = [
-        builtins.storeDir
-        "/etc"
-        "/dev/null"
-        "/var/lib/acme"
-        "/var/run/nscd"
-        "${mcfg.fhsEnv}/bin:/bin"
-        "${mcfg.fhsEnv}/sbin:/sbin"
-        "${mcfg.fhsEnv}/lib:/lib"
-        "${mcfg.fhsEnv}/share:/share"
-      ] ++ (lib.mapCartesianProduct ({ parent, child }: "${mcfg.fhsEnv}${child}:${parent}${child}") {
-        parent = [
-          "/local"
-          "/opt"
-          "/opt/local"
-          "/store"
-          "/store/gnu"
-          "/usr"
-          "/usr/local"
-          "/run/current-system/sw"
-        ];
-        child = [
-          "/bin"
-          "/sbin"
-          "/lib"
-          "/libexec"
-          "/include"
-          "/share"
-        ];
-      });
-      BindPaths = (lib.mapCartesianProduct ({ directoryFn, letter }: "/run/pvv-home-mounts/${letter}:${directoryFn letter}${letter}") {
-        directoryFn = [
-          (_: "/home/pvv/")
-          (l: "/amd/homepvv${l}/")
-        ];
-        letter = mcfg.homeLetters;
-      }) ++ [
-        "/run/httpd-log-processor-access.fifo"
-        "/run/httpd-log-processor-error.fifo"
-      ];
-    };
-  };
-
-  # TODO: create phpfpm pools with php environments that contain packages similar to those present on tom
-}

hosts/temmie/services/userweb/log-processor.nix

@@ -1,167 +0,0 @@
-{ config, lib, pkgs, values, ... }:
-let
-  mcfg = config.services.pvv-userweb;
-in
-{
-  sops.secrets = {
-    "httpd/passwd-ssh-key" = { };
-    "httpd/ssh-known-hosts" = { };
-  };
-
-  systemd.targets.sockets.wants = [
-    "httpd-log-processor@access.socket"
-    "httpd-log-processor@error.socket"
-  ];
-
-  systemd.sockets."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
-    requiredBy = [ "userweb.target" ];
-    socketConfig = {
-      ListenFIFO = "/run/httpd-log-processor-%i.fifo";
-      RemoveOnStop = true;
-
-      SocketUser = "wwwrun";
-      SocketGroup = "wwwrun";
-      SocketMode = "0600";
-    };
-  };
-
-  systemd.services."httpd-log-processor@" = lib.mkIf config.services.httpd.enable {
-    requiredBy = [ "userweb.target" ];
-    serviceConfig = {
-      User = "wwwrun";
-      Group = "wwwrun";
-      Slice = "system-userweb.slice";
-      Restart = "on-failure";
-
-      StandardInput = "socket";
-      StandardOutput = "journal";
-      StandardError = "journal";
-
-      LoadCredential = [
-        "sshkey:${config.sops.secrets."httpd/passwd-ssh-key".path}"
-        "ssh-known-hosts:${config.sops.secrets."httpd/ssh-known-hosts".path}"
-      ];
-      ExecStartPre = let
-        rsyncArgs = lib.cli.toCommandLineShellGNU { } {
-          archive = true;
-          verbose = true;
-          compress = true;
-          rsh = "${lib.getExe' pkgs.openssh "ssh"} -o BatchMode=yes -o UserKnownHostsFile=%d/ssh-known-hosts -i %d/sshkey";
-        };
-        inputDir = "/run/httpd-log-processor-%i/pamunix-in";
-        outputDir = "/run/httpd-log-processor-%i/pamunix-out";
-      in lib.mkForce [
-        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/passwd ${inputDir}/"
-        "${lib.getExe pkgs.rsync} ${rsyncArgs} pvv@smtp.pvv.ntnu.no:/etc/group ${inputDir}/"
-
-        (let
-          args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "${inputDir}/passwd";
-            group-file = "${inputDir}/group";
-            output-dir = outputDir;
-            shadow-file = pkgs.emptyFile;
-
-            output-passwd = true;
-
-            ignore-user-file = toString ./ignore_user_file.txt;
-            ignore-group-file = toString ./ignore_group_file.txt;
-          };
-        in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u ${inputDir}/passwd ${inputDir}/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' ${outputDir}/passwd"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' ${outputDir}/group"
-        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' ${outputDir}/group"
-        "+${lib.getExe' pkgs.coreutils "chown"} root:root ${outputDir}/passwd ${outputDir}/group"
-        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 ${outputDir}/passwd ${outputDir}/group"
-        "+${lib.getExe pkgs.mount} --bind ${outputDir}/passwd /etc/passwd"
-        "+${lib.getExe pkgs.mount} --bind ${outputDir}/group  /etc/group"
-      ];
-
-      ExecStart = "${lib.getExe mcfg.apacheLogProcessorPackage} %i";
-
-      AmbientCapabilities = [ "CAP_SETUID" "CAP_SETGID" ];
-      CapabilityBoundingSet = [ "CAP_SETUID" "CAP_SETGID" ];
-      DeviceAllow = [ "" ];
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      PrivateDevices = true;
-      PrivateIPC = true;
-      PrivateTmp = true;
-      # PrivateUsers = true;
-      ProcSubset = "pid";
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = "tmpfs";
-      ProtectHostname = true;
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectProc = "invisible";
-      ProtectSystem = "strict";
-      ProtectKernelTunables = true;
-      RemoveIPC = true;
-      RestrictAddressFamilies = [
-        "AF_INET"
-        "AF_INET6"
-      ];
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SocketBindDeny = "any";
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-         "@system-service"
-         "@setuid"
-      ];
-      UMask = "0077";
-
-      IPAddressAllow = [
-        "127.0.0.53" # systemd-resolved
-        values.hosts.microbel.ipv4
-        values.hosts.microbel.ipv6
-      ];
-      IPAddressDeny = "any";
-
-      RootDirectory = "/run/httpd-log-processor-%i/root-mnt";
-      MountAPIVFS = true;
-
-      RuntimeDirectoryMode = "0750";
-      RuntimeDirectory = [
-        "httpd-log-processor-%i/root-mnt"
-        "httpd-log-processor-%i/pamunix-in"
-        "httpd-log-processor-%i/pamunix-out"
-      ];
-      BindReadOnlyPaths = [
-        builtins.storeDir
-        "/etc"
-        "/etc/resolv.conf"
-
-        "-/run/httpd-log-processor-%i/pamunix-out/passwd:/etc/passwd"
-        "-/run/httpd-log-processor-%i/pamunix-out/group:/etc/group"
-
-        "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    files
-          group:     files
-          shadow:    files
-          sudoers:   files
-
-          hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
-          networks:  files
-
-          ethers:    files
-          services:  files
-          protocols: files
-          rpc:       files
-
-          subuid:    files
-          subgid:    files
-        ''}:/etc/nsswitch.conf"
-      ] ++ lib.optionals mcfg.debugMode [
-        "/bin"
-      ];
-      BindPaths = map (l: "/run/pvv-home-mounts/${l}:/home/pvv/${l}") mcfg.homeLetters ++ [
-        "/var/log/httpd"
-      ];
-    };
-  };
-}

hosts/temmie/services/userweb/mail.nix

@@ -70,7 +70,6 @@
     serviceConfig = {
       User = "nullmailer-user";
       Group = "nullmailer-user";
-      Slice = "system-userweb.slice";
 
       ReadWritePaths = [
         "/var/spool/nullmailer"

hosts/temmie/services/userweb/module.nix

@@ -1,116 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.pvv-userweb;
-in
-{
-  options.services.pvv-userweb = {
-    enable = lib.mkEnableOption "";
-
-    debugMode = lib.mkEnableOption "";
-
-    apacheLogProcessorPackage = lib.mkOption {
-      type = lib.types.package;
-      default = pkgs.callPackage ./apache-log-processor { };
-    };
-
-    homeLetters = lib.mkOption {
-      type = lib.types.listOf lib.types.str;
-      default = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
-      readOnly = true;
-    };
-
-    packages = lib.mkOption {
-      type = lib.types.listOf lib.types.package;
-      default = _: [ ];
-    };
-
-    php.extensions = lib.mkOption {
-      type = lib.types.functionTo (lib.types.listOf lib.types.package);
-      default = _: [ ];
-    };
-
-    php.options = lib.mkOption {
-      type = lib.types.attrsOf lib.types.str;
-      default = {
-        display_errors = "Off";
-        display_startup_errors = "Off";
-        post_max_size = "40M";
-        upload_max_filesize = "40M";
-      };
-      apply = attrs: lib.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "${k} = ${v}") attrs);
-    };
-
-    # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
-    php.env = lib.mkOption {
-      type = lib.types.package;
-      readOnly = true;
-      default = pkgs.php.buildEnv {
-        extensions = cfg.php.extensions;
-        extraConfig = cfg.php.options;
-      };
-    };
-
-    perl.packages = lib.mkOption {
-      type = lib.types.functionTo (lib.types.listOf lib.types.package);
-      default = _: [ ];
-    };
-
-    perl.env = lib.mkOption {
-      type = lib.types.package;
-      readOnly = true;
-      default = (pkgs.perl.withPackages cfg.perl.packages).overrideAttrs (prev: {
-        # NOTE: `pkgs.perl.propagatedBuildInputs` don't actually propagate through the
-        #       wrapper derivation created by `withPackages`. This should compensate
-        #       for that.
-        postBuild = prev.postBuild + ''
-          cp -r '${pkgs.perl}/nix-support' "$out"/nix-support
-        '';
-      });
-    };
-
-    python3.packages = lib.mkOption {
-      type = lib.types.functionTo (lib.types.listOf lib.types.package);
-      default = _: [ ];
-    };
-
-    python3.env = lib.mkOption {
-      type = lib.types.package;
-      readOnly = true;
-      default = pkgs.python3.buildEnv.override {
-        extraLibs = cfg.python3.packages pkgs.python3Packages;
-        ignoreCollisions = true;
-      };
-    };
-
-    fhsEnv = lib.mkOption {
-      type = lib.types.package;
-      readOnly = true;
-      default = let
-
-      in pkgs.buildEnv {
-        name = "userweb-env";
-        ignoreCollisions = true;
-        paths = with pkgs; [
-          bash
-          config.services.bro.instances.userweb-sendmail.client.package
-          cfg.perl.env
-          cfg.python3.env
-          cfg.php.env
-        ] ++ cfg.packages;
-
-        extraOutputsToInstall = [
-          "man"
-          "doc"
-        ];
-      };
-    };
-  };
-
-  config = lib.mkIf cfg.enable {
-    services.pvv-userweb.packages = lib.mkIf cfg.debugMode (with pkgs; [
-     glibc.getent
-     strace
-     systemd
-   ]);
-  };
-}

hosts/temmie/services/userweb/packages.nix

@@ -1,104 +0,0 @@
-{ pkgs, ... }:
-{
-  services.pvv-userweb = {
-    packages = with pkgs; [
-      # Useful packages for homepages
-      exiftool
-      gnuplot
-      ikiwiki-full
-      imagemagick
-      jhead
-      ruby
-      sbcl
-      sourceHighlight
-
-      # Missing packages from tom
-      # blosxom
-      # pyblosxom
-      # mediawiki (TODO: do people host their own mediawikis in userweb?)
-      # nanoblogger
-
-      # Version control
-      cvs
-      rcs
-      git
-
-      # Compression/Archival
-      bzip2
-      gnutar
-      gzip
-      lz4
-      unzip
-      xz
-      zip
-      zstd
-
-      # Other tools you might expect to find on a normal system
-      acl
-      coreutils-full
-      curl
-      diffutils
-      file
-      findutils
-      gawk
-      gnugrep
-      gnumake
-      gnupg
-      gnused
-      less
-      man
-      util-linux
-      vim
-      wget
-      which
-      xdg-utils
-    ];
-
-    php.extensions = { all, ... }: with all; [
-      bz2
-      curl
-      decimal
-      gd
-      imagick
-      mysqli
-      mysqlnd
-      pgsql
-      posix
-      protobuf sqlite3
-      uuid
-      xml
-      xsl
-      zlib
-      zstd
-
-      pdo
-      pdo_mysql
-      pdo_pgsql
-      pdo_sqlite
-    ];
-
-    perl.packages = perlPkgs: with perlPkgs; [
-      pkgs.exiftool
-      pkgs.ikiwiki
-      pkgs.irssi
-      pkgs.nix.libs.nix-perl-bindings
-
-      CGI
-      DBDPg
-      DBDSQLite
-      DBDmysql
-      DBI
-      Git
-      ImageMagick
-      JSON
-      TemplateToolkit
-    ];
-
-    python3.packages = pythonPkgs: with pythonPkgs; [
-      legacy-cgi
-
-      matplotlib
-      requests
-    ];
-  };
-}

secrets/ildkule/ildkule.yaml

@@ -1,5 +1,6 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
+    postgresql_dibbler_password: ENC[AES256_GCM,data:wP4CVz9qRE3CJrblWWYqOIkcH3LM5H81,iv:j8zr1TRNlPPqIppYlWhDoKlL7m2Ph2wQlU6bvFj8R9A=,tag:Cfp8zbYkoLqI7DTDpOBlJw==,type:str]
 keys:
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
@@ -72,8 +73,8 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-03-16T20:08:18Z"
+    lastmodified: "2026-06-07T12:51:04Z"
-    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
+    mac: ENC[AES256_GCM,data:otGwzc3Bme1PGRU4zWRRf3kmAf5EDjT8sPkDK/zDrO0ve+d3x99Qls4DBZV8G6FsFtUXbeKVo70I5VpqUgaIef+nyMl6zzWkW1wpbKIBYjh5fkaP0xxhLnw+shrB088PAXT0wkP7hJBLO0ZJgTrpprJKhmeqVj2HEXBSJ1wSjk8=,iv:cjqtdbWBVLUfpQdykb3vKDKa/VC/kGBybwYtG/eStqc=,tag:remOj4bX/H/LlAPgcXHAdA==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -96,4 +97,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.9.4
+    version: 3.12.2

users/vegardbm.nix

@@ -1,4 +1,4 @@
-{ pkgs, config, ... }:
+{ pkgs, ... }:
 {
   users.users.vegardbm = {
     isNormalUser = true;
@@ -8,14 +8,12 @@
       "drift"
       "nix-builder-users"
     ];
-    shell = if config.programs.zsh.enable then pkgs.zsh else pkgs.bash;
     packages = with pkgs; [
       btop
       eza
     ];
     openssh.authorizedKeys.keys = [
       "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVA3HqEx3je6L1AC+bP8sTxu3ZTKvTCR0npCyOVAYK5 vbm@arch-xeon"
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICrYATHNvBNAHr9G+VwZIaAQPe02iRgAjqtZkW4x/dje vbm@talos"
     ];
   };
 }