prometheus for dibbler

base/default.nix

@@ -40,6 +40,7 @@
     ./services/scrutiny-collector.nix
     ./services/smartd.nix
     ./services/thermald.nix
+    ./services/timesyncd.nix
     ./services/uptimed.nix
     ./services/userborn.nix
     ./services/userdbd.nix

base/services/timesyncd.nix

@@ -0,0 +1,12 @@
+{ ... }:
+{
+  services.timesyncd = {
+    servers = [ "ntp.ntnu.no" ];
+    fallbackServers = [
+      "0.pool.ntp.org"
+      "1.pool.ntp.org"
+      "0.no.pool.ntp.org"
+    ];
+  };
+}
+

hosts/ildkule/services/monitoring/prometheus/default.nix

@@ -8,6 +8,7 @@ in {
     ./matrix-synapse.nix
     ./mysqld.nix
     ./postgres.nix
+    ./dibbler.nix
   ];
 
   services.prometheus = {

hosts/ildkule/services/monitoring/prometheus/dibbler.nix

@@ -0,0 +1,96 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.exporters.sql;
+  configFile =
+    if cfg.configFile != null then
+      cfg.configFile
+    else
+      let
+        nameInline = lib.mapAttrsToList (k: v: v // { name = k; });
+        renameStartupSql = j: removeAttrs (j // { startup_sql = j.startupSql; }) [ "startupSql" ];
+        configuration = {
+          jobs = map renameStartupSql (
+            nameInline (lib.mapAttrs (k: v: (v // { queries = nameInline v.queries; })) cfg.configuration.jobs)
+          );
+        };
+      in
+      builtins.toFile "config.yaml" (builtins.toJSON configuration);
+in
+{
+  sops.secrets."config/postgresql_dibbler_password" = { };
+
+  services.prometheus.scrapeConfigs = [
+    {
+      job_name = "sql_exporter";
+      scrape_interval = "1m";
+      scheme = "http";
+
+      static_configs = [
+        {
+          targets = [ "localhost:9237" ];
+        }
+      ];
+    }
+  ];
+
+  services.prometheus.exporters.sql = {
+    enable = true;
+    configuration = {
+      jobs.dibbler = {
+        interval = "1m";
+
+        queries."daily_purchase_sum" = {
+          help = "Sum of purchases for the current day.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(price) FROM purchases GROUP BY DATE(time) ORDER BY DATE(time) DESC LIMIT 1";
+        };
+
+        queries."total_purchase_sum" = {
+          help = "Sum of all purchases.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(price) FROM purchases";
+        };
+
+        queries."total_stock_value" = {
+          help = "The value of all stock in dibbler.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(price * stock) FROM products";
+        };
+
+        queries."user_credit_sum" = {
+          help = "The sum of all user credit.";
+          labels = [ "thing" ];
+          values = [ "sum" ];
+          query = "SELECT SUM(credit) FROM users";
+        };
+      };
+    };
+  };
+
+  systemd.services."prometheus-sql-exporter".serviceConfig = {
+    RuntimeDirectory = "prometheus-sql-exporter";
+    LoadCredential = "postgresql_dibbler_password:${
+      config.sops.secrets."config/postgresql_dibbler_password".path
+    }";
+    ExecStartPre = ''
+      |${lib.getExe pkgs.jq} \
+      --null-input \
+      --compact-output \
+      --slurpfile config '${configFile}' \
+      --rawfile pw '%d/postgresql_dibbler_password' \
+      --from-file '${pkgs.writeText "prometheus-sql-exec-start-jq-filter" ''
+        ("postgres://pvv_vv:\($pw | gsub("\n"; ""))@postgres.pvv.ntnu.no") as $pg_uri
+        | $config[0]
+        | .jobs[0].connections[0] = $pg_uri
+      ''}' > /run/prometheus-sql-exporter/config.yaml
+    '';
+  };
+}

hosts/temmie/services/userweb/apache-log-processor/.gitignore

@@ -0,0 +1 @@
+target

hosts/temmie/services/userweb/apache-log-processor/Cargo.lock

@@ -0,0 +1,171 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "apache-log-processor"
+version = "0.1.0"
+dependencies = [
+ "nix",
+ "time",
+]
+
+[[package]]
+name = "bitflags"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
+name = "deranged"
+version = "0.5.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+dependencies = [
+ "powerfmt",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "nix"
+version = "0.31.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "cfg_aliases",
+ "libc",
+]
+
+[[package]]
+name = "num-conv"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
+
+[[package]]
+name = "num_threads"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "time"
+version = "0.3.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"
+dependencies = [
+ "deranged",
+ "itoa",
+ "libc",
+ "num-conv",
+ "num_threads",
+ "powerfmt",
+ "serde_core",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
+
+[[package]]
+name = "time-macros"
+version = "0.2.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"
+dependencies = [
+ "num-conv",
+ "time-core",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"

hosts/temmie/services/userweb/apache-log-processor/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "apache-log-processor"
+version = "0.1.0"
+edition = "2024"
+autobins = false
+license = "MIT"
+authors = [
+  "projects@pvv.ntnu.no",
+]
+
+[dependencies]
+nix = { version = "0.31.3", features = ["event", "fs", "user"] }
+time = { version = "0.3.47", features = ["formatting", "local-offset"] }
+
+
+[[bin]]
+name = "apache-log-processor"
+bench = false
+path = "src/main.rs"

hosts/temmie/services/userweb/apache-log-processor/default.nix

@@ -0,0 +1,33 @@
+{
+  lib
+, rustPlatform
+, stdenv
+}:
+let
+  cargoToml = fromTOML (builtins.readFile ./Cargo.toml);
+  cargoLock = ./Cargo.lock;
+  mainProgram = (lib.head cargoToml.bin).name;
+  pname = cargoToml.package.name;
+in
+rustPlatform.buildRustPackage {
+  inherit pname;
+  inherit (cargoToml.package) version;
+  src = lib.fileset.toSource {
+    root = ./.;
+    fileset = lib.fileset.unions [
+      ./Cargo.toml
+      ./Cargo.lock
+      ./src
+    ];
+  };
+
+  cargoLock.lockFile = cargoLock;
+
+  doCheck = true;
+
+  meta = with lib; {
+    license = licenses.mit;
+    platforms = platforms.linux;
+    inherit mainProgram;
+  };
+}

hosts/temmie/services/userweb/apache-log-processor/src/main.rs

@@ -0,0 +1,321 @@
+use nix::{
+    errno::Errno,
+    fcntl::{FcntlArg, OFlag, fcntl, open},
+    sys::{
+        epoll::{Epoll, EpollCreateFlags, EpollEvent, EpollFlags, EpollTimeout},
+        stat::Mode,
+    },
+    unistd::{User, getegid, geteuid, read, setegid, seteuid, write},
+};
+use std::{
+    collections::VecDeque,
+    os::fd::{AsFd, BorrowedFd, OwnedFd},
+    path::PathBuf,
+    process::exit,
+};
+use time::{OffsetDateTime, format_description};
+
+const READ_BUFFER_SIZE: usize = 8 * 1024;
+
+#[derive(Debug, Clone, Copy)]
+enum LogMode {
+    Access,
+    Error,
+}
+
+fn main() -> Result<(), String> {
+    let log_mode = match std::env::args().nth(1).as_deref() {
+        Some("access") => LogMode::Access,
+        Some("error") => LogMode::Error,
+        Some(other) => {
+            return Err(format!(
+                "invalid log mode `{other}`; expected `access` or `error`"
+            ));
+        }
+        None => return Err("missing log mode argument; expected `access` or `error`".to_string()),
+    };
+
+    let tee_file = match log_mode {
+        LogMode::Access => None,
+        LogMode::Error => Some(
+            open(
+                &PathBuf::from("/var/log/httpd/error.log"),
+                OFlag::O_WRONLY | OFlag::O_APPEND | OFlag::O_CREAT | OFlag::O_CLOEXEC,
+                Mode::S_IRUSR | Mode::S_IWUSR,
+            )
+            .map_err(|error| format!("failed to open error log for teeing: {error}"))?,
+        ),
+    };
+
+    let stdin = std::io::stdin();
+
+    fcntl(stdin.as_fd(), FcntlArg::F_GETFL)
+        .map(OFlag::from_bits_retain)
+        .map(|flags| FcntlArg::F_SETFL(flags | OFlag::O_NONBLOCK))
+        .and_then(|flags| fcntl(stdin.as_fd(), flags))
+        .map_err(|error| format!("failed to make stdin nonblocking: {error}"))?;
+
+    let epoll = Epoll::new(EpollCreateFlags::EPOLL_CLOEXEC)
+        .map_err(|error| format!("failed to create epoll instance: {error}"))?;
+
+    epoll
+        .add(
+            stdin.as_fd(),
+            EpollEvent::new(
+                EpollFlags::EPOLLIN | EpollFlags::EPOLLERR | EpollFlags::EPOLLHUP,
+                0,
+            ),
+        )
+        .map_err(|error| format!("failed to register stdin with epoll: {error}"))?;
+
+    if let Err(error) = event_loop(log_mode, epoll, stdin.as_fd(), tee_file) {
+        eprintln!("Error: {error}");
+        exit(1);
+    }
+
+    Ok(())
+}
+
+fn event_loop(
+    log_mode: LogMode,
+    epoll: Epoll,
+    stdin_fd: BorrowedFd<'_>,
+    mut tee_file: Option<OwnedFd>,
+) -> Result<(), String> {
+    let mut events = [EpollEvent::empty(); 1];
+    let mut pending = VecDeque::new();
+
+    loop {
+        let ready = loop {
+            match epoll.wait(&mut events, EpollTimeout::NONE) {
+                Ok(ready) => break ready,
+                Err(Errno::EINTR) => continue,
+                Err(error) => {
+                    return Err(format!("epoll wait failed: {error}"));
+                }
+            }
+        };
+
+        if ready == 0 {
+            continue;
+        }
+
+        let mut scratch = [0u8; READ_BUFFER_SIZE];
+
+        let eof = loop {
+            match read(stdin_fd, &mut scratch) {
+                Ok(0) => break true,
+                Ok(read_bytes) => pending.extend(scratch[..read_bytes].iter().copied()),
+                Err(Errno::EINTR) => continue,
+                Err(Errno::EAGAIN) => break false,
+                Err(error) => {
+                    return Err(format!("failed to read from stdin: {error}"));
+                }
+            }
+        };
+
+        while let Some(newline_index) = pending.iter().position(|byte| *byte == b'\n') {
+            let line = pending.make_contiguous();
+            process_line(log_mode, &line[..=newline_index], &mut tee_file)?;
+            pending.drain(..=newline_index);
+        }
+
+        if eof {
+            if !pending.is_empty() {
+                process_line(log_mode, pending.make_contiguous(), &mut tee_file)?;
+                pending.clear();
+            }
+            return Ok(());
+        }
+    }
+}
+
+fn process_line(
+    log_mode: LogMode,
+    line: &[u8],
+    tee_file: &mut Option<OwnedFd>,
+) -> Result<(), String> {
+    if let Some(tee_file) = tee_file.as_ref() {
+        write_all_fd(tee_file, line).map_err(|error| {
+            format!("failed to append to APACHE_LOG_PROCESSOR_TEE_FILE: {error}")
+        })?;
+    }
+
+    if let Some(user) =
+        parse_username_from_line(line).and_then(|name| User::from_name(name).ok().flatten())
+    {
+        let identity = EffectiveIdentity::switch_to(&user).map_err(|error| {
+            format!(
+                "failed to switch effective identity to {} (uid {}, gid {}): {error}",
+                user.name, user.uid, user.gid
+            )
+        })?;
+
+        let result: Result<(), String> = (|| {
+            let dir = user.dir.join("nobackup/weblogs");
+
+            if !dir.is_dir() {
+                return Err(format!(
+                    "logs directory {} does not exist for user {}",
+                    dir.display(),
+                    user.name
+                ));
+            }
+
+            let now = OffsetDateTime::now_local()
+                .unwrap_or_else(|_| OffsetDateTime::now_utc())
+                .format(&format_description::parse("[year]-[month]-[day]").unwrap())
+                .map_err(|error| {
+                    format!("failed to format current date for log file name: {error}")
+                })?;
+
+            let logfile = dir.join(match log_mode {
+                LogMode::Access => format!("access-{now}.log"),
+                LogMode::Error => format!("error-{now}.log"),
+            });
+
+            let fd = open(
+                &logfile,
+                OFlag::O_WRONLY | OFlag::O_APPEND | OFlag::O_CREAT | OFlag::O_CLOEXEC,
+                Mode::S_IRUSR
+                    | Mode::S_IWUSR
+                    | Mode::S_IRGRP
+                    | Mode::S_IROTH
+                    | Mode::S_IWGRP
+                    | Mode::S_IWOTH,
+            )
+            .map_err(|error| format!("failed to open log file for user {}: {error}", user.name))?;
+
+            write_all_fd(fd.as_fd(), line).map_err(|error| {
+                format!(
+                    "failed to append to log file for user {}: {error}",
+                    user.name
+                )
+            })?;
+
+            Ok(())
+        })();
+
+        if let Err(error) = result {
+            eprintln!("Error processing log line for user {}: {error}", user.name);
+        }
+
+        identity.restore().map_err(|error| {
+            format!(
+                "failed to restore original effective identity after handling {}: {error}",
+                user.name
+            )
+        })?;
+    }
+
+    Ok(())
+}
+
+fn parse_username_from_line(line: &[u8]) -> Option<&str> {
+    line.splitn(8, |&b| b == b' ')
+        .nth(6)
+        .and_then(|path| {
+            path.strip_prefix(b"/~")
+                .and_then(|rest| rest.split(|&b| b == b'/').next())
+        })
+        .or_else(|| {
+            line.windows(b"/home/pvv/".len())
+                .enumerate()
+                .find_map(|(start, window)| {
+                    (window == b"/home/pvv/")
+                        .then_some(start + b"/home/pvv/".len())
+                        .and_then(|start| line.get(start..))
+                        .filter(|rest| rest.get(1) == Some(&b'/'))
+                        .and_then(|rest| rest.get(2..))
+                        .and_then(|rest| rest.split(|&b| b == b'/').next())
+                })
+        })
+        .filter(|segment| !segment.is_empty())
+        .and_then(|segment| std::str::from_utf8(segment).ok())
+}
+
+fn write_all_fd<Fd: AsFd>(fd: Fd, mut buffer: &[u8]) -> nix::Result<()> {
+    while !buffer.is_empty() {
+        match write(fd.as_fd(), buffer) {
+            Ok(0) => return Err(Errno::EIO),
+            Ok(written) => buffer = &buffer[written..],
+            Err(Errno::EINTR) => continue,
+            Err(error) => return Err(error),
+        }
+    }
+
+    Ok(())
+}
+
+struct EffectiveIdentity {
+    saved_euid: nix::unistd::Uid,
+    saved_egid: nix::unistd::Gid,
+    restored: bool,
+}
+
+impl EffectiveIdentity {
+    fn switch_to(user: &User) -> nix::Result<Self> {
+        let guard = Self {
+            saved_euid: geteuid(),
+            saved_egid: getegid(),
+            restored: false,
+        };
+
+        setegid(user.gid)?;
+        if let Err(error) = seteuid(user.uid) {
+            let _ = setegid(guard.saved_egid);
+            return Err(error);
+        }
+
+        Ok(guard)
+    }
+
+    fn restore(mut self) -> nix::Result<()> {
+        let restore_uid = seteuid(self.saved_euid);
+        let restore_gid = setegid(self.saved_egid);
+        self.restored = true;
+
+        restore_uid?;
+        restore_gid?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_user_from_access_log() {
+        let inputs = [(
+            "1.2.3.4 - - [25/May/2026:10:07:24 +0200] \"GET /~oysteikt/ HTTP/2.0\" 200 3708",
+            "oysteikt",
+        )];
+
+        for (line, expected_user) in inputs {
+            let parsed_user = parse_username_from_line(line.as_bytes());
+            assert_eq!(
+                parsed_user,
+                Some(expected_user),
+                "Failed to parse user from line: {line}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_parse_user_from_error_log() {
+        let inputs = [(
+            "[Sat May 09 20:45:21.480016 2026] [authz_core:error] [pid 3555:tid 3617] [remote 1::2:42000] AH01630: client denied by server configuration: /home/pvv/d/oysteikt/web-docs/.git",
+            "oysteikt",
+        )];
+
+        for (line, expected_user) in inputs {
+            let parsed_user = parse_username_from_line(line.as_bytes());
+            assert_eq!(
+                parsed_user,
+                Some(expected_user),
+                "Failed to parse user from line: {line}"
+            );
+        }
+    }
+}

hosts/temmie/services/userweb/default.nix

@@ -2,6 +2,9 @@
 let
   cfg = config.services.httpd;
 
+  # NOTE Enable this if you want to strace stuff in the sandbox...
+  debug = false;
+
   homeLetters = [ "a" "b" "c" "d" "h" "i" "j" "k" "l" "m" "z" ];
 
   phpOptions = lib.concatStringsSep "\n" (lib.mapAttrsToList (k: v: "${k} = ${v}"){
@@ -11,6 +14,8 @@ let
     upload_max_filesize = "40M";
   });
 
+  apache-log-processor = pkgs.callPackage ./apache-log-processor { };
+
   # https://nixos.org/manual/nixpkgs/stable/#ssec-php-user-guide-installing-with-extensions
   phpEnv = pkgs.php.buildEnv {
     extensions = { all, ... }: with all; [
@@ -130,9 +135,6 @@ let
       file
       findutils
       gawk
-      glibc.getent
-      strace
-      systemd
       gnugrep
       gnumake
       gnupg
@@ -144,6 +146,10 @@ let
       wget
       which
       xdg-utils
+    ] ++ lib.optionals debug [
+      glibc.getent
+      strace
+      systemd
     ];
 
     extraOutputsToInstall = [
@@ -201,10 +207,11 @@ in
       }
     ];
 
+    logPerVirtualHost = false;
+
     extraConfig = ''
       TraceEnable on
       LogLevel warn rewrite:trace3
-      ScriptLog ${cfg.logDir}/cgi.log
     '';
 
     virtualHosts."temmie.pvv.ntnu.no" = {
@@ -216,6 +223,11 @@ in
       ];
 
       extraConfig = ''
+        CustomLog "${cfg.logDir}/access.log" combined
+        CustomLog "|${lib.getExe apache-log-processor} access" combined
+        ErrorLog "|${lib.getExe apache-log-processor} error"
+        ScriptLog "${cfg.logDir}/cgi.log"
+
         UserDir ${lib.concatMapStringsSep " " (l: "/home/pvv/${l}/*/web-docs") homeLetters}
         UserDir disabled root
         AddHandler cgi-script .cgi
@@ -298,19 +310,15 @@ in
             done
           '';
         })}"
-        # "${pkgs.systemd}/bin/resolvectl query smtp.pvv.ntnu.no"
+
-        "${pkgs.strace}/bin/strace ${pkgs.glibc.getent}/bin/getent ahosts smtp.pvv.ntnu.no"
+        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-in/"
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/passwd /run/httpd/pamunix-sync/"
+        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-in/"
-        "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/group /run/httpd/pamunix-sync/"
+
-        # "+|echo 'wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/nologin' >> /run/httpd/pamunix-sync/passwd"
-        # "+|echo 'root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' >> /run/httpd/pamunix-sync/passwd"
-        # "+|echo 'wwwrun:x:54:' >> /run/httpd/pamunix-sync/group"
-        # "${rsyncCommand} pvv@smtp.pvv.ntnu.no:/etc/shadow /run/httpd/pamunix-sync/"
         (let
           args = lib.cli.toCommandLineShellGNU { } {
-            passwd-file = "/run/httpd/pamunix-sync/passwd";
+            passwd-file = "/run/httpd/pamunix-in/passwd";
-            group-file = "/run/httpd/pamunix-sync/group";
+            group-file = "/run/httpd/pamunix-in/group";
-            output-dir = "/run/httpd/systemd-userdb";
+            output-dir = "/run/httpd/pamunix-out";
             shadow-file = pkgs.emptyFile;
 
             output-passwd = true;
@@ -319,7 +327,16 @@ in
             ignore-group-file = toString ./ignore_group_file.txt;
           };
         in ''${lib.getExe pkgs.passwd2systemd-users} ${args}'')
-        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-sync/passwd /run/httpd/pamunix-sync/group"
+        "${lib.getExe' pkgs.coreutils "shred"} -u /run/httpd/pamunix-in/passwd /run/httpd/pamunix-in/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:0:System administrator:/root:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:54:Apache httpd user:/var/empty:/run/current-system/sw/bin/bash' /run/httpd/pamunix-out/passwd"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\root:x:0:' /run/httpd/pamunix-out/group"
+        ":${lib.getExe pkgs.gnused} -i '$ a\\\\wwwrun:x:54:' /run/httpd/pamunix-out/group"
+        "${lib.getExe' pkgs.coreutils "cat"} /run/httpd/pamunix-out/passwd"
+        "+${lib.getExe' pkgs.coreutils "chown"} root:root /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
+        "+${lib.getExe' pkgs.coreutils "chmod"} 0644 /run/httpd/pamunix-out/passwd /run/httpd/pamunix-out/group"
+        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/passwd /etc/passwd"
+        "+${lib.getExe pkgs.mount} --bind /run/httpd/pamunix-out/group  /etc/group"
       ];
       ExecStart = lib.mkForce "${cfg.package}/bin/httpd -D FOREGROUND -f /etc/httpd/httpd.conf -k start";
       ExecReload = lib.mkForce "${cfg.package}/bin/httpd -f /etc/httpd/httpd.conf -k graceful";
@@ -335,9 +352,9 @@ in
       LogsDirectory = [ "httpd" ];
       LogsDirectoryMode = "0700";
 
-      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SYS_PTRACE" ];
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SYS_PTRACE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" "CAP_SETUID" "CAP_SETGID" ] ++ lib.optionals debug [ "CAP_SYS_PTRACE" ];
-      # LockPersonality = true;
+      LockPersonality = !debug;
       PrivateDevices = true;
       PrivateTmp = true;
       # NOTE: this removes CAP_NET_BIND_SERVICE...
@@ -364,15 +381,17 @@ in
         "tcp:443"
       ];
       SystemCallArchitectures = "native";
-      # SystemCallFilter = [
+      SystemCallFilter = lib.mkIf (!debug) [
-      #    "@system-service"
+         "@system-service"
-      # ];
+         "@setuid"
+      ];
       UMask = "0077";
 
+      RuntimeDirectoryMode = "0750";
       RuntimeDirectory = [
         "httpd/root-mnt"
-        "httpd/pamunix-sync"
+        "httpd/pamunix-in"
-        "httpd/systemd-userdb"
+        "httpd/pamunix-out"
       ];
       RootDirectory = "/run/httpd/root-mnt";
       MountAPIVFS = true;
@@ -380,16 +399,16 @@ in
         builtins.storeDir
         "/etc"
         "/dev/null"
-        # NCSD socket
-        # "/var/run"
-        # "/var/run/systemd/resolve"
         "/etc/resolv.conf"
         "/var/lib/acme"
-        "/run/httpd/systemd-userdb:/etc/userdb"
+
+        "-/run/httpd/pamunix-out/passwd:/etc/passwd"
+        "-/run/httpd/pamunix-out/group:/etc/group"
+
         "${pkgs.writeText "userweb-fake-nsswitch.conf" ''
-          passwd:    systemd files
+          passwd:    files
-          group:     systemd files
+          group:     files
-          shadow:    systemd files
+          shadow:    files
           sudoers:   files
 
           hosts:     mymachines resolve [!UNAVAIL=return] files myhostname dns
@@ -417,6 +436,7 @@ in
           "/store/gnu"
           "/usr"
           "/usr/local"
+          "/run/current-system/sw"
         ];
         child = [
           "/bin"

secrets/ildkule/ildkule.yaml

@@ -1,5 +1,6 @@
 config:
     mysqld_exporter_password: ENC[AES256_GCM,data:I9K+QMqaN3FOOVKzeOR9Q6UERStXX0P8WEHyN1jzzbM=,iv:UxvIdlfAyJvNuxPkU4+guKPa0fiD0vVLzHOTYktcmso=,tag:ltnIqEwESYx9HBu8UN0ZLw==,type:str]
+    postgresql_dibbler_password: ENC[AES256_GCM,data:wP4CVz9qRE3CJrblWWYqOIkcH3LM5H81,iv:j8zr1TRNlPPqIppYlWhDoKlL7m2Ph2wQlU6bvFj8R9A=,tag:Cfp8zbYkoLqI7DTDpOBlJw==,type:str]
 keys:
     grafana:
         secret_key: ENC[AES256_GCM,data:+WoAJbDBEgKs0RoHT+7oEELAVQ+/2Xt+5RTMSXg23moCqVRx+Gzll9P5Drw=,iv:AkRn/Y20iEe5i1T+84wAgLCTFtAox2G3giyawAkltAw=,tag:BZbt5Wb5lYLIJBm/pfP4GQ==,type:str]
@@ -72,8 +73,8 @@ sops:
             dC9meDZlc3d3aUJEVjc4REF0Y1BLcGcK79LbJzc5KVgEgyJR11crGuX8YcVoJBbT
             Fin7Zoon06L7qx0Zw5u27wV7RKMnYT7hOMiWs6660ZTLcYJ5M1aEZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2025-03-16T20:08:18Z"
+    lastmodified: "2026-06-07T12:51:04Z"
-    mac: ENC[AES256_GCM,data:C2tpWppc13jKJq5d4nmAKQOaNWHm27TKwxAxm1fi2lejN1lqUaoz5bHfTBA7MfaWvuP5uZnfbtG32eeu48mnlWpo58XRUFFecAhb9JUpW9s5IR3/nbzLNkGU7H5C0oWPrxI4thd+bAVduIgBjjFyGj1pe6J9db3c0yUWRwNlwGU=,iv:YpoQ4psiFYOWLGipxv1QvRvr034XFsyn2Bhyy39HmOo=,tag:ByiCWygFC/VokVTbdLoLgg==,type:str]
+    mac: ENC[AES256_GCM,data:otGwzc3Bme1PGRU4zWRRf3kmAf5EDjT8sPkDK/zDrO0ve+d3x99Qls4DBZV8G6FsFtUXbeKVo70I5VpqUgaIef+nyMl6zzWkW1wpbKIBYjh5fkaP0xxhLnw+shrB088PAXT0wkP7hJBLO0ZJgTrpprJKhmeqVj2HEXBSJ1wSjk8=,iv:cjqtdbWBVLUfpQdykb3vKDKa/VC/kGBybwYtG/eStqc=,tag:remOj4bX/H/LlAPgcXHAdA==,type:str]
     pgp:
         - created_at: "2026-05-20T17:35:58Z"
           enc: |-
@@ -96,4 +97,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.9.4
+    version: 3.12.2

secrets/skrot/skrot.yaml

@@ -1,13 +1,12 @@
 dibbler:
     postgresql:
-        password: ENC[AES256_GCM,data:3X9A3jOpFVRuBg0gRiCEsZVKfLI=,iv:XC7LBNUhALk9IEhItV8fO5p/m7VKL0REBY1W2IZt7G4=,tag:l18R7EhbOlucZHFQiEvpHw==,type:str]
+        password: ENC[AES256_GCM,data:ZeNKipcCB+z8QVGeg1iV3MUXqALjotVz,iv:xCtgOoe6Pkr6Cq3vL+T4L+GW1KAcgP/xUz3YbHs5bCc=,tag:/X5phRYDAws8Aam1j+UaTw==,type:str]
 worblehat:
     postgresql:
         password: ENC[AES256_GCM,data:WpJR6MumY+7WUYdVVgAqv1af+NmqecTMO9aP5lidSpE=,iv:7aoN8mjXckd81LxasMSG3R2vqj0SvzSl7wrEQ1LwToo=,tag:zeeNcEpkYnqyd8be0ZS+kQ==,type:str]
 sops:
     age:
-        - recipient: age1hzkvnktkr8t5gvtq0ccw69e44z5z6wf00n3xhk3hj24emf07je5s6q2evr
+        - enc: |
-          enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvTk5YU3Z2Yy9HS1R4ME5I
             UU1PRWVncHJYcXY5RlFpOWVQUWZsdy93ZDFBCnlxWkpaL1g5WmNSckNYd202WE40
@@ -15,8 +14,8 @@ sops:
             ZnllQzJiK1ZkRmFndmtYdW9IclFWY1EK82f1iGt3nt8dJnEQlMujNqConf6Qq6GX
             hqoqPoc2EM4kun28Bbpq4pAY7eEPRrWFqOkjYVvgIRoS88D7xT3LWg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
+          recipient: age1hzkvnktkr8t5gvtq0ccw69e44z5z6wf00n3xhk3hj24emf07je5s6q2evr
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5WTJIOUcxRlBuNmRrNUZo
             MXFxeVJBTEhDK00yTUw1U2dHckNFYWZKWkhNCnYxYmtrUEVvd1RaYUI5WTRTRW16
@@ -24,8 +23,8 @@ sops:
             eTB4WldMNW9GNUwwaEUzRThsemxRVzQKGpa0J2PBzDRdHijm0e3nFAaxQCHUjz+L
             KataXJEMCijJ6k+7vpb5QMxe2jB1J2PMxNGFp0bWAy2Al3p/Ez2Kww==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
+          recipient: age1ug30gg4y7ftuya0wdv7q0vh4egn00wlv2th7mt7cgc2ze46wmvyq9lq6ge
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZaW1ZSXhVeFVTQW9WYzVh
             WkVUM2JkOU5VNU9oQXE2Y2pvcFlOWTdvbnpJClduS0RHL2xja291a2doQ0wzbzhQ
@@ -33,8 +32,8 @@ sops:
             ZUdnS2RvOXI1dGNYQTl6ZHE1cUdMWHMK4ycAJQLyKCgJIzjQ02bPjz4Ct9eO6ivw
             kfWhyMaoWwM9PhFcwSak0cLpX0C/IOzSzO78pf3WhG16pV7aXapdog==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
+          recipient: age1mrnldl334l2nszuta6ywvewng0fswv2dz9l5g4qcwe3nj4yxf92qjskdx6
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqaml0OVlhcUJSU1hSY3lP
             bkM0cUV4Z2ZLeERHZ3BUNExuYS9KSU5CekQ4CmQ3SE1vdDBtdFJ6czZYR3U5Tk1X
@@ -42,8 +41,8 @@ sops:
             Sy9XbjhwOFR6SFpaNHZLd3ZxdmxOVUEKBBbGmdVVlKHxO+/iODznLP3+dJGppybW
             +1k9uenVHzie+pDKcrQpSyX2WDnmgg7hUAUiXPuz1eEWmwbRJnU/5w==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
+          recipient: age1hmpdk4h69wxpwqk9tkud39f66hprhehxtzhgw97r6dvr7v0mx5jscsuhkn
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXK01vOVV5YlhsZ2ljYS91
             OUVEaEpTbXFKOHVNVDVoMTlrS05wRmsyM2dvCjZHOXlCUGowd0J4UlQzSzM5dWJ0
@@ -51,8 +50,8 @@ sops:
             RUR6Yi9SUDFCUkZmRk5hYTVFeGloZXcKY/XtaSoW8Pu2wS4oistLSc0T5JvMnt+w
             s3yfe/zx9/1K6OtbeljF9FZVOB/dOamvk+Qlfl0T5qush7/WgGzErA==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
+          recipient: age1wrssr4z4g6vl3fd3qme5cewchmmhm0j2xe6wf2meu4r6ycn37anse98mfs
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOM0pFb2tRTURtWmp6elRN
             M0xtajlzMTNPMnppcGhJMVlsNHdwWmNGbFVFCnlxM1JQTkR2elAvdytKUEJ3djBS
@@ -60,8 +59,8 @@ sops:
             eWlyWGhaS1JCNitUSVVScFk2WGEvOG8K2rpYPGx5jhyyRK4UkeJR96wDFr4Frzsr
             QWz7fYZRWKWf0H0qn+bm9IfVJiBAlS5i16D1FnipZVmdWefFaZSEPg==
             -----END AGE ENCRYPTED FILE-----
-        - recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
+          recipient: age1zhxul786an743u0fascv4wtc5xduu7qfy803lfs539yzhgmlq5ds2lznt5
-          enc: |
+        - enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
             YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJVFV0WVZrK0wzbnhkcmcz
             c2lIdVlKcFpoYjZIWlNPN0M5N2g2WG9YdlRJCjg5YlNoSzQ5YW5yRUVSeTEzRThY
@@ -69,8 +68,9 @@ sops:
             MmxPMWNPYzJiOFRqY2VYczhvRm5IR3cKpUVV+zsMolsHI2YK9YqC6ecNT6QXv0TV
             d1SpXRAexZBeWCCHBjSdvQBl8AT4EwrAIP2M2o++6i5DaGoGiEIWZQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-03-18T14:56:22Z"
+          recipient: age1sqs7urnzsdy64efmd0zukzv3gs5pnjksuxd7nqmdwdy5l0nqnunq6hyune
-    mac: ENC[AES256_GCM,data:nBKtFmFKx/Mt9TIFnKuuznsPAXCQpc3+WIspNu5TN9TpIqw75nzYXpxIb2hxRfRu0nbjHXpBy4bkzeMi41BGkvkvV57CZyq11J5i/iIKwuvllaB1IWrdDT2u+6RH3jIspp3KoyxFWdRqcGfNma9dSmtI+1Dd5z7XaxVaoVK2QMI=,iv:6joviyJ2cXmGh/9HH7VEcoK3+4GK5I6i2N/1d65PAN0=,tag:0BFVPWL3BByJH8HbrBTKOw==,type:str]
+    lastmodified: "2026-06-07T08:58:13Z"
+    mac: ENC[AES256_GCM,data:rchs8pGkw7dthGOQNDB5p/kgQdfdystaC+jRr0bZnA4Q41+PVMu+vBSMIZ+9zZek6oENgchmV5rRS0CEeb9UQMMiPXCk2Q2jMaDfiNCmOmjf0YYeFWRM6g2lA+IZ3RgKjwhXa6i5JOeNrNewMjtx7MFcHTn3EBlg2mztyn1xbT0=,iv:0yvoFPDxpugaBmTtXSmhNz9XusJHrU3E02tBm1hVsZo=,tag:hZ1BzqfmlT1OhPSsn+CCTg==,type:str]
     pgp:
         - created_at: "2026-02-10T20:01:32Z"
           enc: |-
@@ -93,4 +93,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.12.1
+    version: 3.13.0