bicep/mysql: enable hugepages

flake.nix

@@ -197,6 +197,7 @@
             inputs.pvv-calendar-bot.nixosModules.default
             inputs.minecraft-heatmap.nixosModules.default
             self.nixosModules.gickup
+            self.nixosModules.hugepages
             self.nixosModules.matrix-ooye
           ];
           overlays = [
@@ -309,6 +310,7 @@
       bluemap = ./modules/bluemap.nix;
       drumknotty = ./modules/drumknotty;
       gickup = ./modules/gickup;
+      hugepages = ./modules/hugepages.nix;
       matrix-ooye = ./modules/matrix-ooye.nix;
       python-http-handlers = ./modules/python-http-handlers.nix;
       robots-txt = ./modules/robots-txt.nix;

hosts/bicep/services/mysql/backup.nix

@@ -41,38 +41,24 @@ in
     path = with pkgs; [
       cfg.package
       coreutils
-      diffutils
       zstd
     ];
 
-    script = ''
+    script = let
+      rotations = 2;
+    in ''
       set -euo pipefail
 
-      dump() {
+      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
-        local name="$1" out tmp
-        out="$STATE_DIRECTORY/$name.sql.zst"
-        tmp="$out.tmp"
-        shift
-        "$@" | zstd -9 --rsyncable -f -o "$tmp"
-        if cmp -s "$tmp" "$out" 2>/dev/null; then
-          rm -f "$tmp"
-        else
-          mv -f "$tmp" "$out"
-        fi
-      }
 
-      declare -A keep
+      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
-      while IFS= read -r db; do
-        [ -n "$db" ] || continue
-        dump "$db" mysqldump --skip-dump-date --databases "$db"
-        keep["$db.sql.zst"]=1
-      done < <(mysql -N -e 'SHOW DATABASES' | grep -vE '^(information_schema|performance_schema)$')
 
-      # drop dumps of databases that no longer exist
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
-      for f in "$STATE_DIRECTORY"/*.sql.zst; do
+      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
-        [ -e "$f" ] || continue
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
-        base="$(basename "$f")"
+
-        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
       done
     '';
 

hosts/bicep/services/mysql/default.nix

@@ -2,6 +2,8 @@
 let
   cfg = config.services.mysql;
   dataDir = "/data/mysql";
+
+  innodbBufferPoolMB = 128;
 in
 {
   imports = [ ./backup.nix ];
@@ -26,6 +28,10 @@ in
         # Useful for the mysqld prometheus exporter
         userstat = 1;
 
+        # Memory settings
+        innodb_buffer_pool_size = "${toString innodbBufferPoolMB}M";
+        "large-pages" = 1;
+
         # This was needed in order to be able to use all of the old users
         # during migration from knakelibrak to bicep in Sep. 2023
         secure_auth = 0;
@@ -47,6 +53,10 @@ in
     }];
   };
 
+  boot.kernel.hugepages.reservations.mysql = lib.mkIf cfg.enable (
+    builtins.ceil (innodbBufferPoolMB / config.boot.kernel.hugepages.size)
+  );
+
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 3306 ];
 
   systemd.tmpfiles.settings."10-mysql".${dataDir}.d = lib.mkIf cfg.enable {

hosts/bicep/services/postgresql/backup.nix

@@ -41,42 +41,25 @@ in
 
     path = with pkgs; [
       coreutils
-      diffutils
       zstd
       cfg.package
     ];
 
-    script = ''
+    script = let
+      rotations = 2;
+    in ''
       set -euo pipefail
 
-      dump() {
+      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
-        local name="$1" out tmp
-        out="$STATE_DIRECTORY/$name.sql.zst"
-        tmp="$out.tmp"
-        shift
-        "$@" | zstd -9 --rsyncable -f -o "$tmp"
-        if cmp -s "$tmp" "$out" 2>/dev/null; then
-          rm -f "$tmp"
-        else
-          mv -f "$tmp" "$out"
-        fi
-      }
 
-      declare -A keep
+      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
-      dump globals pg_dumpall -U postgres --globals-only --restrict-key=backup
-      keep[globals.sql.zst]=1
 
-      while IFS= read -r db; do
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
-        [ -n "$db" ] || continue
+      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
-        dump "$db" pg_dump -U postgres -C -d "$db" --restrict-key=backup
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
-        keep["$db.sql.zst"]=1
-      done < <(psql -U postgres -tAc "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname")
 
-      # drop dumps of databases that no longer exist
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
-      for f in "$STATE_DIRECTORY"/*.sql.zst; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
-        [ -e "$f" ] || continue
-        base="$(basename "$f")"
-        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
       done
     '';
 

hosts/bicep/services/postgresql/default.nix

@@ -1,6 +1,8 @@
 { config, lib, pkgs, values, ... }:
 let
   cfg = config.services.postgresql;
+
+  sharedBuffersMB = 8192;
 in
 {
   imports = [
@@ -29,7 +31,7 @@ in
       superuser_reserved_connections = 3;
 
       # Memory Settings
-      shared_buffers = "8192 MB";
+      shared_buffers = "${toString sharedBuffersMB} MB";
       work_mem = "32 MB";
       maintenance_work_mem = "420 MB";
       effective_cache_size = "22 GB";
@@ -93,6 +95,10 @@ in
     };
   };
 
+  boot.kernel.hugepages.reservations.postgresql = lib.mkIf cfg.enable (
+    builtins.ceil (sharedBuffersMB / config.boot.kernel.hugepages.size)
+  );
+
   systemd.tmpfiles.settings."10-postgresql"."/data/postgresql".d = lib.mkIf cfg.enable {
     user = config.systemd.services.postgresql.serviceConfig.User;
     group = config.systemd.services.postgresql.serviceConfig.Group;

modules/hugepages.nix

@@ -0,0 +1,38 @@
+{ config, lib, ... }:
+let
+  cfg = config.boot.kernel.hugepages;
+in
+{
+  options.boot.kernel.hugepages = {
+    size = lib.mkOption {
+      type = lib.types.enum [ 2 1024 ];
+      default = 2;
+      description = ''
+        Hugepage size in MB.
+
+        You can use this value to calculate the amount of memory you will have available as hugepages.
+      '';
+    };
+
+    reservations = lib.mkOption {
+      type = lib.types.attrsOf lib.types.ints.unsigned;
+      default = { };
+      description = ''
+        Number of hugepages each service wants reserved in vm.nr_hugepages,
+        keyed by service name.
+      '';
+    };
+  };
+
+  config = {
+    boot.kernelParams = let
+      num = {
+        "2" = "2M";
+        "1024" = "1G";
+      }.${toString cfg.size};
+    in [ "hugepagesz=${num}" ];
+
+    boot.kernel.sysctl."vm.nr_hugepages" =
+      lib.foldl' (a: b: a + b) 0 (lib.attrValues cfg.reservations);
+  };
+}