ildkule/grafana: update a bunch of dashboards

flake.lock

@@ -7,11 +7,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1769400154,
-        "narHash": "sha256-K0OeXzFCUZTkCBxUDr3U3ah0odS/urtNVG09WDl+HAA=",
+        "lastModified": 1769510541,
+        "narHash": "sha256-jxuQY0anT3YpwpnYB5w7p6EPS6UWIj4vGxzfsOJvC1I=",
         "ref": "main",
-        "rev": "8e84669d9bf963d5e46bac37fe9b0aa8e8be2d01",
-        "revCount": 230,
+        "rev": "ec43f67e58f049a709fa2c19601b8c637f38126f",
+        "revCount": 232,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/dibbler.git"
       },
@@ -174,11 +174,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1768749374,
-        "narHash": "sha256-dhXYLc64d7TKCnRPW4TlHGl6nLRNdabJB2DpJ8ffUw0=",
+        "lastModified": 1769500363,
+        "narHash": "sha256-vFxmdsLBPdTy5j2bf54gbTQi1XnWbZDmeR/BBh8MFrw=",
         "ref": "main",
-        "rev": "040294f2e1df46e33d995add6944b25859654097",
-        "revCount": 37,
+        "rev": "2618e434e40e109eaab6a0693313c7e0de7324a3",
+        "revCount": 47,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/minecraft-kartverket.git"
       },
@@ -217,11 +217,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1768955766,
-        "narHash": "sha256-V9ns1OII7sWSbIDwPkiqmJ3Xu/bHgQzj+asgH9cTpOo=",
+        "lastModified": 1769018862,
+        "narHash": "sha256-x3eMpPQhZwEDunyaUos084Hx41XwYTi2uHY4Yc4YNlk=",
         "owner": "oddlama",
         "repo": "nix-topology",
-        "rev": "71f27de56a03f6d8a1a72cf4d0dfd780bcc075bc",
+        "rev": "a15cac71d3399a4c2d1a3482ae62040a3a0aa07f",
         "type": "github"
       },
       "original": {
@@ -233,11 +233,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1768877948,
-        "narHash": "sha256-Bq9Hd6DWCBaZ2GkwvJCWGnpGOchaD6RWPSCFxmSmupw=",
-        "rev": "43b2e61c9d09cf6c1c9c192fe6da08accc9bfb1d",
+        "lastModified": 1769484787,
+        "narHash": "sha256-ufhG9uSA8cCEk/97D/7xQEKcO/ftr4IPRH+HQFaKNdE=",
+        "rev": "999ca0e5484922624254294ea1adc2b90081579e",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/25.11-small/nixos-25.11.4368.43b2e61c9d09/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/25.11-small/nixos-25.11.4804.999ca0e54849/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -261,11 +261,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1768886240,
-        "narHash": "sha256-HUAAI7AF+/Ov1u3Vvjs4DL91zTxMkWLC4xJgQ9QxOUQ=",
-        "rev": "80e4adbcf8992d3fd27ad4964fbb84907f9478b0",
+        "lastModified": 1769434638,
+        "narHash": "sha256-u19M4QdjvjEySkGhP4fUNyY6rqAbPCdQf/AFw04CkQU=",
+        "rev": "9c2822d7024c032e66000a8b8a47e91b4e63ffc8",
         "type": "tarball",
-        "url": "https://releases.nixos.org/nixos/unstable-small/nixos-26.05pre930839.80e4adbcf899/nixexprs.tar.xz"
+        "url": "https://releases.nixos.org/nixos/unstable-small/nixos-26.05pre935000.9c2822d7024c/nixexprs.tar.xz"
       },
       "original": {
         "type": "tarball",
@@ -300,11 +300,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1768636400,
-        "narHash": "sha256-AiSKT4/25LS1rUlPduBMogf4EbdMQYDY1rS7AvHFcxk=",
+        "lastModified": 1769009806,
+        "narHash": "sha256-52xTtAOc9B+MBRMRZ8HI6ybNsRLMlHHLh+qwAbaJjRY=",
         "ref": "main",
-        "rev": "3a8f82b12a44e6c4ceacd6955a290a52d1ee2856",
-        "revCount": 573,
+        "rev": "aa8adfc6a4d5b6222752e2d15d4a6d3b3b85252e",
+        "revCount": 575,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/nettsiden.git"
       },
@@ -364,11 +364,11 @@
         "rust-overlay": "rust-overlay_3"
       },
       "locked": {
-        "lastModified": 1768140181,
-        "narHash": "sha256-HfZzup5/jlu8X5vMUglTovVTSwhHGHwwV1YOFIL/ksA=",
+        "lastModified": 1769325266,
+        "narHash": "sha256-q2G2NG7I1tvfFK4GDnn3vt1CCg0GN4ncdo0NSY+Q2Nc=",
         "ref": "main",
-        "rev": "834463ed64773939798589ee6fd4adfe3a97dddd",
-        "revCount": 43,
+        "rev": "23b163e828901cb981eec6f3262e922f437f850b",
+        "revCount": 45,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/roowho2.git"
       },
@@ -428,11 +428,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1767322002,
-        "narHash": "sha256-yHKXXw2OWfIFsyTjduB4EyFwR0SYYF0hK8xI9z4NIn0=",
+        "lastModified": 1769309768,
+        "narHash": "sha256-AbOIlNO+JoqRJkK1VrnDXhxuX6CrdtIu2hSuy4pxi3g=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "03c6e38661c02a27ca006a284813afdc461e9f7e",
+        "rev": "140c9dc582cb73ada2d63a2180524fcaa744fad5",
         "type": "github"
       },
       "original": {
@@ -448,11 +448,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1768863606,
-        "narHash": "sha256-1IHAeS8WtBiEo5XiyJBHOXMzECD6aaIOJmpQKzRRl64=",
+        "lastModified": 1769469829,
+        "narHash": "sha256-wFcr32ZqspCxk4+FvIxIL0AZktRs6DuF8oOsLt59YBU=",
         "owner": "Mic92",
         "repo": "sops-nix",
-        "rev": "c7067be8db2c09ab1884de67ef6c4f693973f4a2",
+        "rev": "c5eebd4eb2e3372fe12a8d70a248a6ee9dd02eff",
         "type": "github"
       },
       "original": {

flake.nix

@@ -281,7 +281,16 @@
     };
 
     devShells = forAllSystems (system: {
-      default = nixpkgs-unstable.legacyPackages.${system}.callPackage ./shell.nix { };
+      default = let
+        pkgs = import nixpkgs-unstable {
+          inherit system;
+          overlays = [
+            (final: prev: {
+              inherit (inputs.disko.packages.${system}) disko;
+            })
+          ];
+        };
+      in pkgs.callPackage ./shell.nix { };
       cuda = let
         cuda-pkgs = import nixpkgs-unstable {
           inherit system;

hosts/bekkalokk/services/mediawiki/default.nix

@@ -52,7 +52,7 @@ in {
   services.rsync-pull-targets = {
     enable = true;
     locations.${cfg.uploadsDir} = {
-      user = config.services.root;
+      user = "root";
       rrsyncArgs.ro = true;
       authorizedKeysAttrs = [
         "restrict"
@@ -61,9 +61,7 @@ in {
         "no-pty"
         "no-X11-forwarding"
       ];
-      # TODO: create new key on principal
-      enable = false;
-      publicKey = "";
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICHFHa3Iq1oKPhbKCAIHgOoWOTkLmIc7yqxeTbut7ig/ mediawiki rsync backup";
     };
   };
 

hosts/bekkalokk/services/vaultwarden.nix

@@ -99,4 +99,20 @@ in {
       ];
     };
   };
+
+  services.rsync-pull-targets = {
+    enable = true;
+    locations."/var/lib/vaultwarden" = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB2cDaW52gBtLVaNqoGijvN2ZAVkAWlII5AXUzT3Dswj vaultwarden rsync backup";
+    };
+  };
 }

hosts/bekkalokk/services/webmail/snappymail.nix

@@ -14,5 +14,20 @@ in {
     enableACME = true;
     kTLS = true;
   };
-}
 
+  services.rsync-pull-targets = {
+    enable = true;
+    locations.${cfg.dataDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJENMnuNsHEeA91oX+cj7Qpex2defSXP/lxznxCAqV03 snappymail rsync backup";
+    };
+  };
+}

hosts/bicep/configuration.nix

@@ -9,8 +9,8 @@
     ./services/calendar-bot.nix
     #./services/git-mirrors
     ./services/minecraft-heatmap.nix
-    ./services/mysql.nix
-    ./services/postgres.nix
+    ./services/mysql
+    ./services/postgresql
 
     ./services/matrix
   ];

hosts/bicep/services/matrix/default.nix

@@ -1,8 +1,9 @@
 { config, ... }:
 {
   imports = [
-    ./synapse.nix
     ./synapse-admin.nix
+    ./synapse-auto-compressor.nix
+    ./synapse.nix
     ./element.nix
     ./coturn.nix
     ./livekit.nix

hosts/bicep/services/matrix/synapse-auto-compressor.nix

@@ -0,0 +1,56 @@
+{ config, lib, utils, ... }:
+let
+  cfg = config.services.synapse-auto-compressor;
+in
+{
+  services.synapse-auto-compressor = {
+    # enable = true;
+    postgresUrl = "postgresql://matrix-synapse@/synapse?host=/run/postgresql";
+  };
+
+  # NOTE: nixpkgs has some broken asserts, vendored the entire unit
+  systemd.services.synapse-auto-compressor = {
+    description = "synapse-auto-compressor";
+    requires = [
+      "postgresql.target"
+    ];
+    inherit (cfg) startAt;
+    serviceConfig = {
+      Type = "oneshot";
+      DynamicUser = true;
+      User = "matrix-synapse";
+      PrivateTmp = true;
+      ExecStart = utils.escapeSystemdExecArgs [
+        "${cfg.package}/bin/synapse_auto_compressor"
+        "-p"
+        cfg.postgresUrl
+        "-c"
+        cfg.settings.chunk_size
+        "-n"
+        cfg.settings.chunks_to_compress
+        "-l"
+        (lib.concatStringsSep "," (map toString cfg.settings.levels))
+      ];
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      PrivateDevices = true;
+      PrivateMounts = true;
+      PrivateUsers = true;
+      RemoveIPC = true;
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true;
+      ProcSubset = "pid";
+      ProtectProc = "invisible";
+      ProtectSystem = "strict";
+      ProtectHome = true;
+      ProtectHostname = true;
+      ProtectClock = true;
+      ProtectKernelTunables = true;
+      ProtectKernelModules = true;
+      ProtectKernelLogs = true;
+      ProtectControlGroups = true;
+    };
+  };
+}

hosts/bicep/services/matrix/synapse.nix

@@ -30,7 +30,7 @@ in {
   services.rsync-pull-targets = {
     enable = true;
     locations.${cfg.settings.media_store_path} = {
-      user = config.services.root;
+      user = "root";
       rrsyncArgs.ro = true;
       authorizedKeysAttrs = [
         "restrict"
@@ -39,9 +39,7 @@ in {
         "no-pty"
         "no-X11-forwarding"
       ];
-      # TODO: create new key on principal
-      enable = false;
-      publicKey = "";
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIASnjI9b3j4ZS3BL/D1ggHfws1BkE8iS0v0cGpEmbG+k matrix_media_store rsync backup";
     };
   };
 

hosts/bicep/services/minecraft-heatmap.nix

@@ -22,7 +22,7 @@ in
     };
   };
 
-  systemd.services.minecraft-heatmap-ingest-logs = {
+  systemd.services.minecraft-heatmap-ingest-logs = lib.mkIf cfg.enable {
     serviceConfig.LoadCredential = [
       "sshkey:${config.sops.secrets."minecraft-heatmap/ssh-key/private".path}"
     ];

hosts/bicep/services/mysql/backup.nix

@@ -0,0 +1,82 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.mysql;
+  backupDir = "/data/mysql-backups";
+in
+{
+  # services.mysqlBackup = lib.mkIf cfg.enable {
+  #   enable = true;
+  #   location = "/var/lib/mysql-backups";
+  # };
+
+  systemd.tmpfiles.settings."10-mysql-backups".${backupDir}.d = {
+	  user = "mysql";
+	  group = "mysql";
+	  mode = "700";
+	};
+
+  services.rsync-pull-targets = lib.mkIf cfg.enable {
+    enable = true;
+    locations.${backupDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgj55/7Cnj4cYMJ5sIkl+OwcGeBe039kXJTOf2wvo9j mysql rsync backup";
+    };
+  };
+
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves.
+  systemd.services."backup-mysql" = lib.mkIf cfg.enable {
+    description = "Backup MySQL data";
+    requires = [ "mysql.service" ];
+
+    path = with pkgs; [
+      cfg.package
+      coreutils
+      zstd
+    ];
+
+    script = let
+      rotations = 2;
+    in ''
+      set -euo pipefail
+
+      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
+
+      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
+
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt ${toString (rotations + 1)} ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      done
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "mysql";
+      Group = "mysql";
+      UMask = "0077";
+
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      StateDirectory = [ "mysql-backups" ];
+      BindPaths = [ "${backupDir}:/var/lib/mysql-backups" ];
+
+      # TODO: hardening
+    };
+
+    startAt = "*-*-* 02:15:00";
+  };
+}

hosts/bicep/services/mysql/default.nix

@@ -4,6 +4,8 @@ let
   dataDir = "/data/mysql";
 in
 {
+  imports = [ ./backup.nix ];
+
   sops.secrets."mysql/password" = {
     owner = "mysql";
     group = "mysql";
@@ -13,7 +15,7 @@ in
 
   services.mysql = {
     enable = true;
-    package = pkgs.mariadb;
+    package = pkgs.mariadb_118;
     settings = {
       mysqld = {
         # PVV allows a lot of connections at the same time
@@ -24,6 +26,9 @@ in
         # This was needed in order to be able to use all of the old users
         # during migration from knakelibrak to bicep in Sep. 2023
         secure_auth = 0;
+
+        slow-query-log = 1;
+        slow-query-log-file = "/var/log/mysql/mysql-slow.log";
       };
     };
 
@@ -39,11 +44,6 @@ in
     }];
   };
 
-  services.mysqlBackup = lib.mkIf cfg.enable {
-    enable = true;
-    location = "/var/lib/mysql/backups";
-  };
-
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 3306 ];
 
   systemd.tmpfiles.settings."10-mysql".${dataDir}.d = lib.mkIf cfg.enable {
@@ -60,6 +60,8 @@ in
     serviceConfig = {
       BindPaths = [ "${dataDir}:${cfg.dataDir}" ];
 
+      LogsDirectory = "mysql";
+
       IPAddressDeny = "any";
       IPAddressAllow = [
         values.ipv4-space

hosts/bicep/services/postgresql/backup.nix

@@ -0,0 +1,83 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.postgresql;
+  backupDir = "/data/postgresql-backups";
+in
+{
+  # services.postgresqlBackup = lib.mkIf cfg.enable {
+  #   enable = true;
+  #   location = "/var/lib/postgresql-backups";
+  #   backupAll = true;
+  # };
+
+  systemd.tmpfiles.settings."10-postgresql-backups".${backupDir}.d = {
+	  user = "postgres";
+	  group = "postgres";
+	  mode = "700";
+	};
+
+  services.rsync-pull-targets = lib.mkIf cfg.enable {
+    enable = true;
+    locations.${backupDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvO7QX7QmwSiGLXEsaxPIOpAqnJP3M+qqQRe5dzf8gJ postgresql rsync backup";
+    };
+  };
+
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves
+  systemd.services."backup-postgresql" = {
+    description = "Backup PostgreSQL data";
+    requires = [ "postgresql.service" ];
+
+    path = with pkgs; [
+      coreutils
+      zstd
+      cfg.package
+    ];
+
+    script = let
+      rotations = 2;
+    in ''
+      set -euo pipefail
+
+      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
+
+      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
+
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt ${toString (rotations + 1)} ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      done
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "postgres";
+      Group = "postgres";
+      UMask = "0077";
+
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      StateDirectory = [ "postgresql-backups" ];
+      BindPaths = [ "${backupDir}:/var/lib/postgresql-backups" ];
+
+      # TODO: hardening
+    };
+
+    startAt = "*-*-* 01:15:00";
+  };
+}

hosts/bicep/services/postgresql/default.nix

@@ -1,8 +1,13 @@
-{ config, pkgs, values, ... }:
+{ config, lib, pkgs, values, ... }:
+let
+  cfg = config.services.postgresql;
+in
 {
+  imports = [ ./backup.nix ];
+
   services.postgresql = {
     enable = true;
-    package = pkgs.postgresql_15;
+    package = pkgs.postgresql_18;
     enableTCPIP = true;
 
     authentication = ''
@@ -74,13 +79,13 @@
     };
   };
 
-  systemd.tmpfiles.settings."10-postgresql"."/data/postgresql".d = {
+  systemd.tmpfiles.settings."10-postgresql"."/data/postgresql".d = lib.mkIf cfg.enable {
     user = config.systemd.services.postgresql.serviceConfig.User;
     group = config.systemd.services.postgresql.serviceConfig.Group;
     mode = "0700";
   };
 
-  systemd.services.postgresql-setup = {
+  systemd.services.postgresql-setup = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
       "systemd-tmpfiles-resetup.service"
@@ -95,7 +100,7 @@
     };
   };
 
-  systemd.services.postgresql = {
+  systemd.services.postgresql = lib.mkIf cfg.enable {
     after = [
       "systemd-tmpfiles-setup.service"
       "systemd-tmpfiles-resetup.service"
@@ -110,18 +115,12 @@
     };
   };
 
-  environment.snakeoil-certs."/etc/certs/postgres" = {
+  environment.snakeoil-certs."/etc/certs/postgres" = lib.mkIf cfg.enable {
     owner = "postgres";
     group = "postgres";
     subject = "/C=NO/O=Programvareverkstedet/CN=postgres.pvv.ntnu.no/emailAddress=drift@pvv.ntnu.no";
   };
 
-  networking.firewall.allowedTCPPorts = [ 5432 ];
-  networking.firewall.allowedUDPPorts = [ 5432 ];
-
-  services.postgresqlBackup = {
-    enable = true;
-    location = "/var/lib/postgres/backups";
-    backupAll = true;
-  };
+  networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 5432 ];
+  networking.firewall.allowedUDPPorts = lib.mkIf cfg.enable [ 5432 ];
 }

hosts/ildkule/services/monitoring/dashboards/mysql.json

@@ -13,7 +13,7 @@
     ]
   },
   "description": "",
-  "editable": true,
+  "editable": false,
   "gnetId": 11323,
   "graphTooltip": 1,
   "id": 31,
@@ -1899,7 +1899,7 @@
       "dashes": false,
       "datasource": "$datasource",
       "decimals": 0,
-      "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool,  TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB 's internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.",
+      "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool,  TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB ‘s internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -3690,7 +3690,7 @@
         },
         "hide": 0,
         "includeAll": false,
-        "label": "Data Source",
+        "label": "Data source",
         "multi": false,
         "name": "datasource",
         "options": [],
@@ -3713,12 +3713,12 @@
         "definition": "label_values(mysql_up, job)",
         "hide": 0,
         "includeAll": true,
-        "label": "job",
+        "label": "Job",
         "multi": true,
         "name": "job",
         "options": [],
         "query": "label_values(mysql_up, job)",
-        "refresh": 1,
+        "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
         "sort": 0,
@@ -3742,12 +3742,12 @@
         "definition": "label_values(mysql_up, instance)",
         "hide": 0,
         "includeAll": true,
-        "label": "instance",
+        "label": "Instance",
         "multi": true,
         "name": "instance",
         "options": [],
         "query": "label_values(mysql_up, instance)",
-        "refresh": 1,
+        "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
         "sort": 0,

hosts/ildkule/services/monitoring/dashboards/postgres.json

@@ -328,7 +328,7 @@
         "rgba(50, 172, 45, 0.97)"
       ],
       "datasource": "${DS_PROMETHEUS}",
-      "format": "decbytes",
+      "format": "short",
       "gauge": {
         "maxValue": 100,
         "minValue": 0,
@@ -411,7 +411,7 @@
         "rgba(50, 172, 45, 0.97)"
       ],
       "datasource": "${DS_PROMETHEUS}",
-      "format": "decbytes",
+      "format": "short",
       "gauge": {
         "maxValue": 100,
         "minValue": 0,
@@ -1410,7 +1410,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "pg_settings_seq_page_cost",
+          "expr": "pg_settings_seq_page_cost{instance=\"$instance\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1872,7 +1872,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1966,7 +1966,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2060,7 +2060,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2251,7 +2251,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2439,7 +2439,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2589,35 +2589,35 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_backend",
           "refId": "A"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_alloc",
           "refId": "B"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "backend_fsync",
           "refId": "C"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_checkpoint",
           "refId": "D"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_clean",
@@ -2886,14 +2886,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
           "refId": "B"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",

hosts/ildkule/services/monitoring/grafana.nix

@@ -47,13 +47,13 @@ in {
         {
           name = "Node Exporter Full";
           type = "file";
-          url = "https://grafana.com/api/dashboards/1860/revisions/29/download";
+          url = "https://grafana.com/api/dashboards/1860/revisions/42/download";
           options.path = dashboards/node-exporter-full.json;
         }
         {
           name = "Matrix Synapse";
           type = "file";
-          url = "https://raw.githubusercontent.com/matrix-org/synapse/develop/contrib/grafana/synapse.json";
+          url = "https://github.com/element-hq/synapse/raw/refs/heads/develop/contrib/grafana/synapse.json";
           options.path = dashboards/synapse.json;
         }
         {
@@ -65,15 +65,9 @@ in {
         {
           name = "Postgresql";
           type = "file";
-          url = "https://grafana.com/api/dashboards/9628/revisions/7/download";
+          url = "https://grafana.com/api/dashboards/9628/revisions/8/download";
           options.path = dashboards/postgres.json;
         }
-        {
-          name = "Go Processes (gogs)";
-          type = "file";
-          url = "https://grafana.com/api/dashboards/240/revisions/3/download";
-          options.path = dashboards/go-processes.json;
-        }
         {
           name = "Gitea Dashboard";
           type = "file";

hosts/kommode/disks.nix

@@ -21,11 +21,11 @@
                 #     name = lib.replaceString "/" "-" subvolPath;
                 #   in {
                 #     "@${name}/active" = {
-                #       mountPoint = subvolPath;
+                #       mountpoint = subvolPath;
                 #       inherit mountOptions;
                 #     };
                 #     "@${name}/snapshots" = {
-                #       mountPoint = "${subvolPath}/.snapshots";
+                #       mountpoint = "${subvolPath}/.snapshots";
                 #       inherit mountOptions;
                 #     };
                 #   };

hosts/kommode/services/gitea/default.nix

@@ -195,6 +195,22 @@ in {
 
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
+  services.rsync-pull-targets = {
+    enable = true;
+    locations.${cfg.dump.backupDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGpMVrOppyqYaDiAhqmAuOaRsubFvcQGBGyz+NHB6+0o gitea rsync backup";
+    };
+  };
+
   systemd.services.gitea-dump = {
     serviceConfig.ExecStart = let
       args = lib.cli.toGNUCommandLineShell { } {

modules/rsync-pull-targets.nix

@@ -124,16 +124,22 @@ in
     services.openssh.enable = true;
     users.users = lib.pipe cfg.locations [
       (lib.filterAttrs (_: value: value.enable))
-      (lib.mapAttrs' (_: { user, location, rrsyncPackage, rrsyncArgs, authorizedKeysAttrs, publicKey, ... }: let
+
+      lib.attrValues
+
+      # Index locations by SSH user
+      (lib.foldl (acc: location: acc // {
+        ${location.user} = (acc.${location.user} or [ ]) ++ [ location ];
+      }) { })
+
+      (lib.mapAttrs (_name: locations: {
+        openssh.authorizedKeys.keys = map ({ user, location, rrsyncPackage, rrsyncArgs, authorizedKeysAttrs, publicKey, ... }: let
           rrsyncArgString = lib.cli.toCommandLineShellGNU {
             isLong = _: false;
           } rrsyncArgs;
           # TODO: handle " in location
-      in {
-        name = user;
-        value.openssh.authorizedKeys.keys = [
-          "command=\"${lib.getExe rrsyncPackage} ${rrsyncArgString} ${location}\",${lib.concatStringsSep "," authorizedKeysAttrs} ${publicKey}"
-        ];
+        in "command=\"${lib.getExe rrsyncPackage} ${rrsyncArgString} ${location}\",${lib.concatStringsSep "," authorizedKeysAttrs} ${publicKey}"
+        ) locations;
       }))
     ];
   };

packages/mediawiki-extensions/default.nix

@@ -33,63 +33,63 @@ in
 lib.mergeAttrsList [
   (mw-ext {
     name = "CodeEditor";
-    commit = "6e5b06e8cf2d040c0abb53ac3735f9f3c96a7a4f";
-    hash = "sha256-Jee+Ws9REUohywhbuemixXKaTRc54+cIlyUNDCyYcEM=";
+    commit = "83e1d0c13f34746f0d7049e38b00e9ab0a47c23f";
+    hash = "sha256-qH9fSQZGA+z6tBSh1DaTKLcujqA6K/vQmZML9w5X8mU=";
   })
   (mw-ext {
     name = "CodeMirror";
-    commit = "da9c5d4f03e6425f6f2cf68b75d21311e0f7e77e";
-    hash = "sha256-aL+v9xeqKHGmQVUWVczh54BkReu+fP49PT1NP7eTC6k=";
+    commit = "af2b08b9ad2b89a64b2626cf80b026c5b45e9922";
+    hash = "sha256-CxXPwCKUlF9Tg4JhwLaKQyvt43owq75jCugVtb3VX+I=";
   })
   (mw-ext {
     name = "DeleteBatch";
-    commit = "122072bbfb4eab96ed8c1451a3e74b5557054c58";
-    hash = "sha256-L6AXoyFJEZoAQpLO6knJvYtQ6JJPMtaa+WhpnwbJeNU=";
+    commit = "3d6f2fd0e3efdae1087dd0cc8b1f96fe0edf734f";
+    hash = "sha256-iD9EjDIW7AGpZan74SIRcr54dV8W7xMKIDjatjdVkKs=";
   })
   (mw-ext {
     name = "PluggableAuth";
-    commit = "5caf605b9dfdd482cb439d1ba2000cba37f8b018";
-    hash = "sha256-TYJqR9ZvaWJ7i1t0XfgUS05qqqCgxAH8tRTklz/Bmlg=";
+    commit = "85e96acd1ac0ebcdaa29c20eae721767a938f426";
+    hash = "sha256-bMVhrg8FsfWhXF605Cj5TgI0A6Jy/MIQ5aaUcLQQ0Ss=";
   })
   (mw-ext {
     name = "Popups";
-    commit = "7ed940a09f83f869cbc0bc20f3ca92f85b534951";
-    hash = "sha256-pcDPcu4kSvMHfSOuShrod694TKI9Oo3AEpMP9DXp9oY=";
+    commit = "410e2343c32a7b18dcdc2bbd995b0bfdf3bf5f37";
+    hash = "sha256-u2AlR75x54rCpiK9Mz00D9odJCn8fmi6DRU4QKmKqSc=";
   })
   (mw-ext {
     name = "Scribunto";
-    commit = "e755852a8e28a030a21ded2d5dd7270eb933b683";
-    hash = "sha256-zyI5nSE+KuodJOWyV0CQM7G0GfkKEgfoF/czi2/qk98=";
+    commit = "904f323f343dba5ff6a6cdd143c4a8ef5b7d2c55";
+    hash = "sha256-ZOVYhjMMyWbqwZOBb39hMIRmzzCPEnz2y8Q2jgyeERw=";
   })
   (mw-ext {
     name = "SimpleSAMLphp";
     kebab-name = "simple-saml-php";
-    commit = "d41b4efd3cc44ca3f9f12e35385fc64337873c2a";
-    hash = "sha256-wfzXtsEEEjQlW5QE4Rf8pasAW/KSJsLkrez13baxeqA=";
+    commit = "a2f77374713473d594e368de24539aebcc1a800a";
+    hash = "sha256-5+t3VQFKcrIffDNPJ4RWBIWS6K1gTOcEleYWmM6xWms=";
   })
   (mw-ext {
     name = "TemplateData";
-    commit = "fd7cf4d95a70ef564130266f2a6b18f33a2a2ff9";
-    hash = "sha256-5OhDPFhIi55Eh5+ovMP1QTjNBb9Sm/3vyArNCApAgSw=";
+    commit = "76a6a04bd13a606923847ba68750b5d98372cacd";
+    hash = "sha256-X2+U5PMqzkSljw2ypIvJUSaPDaonTkQx89OgKzf5scw=";
   })
   (mw-ext {
     name = "TemplateStyles";
-    commit = "0f7b94a0b094edee1c2a9063a3c42a1bdc0282d9";
-    hash = "sha256-R406FgNcIip9St1hurtZoPPykRQXBrkJRKA9hapG81I=";
+    commit = "7de60a8da6576d7930f293d19ef83529abf52704";
+    hash = "sha256-iPmFDoO5V4964CVyd1mBSQcNlW34odbvpm2CfDBlPBU=";
   })
   (mw-ext {
     name = "UserMerge";
-    commit = "d1917817dd287e7d883e879459d2d2d7bc6966f2";
-    hash = "sha256-la3/AQ38DMsrZ2f24T/z3yKzIrbyi3w6FIB5YfxGK9U=";
+    commit = "71eb53ff4289ac4efaa31685ab8b6483c165a584";
+    hash = "sha256-OfKSEPgctfr659oh5jf99T0Rzqn+60JhNaZq+2gfubk=";
   })
   (mw-ext {
     name = "VisualEditor";
-    commit = "032364cfdff33818e6ae0dfa251fe3973b0ae4f3";
-    hash = "sha256-AQDdq9r6rSo8h4u1ERonH14/1i1BgLGdzANEiQ065PU=";
+    commit = "a6a63f53605c4d596c3df1dcc2583ffd3eb8d929";
+    hash = "sha256-4d8picO66uzKoxh1TdyvKLHebc6ZL7N2DdXLV2vgBL4=";
   })
   (mw-ext {
     name = "WikiEditor";
-    commit = "cb9f7e06a9c59b6d3b31c653e5886b7f53583d01";
-    hash = "sha256-UWi3Ac+LCOLliLkXnS8YL0rD/HguuPH5MseqOm0z7s4=";
+    commit = "0a5719bb95326123dd0fee1f88658358321ed7be";
+    hash = "sha256-eQMyjhdm1E6TkktIHad1NMeMo8QNoO8z4A05FYOMCwQ=";
   })
 ]

shell.nix

@@ -1,15 +1,16 @@
 { pkgs ? import <nixpkgs> {} }:
 pkgs.mkShellNoCC {
   packages = with pkgs; [
-    just
-    jq
+    disko
+    editorconfig-checker
+    gnupg
     gum
+    jq
+    just
+    openstackclient
     sops
     ssh-to-age
-    gnupg
     statix
-    openstackclient
-    editorconfig-checker
   ];
 
   env = {