ildkule/grafana: update a bunch of dashboards

hosts/bekkalokk/services/vaultwarden.nix

@@ -99,4 +99,20 @@ in {
       ];
     };
   };
+
+  services.rsync-pull-targets = {
+    enable = true;
+    locations."/var/lib/vaultwarden" = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB2cDaW52gBtLVaNqoGijvN2ZAVkAWlII5AXUzT3Dswj vaultwarden rsync backup";
+    };
+  };
 }

hosts/bekkalokk/services/webmail/snappymail.nix

@@ -14,5 +14,20 @@ in {
     enableACME = true;
     kTLS = true;
   };
-}
 
+  services.rsync-pull-targets = {
+    enable = true;
+    locations.${cfg.dataDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJENMnuNsHEeA91oX+cj7Qpex2defSXP/lxznxCAqV03 snappymail rsync backup";
+    };
+  };
+}

hosts/bicep/configuration.nix

@@ -9,8 +9,8 @@
     ./services/calendar-bot.nix
     #./services/git-mirrors
     ./services/minecraft-heatmap.nix
-    ./services/mysql.nix
+    ./services/mysql
-    ./services/postgres.nix
+    ./services/postgresql
 
     ./services/matrix
   ];

hosts/bicep/services/mysql/backup.nix

@@ -0,0 +1,82 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.mysql;
+  backupDir = "/data/mysql-backups";
+in
+{
+  # services.mysqlBackup = lib.mkIf cfg.enable {
+  #   enable = true;
+  #   location = "/var/lib/mysql-backups";
+  # };
+
+  systemd.tmpfiles.settings."10-mysql-backups".${backupDir}.d = {
+	  user = "mysql";
+	  group = "mysql";
+	  mode = "700";
+	};
+
+  services.rsync-pull-targets = lib.mkIf cfg.enable {
+    enable = true;
+    locations.${backupDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgj55/7Cnj4cYMJ5sIkl+OwcGeBe039kXJTOf2wvo9j mysql rsync backup";
+    };
+  };
+
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves.
+  systemd.services."backup-mysql" = lib.mkIf cfg.enable {
+    description = "Backup MySQL data";
+    requires = [ "mysql.service" ];
+
+    path = with pkgs; [
+      cfg.package
+      coreutils
+      zstd
+    ];
+
+    script = let
+      rotations = 2;
+    in ''
+      set -euo pipefail
+
+      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
+
+      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
+
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt ${toString (rotations + 1)} ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      done
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "mysql";
+      Group = "mysql";
+      UMask = "0077";
+
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      StateDirectory = [ "mysql-backups" ];
+      BindPaths = [ "${backupDir}:/var/lib/mysql-backups" ];
+
+      # TODO: hardening
+    };
+
+    startAt = "*-*-* 02:15:00";
+  };
+}

hosts/bicep/services/mysql/default.nix

@@ -4,6 +4,8 @@ let
   dataDir = "/data/mysql";
 in
 {
+  imports = [ ./backup.nix ];
+
   sops.secrets."mysql/password" = {
     owner = "mysql";
     group = "mysql";
@@ -42,27 +44,6 @@ in
     }];
   };
 
-  services.mysqlBackup = lib.mkIf cfg.enable {
-    enable = true;
-    location = "/var/lib/mysql-backups";
-  };
-
-  services.rsync-pull-targets = lib.mkIf cfg.enable {
-    enable = true;
-    locations.${config.services.mysqlBackup.location} = {
-      user = "root";
-      rrsyncArgs.ro = true;
-      authorizedKeysAttrs = [
-        "restrict"
-        "no-agent-forwarding"
-        "no-port-forwarding"
-        "no-pty"
-        "no-X11-forwarding"
-      ];
-      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgj55/7Cnj4cYMJ5sIkl+OwcGeBe039kXJTOf2wvo9j mysql rsync backup";
-    };
-  };
-
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 3306 ];
 
   systemd.tmpfiles.settings."10-mysql".${dataDir}.d = lib.mkIf cfg.enable {

hosts/bicep/services/postgresql/backup.nix

@@ -0,0 +1,83 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.postgresql;
+  backupDir = "/data/postgresql-backups";
+in
+{
+  # services.postgresqlBackup = lib.mkIf cfg.enable {
+  #   enable = true;
+  #   location = "/var/lib/postgresql-backups";
+  #   backupAll = true;
+  # };
+
+  systemd.tmpfiles.settings."10-postgresql-backups".${backupDir}.d = {
+	  user = "postgres";
+	  group = "postgres";
+	  mode = "700";
+	};
+
+  services.rsync-pull-targets = lib.mkIf cfg.enable {
+    enable = true;
+    locations.${backupDir} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvO7QX7QmwSiGLXEsaxPIOpAqnJP3M+qqQRe5dzf8gJ postgresql rsync backup";
+    };
+  };
+
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves
+  systemd.services."backup-postgresql" = {
+    description = "Backup PostgreSQL data";
+    requires = [ "postgresql.service" ];
+
+    path = with pkgs; [
+      coreutils
+      zstd
+      cfg.package
+    ];
+
+    script = let
+      rotations = 2;
+    in ''
+      set -euo pipefail
+
+      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
+
+      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+
+      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
+      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
+
+      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt ${toString (rotations + 1)} ]; do
+        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      done
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "postgres";
+      Group = "postgres";
+      UMask = "0077";
+
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      StateDirectory = [ "postgresql-backups" ];
+      BindPaths = [ "${backupDir}:/var/lib/postgresql-backups" ];
+
+      # TODO: hardening
+    };
+
+    startAt = "*-*-* 01:15:00";
+  };
+}

hosts/bicep/services/postgresql/default.nix

@@ -3,6 +3,8 @@ let
   cfg = config.services.postgresql;
 in
 {
+  imports = [ ./backup.nix ];
+
   services.postgresql = {
     enable = true;
     package = pkgs.postgresql_18;
@@ -121,26 +123,4 @@ in
 
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 5432 ];
   networking.firewall.allowedUDPPorts = lib.mkIf cfg.enable [ 5432 ];
-
-  services.postgresqlBackup = lib.mkIf cfg.enable {
-    enable = true;
-    location = "/var/lib/postgres-backups";
-    backupAll = true;
-  };
-
-  services.rsync-pull-targets = lib.mkIf cfg.enable {
-    enable = true;
-    locations.${config.services.postgresqlBackup.location} = {
-      user = "root";
-      rrsyncArgs.ro = true;
-      authorizedKeysAttrs = [
-        "restrict"
-        "no-agent-forwarding"
-        "no-port-forwarding"
-        "no-pty"
-        "no-X11-forwarding"
-      ];
-      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvO7QX7QmwSiGLXEsaxPIOpAqnJP3M+qqQRe5dzf8gJ postgresql rsync backup";
-    };
-  };
 }

hosts/ildkule/services/monitoring/dashboards/mysql.json

@@ -13,7 +13,7 @@
     ]
   },
   "description": "",
-  "editable": true,
+  "editable": false,
   "gnetId": 11323,
   "graphTooltip": 1,
   "id": 31,
@@ -1899,7 +1899,7 @@
       "dashes": false,
       "datasource": "$datasource",
       "decimals": 0,
-      "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool,  TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB 's internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.",
+      "description": "***System Memory***: Total Memory for the system.\\\n***InnoDB Buffer Pool Data***: InnoDB maintains a storage area called the buffer pool for caching data and indexes in memory.\\\n***TokuDB Cache Size***: Similar in function to the InnoDB Buffer Pool,  TokuDB will allocate 50% of the installed RAM for its own cache.\\\n***Key Buffer Size***: Index blocks for MYISAM tables are buffered and are shared by all threads. key_buffer_size is the size of the buffer used for index blocks.\\\n***Adaptive Hash Index Size***: When InnoDB notices that some index values are being accessed very frequently, it builds a hash index for them in memory on top of B-Tree indexes.\\\n ***Query Cache Size***: The query cache stores the text of a SELECT statement together with the corresponding result that was sent to the client. The query cache has huge scalability problems in that only one thread can do an operation in the query cache at the same time.\\\n***InnoDB Dictionary Size***: The data dictionary is InnoDB ‘s internal catalog of tables. InnoDB stores the data dictionary on disk, and loads entries into memory while the server is running.\\\n***InnoDB Log Buffer Size***: The MySQL InnoDB log buffer allows transactions to run without having to write the log to disk before the transactions commit.",
       "editable": true,
       "error": false,
       "fieldConfig": {
@@ -3690,7 +3690,7 @@
         },
         "hide": 0,
         "includeAll": false,
-        "label": "Data Source",
+        "label": "Data source",
         "multi": false,
         "name": "datasource",
         "options": [],
@@ -3713,12 +3713,12 @@
         "definition": "label_values(mysql_up, job)",
         "hide": 0,
         "includeAll": true,
-        "label": "job",
+        "label": "Job",
         "multi": true,
         "name": "job",
         "options": [],
         "query": "label_values(mysql_up, job)",
-        "refresh": 1,
+        "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
         "sort": 0,
@@ -3742,12 +3742,12 @@
         "definition": "label_values(mysql_up, instance)",
         "hide": 0,
         "includeAll": true,
-        "label": "instance",
+        "label": "Instance",
         "multi": true,
         "name": "instance",
         "options": [],
         "query": "label_values(mysql_up, instance)",
-        "refresh": 1,
+        "refresh": 2,
         "regex": "",
         "skipUrlSync": false,
         "sort": 0,

hosts/ildkule/services/monitoring/dashboards/postgres.json

@@ -328,7 +328,7 @@
         "rgba(50, 172, 45, 0.97)"
       ],
       "datasource": "${DS_PROMETHEUS}",
-      "format": "decbytes",
+      "format": "short",
       "gauge": {
         "maxValue": 100,
         "minValue": 0,
@@ -411,7 +411,7 @@
         "rgba(50, 172, 45, 0.97)"
       ],
       "datasource": "${DS_PROMETHEUS}",
-      "format": "decbytes",
+      "format": "short",
       "gauge": {
         "maxValue": 100,
         "minValue": 0,
@@ -1410,7 +1410,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "pg_settings_seq_page_cost",
+          "expr": "pg_settings_seq_page_cost{instance=\"$instance\"}",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -1872,7 +1872,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -1966,7 +1966,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2060,7 +2060,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2251,7 +2251,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2439,7 +2439,7 @@
       },
       "yaxes": [
         {
-          "format": "bytes",
+          "format": "short",
           "label": null,
           "logBase": 1,
           "max": null,
@@ -2589,35 +2589,35 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_backend{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_backend",
           "refId": "A"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_alloc_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_alloc",
           "refId": "B"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "backend_fsync",
           "refId": "C"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_checkpoint_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_checkpoint",
           "refId": "D"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_buffers_clean{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_buffers_clean_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "buffers_clean",
@@ -2886,14 +2886,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_write_time_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "write_time - Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk.",
           "refId": "B"
         },
         {
-          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time{instance=\"$instance\"}[5m])",
+          "expr": "irate(pg_stat_bgwriter_checkpoint_sync_time_total{instance=\"$instance\"}[5m])",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "sync_time - Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk.",

hosts/ildkule/services/monitoring/grafana.nix

@@ -47,13 +47,13 @@ in {
         {
           name = "Node Exporter Full";
           type = "file";
-          url = "https://grafana.com/api/dashboards/1860/revisions/29/download";
+          url = "https://grafana.com/api/dashboards/1860/revisions/42/download";
           options.path = dashboards/node-exporter-full.json;
         }
         {
           name = "Matrix Synapse";
           type = "file";
-          url = "https://raw.githubusercontent.com/matrix-org/synapse/develop/contrib/grafana/synapse.json";
+          url = "https://github.com/element-hq/synapse/raw/refs/heads/develop/contrib/grafana/synapse.json";
           options.path = dashboards/synapse.json;
         }
         {
@@ -65,15 +65,9 @@ in {
         {
           name = "Postgresql";
           type = "file";
-          url = "https://grafana.com/api/dashboards/9628/revisions/7/download";
+          url = "https://grafana.com/api/dashboards/9628/revisions/8/download";
           options.path = dashboards/postgres.json;
         }
-        {
-          name = "Go Processes (gogs)";
-          type = "file";
-          url = "https://grafana.com/api/dashboards/240/revisions/3/download";
-          options.path = dashboards/go-processes.json;
-        }
         {
           name = "Gitea Dashboard";
           type = "file";

packages/mediawiki-extensions/default.nix

@@ -33,63 +33,63 @@ in
 lib.mergeAttrsList [
   (mw-ext {
     name = "CodeEditor";
-    commit = "6e5b06e8cf2d040c0abb53ac3735f9f3c96a7a4f";
+    commit = "83e1d0c13f34746f0d7049e38b00e9ab0a47c23f";
-    hash = "sha256-Jee+Ws9REUohywhbuemixXKaTRc54+cIlyUNDCyYcEM=";
+    hash = "sha256-qH9fSQZGA+z6tBSh1DaTKLcujqA6K/vQmZML9w5X8mU=";
   })
   (mw-ext {
     name = "CodeMirror";
-    commit = "da9c5d4f03e6425f6f2cf68b75d21311e0f7e77e";
+    commit = "af2b08b9ad2b89a64b2626cf80b026c5b45e9922";
-    hash = "sha256-aL+v9xeqKHGmQVUWVczh54BkReu+fP49PT1NP7eTC6k=";
+    hash = "sha256-CxXPwCKUlF9Tg4JhwLaKQyvt43owq75jCugVtb3VX+I=";
   })
   (mw-ext {
     name = "DeleteBatch";
-    commit = "122072bbfb4eab96ed8c1451a3e74b5557054c58";
+    commit = "3d6f2fd0e3efdae1087dd0cc8b1f96fe0edf734f";
-    hash = "sha256-L6AXoyFJEZoAQpLO6knJvYtQ6JJPMtaa+WhpnwbJeNU=";
+    hash = "sha256-iD9EjDIW7AGpZan74SIRcr54dV8W7xMKIDjatjdVkKs=";
   })
   (mw-ext {
     name = "PluggableAuth";
-    commit = "5caf605b9dfdd482cb439d1ba2000cba37f8b018";
+    commit = "85e96acd1ac0ebcdaa29c20eae721767a938f426";
-    hash = "sha256-TYJqR9ZvaWJ7i1t0XfgUS05qqqCgxAH8tRTklz/Bmlg=";
+    hash = "sha256-bMVhrg8FsfWhXF605Cj5TgI0A6Jy/MIQ5aaUcLQQ0Ss=";
   })
   (mw-ext {
     name = "Popups";
-    commit = "7ed940a09f83f869cbc0bc20f3ca92f85b534951";
+    commit = "410e2343c32a7b18dcdc2bbd995b0bfdf3bf5f37";
-    hash = "sha256-pcDPcu4kSvMHfSOuShrod694TKI9Oo3AEpMP9DXp9oY=";
+    hash = "sha256-u2AlR75x54rCpiK9Mz00D9odJCn8fmi6DRU4QKmKqSc=";
   })
   (mw-ext {
     name = "Scribunto";
-    commit = "e755852a8e28a030a21ded2d5dd7270eb933b683";
+    commit = "904f323f343dba5ff6a6cdd143c4a8ef5b7d2c55";
-    hash = "sha256-zyI5nSE+KuodJOWyV0CQM7G0GfkKEgfoF/czi2/qk98=";
+    hash = "sha256-ZOVYhjMMyWbqwZOBb39hMIRmzzCPEnz2y8Q2jgyeERw=";
   })
   (mw-ext {
     name = "SimpleSAMLphp";
     kebab-name = "simple-saml-php";
-    commit = "d41b4efd3cc44ca3f9f12e35385fc64337873c2a";
+    commit = "a2f77374713473d594e368de24539aebcc1a800a";
-    hash = "sha256-wfzXtsEEEjQlW5QE4Rf8pasAW/KSJsLkrez13baxeqA=";
+    hash = "sha256-5+t3VQFKcrIffDNPJ4RWBIWS6K1gTOcEleYWmM6xWms=";
   })
   (mw-ext {
     name = "TemplateData";
-    commit = "fd7cf4d95a70ef564130266f2a6b18f33a2a2ff9";
+    commit = "76a6a04bd13a606923847ba68750b5d98372cacd";
-    hash = "sha256-5OhDPFhIi55Eh5+ovMP1QTjNBb9Sm/3vyArNCApAgSw=";
+    hash = "sha256-X2+U5PMqzkSljw2ypIvJUSaPDaonTkQx89OgKzf5scw=";
   })
   (mw-ext {
     name = "TemplateStyles";
-    commit = "0f7b94a0b094edee1c2a9063a3c42a1bdc0282d9";
+    commit = "7de60a8da6576d7930f293d19ef83529abf52704";
-    hash = "sha256-R406FgNcIip9St1hurtZoPPykRQXBrkJRKA9hapG81I=";
+    hash = "sha256-iPmFDoO5V4964CVyd1mBSQcNlW34odbvpm2CfDBlPBU=";
   })
   (mw-ext {
     name = "UserMerge";
-    commit = "d1917817dd287e7d883e879459d2d2d7bc6966f2";
+    commit = "71eb53ff4289ac4efaa31685ab8b6483c165a584";
-    hash = "sha256-la3/AQ38DMsrZ2f24T/z3yKzIrbyi3w6FIB5YfxGK9U=";
+    hash = "sha256-OfKSEPgctfr659oh5jf99T0Rzqn+60JhNaZq+2gfubk=";
   })
   (mw-ext {
     name = "VisualEditor";
-    commit = "032364cfdff33818e6ae0dfa251fe3973b0ae4f3";
+    commit = "a6a63f53605c4d596c3df1dcc2583ffd3eb8d929";
-    hash = "sha256-AQDdq9r6rSo8h4u1ERonH14/1i1BgLGdzANEiQ065PU=";
+    hash = "sha256-4d8picO66uzKoxh1TdyvKLHebc6ZL7N2DdXLV2vgBL4=";
   })
   (mw-ext {
     name = "WikiEditor";
-    commit = "cb9f7e06a9c59b6d3b31c653e5886b7f53583d01";
+    commit = "0a5719bb95326123dd0fee1f88658358321ed7be";
-    hash = "sha256-UWi3Ac+LCOLliLkXnS8YL0rD/HguuPH5MseqOm0z7s4=";
+    hash = "sha256-eQMyjhdm1E6TkktIHad1NMeMo8QNoO8z4A05FYOMCwQ=";
   })
 ]