WIP: backup mysql

Reviewed-on: https://git.pvv.ntnu.no/Drift/pvv-nixos-config/pulls/64
Reviewed-by: Oystein Kristoffer Tveit <oysteikt@pvv.ntnu.no>

.gitignore

@@ -1,3 +1,4 @@
 result*
 /configuration.nix
 /.direnv/
+*.qcow2

base.nix

@@ -139,4 +139,12 @@
     acceptTerms = true;
     defaults.email = "drift@pvv.ntnu.no";
   };
+  # Let's not spam LetsEncrypt in `nixos-rebuild build-vm` mode:
+  virtualisation.vmVariant = {
+    security.acme.defaults.server = "https://127.0.0.1";
+    security.acme.preliminarySelfsigned = true;
+
+    users.users.root.initialPassword = "root";
+  };
+
 }

flake.lock

@@ -194,11 +194,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1693136143,
-        "narHash": "sha256-amHprjftc3y/bg8yf4hITCLa+ez5HIi0yGfR7TU6UIc=",
+        "lastModified": 1723850344,
+        "narHash": "sha256-aT37O9l9eclWEnqxASVNBL1dKwDHZUOqdbA4VO9DJvw=",
         "ref": "refs/heads/main",
-        "rev": "a32894b305f042d561500f5799226afd1faf5abb",
-        "revCount": 9,
+        "rev": "38b66677ab8c01aee10cd59e745af9ce3ea88092",
+        "revCount": 19,
         "type": "git",
         "url": "https://git.pvv.ntnu.no/Projects/calendar-bot.git"
       },

hosts/bicep/configuration.nix

@@ -12,8 +12,7 @@
     ./services/mysql.nix
     ./services/postgres.nix
     ./services/mysql.nix
-    # TODO: fix the calendar bot
-    # ./services/calendar-bot.nix
+    ./services/calendar-bot.nix
 
     ./services/matrix
   ];

hosts/bicep/services/calendar-bot.nix

@@ -2,11 +2,19 @@
 let
   cfg = config.services.pvv-calendar-bot;
 in {
-  sops.secrets."calendar-bot/matrix_token" = {
-    sopsFile = ../../../secrets/bicep/bicep.yaml;
-    key = "calendar-bot/matrix_token";
-    owner = cfg.user;
-    group = cfg.group;
+  sops.secrets = {
+    "calendar-bot/matrix_token" = {
+      sopsFile = ../../../secrets/bicep/bicep.yaml;
+      key = "calendar-bot/matrix_token";
+      owner = cfg.user;
+      group = cfg.group;
+    };
+    "calendar-bot/mysql_password" = {
+      sopsFile = ../../../secrets/bicep/bicep.yaml;
+      key = "calendar-bot/mysql_password";
+      owner = cfg.user;
+      group = cfg.group;
+    };
   };
 
   services.pvv-calendar-bot = {
@@ -18,6 +26,11 @@ in {
         user = "@bot_calendar:pvv.ntnu.no";
         channel = "!gkNLUIhYVpEyLatcRz:pvv.ntnu.no";
       };
+      database = {
+        host = "mysql.pvv.ntnu.no";
+        user = "calendar-bot";
+        passwordFile = config.sops.secrets."calendar-bot/mysql_password".path;
+      };
       secretsFile = config.sops.secrets."calendar-bot/matrix_token".path;
       onCalendar = "*-*-* 09:00:00";
     };

hosts/bicep/services/mysql.nix

@@ -1,4 +1,7 @@
 { pkgs, lib, config, values, ... }:
+let
+  backupDir = "/var/lib/mysql/backups";
+in
 {
   sops.secrets."mysql/password" = {
     owner = "mysql";
@@ -36,11 +39,6 @@
     }];
   };
 
-  services.mysqlBackup = {
-    enable = true;
-    location = "/var/lib/mysql/backups";
-  };
-
   networking.firewall.allowedTCPPorts = [ 3306 ];
 
   systemd.services.mysql.serviceConfig = {
@@ -50,4 +48,51 @@
       values.ipv6-space
     ];
   };
+
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves
+  systemd.services."backup-mysql" = {
+    description = "Backup MySQL data";
+    requires = [ "mysql.service" ];
+
+    path = [
+      pkgs.coreutils
+      pkgs.rsync
+      pkgs.gzip
+      config.services.mysql.package
+    ];
+
+    script = let
+      rotations = 10;
+      sshTarget1 = "root@isvegg.pvv.ntnu.no:/mnt/backup1/bicep/mysql";
+      sshTarget2 = "root@isvegg.pvv.ntnu.no:/mnt/backup2/bicep/mysql";
+    in ''
+      set -eo pipefail
+
+      mysqldump | gzip -c -9 --rsyncable > "${backupDir}/$(date --iso-8601)-dump.sql.gz"
+
+      while [ $(ls -1 "${backupDir}" | wc -l) -gt ${toString rotations} ]; do
+        rm $(find "${backupDir}" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)
+      done
+
+      rsync -avz --delete "${backupDir}" '${sshTarget1}'
+      rsync -avz --delete "${backupDir}" '${sshTarget2}'
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "mysql";
+      Group = "mysql";
+      UMask = "0077";
+      ReadWritePaths = [ backupDir ];
+    };
+
+    startAt = "*-*-* 02:15:00";
+  };
+
+  systemd.tmpfiles.settings."10-mysql-backup".${backupDir}.d = {
+    user = "mysql";
+    group = "mysql";
+    mode = "700";
+  };
 }

hosts/bicep/services/postgres.nix

@@ -1,6 +1,7 @@
-{ config, pkgs, ... }:
+{ config, pkgs, lib, ... }:
 let
   sslCert = config.security.acme.certs."postgres.pvv.ntnu.no";
+  backupDir = "/var/lib/postgresql/backups";
 in
 {
   services.postgresql = {
@@ -89,9 +90,50 @@ in
   networking.firewall.allowedTCPPorts = [ 5432 ];
   networking.firewall.allowedUDPPorts = [ 5432 ];
 
-  services.postgresqlBackup = {
-    enable = true;
-    location = "/var/lib/postgres/backups";
-    backupAll = true;
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves
+  systemd.services."backup-postgresql" = {
+    description = "Backup PostgreSQL data";
+    requires = [ "postgresql.service" ];
+
+    path = [
+      pkgs.coreutils
+      pkgs.rsync
+      pkgs.gzip
+      config.services.postgresql.package
+    ];
+
+    script = let
+      rotations = 10;
+      sshTarget1 = "root@isvegg.pvv.ntnu.no:/mnt/backup1/bicep/postgresql";
+      sshTarget2 = "root@isvegg.pvv.ntnu.no:/mnt/backup2/bicep/postgresql";
+    in ''
+      set -eo pipefail
+
+      pg_dumpall -U postgres | gzip -c -9 --rsyncable > "${backupDir}/$(date --iso-8601)-dump.sql.gz"
+
+      while [ $(ls -1 "${backupDir}" | wc -l) -gt ${toString rotations} ]; do
+        rm $(find "${backupDir}" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)
+      done
+
+      rsync -avz --delete "${backupDir}" '${sshTarget1}'
+      rsync -avz --delete "${backupDir}" '${sshTarget2}'
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "postgres";
+      Group = "postgres";
+      UMask = "0077";
+      ReadWritePaths = [ backupDir ];
+    };
+
+    startAt = "*-*-* 01:15:00";
+  };
+
+  systemd.tmpfiles.settings."10-postgresql-backup".${backupDir}.d = {
+    user = "postgres";
+    group = "postgres";
+    mode = "700";
   };
 }

justfile

@@ -10,6 +10,10 @@ check:
 build-machine machine=`just _a_machine`:
   {{nom}} build .#nixosConfigurations.{{ machine }}.config.system.build.toplevel
 
+run-vm machine=`just _a_machine`:
+  nixos-rebuild build-vm --flake .#{{ machine }}
+  QEMU_NET_OPTS="hostfwd=tcp::8080-:80,hostfwd=tcp::8081-:443,hostfwd=tcp::2222-:22" ./result/bin/run-*-vm
+
 @update-inputs:
   nix eval .#inputs --apply builtins.attrNames --json \
     | jq '.[]' -r \

secrets/bicep/bicep.yaml

@@ -1,5 +1,6 @@
 calendar-bot:
     matrix_token: ENC[AES256_GCM,data:zJv9sw6pEzb9hxKT682wsD87HC9iejbps2wl2Z5QW1XZUSBHdcqyg1pxd+jFKTeKGQ==,iv:zDbvF1H98NsECjCtGXS+Y9HIhXowzz9HF9mltqnArog=,tag:/ftcOSQ13ElkVJBxYIMUGQ==,type:str]
+    mysql_password: ENC[AES256_GCM,data:Gqag8yOgPH3ntoT5TmaqJWv1j+si2qIyz5Ryfw5E2A==,iv:kQDcxnPfwJQcFovI4f87UDt18F8ah3z5xeY86KmdCyY=,tag:A1sCSNXJziAmtUWohqwJgg==,type:str]
 mysql:
     password: ENC[AES256_GCM,data:KqEe0TVdeMIzPKsmFg9x0X9xWijnOk306ycyXTm2Tpqo/O0F,iv:Y+hlQ8n1ZIP9ncXBzd2kCSs/DWVTWhiEluFVwZFKRCA=,tag:xlaUk0Wftk62LpYE5pKNQw==,type:str]
 sops:
@@ -62,8 +63,8 @@ sops:
             cTh5bnJ3WW90aXRCSUp6NHFYeU1tZ0kK4afdtJwGNu6wLRI0fuu+mBVeqVeB0rgX
             0q5hwyzjiRnHnyjF38CmcGgydSfDRmF6P+WIMbCwXC6LwfRhAmBGPg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-09-05T23:28:56Z"
-    mac: ENC[AES256_GCM,data:pCWTkmCQgBOqhejK2sCLQ3H8bRXmXlToQxYmOG0IWDo2eGiZOLuIkZ1/1grYgfxAGiD4ysJod0nJuvo+eAsMeYAy6QJVtrOqO2d9V2NEdzLckXyYvwyJyZoFbNC5EW9471V0m4jLRSh5821ckNo/wtWFR11wfO15tI3MqtD1rtA=,iv:QDnckPl0LegaH0b7V4WAtmVXaL4LN+k3uKHQI2dkW7E=,tag:mScUQBR0ZHl1pi/YztrvFg==,type:str]
+    lastmodified: "2024-08-15T21:18:33Z"
+    mac: ENC[AES256_GCM,data:uR5HgeDAYqoqB9kk1V6p0T30+v6WpQJi4+qIeCDRnoUPnQKUVR10hvBhICck+E+Uh8p+tGhM6Uf3YrAJAV0ZCUiNJjtwDJQQLUDT53vdOAXN4xADCQqNuhgVwVMaruoTheEiwOswRuhFeEwy0gBj3Ze2pu47lueHYclmEzumLeQ=,iv:t0UyXN2YaR2m7M/pV2wTLJG5wVfqTIUs7wSQMmyeTVw=,tag:O7dIffzrDAXz3kGx5uazhw==,type:str]
     pgp:
         - created_at: "2024-08-04T00:03:40Z"
           enc: |-
@@ -86,4 +87,4 @@ sops:
             -----END PGP MESSAGE-----
           fp: F7D37890228A907440E1FD4846B9228E814A2AAC
     unencrypted_suffix: _unencrypted
-    version: 3.7.3
+    version: 3.9.0