bicep/mysql: split backups into one file per db

hosts/bekkalokk/services/well-known/default.nix

@@ -1,70 +1,25 @@
 { lib, ... }:
 {
-  # TODO: move this to base so that all virtualHosts take effect on their respecitve hosts
+  services.nginx.virtualHosts = lib.genAttrs [
-
+    "pvv.ntnu.no"
-  # NOTE: automatically hosting well-known files by looping over all existing `virtualHosts`
+    "www.pvv.ntnu.no"
-  #       unfortunately causes infinite recursuion due to submodule usage within the nginx
+    "pvv.org"
-  #       module. For now, the easiest solution was to manually specify a list of virtualHosts
+    "www.pvv.org"
-  #       here, but it would be nice to find a better solution in the future.
+  ] (_: {
-  services.nginx.virtualHosts = lib.mkMerge [
+    locations = {
-    (lib.genAttrs [
+      "^~ /.well-known/" = {
-      "pvv.ntnu.no"
+        alias = (toString ./root) + "/";
-      "pvv.org"
-
-      "www.pvv.ntnu.no"
-      "www.pvv.org"
-      "www2.pvv.ntnu.no"
-      "www2.pvv.org"
-
-      # NOTE: this list is probably not complete
-      "alps.pvv.ntnu.no"
-      "chat.pvv.ntnu.no"
-      "grafana.pvv.ntnu.no"
-      "status.pvv.ntnu.no"
-      "matrix.pvv.ntnu.no"
-      "mirrors.pvv.ntnu.no"
-      "pages.pvv.ntnu.no"
-      "ooye.pvv.ntnu.no"
-      "ooye.pvv.ntnu.no"
-      "dav.pvv.ntnu.no"
-      "git.pvv.ntnu.no"
-      "idp.pvv.ntnu.no"
-      "minecraft.pvv.ntnu.no"
-      "pw.pvv.ntnu.no"
-      "snappymail.pvv.ntnu.no"
-      "webmail.pvv.ntnu.no"
-      "wiki.pvv.ntnu.no"
-    ] (_: {
-      locations."^~ /.well-known/security.txt" = {
-        alias = toString ./root/security.txt;
       };
-    }))
 
-    (lib.genAttrs [
+      # Proxy the matrix well-known files
-      "pvv.ntnu.no"
+      # Host has be set before proxy_pass
-      "pvv.org"
+      # The header must be set so nginx on the other side routes it to the right place
-      "mail.pvv.ntnu.no"
+      "^~ /.well-known/matrix/" = {
-      "mail.pvv.org"
-      "smtp.pvv.ntnu.no"
-      "smtp.pvv.org"
-    ] (_: {
-      locations."^~ /.well-known/autoconfig/mail/" = {
-        root = toString ./root/autoconfig/mail;
-      };
-    }))
-
-    (lib.genAttrs [
-      "pvv.ntnu.no"
-      "pvv.org"
-      "www.pvv.ntnu.no"
-      "www.pvv.org"
-    ] (_: {
-      locations."^~ /.well-known/matrix/" = {
         extraConfig = ''
           proxy_set_header Host matrix.pvv.ntnu.no;
           proxy_pass https://matrix.pvv.ntnu.no/.well-known/matrix/;
         '';
       };
-    }))
+    };
-  ];
+  });
 }

hosts/bicep/services/mysql/backup.nix

@@ -41,24 +41,38 @@ in
     path = with pkgs; [
       cfg.package
       coreutils
+      diffutils
       zstd
     ];
 
-    script = let
+    script = ''
-      rotations = 2;
-    in ''
       set -euo pipefail
 
-      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
+      dump() {
+        local name="$1" out tmp
+        out="$STATE_DIRECTORY/$name.sql.zst"
+        tmp="$out.tmp"
+        shift
+        "$@" | zstd -9 --rsyncable -f -o "$tmp"
+        if cmp -s "$tmp" "$out" 2>/dev/null; then
+          rm -f "$tmp"
+        else
+          mv -f "$tmp" "$out"
+        fi
+      }
 
-      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+      declare -A keep
+      while IFS= read -r db; do
+        [ -n "$db" ] || continue
+        dump "$db" mysqldump --skip-dump-date --databases "$db"
+        keep["$db.sql.zst"]=1
+      done < <(mysql -N -e 'SHOW DATABASES' | grep -vE '^(information_schema|performance_schema)$')
 
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      # drop dumps of databases that no longer exist
-      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
+      for f in "$STATE_DIRECTORY"/*.sql.zst; do
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
+        [ -e "$f" ] || continue
-
+        base="$(basename "$f")"
-      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
-        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
       done
     '';
 

hosts/bicep/services/postgresql/backup.nix

@@ -41,25 +41,42 @@ in
 
     path = with pkgs; [
       coreutils
+      diffutils
       zstd
       cfg.package
     ];
 
-    script = let
+    script = ''
-      rotations = 2;
-    in ''
       set -euo pipefail
 
-      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
+      dump() {
+        local name="$1" out tmp
+        out="$STATE_DIRECTORY/$name.sql.zst"
+        tmp="$out.tmp"
+        shift
+        "$@" | zstd -9 --rsyncable -f -o "$tmp"
+        if cmp -s "$tmp" "$out" 2>/dev/null; then
+          rm -f "$tmp"
+        else
+          mv -f "$tmp" "$out"
+        fi
+      }
 
-      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+      declare -A keep
+      dump globals pg_dumpall -U postgres --globals-only --restrict-key=backup
+      keep[globals.sql.zst]=1
 
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      while IFS= read -r db; do
-      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
+        [ -n "$db" ] || continue
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
+        dump "$db" pg_dump -U postgres -C -d "$db" --restrict-key=backup
+        keep["$db.sql.zst"]=1
+      done < <(psql -U postgres -tAc "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname")
 
-      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+      # drop dumps of databases that no longer exist
-        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      for f in "$STATE_DIRECTORY"/*.sql.zst; do
+        [ -e "$f" ] || continue
+        base="$(basename "$f")"
+        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
       done
     '';