bicep/mysql: split backups into one file per db

hosts/bicep/services/mysql/backup.nix

@@ -41,24 +41,38 @@ in
     path = with pkgs; [
       cfg.package
       coreutils
+      diffutils
       zstd
     ];
 
-    script = let
+    script = ''
-      rotations = 2;
-    in ''
       set -euo pipefail
 
-      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
+      dump() {
+        local name="$1" out tmp
+        out="$STATE_DIRECTORY/$name.sql.zst"
+        tmp="$out.tmp"
+        shift
+        "$@" | zstd -9 --rsyncable -f -o "$tmp"
+        if cmp -s "$tmp" "$out" 2>/dev/null; then
+          rm -f "$tmp"
+        else
+          mv -f "$tmp" "$out"
+        fi
+      }
 
-      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+      declare -A keep
+      while IFS= read -r db; do
+        [ -n "$db" ] || continue
+        dump "$db" mysqldump --skip-dump-date --databases "$db"
+        keep["$db.sql.zst"]=1
+      done < <(mysql -N -e 'SHOW DATABASES' | grep -vE '^(information_schema|performance_schema)$')
 
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      # drop dumps of databases that no longer exist
-      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
+      for f in "$STATE_DIRECTORY"/*.sql.zst; do
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
+        [ -e "$f" ] || continue
-
+        base="$(basename "$f")"
-      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
-        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
       done
     '';
 

hosts/bicep/services/postgresql/backup.nix

@@ -41,25 +41,42 @@ in
 
     path = with pkgs; [
       coreutils
+      diffutils
       zstd
       cfg.package
     ];
 
-    script = let
+    script = ''
-      rotations = 2;
-    in ''
       set -euo pipefail
 
-      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
+      dump() {
+        local name="$1" out tmp
+        out="$STATE_DIRECTORY/$name.sql.zst"
+        tmp="$out.tmp"
+        shift
+        "$@" | zstd -9 --rsyncable -f -o "$tmp"
+        if cmp -s "$tmp" "$out" 2>/dev/null; then
+          rm -f "$tmp"
+        else
+          mv -f "$tmp" "$out"
+        fi
+      }
 
-      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+      declare -A keep
+      dump globals pg_dumpall -U postgres --globals-only --restrict-key=backup
+      keep[globals.sql.zst]=1
 
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      while IFS= read -r db; do
-      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
+        [ -n "$db" ] || continue
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
+        dump "$db" pg_dump -U postgres -C -d "$db" --restrict-key=backup
+        keep["$db.sql.zst"]=1
+      done < <(psql -U postgres -tAc "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname")
 
-      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+      # drop dumps of databases that no longer exist
-        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      for f in "$STATE_DIRECTORY"/*.sql.zst; do
+        [ -e "$f" ] || continue
+        base="$(basename "$f")"
+        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
       done
     '';
 

modules/drumknotty/default.nix

@@ -138,15 +138,6 @@ in
 
         ExecStart =
           let
-            mkRespawningCommand = name: command: pkgs.writeShellScript "drumknotty-${name}-watchdog" ''
-              while true; do
-                ${command}
-                echo
-                echo "${name} exited, restarting in 3s..." >&2
-                sleep 3
-              done
-            '';
-
             screenrc = let
               convertToFile = lines: lib.pipe lines [
                 lib.concatLists
@@ -166,7 +157,8 @@ in
                   config = "/etc/dibbler/dibbler.toml";
                 };
               in lib.optionals cfg.dibbler.enable [
-                "screen -t dibbler ${mkRespawningCommand "dibbler" "${lib.getExe cfg.dibbler.package} ${dibblerArgs} loop"}"
+                "screen -t dibbler ${lib.getExe cfg.dibbler.package} ${dibblerArgs} loop"
+
               ])
 
               (let
@@ -174,7 +166,7 @@ in
                   config = "/etc/worblehat/config.toml";
                 };
               in lib.optionals cfg.worblehat.enable [
-                "screen -t worblehat ${mkRespawningCommand "worblehat" "${lib.getExe cfg.worblehat.package} ${worblehatArgs} cli"}"
+                "screen -t worblehat ${lib.getExe cfg.worblehat.package} ${worblehatArgs} cli"
               ])
 
               [ "select 0" ]