bicep/mysql: split backups into one file per db

hosts/bekkalokk/services/idp-simplesamlphp/default.nix

@@ -177,11 +177,6 @@ in
       };
     };
 
-    systemd.services."phpfpm-idp" = {
-      after = [ "sops-install-secrets.service" ];
-      requires = [ "sops-install-secrets.service" ];
-    };
-
     services.nginx.virtualHosts."idp.pvv.ntnu.no" = {
       forceSSL = true;
       enableACME = true;

hosts/bekkalokk/services/mediawiki/default.nix

@@ -287,7 +287,6 @@ in {
 
   systemd.services.mediawiki-init = lib.mkIf cfg.enable {
     after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
     serviceConfig = {
       UMask = lib.mkForce "0007";
     };
@@ -295,7 +294,6 @@ in {
 
   systemd.services.phpfpm-mediawiki = lib.mkIf cfg.enable {
     after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
     serviceConfig = {
       UMask = lib.mkForce "0007";
     };

hosts/bekkalokk/services/vaultwarden.nix

@@ -61,11 +61,6 @@ in {
     };
   };
 
-  systemd.services.vaultwarden = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   services.nginx.virtualHosts."${domain}" = {
     forceSSL = true;
     enableACME = true;

hosts/bekkalokk/services/webmail/roundcube.nix

@@ -49,11 +49,6 @@ in
     '';
   };
 
-  systemd.services."phpfpm-roundcube" = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   # TODO: move this back to `webmail.pvv.ntnu.no/roundcube` subpath
 
   services.nginx.virtualHosts.${domain} = {

hosts/bekkalokk/services/website/default.nix

@@ -116,11 +116,6 @@ in {
     ];
   };
 
-  systemd.services."phpfpm-pvv-nettsiden" = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   services.nginx.virtualHosts."pvv.ntnu.no" = {
     globalRedirect = cfg.domainName;
     redirectCode = 307;

hosts/bicep/services/calendar-bot.nix

@@ -35,9 +35,4 @@ in {
       onCalendar = "*-*-* 09:00:00";
     };
   };
-
-  systemd.services.pvv-calendar-bot = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
 }

hosts/bicep/services/git-mirrors/default.nix

@@ -20,8 +20,6 @@ in
       lfs = false;
     };
 
-    # TODO: add a module setting to specify additional systemd dependencies
-    #       and then mark "sops-install-secrets.service" as a dependency here
     instances = let
       defaultGithubConfig = {
         settings.token_file = config.sops.secrets."gickup/github-token".path;

hosts/bicep/services/matrix/coturn.nix

@@ -55,11 +55,6 @@
   # It needs this to be allowed to access the files with the acme group
   systemd.services.coturn.serviceConfig.PrivateUsers = lib.mkForce false;
 
-  systemd.services.coturn = {
-    requires = [ "sops-install-secrets.service" ];
-    after = [ "sops-install-secrets.service" ];
-  };
-
   systemd.services."acme-${config.services.coturn.realm}".serviceConfig = {
     AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
   };

hosts/bicep/services/matrix/discord.nix

@@ -39,8 +39,6 @@ in
   };
 
   systemd.services.mx-puppet-discord = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
     serviceConfig.SupplementaryGroups = [
       config.users.groups.keys-matrix-registrations.name
     ];

hosts/bicep/services/matrix/hookshot/default.nix

@@ -183,7 +183,6 @@ in
   };
 
   services.matrix-hookshot.serviceDependencies = [
-    "sops-install-secrets.service"
     "matrix-synapse.target"
     "nginx.service"
   ];

hosts/bicep/services/matrix/livekit.nix

@@ -43,12 +43,7 @@ in
     keyFile = config.sops.templates."matrix-livekit-keyfile".path;
   };
 
-  systemd.services.lk-jwt-service = lib.mkIf cfg.enable {
+  systemd.services.lk-jwt-service.environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = lib.mkIf cfg.enable (builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ]);
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-
-    environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ];
-  };
 
   services.nginx.virtualHosts.${matrixDomain} = lib.mkIf cfg.enable {
     locations."^~ /livekit/jwt/" = {

hosts/bicep/services/matrix/mjolnir.nix

@@ -55,57 +55,52 @@
     pantalaimon.username = "bot_admin";
   };
 
-  systemd.services.mjolnir = {
+  systemd.services.mjolnir.serviceConfig = {
-    requires = [ "sops-install-secrets.service" ];
+    DynamicUser = true;
-    after = [ "sops-install-secrets.service" ];
+    RuntimeDirectory = [ "mjolnir/root-mnt" ];
+    RootDirectory = "/run/mjolnir/root-mnt";
+    BindReadOnlyPaths = [
+      config.sops.secrets."matrix/mjolnir/access_token".path
+      builtins.storeDir
+      "/etc"
+      "/run/nscd"
+      "/var/run/nscd"
+    ];
 
-    serviceConfig = {
+    AmbientCapabilities = "";
-      DynamicUser = true;
+    CapabilityBoundingSet = "";
-      RuntimeDirectory = [ "mjolnir/root-mnt" ];
+    LockPersonality = true;
-      RootDirectory = "/run/mjolnir/root-mnt";
+    MemoryDenyWriteExecute = false; # node needs this
-      BindReadOnlyPaths = [
+    NoNewPrivileges = true;
-        config.sops.secrets."matrix/mjolnir/access_token".path
+    PrivateDevices = true;
-        builtins.storeDir
+    PrivateMounts = true;
-        "/etc"
+    PrivateTmp = true;
-        "/run/nscd"
+    PrivateUsers = true;
-        "/var/run/nscd"
+    ProcSubset = "pid";
-      ];
+    ProtectClock = true;
-
+    ProtectControlGroups = true;
-      AmbientCapabilities = "";
+    ProtectHome = true;
-      CapabilityBoundingSet = "";
+    ProtectHostname = true;
-      LockPersonality = true;
+    ProtectKernelLogs = true;
-      MemoryDenyWriteExecute = false; # node needs this
+    ProtectKernelModules = true;
-      NoNewPrivileges = true;
+    ProtectKernelTunables = true;
-      PrivateDevices = true;
+    ProtectProc = "invisible";
-      PrivateMounts = true;
+    ProtectSystem = "strict";
-      PrivateTmp = true;
+    RemoveIPC = true;
-      PrivateUsers = true;
+    RestrictAddressFamilies = [
-      ProcSubset = "pid";
+      "AF_INET"
-      ProtectClock = true;
+      "AF_INET6"
-      ProtectControlGroups = true;
+      "AF_UNIX"
-      ProtectHome = true;
+    ];
-      ProtectHostname = true;
+    RestrictNamespaces = true;
-      ProtectKernelLogs = true;
+    RestrictRealtime = true;
-      ProtectKernelModules = true;
+    RestrictSUIDSGID = true;
-      ProtectKernelTunables = true;
+    SystemCallArchitectures = "native";
-      ProtectProc = "invisible";
+    SystemCallFilter = [
-      ProtectSystem = "strict";
+      "@system-service"
-      RemoveIPC = true;
+      "~@privileged"
-      RestrictAddressFamilies = [
+      "~@resources"
-        "AF_INET"
+    ];
-        "AF_INET6"
+    UMask = "0077";
-        "AF_UNIX"
-      ];
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-        "@system-service"
-        "~@privileged"
-        "~@resources"
-      ];
-      UMask = "0077";
-    };
   };
 }

hosts/bicep/services/matrix/out-of-your-element.nix

@@ -56,15 +56,7 @@ in
     enableSynapseIntegration = false;
   };
 
-  systemd.services."matrix-ooye-pre-start" = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   systemd.services."matrix-ooye" = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-
     serviceConfig = {
       RuntimeDirectory = [ "matrix-ooye/root-mnt" ];
       RootDirectory = "/run/matrix-ooye/root-mnt";

hosts/bicep/services/matrix/synapse.nix

@@ -44,11 +44,6 @@ in {
     };
   };
 
-  systemd.targets."matrix-synapse" = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   services.matrix-synapse-next = {
     enable = true;
 
@@ -79,7 +74,7 @@ in {
         name = "psycopg2";
         args = {
           host = "/var/run/postgresql";
-          dbname = "synapse";
+          database = "synapse";
           user = "matrix-synapse";
           cp_min = 1;
           cp_max = 5;

hosts/bicep/services/minecraft-heatmap.nix

@@ -23,15 +23,6 @@ in
   };
 
   systemd.services.minecraft-heatmap-ingest-logs = lib.mkIf cfg.enable {
-    after = [
-      "sops-install-secrets.service"
-      "network-online.target"
-    ];
-    requires = [
-      "sops-install-secrets.service"
-      "network-online.target"
-    ];
-
     serviceConfig = {
       LoadCredential = [
         "sshkey:${config.sops.secrets."minecraft-heatmap/ssh-key/private".path}"

hosts/bicep/services/mysql/backup.nix

@@ -41,24 +41,38 @@ in
     path = with pkgs; [
       cfg.package
       coreutils
+      diffutils
       zstd
     ];
 
-    script = let
+    script = ''
-      rotations = 2;
-    in ''
       set -euo pipefail
 
-      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
+      dump() {
+        local name="$1" out tmp
+        out="$STATE_DIRECTORY/$name.sql.zst"
+        tmp="$out.tmp"
+        shift
+        "$@" | zstd -9 --rsyncable -f -o "$tmp"
+        if cmp -s "$tmp" "$out" 2>/dev/null; then
+          rm -f "$tmp"
+        else
+          mv -f "$tmp" "$out"
+        fi
+      }
 
-      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+      declare -A keep
+      while IFS= read -r db; do
+        [ -n "$db" ] || continue
+        dump "$db" mysqldump --skip-dump-date --databases "$db"
+        keep["$db.sql.zst"]=1
+      done < <(mysql -N -e 'SHOW DATABASES' | grep -vE '^(information_schema|performance_schema)$')
 
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      # drop dumps of databases that no longer exist
-      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
+      for f in "$STATE_DIRECTORY"/*.sql.zst; do
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
+        [ -e "$f" ] || continue
-
+        base="$(basename "$f")"
-      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
-        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
       done
     '';
 

hosts/bicep/services/mysql/default.nix

@@ -56,13 +56,9 @@ in
 
   systemd.services.mysql = lib.mkIf cfg.enable {
     after = [
-      "sops-install-secrets.service"
       "systemd-tmpfiles-setup.service"
       "systemd-tmpfiles-resetup.service"
     ];
-    requires = [
-      "sops-install-secrets.service"
-    ];
 
     serviceConfig = {
       BindPaths = [ "${dataDir}:${cfg.dataDir}" ];

hosts/bicep/services/postgresql/backup.nix

@@ -41,25 +41,42 @@ in
 
     path = with pkgs; [
       coreutils
+      diffutils
       zstd
       cfg.package
     ];
 
-    script = let
+    script = ''
-      rotations = 2;
-    in ''
       set -euo pipefail
 
-      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
+      dump() {
+        local name="$1" out tmp
+        out="$STATE_DIRECTORY/$name.sql.zst"
+        tmp="$out.tmp"
+        shift
+        "$@" | zstd -9 --rsyncable -f -o "$tmp"
+        if cmp -s "$tmp" "$out" 2>/dev/null; then
+          rm -f "$tmp"
+        else
+          mv -f "$tmp" "$out"
+        fi
+      }
 
-      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
+      declare -A keep
+      dump globals pg_dumpall -U postgres --globals-only --restrict-key=backup
+      keep[globals.sql.zst]=1
 
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
+      while IFS= read -r db; do
-      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
+        [ -n "$db" ] || continue
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
+        dump "$db" pg_dump -U postgres -C -d "$db" --restrict-key=backup
+        keep["$db.sql.zst"]=1
+      done < <(psql -U postgres -tAc "SELECT datname FROM pg_database WHERE datallowconn ORDER BY datname")
 
-      while [ "$(find "$STATE_DIRECTORY" -type f -printf '.' | wc -c)" -gt '${toString (rotations + 1)}' ]; do
+      # drop dumps of databases that no longer exist
-        rm "$(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)"
+      for f in "$STATE_DIRECTORY"/*.sql.zst; do
+        [ -e "$f" ] || continue
+        base="$(basename "$f")"
+        [ -n "''${keep[$base]:-}" ] || rm -f "$f"
       done
     '';
 

hosts/gluttony/services/bluemap.nix

@@ -82,15 +82,6 @@ in {
   };
 
   systemd.services."render-bluemap-maps" = {
-    after = [
-      "sops-install-secrets.service"
-      "network-online.target"
-    ];
-    requires = [
-      "sops-install-secrets.service"
-      "network-online.target"
-    ];
-
     serviceConfig = {
       StateDirectory = [ "bluemap/world" ];
       ExecStartPre = let

hosts/ildkule/services/monitoring/grafana.nix

@@ -79,11 +79,6 @@ in {
     };
   };
 
-  systemd.services.grafana = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   services.nginx.virtualHosts.${cfg.settings.server.domain} = {
     enableACME = true;
     forceSSL = true;

hosts/ildkule/services/monitoring/prometheus/postgres.nix

@@ -37,15 +37,9 @@ in {
     };
   };
 
-  systemd.services.prometheus-postgres-exporter-knakelibrak = {
+  systemd.services.prometheus-postgres-exporter-knakelibrak.serviceConfig = let
-    after = [ "sops-install-secrets.service" ];
+    localCfg = config.services.prometheus.exporters.postgres;
-    requires = [ "sops-install-secrets.service" ];
+  in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
-
-    # TODO: is it really necessary to use recursiveUpdate on the entire attrset here?
-    #       Why don't we just update the relevant attrs directly?
-    serviceConfig = let
-      localCfg = config.services.prometheus.exporters.postgres;
-    in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
       EnvironmentFile = config.sops.secrets."keys/postgres/postgres_exporter_knakelibrak_env".path;
       ExecStart = ''
         ${pkgs.prometheus-postgres-exporter}/bin/postgres_exporter \
@@ -54,5 +48,4 @@ in {
           ${lib.concatStringsSep " \\\n  " localCfg.extraFlags}
       '';
     };
-  };
 }

hosts/kommode/services/gitea/default.nix

@@ -154,14 +154,8 @@ in {
   environment.systemPackages = [ cfg.package ];
 
   systemd.services.gitea = lib.mkIf cfg.enable {
-    after = [
+    wants = [ "redis-gitea.service" ];
-      "sops-install-secrets.service"
+    after = [ "redis-gitea.service" ];
-      "redis-gitea.service"
-    ];
-    requires = [
-      "sops-install-secrets.service"
-      "redis-gitea.service"
-    ];
 
     serviceConfig = {
       CPUSchedulingPolicy = "batch";
@@ -222,9 +216,6 @@ in {
   };
 
   systemd.services.gitea-dump = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-
     serviceConfig.ExecStart = let
       args = lib.cli.toCommandLineShellGNU { } {
         type = cfg.dump.type;

hosts/kommode/services/gitea/gpg.nix

@@ -32,11 +32,7 @@ in
 
   systemd.services.gitea-ensure-gnupg-homedir = {
     description = "Import gpg key for gitea";
-
     before = [ "gitea.service" ];
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-
     environment = { inherit GNUPGHOME; };
     serviceConfig = {
       Type = "oneshot";

hosts/kommode/services/gitea/import-users/default.nix

@@ -12,16 +12,6 @@ in
   systemd.services.gitea-import-users = lib.mkIf cfg.enable {
     enable = true;
     environment.PASSWD_FILE_PATH = "/run/gitea-import-users/passwd";
-    after = [
-      "sops-install-secrets.service"
-      "gitea.service"
-      "network-online.target"
-    ];
-    requires = [
-      "sops-install-secrets.service"
-      "gitea.service"
-      "network-online.target"
-    ];
     serviceConfig = {
       ExecStartPre = ''${pkgs.rsync}/bin/rsync -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=$CREDENTIALS_DIRECTORY/ssh-known-hosts -i $CREDENTIALS_DIRECTORY/sshkey" -a pvv@smtp.pvv.ntnu.no:/etc/passwd /run/gitea-import-users/passwd'';
       ExecStart = pkgs.writers.writePython3 "gitea-import-users" {

hosts/kommode/services/gitea/web-secret-provider/default.nix

@@ -48,16 +48,7 @@ in
   # %d - secrets directory
   systemd.services."gitea-web-secret-provider@" = {
     description = "Ensure all repos in %i has an SSH key to push web content";
-    after = [
+    requires = [ "gitea.service" "network.target" ];
-      "sops-install-secrets.service"
-      "gitea.service"
-      "network-online.target"
-    ];
-    requires = [
-      "sops-install-secrets.service"
-      "gitea.service"
-      "network-online.target"
-    ];
     serviceConfig = {
       Slice = "system-giteaweb.slice";
       Type = "oneshot";

hosts/lupine/services/openvpn.nix

@@ -94,11 +94,6 @@ in
     };
   };
 
-  systemd.services."openvpn-ov-firewall" = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   networking.firewall = {
     allowedTCPPorts = [ 1194 ];
     allowedUDPPorts = [ 1194 ];

hosts/skrot/services/drumknotty.nix

@@ -53,10 +53,5 @@
     };
   };
 
-  systemd.services.drumknotty-screen-session = {
-    after = [ "sops-install-secrets.service" ];
-    requires = [ "sops-install-secrets.service" ];
-  };
-
   services.roowho2.settings.rwhod.ignoreUsers = [ "drumknotty" ];
 }

hosts/temmie/services/userweb/passwd-sync.nix

@@ -29,15 +29,9 @@ in
     systemd.services."httpd-passwd-sync" = {
       requiredBy = [ "userweb.target" ];
       after = [
-        "network-online.target"
-        "sops-install-secrets.service"
         "systemd-tmpfiles-setup.service"
         "systemd-tmpfiles-resetup.service"
       ];
-      requires = [
-        "network-online.target"
-        "sops-install-secrets.service"
-      ];
       serviceConfig = {
         Type = "oneshot";
         Slice = "system-userweb.slice";

modules/matrix-ooye.nix

@@ -159,6 +159,7 @@ in
         description = "Out of Your Element - a Discord bridge for Matrix.";
 
         wants = [
+          "network-online.target"
           "matrix-synapse.service"
           "conduit.service"
           "dendrite.service"
@@ -167,10 +168,7 @@ in
           "matrix-ooye-pre-start.service"
           "network-online.target"
         ];
-        requires = [
+        requires = [ "matrix-ooye-pre-start.service" ];
-          "network-online.target"
-          "matrix-ooye-pre-start.service"
-        ];
         wantedBy = [ "multi-user.target" ];
 
         startLimitIntervalSec = 5;