treewide: add missing systemd ordering directives

In particular, we were missing `sops-install-secrets.service` in a ton
of places, and `network-online.target` for some homebrewn services.

This has caused failure of some services on machine startup.

hosts/bekkalokk/services/idp-simplesamlphp/default.nix

@@ -177,6 +177,11 @@ in
       };
     };
 
+    systemd.services."phpfpm-idp" = {
+      after = [ "sops-install-secrets.service" ];
+      requires = [ "sops-install-secrets.service" ];
+    };
+
     services.nginx.virtualHosts."idp.pvv.ntnu.no" = {
       forceSSL = true;
       enableACME = true;

hosts/bekkalokk/services/mediawiki/default.nix

@@ -287,6 +287,7 @@ in {
 
   systemd.services.mediawiki-init = lib.mkIf cfg.enable {
     after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
     serviceConfig = {
       UMask = lib.mkForce "0007";
     };
@@ -294,6 +295,7 @@ in {
 
   systemd.services.phpfpm-mediawiki = lib.mkIf cfg.enable {
     after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
     serviceConfig = {
       UMask = lib.mkForce "0007";
     };

hosts/bekkalokk/services/vaultwarden.nix

@@ -61,6 +61,11 @@ in {
     };
   };
 
+  systemd.services.vaultwarden = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   services.nginx.virtualHosts."${domain}" = {
     forceSSL = true;
     enableACME = true;

hosts/bekkalokk/services/webmail/roundcube.nix

@@ -49,6 +49,11 @@ in
     '';
   };
 
+  systemd.services."phpfpm-roundcube" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   # TODO: move this back to `webmail.pvv.ntnu.no/roundcube` subpath
 
   services.nginx.virtualHosts.${domain} = {

hosts/bekkalokk/services/website/default.nix

@@ -116,6 +116,11 @@ in {
     ];
   };
 
+  systemd.services."phpfpm-pvv-nettsiden" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   services.nginx.virtualHosts."pvv.ntnu.no" = {
     globalRedirect = cfg.domainName;
     redirectCode = 307;

hosts/bekkalokk/services/well-known/default.nix

@@ -1,70 +1,25 @@
 { lib, ... }:
 {
-  # TODO: move this to base so that all virtualHosts take effect on their respecitve hosts
+  services.nginx.virtualHosts = lib.genAttrs [
-
-  # NOTE: automatically hosting well-known files by looping over all existing `virtualHosts`
-  #       unfortunately causes infinite recursuion due to submodule usage within the nginx
-  #       module. For now, the easiest solution was to manually specify a list of virtualHosts
-  #       here, but it would be nice to find a better solution in the future.
-  services.nginx.virtualHosts = lib.mkMerge [
-    (lib.genAttrs [
     "pvv.ntnu.no"
-      "pvv.org"
-
     "www.pvv.ntnu.no"
-      "www.pvv.org"
-      "www2.pvv.ntnu.no"
-      "www2.pvv.org"
-
-      # NOTE: this list is probably not complete
-      "alps.pvv.ntnu.no"
-      "chat.pvv.ntnu.no"
-      "grafana.pvv.ntnu.no"
-      "status.pvv.ntnu.no"
-      "matrix.pvv.ntnu.no"
-      "mirrors.pvv.ntnu.no"
-      "pages.pvv.ntnu.no"
-      "ooye.pvv.ntnu.no"
-      "ooye.pvv.ntnu.no"
-      "dav.pvv.ntnu.no"
-      "git.pvv.ntnu.no"
-      "idp.pvv.ntnu.no"
-      "minecraft.pvv.ntnu.no"
-      "pw.pvv.ntnu.no"
-      "snappymail.pvv.ntnu.no"
-      "webmail.pvv.ntnu.no"
-      "wiki.pvv.ntnu.no"
-    ] (_: {
-      locations."^~ /.well-known/security.txt" = {
-        alias = toString ./root/security.txt;
-      };
-    }))
-
-    (lib.genAttrs [
-      "pvv.ntnu.no"
     "pvv.org"
-      "mail.pvv.ntnu.no"
-      "mail.pvv.org"
-      "smtp.pvv.ntnu.no"
-      "smtp.pvv.org"
-    ] (_: {
-      locations."^~ /.well-known/autoconfig/mail/" = {
-        root = toString ./root/autoconfig/mail;
-      };
-    }))
-
-    (lib.genAttrs [
-      "pvv.ntnu.no"
-      "pvv.org"
-      "www.pvv.ntnu.no"
     "www.pvv.org"
   ] (_: {
-      locations."^~ /.well-known/matrix/" = {
+    locations = {
+      "^~ /.well-known/" = {
+        alias = (toString ./root) + "/";
+      };
+
+      # Proxy the matrix well-known files
+      # Host has be set before proxy_pass
+      # The header must be set so nginx on the other side routes it to the right place
+      "^~ /.well-known/matrix/" = {
         extraConfig = ''
           proxy_set_header Host matrix.pvv.ntnu.no;
           proxy_pass https://matrix.pvv.ntnu.no/.well-known/matrix/;
         '';
       };
-    }))
+    };
-  ];
+  });
 }

hosts/bicep/services/calendar-bot.nix

@@ -35,4 +35,9 @@ in {
       onCalendar = "*-*-* 09:00:00";
     };
   };
+
+  systemd.services.pvv-calendar-bot = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
 }

hosts/bicep/services/git-mirrors/default.nix

@@ -20,6 +20,8 @@ in
       lfs = false;
     };
 
+    # TODO: add a module setting to specify additional systemd dependencies
+    #       and then mark "sops-install-secrets.service" as a dependency here
     instances = let
       defaultGithubConfig = {
         settings.token_file = config.sops.secrets."gickup/github-token".path;

hosts/bicep/services/matrix/coturn.nix

@@ -55,6 +55,11 @@
   # It needs this to be allowed to access the files with the acme group
   systemd.services.coturn.serviceConfig.PrivateUsers = lib.mkForce false;
 
+  systemd.services.coturn = {
+    requires = [ "sops-install-secrets.service" ];
+    after = [ "sops-install-secrets.service" ];
+  };
+
   systemd.services."acme-${config.services.coturn.realm}".serviceConfig = {
     AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
   };

hosts/bicep/services/matrix/discord.nix

@@ -39,6 +39,8 @@ in
   };
 
   systemd.services.mx-puppet-discord = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
     serviceConfig.SupplementaryGroups = [
       config.users.groups.keys-matrix-registrations.name
     ];

hosts/bicep/services/matrix/hookshot/default.nix

@@ -183,6 +183,7 @@ in
   };
 
   services.matrix-hookshot.serviceDependencies = [
+    "sops-install-secrets.service"
     "matrix-synapse.target"
     "nginx.service"
   ];

hosts/bicep/services/matrix/livekit.nix

@@ -43,7 +43,12 @@ in
     keyFile = config.sops.templates."matrix-livekit-keyfile".path;
   };
 
-  systemd.services.lk-jwt-service.environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = lib.mkIf cfg.enable (builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ]);
+  systemd.services.lk-jwt-service = lib.mkIf cfg.enable {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
+    environment.LIVEKIT_FULL_ACCESS_HOMESERVERS = builtins.concatStringsSep "," [ "pvv.ntnu.no" "dodsorf.as" ];
+  };
 
   services.nginx.virtualHosts.${matrixDomain} = lib.mkIf cfg.enable {
     locations."^~ /livekit/jwt/" = {

hosts/bicep/services/matrix/mjolnir.nix

@@ -55,7 +55,11 @@
     pantalaimon.username = "bot_admin";
   };
 
-  systemd.services.mjolnir.serviceConfig = {
+  systemd.services.mjolnir = {
+    requires = [ "sops-install-secrets.service" ];
+    after = [ "sops-install-secrets.service" ];
+
+    serviceConfig = {
       DynamicUser = true;
       RuntimeDirectory = [ "mjolnir/root-mnt" ];
       RootDirectory = "/run/mjolnir/root-mnt";
@@ -103,4 +107,5 @@
       ];
       UMask = "0077";
     };
+  };
 }

hosts/bicep/services/matrix/out-of-your-element.nix

@@ -56,7 +56,15 @@ in
     enableSynapseIntegration = false;
   };
 
+  systemd.services."matrix-ooye-pre-start" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   systemd.services."matrix-ooye" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
     serviceConfig = {
       RuntimeDirectory = [ "matrix-ooye/root-mnt" ];
       RootDirectory = "/run/matrix-ooye/root-mnt";

hosts/bicep/services/matrix/synapse.nix

@@ -44,6 +44,11 @@ in {
     };
   };
 
+  systemd.targets."matrix-synapse" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   services.matrix-synapse-next = {
     enable = true;
 
@@ -74,7 +79,7 @@ in {
         name = "psycopg2";
         args = {
           host = "/var/run/postgresql";
-          database = "synapse";
+          dbname = "synapse";
           user = "matrix-synapse";
           cp_min = 1;
           cp_max = 5;

hosts/bicep/services/minecraft-heatmap.nix

@@ -23,6 +23,15 @@ in
   };
 
   systemd.services.minecraft-heatmap-ingest-logs = lib.mkIf cfg.enable {
+    after = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+
     serviceConfig = {
       LoadCredential = [
         "sshkey:${config.sops.secrets."minecraft-heatmap/ssh-key/private".path}"

hosts/bicep/services/mysql/default.nix

@@ -56,9 +56,13 @@ in
 
   systemd.services.mysql = lib.mkIf cfg.enable {
     after = [
+      "sops-install-secrets.service"
       "systemd-tmpfiles-setup.service"
       "systemd-tmpfiles-resetup.service"
     ];
+    requires = [
+      "sops-install-secrets.service"
+    ];
 
     serviceConfig = {
       BindPaths = [ "${dataDir}:${cfg.dataDir}" ];

hosts/gluttony/services/bluemap.nix

@@ -82,6 +82,15 @@ in {
   };
 
   systemd.services."render-bluemap-maps" = {
+    after = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "network-online.target"
+    ];
+
     serviceConfig = {
       StateDirectory = [ "bluemap/world" ];
       ExecStartPre = let

hosts/ildkule/services/monitoring/grafana.nix

@@ -79,6 +79,11 @@ in {
     };
   };
 
+  systemd.services.grafana = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   services.nginx.virtualHosts.${cfg.settings.server.domain} = {
     enableACME = true;
     forceSSL = true;

hosts/ildkule/services/monitoring/prometheus/postgres.nix

@@ -37,7 +37,13 @@ in {
     };
   };
 
-  systemd.services.prometheus-postgres-exporter-knakelibrak.serviceConfig = let
+  systemd.services.prometheus-postgres-exporter-knakelibrak = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
+    # TODO: is it really necessary to use recursiveUpdate on the entire attrset here?
+    #       Why don't we just update the relevant attrs directly?
+    serviceConfig = let
       localCfg = config.services.prometheus.exporters.postgres;
     in lib.recursiveUpdate config.systemd.services.prometheus-postgres-exporter.serviceConfig {
       EnvironmentFile = config.sops.secrets."keys/postgres/postgres_exporter_knakelibrak_env".path;
@@ -48,4 +54,5 @@ in {
           ${lib.concatStringsSep " \\\n  " localCfg.extraFlags}
       '';
     };
+  };
 }

hosts/kommode/services/gitea/default.nix

@@ -154,8 +154,14 @@ in {
   environment.systemPackages = [ cfg.package ];
 
   systemd.services.gitea = lib.mkIf cfg.enable {
-    wants = [ "redis-gitea.service" ];
+    after = [
-    after = [ "redis-gitea.service" ];
+      "sops-install-secrets.service"
+      "redis-gitea.service"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "redis-gitea.service"
+    ];
 
     serviceConfig = {
       CPUSchedulingPolicy = "batch";
@@ -216,6 +222,9 @@ in {
   };
 
   systemd.services.gitea-dump = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
     serviceConfig.ExecStart = let
       args = lib.cli.toCommandLineShellGNU { } {
         type = cfg.dump.type;

hosts/kommode/services/gitea/gpg.nix

@@ -32,7 +32,11 @@ in
 
   systemd.services.gitea-ensure-gnupg-homedir = {
     description = "Import gpg key for gitea";
+
     before = [ "gitea.service" ];
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+
     environment = { inherit GNUPGHOME; };
     serviceConfig = {
       Type = "oneshot";

hosts/kommode/services/gitea/import-users/default.nix

@@ -12,6 +12,16 @@ in
   systemd.services.gitea-import-users = lib.mkIf cfg.enable {
     enable = true;
     environment.PASSWD_FILE_PATH = "/run/gitea-import-users/passwd";
+    after = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
     serviceConfig = {
       ExecStartPre = ''${pkgs.rsync}/bin/rsync -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=$CREDENTIALS_DIRECTORY/ssh-known-hosts -i $CREDENTIALS_DIRECTORY/sshkey" -a pvv@smtp.pvv.ntnu.no:/etc/passwd /run/gitea-import-users/passwd'';
       ExecStart = pkgs.writers.writePython3 "gitea-import-users" {

hosts/kommode/services/gitea/web-secret-provider/default.nix

@@ -48,7 +48,16 @@ in
   # %d - secrets directory
   systemd.services."gitea-web-secret-provider@" = {
     description = "Ensure all repos in %i has an SSH key to push web content";
-    requires = [ "gitea.service" "network.target" ];
+    after = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
+    requires = [
+      "sops-install-secrets.service"
+      "gitea.service"
+      "network-online.target"
+    ];
     serviceConfig = {
       Slice = "system-giteaweb.slice";
       Type = "oneshot";

hosts/lupine/services/openvpn.nix

@@ -94,6 +94,11 @@ in
     };
   };
 
+  systemd.services."openvpn-ov-firewall" = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   networking.firewall = {
     allowedTCPPorts = [ 1194 ];
     allowedUDPPorts = [ 1194 ];

hosts/skrot/services/drumknotty.nix

@@ -53,5 +53,10 @@
     };
   };
 
+  systemd.services.drumknotty-screen-session = {
+    after = [ "sops-install-secrets.service" ];
+    requires = [ "sops-install-secrets.service" ];
+  };
+
   services.roowho2.settings.rwhod.ignoreUsers = [ "drumknotty" ];
 }

hosts/temmie/services/userweb/passwd-sync.nix

@@ -29,9 +29,15 @@ in
     systemd.services."httpd-passwd-sync" = {
       requiredBy = [ "userweb.target" ];
       after = [
+        "network-online.target"
+        "sops-install-secrets.service"
         "systemd-tmpfiles-setup.service"
         "systemd-tmpfiles-resetup.service"
       ];
+      requires = [
+        "network-online.target"
+        "sops-install-secrets.service"
+      ];
       serviceConfig = {
         Type = "oneshot";
         Slice = "system-userweb.slice";

modules/matrix-ooye.nix

@@ -159,7 +159,6 @@ in
         description = "Out of Your Element - a Discord bridge for Matrix.";
 
         wants = [
-          "network-online.target"
           "matrix-synapse.service"
           "conduit.service"
           "dendrite.service"
@@ -168,7 +167,10 @@ in
           "matrix-ooye-pre-start.service"
           "network-online.target"
         ];
-        requires = [ "matrix-ooye-pre-start.service" ];
+        requires = [
+          "network-online.target"
+          "matrix-ooye-pre-start.service"
+        ];
         wantedBy = [ "multi-user.target" ];
 
         startLimitIntervalSec = 5;