WIP

This is whitelisted to just bicep

As a side-effect it's also much easier to use synapse-admin now

base.nix

@@ -1,190 +0,0 @@
-{ config, lib, pkgs, inputs, values, ... }:
-
-{
-  imports = [
-    ./users
-    ./modules/snakeoil-certs.nix
-  ];
-
-  networking.domain = "pvv.ntnu.no";
-  networking.useDHCP = false;
-  # networking.search = [ "pvv.ntnu.no" "pvv.org" ];
-  # networking.nameservers = lib.mkDefault [ "129.241.0.200" "129.241.0.201" ];
-  # networking.tempAddresses = lib.mkDefault "disabled";
-  # networking.defaultGateway = values.hosts.gateway;
-
-  systemd.network.enable = true;
-
-  services.resolved = {
-    enable = lib.mkDefault true;
-    dnssec = "false"; # Supposdly this keeps breaking and the default is to allow downgrades anyways...
-  };
-
-  time.timeZone = "Europe/Oslo";
-
-  i18n.defaultLocale = "en_US.UTF-8";
-  console = {
-    font = "Lat2-Terminus16";
-    keyMap = "no";
-  };
-
-  system.autoUpgrade = {
-    enable = true;
-    flake = "git+https://git.pvv.ntnu.no/Drift/pvv-nixos-config.git";
-    flags = [
-      "--update-input" "nixpkgs"
-      "--update-input" "nixpkgs-unstable"
-      "--no-write-lock-file"
-    ];
-  };
-  nix.gc.automatic = true;
-  nix.gc.options = "--delete-older-than 2d";
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
-
-  /* This makes commandline tools like
-  ** nix run nixpkgs#hello
-  ** and nix-shell -p hello
-  ** use the same channel the system
-  ** was built with
-  */
-  nix.registry = {
-    nixpkgs.flake = inputs.nixpkgs;
-  };
-  nix.nixPath = [ "nixpkgs=${inputs.nixpkgs}" ];
-
-  environment.systemPackages = with pkgs; [
-    file
-    git
-    gnupg
-    htop
-    nano
-    ripgrep
-    rsync
-    screen
-    tmux
-    vim
-    wget
-
-    kitty.terminfo
-  ];
-
-  programs.zsh.enable = true;
-
-  users.groups."drift".name = "drift";
-
-  # Trusted users on the nix builder machines
-  users.groups."nix-builder-users".name = "nix-builder-users";
-
-  # Let's not thermal throttle
-  services.thermald.enable = lib.mkIf (lib.all (x: x) [
-      (config.nixpkgs.system == "x86_64-linux")
-      (!config.boot.isContainer or false)
-    ]) true;
-
-  services.openssh = {
-    enable = true;
-    extraConfig = ''
-      PubkeyAcceptedAlgorithms=+ssh-rsa
-      Match Group wheel
-        PasswordAuthentication no
-      Match All
-    '';
-    settings.PermitRootLogin = "yes";
-  };
-
-  # nginx return 444 for all nonexistent virtualhosts
-
-  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
-
-  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
-    "/etc/certs/nginx" = {
-      owner = "nginx";
-      group = "nginx";
-    };
-  };
-
-  services.nginx = {
-    recommendedTlsSettings = true;
-    recommendedProxySettings = true;
-    recommendedOptimisation = true;
-    recommendedGzipSettings = true;
-
-    appendConfig = ''
-      pcre_jit on;
-      worker_processes auto;
-      worker_rlimit_nofile 100000;
-    '';
-    eventsConfig = ''
-      worker_connections 2048;
-      use epoll;
-      multi_accept on;
-    '';
-  };
-
-  systemd.services.nginx.serviceConfig = lib.mkIf config.services.nginx.enable {
-    LimitNOFILE = 65536;
-  };
-
-  services.nginx.virtualHosts."_" = lib.mkIf config.services.nginx.enable {
-    sslCertificate = "/etc/certs/nginx.crt";
-    sslCertificateKey = "/etc/certs/nginx.key";
-    addSSL = true;
-    extraConfig = "return 444;";
-  };
-
-  # source: https://github.com/logrotate/logrotate/blob/main/examples/logrotate.service
-  systemd.services.logrotate = {
-    documentation = [ "man:logrotate(8)" "man:logrotate.conf(5)" ];
-    unitConfig.RequiresMountsFor = "/var/log";
-    serviceConfig = {
-      Nice = 19;
-      IOSchedulingClass = "best-effort";
-      IOSchedulingPriority = 7;
-
-      ReadWritePaths = [ "/var/log" ];
-
-      AmbientCapabilities = [ "" ];
-      CapabilityBoundingSet = [ "" ];
-      DeviceAllow = [ "" ];
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      NoNewPrivileges = true; # disable for third party rotate scripts
-      PrivateDevices = true;
-      PrivateNetwork = true; # disable for mail delivery
-      PrivateTmp = true;
-      ProtectClock = true;
-      ProtectControlGroups = true;
-      ProtectHome = true; # disable for userdir logs
-      ProtectHostname = true;
-      ProtectKernelLogs = true;
-      ProtectKernelModules = true;
-      ProtectKernelTunables = true;
-      ProtectProc = "invisible";
-      ProtectSystem = "full";
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true; # disable for creating setgid directories
-      SocketBindDeny = [ "any" ];
-      SystemCallArchitectures = "native";
-      SystemCallFilter = [
-        "@system-service"
-      ];
-    };
-  };
-
-  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
-
-  security.acme = {
-    acceptTerms = true;
-    defaults.email = "drift@pvv.ntnu.no";
-  };
-  # Let's not spam LetsEncrypt in `nixos-rebuild build-vm` mode:
-  virtualisation.vmVariant = {
-    security.acme.defaults.server = "https://127.0.0.1";
-    security.acme.preliminarySelfsigned = true;
-
-    users.users.root.initialPassword = "root";
-  };
-
-}

base/default.nix

@@ -0,0 +1,60 @@
+{ pkgs, lib, ... }:
+
+{
+  imports = [
+    ../users
+    ../modules/snakeoil-certs.nix
+
+    ./networking.nix
+    ./nix.nix
+
+    ./services/acme.nix
+    ./services/auto-upgrade.nix
+    ./services/irqbalance.nix
+    ./services/logrotate.nix
+    ./services/nginx.nix
+    ./services/openssh.nix
+    ./services/postfix.nix
+    ./services/smartd.nix
+    ./services/thermald.nix
+  ];
+
+  boot.tmp.cleanOnBoot = lib.mkDefault true;
+
+  time.timeZone = "Europe/Oslo";
+
+  i18n.defaultLocale = "en_US.UTF-8";
+  console = {
+    font = "Lat2-Terminus16";
+    keyMap = "no";
+  };
+
+  environment.systemPackages = with pkgs; [
+    file
+    git
+    gnupg
+    htop
+    nano
+    ripgrep
+    rsync
+    screen
+    tmux
+    vim
+    wget
+
+    kitty.terminfo
+  ];
+
+  programs.zsh.enable = true;
+
+  security.sudo.execWheelOnly = true;
+  security.sudo.extraConfig = ''
+    Defaults lecture = never
+  '';
+
+  users.groups."drift".name = "drift";
+
+  # Trusted users on the nix builder machines
+  users.groups."nix-builder-users".name = "nix-builder-users";
+}
+

base/networking.nix

@@ -0,0 +1,16 @@
+{ lib, values, ... }:
+{
+  networking.domain = "pvv.ntnu.no";
+  networking.useDHCP = false;
+  # networking.search = [ "pvv.ntnu.no" "pvv.org" ];
+  # networking.nameservers = lib.mkDefault [ "129.241.0.200" "129.241.0.201" ];
+  # networking.tempAddresses = lib.mkDefault "disabled";
+  # networking.defaultGateway = values.hosts.gateway;
+
+  systemd.network.enable = true;
+
+  services.resolved = {
+    enable = lib.mkDefault true;
+    dnssec = "false"; # Supposdly this keeps breaking and the default is to allow downgrades anyways...
+  };
+}

base/nix.nix

@@ -0,0 +1,30 @@
+{ inputs, ... }:
+{
+  nix = {
+    gc = {
+      automatic = true;
+      options = "--delete-older-than 2d";
+    };
+
+    settings = {
+      allow-dirty = true;
+      auto-optimise-store = true;
+      builders-use-substitutes = true;
+      experimental-features = [ "nix-command" "flakes" ];
+      log-lines = 50;
+      use-xdg-base-directories = true;
+    };
+
+    /* This makes commandline tools like
+    ** nix run nixpkgs#hello
+    ** and nix-shell -p hello
+    ** use the same channel the system
+    ** was built with
+    */
+    registry = {
+      "nixpkgs".flake = inputs.nixpkgs;
+      "pvv-nix".flake = inputs.self;
+    };
+    nixPath = [ "nixpkgs=${inputs.nixpkgs}" ];
+  };
+}

base/services/acme.nix

@@ -0,0 +1,15 @@
+{ ... }:
+{
+  security.acme = {
+    acceptTerms = true;
+    defaults.email = "drift@pvv.ntnu.no";
+  };
+
+  # Let's not spam LetsEncrypt in `nixos-rebuild build-vm` mode:
+  virtualisation.vmVariant = {
+    security.acme.defaults.server = "https://127.0.0.1";
+    security.acme.preliminarySelfsigned = true;
+
+    users.users.root.initialPassword = "root";
+  };
+}

base/services/auto-upgrade.nix

@@ -0,0 +1,12 @@
+{ ... }:
+{
+  system.autoUpgrade = {
+    enable = true;
+    flake = "git+https://git.pvv.ntnu.no/Drift/pvv-nixos-config.git";
+    flags = [
+      "--update-input" "nixpkgs"
+      "--update-input" "nixpkgs-unstable"
+      "--no-write-lock-file"
+    ];
+  };
+}

base/services/irqbalance.nix

@@ -0,0 +1,4 @@
+{ ... }:
+{
+  services.irqbalance.enable = true;
+}

base/services/logrotate.nix

@@ -0,0 +1,42 @@
+{ ... }:
+{
+  # source: https://github.com/logrotate/logrotate/blob/main/examples/logrotate.service
+  systemd.services.logrotate = {
+    documentation = [ "man:logrotate(8)" "man:logrotate.conf(5)" ];
+    unitConfig.RequiresMountsFor = "/var/log";
+    serviceConfig = {
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      ReadWritePaths = [ "/var/log" ];
+
+      AmbientCapabilities = [ "" ];
+      CapabilityBoundingSet = [ "" ];
+      DeviceAllow = [ "" ];
+      LockPersonality = true;
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true; # disable for third party rotate scripts
+      PrivateDevices = true;
+      PrivateNetwork = true; # disable for mail delivery
+      PrivateTmp = true;
+      ProtectClock = true;
+      ProtectControlGroups = true;
+      ProtectHome = true; # disable for userdir logs
+      ProtectHostname = true;
+      ProtectKernelLogs = true;
+      ProtectKernelModules = true;
+      ProtectKernelTunables = true;
+      ProtectProc = "invisible";
+      ProtectSystem = "full";
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      RestrictSUIDSGID = true; # disable for creating setgid directories
+      SocketBindDeny = [ "any" ];
+      SystemCallArchitectures = "native";
+      SystemCallFilter = [
+        "@system-service"
+      ];
+    };
+  };
+}

base/services/nginx.nix

@@ -0,0 +1,44 @@
+{ config, lib, ... }:
+{
+  # nginx return 444 for all nonexistent virtualhosts
+
+  systemd.services.nginx.after = [ "generate-snakeoil-certs.service" ];
+
+  environment.snakeoil-certs = lib.mkIf config.services.nginx.enable {
+    "/etc/certs/nginx" = {
+      owner = "nginx";
+      group = "nginx";
+    };
+  };
+
+  networking.firewall.allowedTCPPorts = lib.mkIf config.services.nginx.enable [ 80 443 ];
+
+  services.nginx = {
+    recommendedTlsSettings = true;
+    recommendedProxySettings = true;
+    recommendedOptimisation = true;
+    recommendedGzipSettings = true;
+
+    appendConfig = ''
+      pcre_jit on;
+      worker_processes auto;
+      worker_rlimit_nofile 100000;
+    '';
+    eventsConfig = ''
+      worker_connections 2048;
+      use epoll;
+      multi_accept on;
+    '';
+  };
+
+  systemd.services.nginx.serviceConfig = lib.mkIf config.services.nginx.enable {
+    LimitNOFILE = 65536;
+  };
+
+  services.nginx.virtualHosts."_" = lib.mkIf config.services.nginx.enable {
+    sslCertificate = "/etc/certs/nginx.crt";
+    sslCertificateKey = "/etc/certs/nginx.key";
+    addSSL = true;
+    extraConfig = "return 444;";
+  };
+}

base/services/openssh.nix

@@ -0,0 +1,14 @@
+{ ... }:
+{
+  services.openssh = {
+    enable = true;
+    startWhenNeeded = true;
+    extraConfig = ''
+      PubkeyAcceptedAlgorithms=+ssh-rsa
+      Match Group wheel
+        PasswordAuthentication no
+      Match All
+    '';
+    settings.PermitRootLogin = "yes";
+  };
+}

base/services/postfix.nix

@@ -0,0 +1,23 @@
+{ config, pkgs, lib, ... }:
+let
+  cfg = config.services.postfix;
+in
+{
+  services.postfix = {
+    enable = true;
+
+    hostname = "${config.networking.hostName}.pvv.ntnu.no";
+    domain = "pvv.ntnu.no";
+
+    relayHost = "smtp.pvv.ntnu.no";
+    relayPort = 465;
+
+    config = {
+      smtp_tls_wrappermode = "yes";
+      smtp_tls_security_level = "encrypt";
+    };
+
+    # Nothing should be delivered to this machine
+    destination = [ ];
+  };
+}

base/services/smartd.nix

@@ -0,0 +1,8 @@
+{ config, pkgs, lib, ... }:
+{
+  services.smartd.enable = lib.mkDefault true;
+
+  environment.systemPackages = lib.optionals config.services.smartd.enable (with pkgs; [
+    smartmontools
+  ]);
+}

base/services/thermald.nix

@@ -0,0 +1,8 @@
+{ config, lib, ... }:
+{
+  # Let's not thermal throttle
+  services.thermald.enable = lib.mkIf (lib.all (x: x) [
+      (config.nixpkgs.system == "x86_64-linux")
+      (!config.boot.isContainer or false)
+    ]) true;
+}

hosts/bekkalokk/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
     ./hardware-configuration.nix
 
-    ../../base.nix
+    ../../base
     ../../misc/metrics-exporters.nix
 
     ./services/gitea/default.nix
@@ -32,6 +32,8 @@
     address = with values.hosts.bekkalokk; [ (ipv4 + "/25") (ipv6 + "/64") ];
   };
 
+  services.btrfs.autoScrub.enable = true;
+
   # Do not change, even during upgrades.
   # See https://search.nixos.org/options?show=system.stateVersion
   system.stateVersion = "22.11";

hosts/bekkalokk/services/gitea/default.nix

@@ -59,6 +59,7 @@ in {
       service = {
         DISABLE_REGISTRATION = true;
         ENABLE_NOTIFY_MAIL = true;
+        AUTO_WATCH_NEW_REPOS = false;
       };
       admin.DEFAULT_EMAIL_NOTIFICATIONS = "onmention";
       session.COOKIE_SECURE = true;

hosts/bekkalokk/services/gitea/import-users/default.nix

@@ -14,6 +14,9 @@ in
     preStart=''${pkgs.rsync}/bin/rsync -e "${pkgs.openssh}/bin/ssh -o UserKnownHostsFile=$CREDENTIALS_DIRECTORY/ssh-known-hosts -i $CREDENTIALS_DIRECTORY/sshkey" -a pvv@smtp.pvv.ntnu.no:/etc/passwd /tmp/passwd-import'';
     serviceConfig = {
       ExecStart = pkgs.writers.writePython3 "gitea-import-users" {
+        flakeIgnore = [
+          "E501" # Line over 80 chars lol
+        ];
         libraries = with pkgs.python3Packages; [ requests ];
       } (builtins.readFile ./gitea-import-users.py);
       LoadCredential=[

hosts/bekkalokk/services/gitea/import-users/gitea-import-users.py

@@ -2,18 +2,92 @@ import requests
 import secrets
 import os
 
+
 EMAIL_DOMAIN = os.getenv('EMAIL_DOMAIN')
 if EMAIL_DOMAIN is None:
     EMAIL_DOMAIN = 'pvv.ntnu.no'
 
+
 API_TOKEN = os.getenv('API_TOKEN')
 if API_TOKEN is None:
     raise Exception('API_TOKEN not set')
 
+
 GITEA_API_URL = os.getenv('GITEA_API_URL')
 if GITEA_API_URL is None:
     GITEA_API_URL = 'https://git.pvv.ntnu.no/api/v1'
 
+
+def gitea_list_all_users() -> dict[str, dict[str, any]] | None:
+    r = requests.get(
+        GITEA_API_URL + '/admin/users',
+        headers={'Authorization': 'token ' + API_TOKEN}
+    )
+
+    if r.status_code != 200:
+        print('Failed to get users:', r.text)
+        return None
+
+    return {user['login']: user for user in r.json()}
+
+
+def gitea_create_user(username: str, userdata: dict[str, any]) -> bool:
+    r = requests.post(
+        GITEA_API_URL + '/admin/users',
+        json=userdata,
+        headers={'Authorization': 'token ' + API_TOKEN},
+    )
+
+    if r.status_code != 201:
+        print(f'ERR: Failed to create user {username}:', r.text)
+        return False
+
+    return True
+
+
+def gitea_edit_user(username: str, userdata: dict[str, any]) -> bool:
+    r = requests.patch(
+        GITEA_API_URL + f'/admin/users/{username}',
+        json=userdata,
+        headers={'Authorization': 'token ' + API_TOKEN},
+    )
+
+    if r.status_code != 200:
+        print(f'ERR: Failed to update user {username}:', r.text)
+        return False
+
+    return True
+
+
+def gitea_list_teams_for_organization(org: str) -> dict[str, any] | None:
+    r = requests.get(
+        GITEA_API_URL + f'/orgs/{org}/teams',
+        headers={'Authorization': 'token ' + API_TOKEN},
+    )
+
+    if r.status_code != 200:
+        print(f"ERR: Failed to list teams for {org}:", r.text)
+        return None
+
+    return {team['name']: team for team in r.json()}
+
+
+def gitea_add_user_to_organization_team(username: str, team_id: int) -> bool:
+    r = requests.put(
+        GITEA_API_URL + f'/teams/{team_id}/members/{username}',
+        headers={'Authorization': 'token ' + API_TOKEN},
+    )
+
+    if r.status_code != 204:
+        print(f'ERR: Failed to add user {username} to org team {team_id}:', r.text)
+        return False
+
+    return True
+
+
+# If a passwd user has one of the following shells,
+# it is most likely not a PVV user, but rather a system user.
+# Users with these shells should thus be ignored.
 BANNED_SHELLS = [
     "/usr/bin/nologin",
     "/usr/sbin/nologin",
@@ -22,12 +96,36 @@ BANNED_SHELLS = [
     "/bin/msgsh",
 ]
 
-existing_users = {}
+
+# Reads out a passwd-file line for line, and filters out
+# real PVV users (as opposed to system users meant for daemons and such)
+def passwd_file_parser(passwd_path):
+    with open(passwd_path, 'r') as f:
+        for line in f.readlines():
+            uid = int(line.split(':')[2])
+            if uid < 1000:
+                continue
+
+            shell = line.split(':')[-1]
+            if shell in BANNED_SHELLS:
+                continue
+
+            username = line.split(':')[0]
+            name = line.split(':')[4].split(',')[0]
+            yield (username, name)
 
 
-# This function should only ever be called when adding users
-# from the passwd file
-def add_user(username, name):
+# This function either creates a new user in gitea
+# and fills it out with some default information if
+# it does not exist, or ensures that the default information
+# is correct if the user already exists. All user information
+# (including non-default fields) is pulled from gitea and added
+# to the `existing_users` dict
+def add_or_patch_gitea_user(
+    username: str,
+    name: str,
+    existing_users: dict[str, dict[str, any]],
+) -> None:
     user = {
         "full_name": name,
         "username": username,
@@ -41,53 +139,59 @@ def add_user(username, name):
         user["visibility"] = "private"
         user["email"] = username + '@' + EMAIL_DOMAIN
 
-        r = requests.post(GITEA_API_URL + '/admin/users', json=user,
-                          headers={'Authorization': 'token ' + API_TOKEN})
-        if r.status_code != 201:
-            print('ERR: Failed to create user ' + username + ': ' + r.text)
+        if not gitea_create_user(username, user):
             return
 
-        print('Created user ' + username)
+        print('Created user', username)
         existing_users[username] = user
 
     else:
         user["visibility"] = existing_users[username]["visibility"]
-        r = requests.patch(GITEA_API_URL + f'/admin/users/{username}',
-                           json=user,
-                           headers={'Authorization': 'token ' + API_TOKEN})
-        if r.status_code != 200:
-            print('ERR: Failed to update user ' + username + ': ' + r.text)
+
+        if not gitea_edit_user(username, user):
             return
 
-        print('Updated user ' + username)
+        print('Updated user', username)
+
+
+# This function adds a user to a gitea team (part of organization)
+# if the user is not already part of said team.
+def ensure_gitea_user_is_part_of_team(
+    username: str,
+    org: str,
+    team_name: str,
+) -> None:
+    teams = gitea_list_teams_for_organization(org)
+
+    if teams is None:
+        return
+
+    if team_name not in teams:
+        print(f'ERR: could not find team "{team_name}" in organization "{org}"')
+
+    gitea_add_user_to_organization_team(username, teams[team_name]['id'])
+
+    print(f'User {username} is now part of {org}/{team_name}')
+
+
+# List of teams that all users should be part of by default
+COMMON_USER_TEAMS = [
+    ("Projects", "Members"),
+    ("Kurs", "Members"),
+]
 
 
 def main():
-    # Fetch existing users
-    r = requests.get(GITEA_API_URL + '/admin/users',
-                     headers={'Authorization': 'token ' + API_TOKEN})
+    existing_users = gitea_list_all_users()
+    if existing_users is None:
+        exit(1)
 
-    if r.status_code != 200:
-        raise Exception('Failed to get users: ' + r.text)
-
-    for user in r.json():
-        existing_users[user['login']] = user
-
-    # Read the file, add each user
-    with open("/tmp/passwd-import", 'r') as f:
-        for line in f.readlines():
-            uid = int(line.split(':')[2])
-            if uid < 1000:
-                continue
-
-            shell = line.split(':')[-1]
-            if shell in BANNED_SHELLS:
-                continue
-
-            username = line.split(':')[0]
-            name = line.split(':')[4].split(',')[0]
-
-            add_user(username, name)
+    for username, name in passwd_file_parser("/tmp/passwd-import"):
+        print(f"Processing {username}")
+        add_or_patch_gitea_user(username, name, existing_users)
+        for org, team_name in COMMON_USER_TEAMS:
+            ensure_gitea_user_is_part_of_team(username, org, team_name)
+        print()
 
 
 if __name__ == '__main__':

hosts/bicep/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
     ./hardware-configuration.nix
 
-    ../../base.nix
+    ../../base
     ../../misc/metrics-exporters.nix
     ./services/nginx
 

hosts/bicep/services/matrix/mjolnir.nix

@@ -11,7 +11,7 @@
   services.mjolnir = {
     enable = true;
     pantalaimon.enable = false;
-    homeserverUrl = "http://127.0.0.1:8008";
+    homeserverUrl = "https://matrix.pvv.ntnu.no";
     accessTokenFile = config.sops.secrets."matrix/mjolnir/access_token".path;
     managementRoom = "!gsdeCoWjvYRBrzuiRq:pvv.ntnu.no";
     protectedRooms = map (a: "https://matrix.to/#/${a}") [

hosts/bicep/services/matrix/synapse.nix

@@ -157,6 +157,18 @@ in {
       '';
     };
   }
+  {
+    locations."/_synapse/admin" = {
+      proxyPass = "http://$synapse_backend";
+      extraConfig = ''
+        allow 127.0.0.1;
+        allow ::1;
+        allow ${values.hosts.bicep.ipv4};
+        allow ${values.hosts.bicep.ipv6};
+        deny all;
+      '';
+    };
+  }
   {
     locations = let
       connectionInfo = w: matrix-lib.workerConnectionResource "metrics" w;

hosts/bicep/services/mysql.nix

@@ -1,4 +1,7 @@
 { pkgs, lib, config, values, ... }:
+let
+  backupDir = "/var/lib/mysql/backups";
+in
 {
   sops.secrets."mysql/password" = {
     owner = "mysql";
@@ -36,11 +39,6 @@
     }];
   };
 
-  services.mysqlBackup = {
-    enable = true;
-    location = "/var/lib/mysql/backups";
-  };
-
   networking.firewall.allowedTCPPorts = [ 3306 ];
 
   systemd.services.mysql.serviceConfig = {
@@ -50,4 +48,58 @@
       values.ipv6-space
     ];
   };
+
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves
+  systemd.services."backup-mysql" = {
+    description = "Backup MySQL data";
+    requires = [ "mysql.service" ];
+
+    path = [
+      pkgs.coreutils
+      pkgs.rsync
+      pkgs.gzip
+      config.services.mysql.package
+    ];
+
+    script = let
+      rotations = 10;
+      # rsyncTarget = "root@isvegg.pvv.ntnu.no:/mnt/backup1/bicep/mysql";
+      rsyncTarget = "/data/backup/mysql";
+    in ''
+      set -eo pipefail
+
+      mysqldump --all-databases | gzip -c -9 --rsyncable > "${backupDir}/$(date --iso-8601)-dump.sql.gz"
+
+      while [ $(ls -1 "${backupDir}" | wc -l) -gt ${toString rotations} ]; do
+        rm $(find "${backupDir}" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)
+      done
+
+      rsync -avz --delete "${backupDir}" '${rsyncTarget}'
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "mysql";
+      Group = "mysql";
+      UMask = "0077";
+
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      ReadWritePaths = [
+        backupDir
+        "/data/backup/mysql" # NOTE: should not be part of this option once rsyncTarget is remote
+      ];
+    };
+
+    startAt = "*-*-* 02:15:00";
+  };
+
+  systemd.tmpfiles.settings."10-mysql-backup".${backupDir}.d = {
+    user = "mysql";
+    group = "mysql";
+    mode = "700";
+  };
 }

hosts/bicep/services/postgres.nix

@@ -1,4 +1,7 @@
-{ config, pkgs, ... }:
+{ config, pkgs, lib, ... }:
+let
+  backupDir = "/var/lib/postgresql/backups";
+in
 {
   services.postgresql = {
     enable = true;
@@ -90,9 +93,57 @@
   networking.firewall.allowedTCPPorts = [ 5432 ];
   networking.firewall.allowedUDPPorts = [ 5432 ];
 
-  services.postgresqlBackup = {
-    enable = true;
-    location = "/var/lib/postgres/backups";
-    backupAll = true;
+  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
+  #       another unit, it was easier to just make one ourselves
+  systemd.services."backup-postgresql" = {
+    description = "Backup PostgreSQL data";
+    requires = [ "postgresql.service" ];
+
+    path = [
+      pkgs.coreutils
+      pkgs.rsync
+      pkgs.gzip
+      config.services.postgresql.package
+    ];
+
+    script = let
+      rotations = 10;
+      # rsyncTarget = "root@isvegg.pvv.ntnu.no:/mnt/backup1/bicep/postgresql";
+      rsyncTarget = "/data/backup/postgresql";
+    in ''
+      set -eo pipefail
+
+      pg_dumpall -U postgres | gzip -c -9 --rsyncable > "${backupDir}/$(date --iso-8601)-dump.sql.gz"
+
+      while [ $(ls -1 "${backupDir}" | wc -l) -gt ${toString rotations} ]; do
+        rm $(find "${backupDir}" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)
+      done
+
+      rsync -avz --delete "${backupDir}" '${rsyncTarget}'
+    '';
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "postgres";
+      Group = "postgres";
+      UMask = "0077";
+
+      Nice = 19;
+      IOSchedulingClass = "best-effort";
+      IOSchedulingPriority = 7;
+
+      ReadWritePaths = [
+        backupDir
+        "/data/backup/postgresql" # NOTE: should not be part of this option once rsyncTarget is remote
+      ];
+    };
+
+    startAt = "*-*-* 01:15:00";
+  };
+
+  systemd.tmpfiles.settings."10-postgresql-backup".${backupDir}.d = {
+    user = "postgres";
+    group = "postgres";
+    mode = "700";
   };
 }

hosts/bob/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-      ../../base.nix
+      ../../base
       ../../misc/metrics-exporters.nix
       ./disks.nix
 

hosts/brzeczyszczykiewicz/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-      ../../base.nix
+      ../../base
       ../../misc/metrics-exporters.nix
 
       ./services/grzegorz.nix

hosts/buskerud/configuration.nix

@@ -2,7 +2,7 @@
 {
   imports = [
     ./hardware-configuration.nix
-    ../../base.nix
+    ../../base
     ../../misc/metrics-exporters.nix
 
     ./services/libvirt.nix

hosts/georg/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-      ../../base.nix
+      ../../base
       ../../misc/metrics-exporters.nix
 
       ../../modules/grzegorz.nix

hosts/ildkule/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-      ../../base.nix
+      ../../base
       ../../misc/metrics-exporters.nix
 
       ./services/monitoring

hosts/shark/configuration.nix

@@ -3,7 +3,7 @@
   imports = [
       # Include the results of the hardware scan.
       ./hardware-configuration.nix
-      ../../base.nix
+      ../../base
       ../../misc/metrics-exporters.nix
     ];
 

secrets/bekkalokk/bekkalokk.yaml

@@ -6,7 +6,7 @@ gitea:
     email-password: ENC[AES256_GCM,data:KRwC+aL1aPvJuXt91Oq1ttATMnFTnuUy,iv:ats8TygB/2pORkaTZzPOLufZ9UmvVAKoRcWNvYF1z6w=,tag:Do0fA+4cZ3+l7JJyu8hjBg==,type:str]
     passwd-ssh-key: ENC[AES256_GCM,data:L0lF0wvpayss1NU9m3A45cH0bCMQzODTFVrq6EPd1JHx54wIcoaRBYLmxXKXASzBlCg9zlwXMUIk3OQcS3kdzMKL0iqcSL2iicAcKjFIHyrWLqXgwV5pRSP/tRPcVw8KW8gz0bh33EgESs5ReddZ3VZ0Cy1s2YupMRQvBXr89k1+Hv70OWB6P06hvxhv/zKcMGI1N/dWLroMgrQuT9imw4+/Q1RqwzTYeEU+eUn24AM9GjcBg4qf3OI+6g0nXUat/upIYE28iF5J3lbUSmDSmirBLc8xgHLdOyyJPTObWYWYxlSL78T7IqiMm9lI3rtBlpJDDcn/YxZpVqN5bg2154GISNK+uR0TVSLdJ+drdGHIfIX3G78XSxf2L9rbJyRn8MQlgStfdBIQicLavQKVMrmj+XQfvEMez23WbPLjH4oViBQFI+GrOHOGy/f16cz8Sn4n+69OcsOeTxs3tKYdfq6r1XLYSJ/fe/zvxBpaZiyGXljsuyEdIyBL2A8D6uSXe3Nd3/DAdBtceFfIdN1olCdutixzVWgxaJnrel161z5A/4w=,iv:Uy46yY3jFYSvpxrgCHxRMUksnWfhf5DViLMvCXVMMl4=,tag:wFEJ5+icFrOKkc56gY0A5g==,type:str]
     ssh-known-hosts: ENC[AES256_GCM,data:zlRLoelQeumMxGqPmgMTB69X1RVWXIs2jWwc67lk0wrdNOHUs5UzV5TUA1JnQ43RslBU92+js7DkyvE5enGzw7zZE5F1ZYdGv/eCgvkTMC9BoLfzHzP6OzayPLYEt3xJ5PRocN8JUAD55cuu4LgsuebuydHPi2oWOfpbSUBKSeCh6dvk5Pp1XRDprPS5SzGLW8Xjq98QlzmfGv50meI9CDJZVF9Wq/72gkyfgtb3YVdr,iv:AF06TBitHegfWk6w07CdkHklh4ripQCmA45vswDQgss=,tag:zKh7WVXMJN2o9ZIwIkby3Q==,type:str]
-    import-user-env: ENC[AES256_GCM,data:vfaqjGEnUM9VtOPvBurz7nFwzGZt3L2EqijrQej4wiOcGCrRA4tN6kBV6NmhHqlFPsw=,iv:viPGkyOOacCWcgTu25da4qH7DC4wz2qdeC1W2WcMUdI=,tag:BllNqGQoaxqUo3lTz9LGnw==,type:str]
+    import-user-env: ENC[AES256_GCM,data:wArFwTd0ZoB4VXHPpichfnmykxGxN8y2EQsMgOPHv7zsm6A+m2rG9BWDGskQPr5Ns9o=,iv:gPUzYFSNoALJb1N0dsbNlgHIb7+xG7E9ANpmVNZURQ0=,tag:JghfRy2OcDFWKS9zX1XJ9A==,type:str]
     runners:
         alpha: ENC[AES256_GCM,data:gARxCufePz+EMVwEwRsL2iZUfh9HUowWqtb7Juz3fImeeAdbt+k3DvL/Nwgegg==,iv:3fEaWd7v7uLGTy2J7EFQGfN0ztI0uCOJRz5Mw8V5UOU=,tag:Aa6LwWeW2hfDz1SqEhUJpA==,type:str]
         beta: ENC[AES256_GCM,data:DVjS78IKWiWgf+PuijCZKx4ZaEJGhQr7vl+lc7QOg1JlA4p9Kux/tOD8+f2+jA==,iv:tk3Xk7lKWNdZ035+QVIhxXy2iJbHwunI4jRFM4It46E=,tag:9Mr6o//svYEyYhSvzkOXMg==,type:str]
@@ -92,8 +92,8 @@ sops:
             UHpLRkdQTnhkeGlWVG9VS1hkWktyckEKAdwnA9URLYZ50lMtXrU9Q09d0L3Zfsyr
             4UsvjjdnFtsXwEZ9ZzOQrpiN0Oz24s3csw5KckDni6kslaloJZsLGg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-08-13T19:49:24Z"
-    mac: ENC[AES256_GCM,data:AeJ53D+8A8mHYRmVHdqhcS1ZTbqVe5gQqJsJjMk4T/ZlNX8/V4M9mqAW2FB9m/JSdj234gDu+PBHcW70ZrCqeVsoUW/ETVgUX3W2gBmBgYJiRETp8I7/eks/5YEV6vIIxQsZNP/9dZTNX4T2wD74ELl23NSTXA/6k2tyzBlTMYo=,iv:DABafHvw+5w0PHCKqLgpwmQnv0uHOTyj+s8gdnHFTZ4=,tag:SNZ7W+6zdyuuv2AB9ir8eg==,type:str]
+    lastmodified: "2024-08-26T19:38:58Z"
+    mac: ENC[AES256_GCM,data:3FyfZPmJ7znQEul+IwqN1ZaM53n6os3grquJwJ9vfyDSc2h8UZBhqYG+2uW9Znp9DSIjuhCUI8iqGKRJE0M/6IDICeXms/5+ynVFOS9bA2cdzPvWaj0FFAd2x3g4Vhs47+vRlsnIe/tMiKU3IOvzOfI6KAUHc9L2ySrzH7z2+fo=,iv:1iZSR9qOIEtf+fNbtWSwJBIUEQGKadfHSVOnkFzOwq8=,tag:Sk6JEU1B6Rd1GXLYC6rQtQ==,type:str]
     pgp:
         - created_at: "2024-08-04T00:03:28Z"
           enc: |-