kommode/gitea: add robots.txt

flake.nix

@@ -196,6 +196,7 @@
         modules = [
           inputs.nix-gitea-themes.nixosModules.default
           inputs.disko.nixosModules.disko
+          self.nixosModules.robots-txt
         ];
       };
 

hosts/bicep/configuration.nix

@@ -9,8 +9,8 @@
     ./services/calendar-bot.nix
     #./services/git-mirrors
     ./services/minecraft-heatmap.nix
-    ./services/mysql
-    ./services/postgresql
+    ./services/mysql.nix
+    ./services/postgres.nix
 
     ./services/matrix
   ];

hosts/bicep/services/mysql.nix

@@ -4,8 +4,6 @@ let
   dataDir = "/data/mysql";
 in
 {
-  imports = [ ./backup.nix ];
-
   sops.secrets."mysql/password" = {
     owner = "mysql";
     group = "mysql";
@@ -44,6 +42,27 @@ in
     }];
   };
 
+  services.mysqlBackup = lib.mkIf cfg.enable {
+    enable = true;
+    location = "/var/lib/mysql-backups";
+  };
+
+  services.rsync-pull-targets = lib.mkIf cfg.enable {
+    enable = true;
+    locations.${config.services.mysqlBackup.location} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgj55/7Cnj4cYMJ5sIkl+OwcGeBe039kXJTOf2wvo9j mysql rsync backup";
+    };
+  };
+
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 3306 ];
 
   systemd.tmpfiles.settings."10-mysql".${dataDir}.d = lib.mkIf cfg.enable {

hosts/bicep/services/mysql/backup.nix

@@ -1,82 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.mysql;
-  backupDir = "/data/mysql-backups";
-in
-{
-  # services.mysqlBackup = lib.mkIf cfg.enable {
-  #   enable = true;
-  #   location = "/var/lib/mysql-backups";
-  # };
-
-  systemd.tmpfiles.settings."10-mysql-backups".${backupDir}.d = {
-	  user = "mysql";
-	  group = "mysql";
-	  mode = "700";
-	};
-
-  services.rsync-pull-targets = lib.mkIf cfg.enable {
-    enable = true;
-    locations.${backupDir} = {
-      user = "root";
-      rrsyncArgs.ro = true;
-      authorizedKeysAttrs = [
-        "restrict"
-        "no-agent-forwarding"
-        "no-port-forwarding"
-        "no-pty"
-        "no-X11-forwarding"
-      ];
-      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJgj55/7Cnj4cYMJ5sIkl+OwcGeBe039kXJTOf2wvo9j mysql rsync backup";
-    };
-  };
-
-  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
-  #       another unit, it was easier to just make one ourselves.
-  systemd.services."backup-mysql" = lib.mkIf cfg.enable {
-    description = "Backup MySQL data";
-    requires = [ "mysql.service" ];
-
-    path = with pkgs; [
-      cfg.package
-      coreutils
-      zstd
-    ];
-
-    script = let
-      rotations = 2;
-    in ''
-      set -euo pipefail
-
-      OUT_FILE="$STATE_DIRECTORY/mysql-dump-$(date --iso-8601).sql.zst"
-
-      mysqldump --all-databases | zstd --compress -9 --rsyncable -o "$OUT_FILE"
-
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
-      rm "$STATE_DIRECTORY/mysql-dump-latest.sql.zst" ||:
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/mysql-dump-latest.sql.zst"
-
-      while [ $(find -type f -printf '.' "$STATE_DIRECTORY" | wc -c) -gt ${toString (rotations + 1)} ]; do
-        rm $(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)
-      done
-    '';
-
-    serviceConfig = {
-      Type = "oneshot";
-      User = "mysql";
-      Group = "mysql";
-      UMask = "0077";
-
-      Nice = 19;
-      IOSchedulingClass = "best-effort";
-      IOSchedulingPriority = 7;
-
-      StateDirectory = [ "mysql-backups" ];
-      BindPaths = [ "${backupDir}:/var/lib/mysql-backups" ];
-
-      # TODO: hardening
-    };
-
-    startAt = "*-*-* 02:15:00";
-  };
-}

hosts/bicep/services/postgres.nix

@@ -3,8 +3,6 @@ let
   cfg = config.services.postgresql;
 in
 {
-  imports = [ ./backup.nix ];
-
   services.postgresql = {
     enable = true;
     package = pkgs.postgresql_18;
@@ -123,4 +121,26 @@ in
 
   networking.firewall.allowedTCPPorts = lib.mkIf cfg.enable [ 5432 ];
   networking.firewall.allowedUDPPorts = lib.mkIf cfg.enable [ 5432 ];
+
+  services.postgresqlBackup = lib.mkIf cfg.enable {
+    enable = true;
+    location = "/var/lib/postgres-backups";
+    backupAll = true;
+  };
+
+  services.rsync-pull-targets = lib.mkIf cfg.enable {
+    enable = true;
+    locations.${config.services.postgresqlBackup.location} = {
+      user = "root";
+      rrsyncArgs.ro = true;
+      authorizedKeysAttrs = [
+        "restrict"
+        "no-agent-forwarding"
+        "no-port-forwarding"
+        "no-pty"
+        "no-X11-forwarding"
+      ];
+      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvO7QX7QmwSiGLXEsaxPIOpAqnJP3M+qqQRe5dzf8gJ postgresql rsync backup";
+    };
+  };
 }

hosts/bicep/services/postgresql/backup.nix

@@ -1,83 +0,0 @@
-{ config, lib, pkgs, ... }:
-let
-  cfg = config.services.postgresql;
-  backupDir = "/data/postgresql-backups";
-in
-{
-  # services.postgresqlBackup = lib.mkIf cfg.enable {
-  #   enable = true;
-  #   location = "/var/lib/postgresql-backups";
-  #   backupAll = true;
-  # };
-
-  systemd.tmpfiles.settings."10-postgresql-backups".${backupDir}.d = {
-	  user = "postgres";
-	  group = "postgres";
-	  mode = "700";
-	};
-
-  services.rsync-pull-targets = lib.mkIf cfg.enable {
-    enable = true;
-    locations.${backupDir} = {
-      user = "root";
-      rrsyncArgs.ro = true;
-      authorizedKeysAttrs = [
-        "restrict"
-        "no-agent-forwarding"
-        "no-port-forwarding"
-        "no-pty"
-        "no-X11-forwarding"
-      ];
-      publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGvO7QX7QmwSiGLXEsaxPIOpAqnJP3M+qqQRe5dzf8gJ postgresql rsync backup";
-    };
-  };
-
-  # NOTE: instead of having the upstream nixpkgs postgres backup unit trigger
-  #       another unit, it was easier to just make one ourselves
-  systemd.services."backup-postgresql" = {
-    description = "Backup PostgreSQL data";
-    requires = [ "postgresql.service" ];
-
-    path = with pkgs; [
-      coreutils
-      zstd
-      cfg.package
-    ];
-
-    script = let
-      rotations = 2;
-    in ''
-      set -euo pipefail
-
-      OUT_FILE="$STATE_DIRECTORY/postgresql-dump-$(date --iso-8601).sql.zst"
-
-      pg_dumpall -U postgres | zstd --compress -9 --rsyncable -o "$OUT_FILE"
-
-      # NOTE: this needs to be a hardlink for rrsync to allow sending it
-      rm "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst" ||:
-      ln -T "$OUT_FILE" "$STATE_DIRECTORY/postgresql-dump-latest.sql.zst"
-
-      while [ $(find -type f -printf '.' "$STATE_DIRECTORY" | wc -c) -gt ${toString (rotations + 1)} ]; do
-        rm $(find "$STATE_DIRECTORY" -type f -printf '%T+ %p\n' | sort | head -n 1 | cut -d' ' -f2)
-      done
-    '';
-
-    serviceConfig = {
-      Type = "oneshot";
-      User = "postgres";
-      Group = "postgres";
-      UMask = "0077";
-
-      Nice = 19;
-      IOSchedulingClass = "best-effort";
-      IOSchedulingPriority = 7;
-
-      StateDirectory = [ "postgresql-backups" ];
-      BindPaths = [ "${backupDir}:/var/lib/postgresql-backups" ];
-
-      # TODO: hardening
-    };
-
-    startAt = "*-*-* 01:15:00";
-  };
-}

hosts/kommode/services/gitea/default.nix

@@ -193,6 +193,109 @@ in {
     };
   };
 
+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
+
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
+
+          Sourced from:
+          - https://www.vg.no/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = [
+          "AI2Bot"
+          "Ai2Bot-Dolma"
+          "Amazonbot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "Claude-Web"
+          "ClaudeBot"
+          "Crawlspace"
+          "Diffbot"
+          "FacebookBot"
+          "FriendlyCrawler"
+          "GPTBot"
+          "Google-Extended"
+          "ICC-Crawler"
+          "ImagesiftBot"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "OAI-SearchBot"
+          "Omgili"
+          "Omgilibot"
+          "PanguBot"
+          "PerplexityBot"
+          "PetalBot"
+          "Scrapy"
+          "SemrushBot-OCOB"
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+      }
+      {
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
+  };
+
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
   services.rsync-pull-targets = {

packages/mediawiki-extensions/default.nix

@@ -33,63 +33,63 @@ in
 lib.mergeAttrsList [
   (mw-ext {
     name = "CodeEditor";
-    commit = "83e1d0c13f34746f0d7049e38b00e9ab0a47c23f";
-    hash = "sha256-qH9fSQZGA+z6tBSh1DaTKLcujqA6K/vQmZML9w5X8mU=";
+    commit = "6e5b06e8cf2d040c0abb53ac3735f9f3c96a7a4f";
+    hash = "sha256-Jee+Ws9REUohywhbuemixXKaTRc54+cIlyUNDCyYcEM=";
   })
   (mw-ext {
     name = "CodeMirror";
-    commit = "af2b08b9ad2b89a64b2626cf80b026c5b45e9922";
-    hash = "sha256-CxXPwCKUlF9Tg4JhwLaKQyvt43owq75jCugVtb3VX+I=";
+    commit = "da9c5d4f03e6425f6f2cf68b75d21311e0f7e77e";
+    hash = "sha256-aL+v9xeqKHGmQVUWVczh54BkReu+fP49PT1NP7eTC6k=";
   })
   (mw-ext {
     name = "DeleteBatch";
-    commit = "3d6f2fd0e3efdae1087dd0cc8b1f96fe0edf734f";
-    hash = "sha256-iD9EjDIW7AGpZan74SIRcr54dV8W7xMKIDjatjdVkKs=";
+    commit = "122072bbfb4eab96ed8c1451a3e74b5557054c58";
+    hash = "sha256-L6AXoyFJEZoAQpLO6knJvYtQ6JJPMtaa+WhpnwbJeNU=";
   })
   (mw-ext {
     name = "PluggableAuth";
-    commit = "85e96acd1ac0ebcdaa29c20eae721767a938f426";
-    hash = "sha256-bMVhrg8FsfWhXF605Cj5TgI0A6Jy/MIQ5aaUcLQQ0Ss=";
+    commit = "5caf605b9dfdd482cb439d1ba2000cba37f8b018";
+    hash = "sha256-TYJqR9ZvaWJ7i1t0XfgUS05qqqCgxAH8tRTklz/Bmlg=";
   })
   (mw-ext {
     name = "Popups";
-    commit = "410e2343c32a7b18dcdc2bbd995b0bfdf3bf5f37";
-    hash = "sha256-u2AlR75x54rCpiK9Mz00D9odJCn8fmi6DRU4QKmKqSc=";
+    commit = "7ed940a09f83f869cbc0bc20f3ca92f85b534951";
+    hash = "sha256-pcDPcu4kSvMHfSOuShrod694TKI9Oo3AEpMP9DXp9oY=";
   })
   (mw-ext {
     name = "Scribunto";
-    commit = "904f323f343dba5ff6a6cdd143c4a8ef5b7d2c55";
-    hash = "sha256-ZOVYhjMMyWbqwZOBb39hMIRmzzCPEnz2y8Q2jgyeERw=";
+    commit = "e755852a8e28a030a21ded2d5dd7270eb933b683";
+    hash = "sha256-zyI5nSE+KuodJOWyV0CQM7G0GfkKEgfoF/czi2/qk98=";
   })
   (mw-ext {
     name = "SimpleSAMLphp";
     kebab-name = "simple-saml-php";
-    commit = "a2f77374713473d594e368de24539aebcc1a800a";
-    hash = "sha256-5+t3VQFKcrIffDNPJ4RWBIWS6K1gTOcEleYWmM6xWms=";
+    commit = "d41b4efd3cc44ca3f9f12e35385fc64337873c2a";
+    hash = "sha256-wfzXtsEEEjQlW5QE4Rf8pasAW/KSJsLkrez13baxeqA=";
   })
   (mw-ext {
     name = "TemplateData";
-    commit = "76a6a04bd13a606923847ba68750b5d98372cacd";
-    hash = "sha256-X2+U5PMqzkSljw2ypIvJUSaPDaonTkQx89OgKzf5scw=";
+    commit = "fd7cf4d95a70ef564130266f2a6b18f33a2a2ff9";
+    hash = "sha256-5OhDPFhIi55Eh5+ovMP1QTjNBb9Sm/3vyArNCApAgSw=";
   })
   (mw-ext {
     name = "TemplateStyles";
-    commit = "7de60a8da6576d7930f293d19ef83529abf52704";
-    hash = "sha256-iPmFDoO5V4964CVyd1mBSQcNlW34odbvpm2CfDBlPBU=";
+    commit = "0f7b94a0b094edee1c2a9063a3c42a1bdc0282d9";
+    hash = "sha256-R406FgNcIip9St1hurtZoPPykRQXBrkJRKA9hapG81I=";
   })
   (mw-ext {
     name = "UserMerge";
-    commit = "71eb53ff4289ac4efaa31685ab8b6483c165a584";
-    hash = "sha256-OfKSEPgctfr659oh5jf99T0Rzqn+60JhNaZq+2gfubk=";
+    commit = "d1917817dd287e7d883e879459d2d2d7bc6966f2";
+    hash = "sha256-la3/AQ38DMsrZ2f24T/z3yKzIrbyi3w6FIB5YfxGK9U=";
   })
   (mw-ext {
     name = "VisualEditor";
-    commit = "a6a63f53605c4d596c3df1dcc2583ffd3eb8d929";
-    hash = "sha256-4d8picO66uzKoxh1TdyvKLHebc6ZL7N2DdXLV2vgBL4=";
+    commit = "032364cfdff33818e6ae0dfa251fe3973b0ae4f3";
+    hash = "sha256-AQDdq9r6rSo8h4u1ERonH14/1i1BgLGdzANEiQ065PU=";
   })
   (mw-ext {
     name = "WikiEditor";
-    commit = "0a5719bb95326123dd0fee1f88658358321ed7be";
-    hash = "sha256-eQMyjhdm1E6TkktIHad1NMeMo8QNoO8z4A05FYOMCwQ=";
+    commit = "cb9f7e06a9c59b6d3b31c653e5886b7f53583d01";
+    hash = "sha256-UWi3Ac+LCOLliLkXnS8YL0rD/HguuPH5MseqOm0z7s4=";
   })
 ]