kommode/gitea: add robots.txt

flake.lock

@@ -1,27 +1,5 @@
 {
   "nodes": {
-    "bro": {
-      "inputs": {
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "rust-overlay": "rust-overlay"
-      },
-      "locked": {
-        "lastModified": 1779629827,
-        "narHash": "sha256-nrlB50/oelB8oFx9DhOoXI5z0VoTZGEA6XxYvkvpqDA=",
-        "ref": "main",
-        "rev": "7d0f35e12e4dec39f981c08fc33515589f41f4a5",
-        "revCount": 3,
-        "type": "git",
-        "url": "https://git.pvv.ntnu.no/Projects/bro.git"
-      },
-      "original": {
-        "ref": "main",
-        "type": "git",
-        "url": "https://git.pvv.ntnu.no/Projects/bro.git"
-      }
-    },
     "crane": {
       "locked": {
         "lastModified": 1776635034,
@@ -123,7 +101,7 @@
         "nixpkgs": [
           "nixpkgs-unstable"
         ],
-        "rust-overlay": "rust-overlay_2"
+        "rust-overlay": "rust-overlay"
       },
       "locked": {
         "lastModified": 1777019032,
@@ -187,7 +165,7 @@
         "nixpkgs": [
           "nixpkgs"
         ],
-        "rust-overlay": "rust-overlay_3"
+        "rust-overlay": "rust-overlay_2"
       },
       "locked": {
         "lastModified": 1767906976,
@@ -374,7 +352,6 @@
     },
     "root": {
       "inputs": {
-        "bro": "bro",
         "dibbler": "dibbler",
         "disko": "disko",
         "gergle": "gergle",
@@ -400,7 +377,7 @@
         "nixpkgs": [
           "nixpkgs"
         ],
-        "rust-overlay": "rust-overlay_4"
+        "rust-overlay": "rust-overlay_3"
       },
       "locked": {
         "lastModified": 1778600367,
@@ -419,27 +396,6 @@
       }
     },
     "rust-overlay": {
-      "inputs": {
-        "nixpkgs": [
-          "bro",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1779419951,
-        "narHash": "sha256-dMX0PUslUHPajP6o8FEoRdFv9afq/dec4POR0vVfjK4=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "5b5c521d6cae9ef4aa32f888eb2c0ce595c9be52",
-        "type": "github"
-      },
-      "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "type": "github"
-      }
-    },
-    "rust-overlay_2": {
       "inputs": {
         "nixpkgs": [
           "greg-ng",
@@ -460,7 +416,7 @@
         "type": "github"
       }
     },
-    "rust-overlay_3": {
+    "rust-overlay_2": {
       "inputs": {
         "nixpkgs": [
           "minecraft-heatmap",
@@ -481,7 +437,7 @@
         "type": "github"
       }
     },
-    "rust-overlay_4": {
+    "rust-overlay_3": {
       "inputs": {
         "nixpkgs": [
           "roowho2",

flake.nix

@@ -47,9 +47,6 @@
 
     qotd.url = "git+https://git.pvv.ntnu.no/Projects/qotd.git?ref=main";
     qotd.inputs.nixpkgs.follows = "nixpkgs";
-
-    bro.url = "git+https://git.pvv.ntnu.no/Projects/bro.git?ref=main";
-    bro.inputs.nixpkgs.follows = "nixpkgs";
   };
 
   outputs = {
@@ -217,14 +214,7 @@
         };
         shark = stableNixosConfig "shark" {};
         wenche = stableNixosConfig "wenche" {};
-        temmie = stableNixosConfig "temmie" {
+        temmie = stableNixosConfig "temmie" {};
-          overlays = [
-            inputs.bro.overlays.default
-          ];
-          modules = [
-            inputs.bro.nixosModules.default
-          ];
-        };
         gluttony = stableNixosConfig "gluttony" {
           overlays = [
             (final: prev: { bluemap = final.callPackage ./packages/bluemap.nix {}; })
@@ -239,6 +229,7 @@
           modules = [
             inputs.nix-gitea-themes.nixosModules.default
             inputs.disko.nixosModules.disko
+            self.nixosModules.robots-txt
           ];
         };
 

hosts/kommode/services/gitea/default.nix

@@ -193,6 +193,109 @@ in {
     };
   };
 
+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
+
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
+
+          Sourced from:
+          - https://www.vg.no/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = [
+          "AI2Bot"
+          "Ai2Bot-Dolma"
+          "Amazonbot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "Claude-Web"
+          "ClaudeBot"
+          "Crawlspace"
+          "Diffbot"
+          "FacebookBot"
+          "FriendlyCrawler"
+          "GPTBot"
+          "Google-Extended"
+          "ICC-Crawler"
+          "ImagesiftBot"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "OAI-SearchBot"
+          "Omgili"
+          "Omgilibot"
+          "PanguBot"
+          "PerplexityBot"
+          "PetalBot"
+          "Scrapy"
+          "SemrushBot-OCOB"
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+      }
+      {
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
+  };
+
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
   services.rsync-pull-targets = {

hosts/temmie/services/userweb/mail.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, ... }:
+{ config, lib, ... }:
 {
   services.postfix.enable = lib.mkForce false;
 
@@ -9,74 +9,4 @@
       remotes = "mail.pvv.ntnu.no smtp --port=25";
     };
   };
-
-  services.bro = {
-    enable = true;
-
-    instances.userweb-sendmail = {
-      enable = true;
-
-      client = {
-        settings.BRO_FILE_FLAGS = [
-          "-C"
-        ];
-      };
-
-      server = {
-        settings = {
-          executable = lib.getExe pkgs.system-sendmail;
-          # allowed-env = [  ];
-        };
-      };
-    };
-  };
-
-  environment.systemPackages = [
-    (config.services.bro.instances.userweb-sendmail.client.package.overrideAttrs (prev: {
-      postInstall = (prev.postInstall or "") + ''
-        mv "$out/bin/sendmail" "$out/bin/bro-sendmail"
-      '';
-    }))
-  ];
-
-  systemd.services.bro-userweb-sendmail = {
-    serviceConfig = {
-      User = "nobody";
-      Group = "nobody";
-
-      AmbientCapabilities = "";
-      CapabilityBoundingSet = "";
-      NoNewPrivileges = false;
-      ProtectSystem = "strict";
-      ProtectHome = true;
-      PrivateTmp = true;
-      PrivateDevices = true;
-      PrivateUsers = false;
-      ProtectHostname = true;
-      ProtectClock = true;
-      ProtectKernelTunables = true;
-      ProtectKernelModules = true;
-      ProtectKernelLogs = true;
-      ProtectControlGroups = true;
-      RestrictAddressFamilies = [
-        "AF_UNIX"
-        "AF_INET"
-        "AF_INET6"
-        "AF_NETLINK"
-      ];
-      LockPersonality = true;
-      MemoryDenyWriteExecute = true;
-      PrivateMounts = true;
-      ProcSubset = "pid";
-      ProtectProc = "invisible";
-      RemoveIPC = true;
-      RestrictNamespaces = true;
-      RestrictRealtime = true;
-      RestrictSUIDSGID = true;
-      UMask = "0027";
-      SystemCallArchitectures = "native";
-      # SystemCallFilter = [
-      # ];
-    };
-  };
 }