diff --git a/flake.nix b/flake.nix index ed7eaa6..5ed1915 100644 --- a/flake.nix +++ b/flake.nix @@ -105,6 +105,7 @@ modules = [ inputs.nix-gitea-themes.nixosModules.default inputs.pvv-nettsiden.nixosModules.default + self.nixosModules.robots-txt ]; }; bob = stableNixosConfig "bob" { diff --git a/hosts/bekkalokk/services/gitea/default.nix b/hosts/bekkalokk/services/gitea/default.nix index d07cc01..5d74166 100644 --- a/hosts/bekkalokk/services/gitea/default.nix +++ b/hosts/bekkalokk/services/gitea/default.nix @@ -163,5 +163,108 @@ in { }; }; + environment.robots-txt."gitea" = { + virtualHost = domain; + rules = [ + { + pre_comment = '' + Gitea internals + + See these for more information: + - https://gitea.com/robots.txt + - https://codeberg.org/robots.txt + ''; + User-agent = "*"; + Disallow = [ + "/api/*" + "/avatars" + "/*/*/src/commit/*" + "/*/*/commit/*" + "/*/*/*/refs/*" + "/*/*/*/star" + "/*/*/*/watch" + "/*/*/labels" + "/*/*/activity/*" + "/vendor/*" + "/swagger.*.json" + "/repo/create" + "/repo/migrate" + "/org/create" + "/*/*/fork" + "/*/*/watchers" + "/*/*/stargazers" + "/*/*/forks" + "*/.git/" + "/*.git" + "/*.atom" + "/*.rss" + ]; + } + { + pre_comment = "Language Spam"; + Disallow = "/*?lang="; + } + { + pre_comment = '' + AI bots + + Sourced from: + - https://www.vg.no/robots.txt + - https://codeberg.org/robots.txt + ''; + User-agent = [ + "AI2Bot" + "Ai2Bot-Dolma" + "Amazonbot" + "Applebot-Extended" + "Bytespider" + "CCBot" + "ChatGPT-User" + "Claude-Web" + "ClaudeBot" + "Crawlspace" + "Diffbot" + "FacebookBot" + "FriendlyCrawler" + "GPTBot" + "Google-Extended" + "ICC-Crawler" + "ImagesiftBot" + "Kangaroo Bot" + "Meta-ExternalAgent" + "OAI-SearchBot" + "Omgili" + "Omgilibot" + "PanguBot" + "PerplexityBot" + "PetalBot" + "Scrapy" + "SemrushBot-OCOB" + "Sidetrade indexer bot" + "Timpibot" + "VelenPublicWebCrawler" + "Webzio-Extended" + "YouBot" + "anthropic-ai" + "cohere-ai" + "cohere-training-data-crawler" + "facebookexternalhit" + "iaskspider/2.0" + "img2dataset" + "meta-externalagent" + "omgili" + "omgilibot" + ]; + Disallow = "/"; + } + { + Crawl-delay = "2"; + } + { + Sitemap = "https://${domain}/sitemap.xml"; + } + ]; + }; + networking.firewall.allowedTCPPorts = [ sshPort ]; }