felixalb/pvv-nixos-config
1
0
Fork 0
mirror of https://git.pvv.ntnu.no/Drift/pvv-nixos-config.git synced 2025-04-03 12:51:23 +02:00
Code Issues Projects Releases Wiki Activity

bekkalokk/gitea: add robots.txt

Browse Source
This commit is contained in:
h7x4 2025-03-15 14:33:13 +01:00
parent c8bf3b7c01
commit a040ef59a8
No known key found for this signature in database
GPG Key ID: 9F2F7D8250F35146
2 changed files with 104 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
flake.nix
View File

@ -105,6 +105,7 @@
modules = [ modules = [
inputs.nix-gitea-themes.nixosModules.default inputs.nix-gitea-themes.nixosModules.default
inputs.pvv-nettsiden.nixosModules.default inputs.pvv-nettsiden.nixosModules.default
self.nixosModules.robots-txt
]; ];
}; };
bob = stableNixosConfig "bob" { bob = stableNixosConfig "bob" {

103
hosts/bekkalokk/services/gitea/default.nix
View File

@ -163,5 +163,108 @@ in {
}; };
}; };
environment.robots-txt."gitea" = {
virtualHost = domain;
rules = [
{
pre_comment = ''
Gitea internals
See these for more information:
- https://gitea.com/robots.txt
- https://codeberg.org/robots.txt
'';
User-agent = "*";
Disallow = [
"/api/*"
"/avatars"
"/*/*/src/commit/*"
"/*/*/commit/*"
"/*/*/*/refs/*"
"/*/*/*/star"
"/*/*/*/watch"
"/*/*/labels"
"/*/*/activity/*"
"/vendor/*"
"/swagger.*.json"
"/repo/create"
"/repo/migrate"
"/org/create"
"/*/*/fork"
"/*/*/watchers"
"/*/*/stargazers"
"/*/*/forks"
"*/.git/"
"/*.git"
"/*.atom"
"/*.rss"
];
}
{
pre_comment = "Language Spam";
Disallow = "/*?lang=";
}
{
pre_comment = ''
AI bots
Sourced from:
- https://www.vg.no/robots.txt
- https://codeberg.org/robots.txt
'';
User-agent = [
"AI2Bot"
"Ai2Bot-Dolma"
"Amazonbot"
"Applebot-Extended"
"Bytespider"
"CCBot"
"ChatGPT-User"
"Claude-Web"
"ClaudeBot"
"Crawlspace"
"Diffbot"
"FacebookBot"
"FriendlyCrawler"
"GPTBot"
"Google-Extended"
"ICC-Crawler"
"ImagesiftBot"
"Kangaroo Bot"
"Meta-ExternalAgent"
"OAI-SearchBot"
"Omgili"
"Omgilibot"
"PanguBot"
"PerplexityBot"
"PetalBot"
"Scrapy"
"SemrushBot-OCOB"
"Sidetrade indexer bot"
"Timpibot"
"VelenPublicWebCrawler"
"Webzio-Extended"
"YouBot"
"anthropic-ai"
"cohere-ai"
"cohere-training-data-crawler"
"facebookexternalhit"
"iaskspider/2.0"
"img2dataset"
"meta-externalagent"
"omgili"
"omgilibot"
];
Disallow = "/";
}
{
Crawl-delay = "2";
}
{
Sitemap = "https://${domain}/sitemap.xml";
}
];
};
networking.firewall.allowedTCPPorts = [ sshPort ]; networking.firewall.allowedTCPPorts = [ sshPort ];
} }