kommode/gitea: add robots.txt

2026-01-07 07:58:25 +01:00 · 2025-12-30 00:34:26 +09:00
6 changed files with 113 additions and 346 deletions
--- a/flake.lock
+++ b/flake.lock
@@ -1,26 +1,5 @@
 {
  "nodes": {
-    "devshell": {
-      "inputs": {
-        "nixpkgs": [
-          "nix-topology",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1728330715,
-        "narHash": "sha256-xRJ2nPOXb//u1jaBnDP56M7v5ldavjbtR6lfGqSvcKg=",
-        "owner": "numtide",
-        "repo": "devshell",
-        "rev": "dd6b80932022cea34a019e2bb32f6fa9e494dfef",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "devshell",
-        "type": "github"
-      }
-    },
    "disko": {
      "inputs": {
        "nixpkgs": [
@@ -42,40 +21,6 @@
        "type": "github"
      }
    },
-    "flake-compat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1696426674,
-        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1726560853,
-        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
    "gergle": {
      "inputs": {
        "nixpkgs": [
@@ -97,28 +42,6 @@
        "url": "https://git.pvv.ntnu.no/Grzegorz/gergle.git"
      }
    },
-    "gitignore": {
-      "inputs": {
-        "nixpkgs": [
-          "nix-topology",
-          "pre-commit-hooks",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1709087332,
-        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
-        "type": "github"
-      },
-      "original": {
-        "owner": "hercules-ci",
-        "repo": "gitignore.nix",
-        "type": "github"
-      }
-    },
    "greg-ng": {
      "inputs": {
        "nixpkgs": [
@@ -247,30 +170,6 @@
        "url": "https://git.pvv.ntnu.no/Drift/nix-gitea-themes.git"
      }
    },
-    "nix-topology": {
-      "inputs": {
-        "devshell": "devshell",
-        "flake-utils": "flake-utils",
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "pre-commit-hooks": "pre-commit-hooks"
-      },
-      "locked": {
-        "lastModified": 1765969653,
-        "narHash": "sha256-qVpQxyvdByeDfb+d+jhbyNna2Ie+w85iHpt4Qu0rv/E=",
-        "owner": "oddlama",
-        "repo": "nix-topology",
-        "rev": "0ed73e5a1b65eb8ed388d070ebe8dedb9182f466",
-        "type": "github"
-      },
-      "original": {
-        "owner": "oddlama",
-        "ref": "main",
-        "repo": "nix-topology",
-        "type": "github"
-      }
-    },
    "nixpkgs": {
      "locked": {
        "lastModified": 1764806471,
@@ -297,33 +196,6 @@
        "url": "https://nixos.org/channels/nixos-unstable-small/nixexprs.tar.xz"
      }
    },
-    "pre-commit-hooks": {
-      "inputs": {
-        "flake-compat": "flake-compat",
-        "gitignore": "gitignore",
-        "nixpkgs": [
-          "nix-topology",
-          "nixpkgs"
-        ],
-        "nixpkgs-stable": [
-          "nix-topology",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1730797577,
-        "narHash": "sha256-SrID5yVpyUfknUTGWgYkTyvdr9J1LxUym4om3SVGPkg=",
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "rev": "1864030ed24a2b8b4e4d386a5eeaf0c5369e50a9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "cachix",
-        "repo": "pre-commit-hooks.nix",
-        "type": "github"
-      }
-    },
    "pvv-calendar-bot": {
      "inputs": {
        "nixpkgs": [
@@ -376,7 +248,6 @@
        "minecraft-heatmap": "minecraft-heatmap",
        "minecraft-kartverket": "minecraft-kartverket",
        "nix-gitea-themes": "nix-gitea-themes",
-        "nix-topology": "nix-topology",
        "nixpkgs": "nixpkgs",
        "nixpkgs-unstable": "nixpkgs-unstable",
        "pvv-calendar-bot": "pvv-calendar-bot",
@@ -446,21 +317,6 @@
        "repo": "sops-nix",
        "type": "github"
      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
    }
  },
  "root": "root",
--- a/flake.nix
+++ b/flake.nix
@@ -11,9 +11,6 @@
    disko.url = "github:nix-community/disko/v1.11.0";
    disko.inputs.nixpkgs.follows = "nixpkgs";

-    nix-topology.url = "github:oddlama/nix-topology/main";
-    nix-topology.inputs.nixpkgs.follows = "nixpkgs";
-
    pvv-nettsiden.url = "git+https://git.pvv.ntnu.no/Projects/nettsiden.git?ref=main";
    pvv-nettsiden.inputs.nixpkgs.follows = "nixpkgs";

@@ -153,6 +150,12 @@
          self.nixosModules.bluemap
        ];
      };
+      bob = stableNixosConfig "bob" {
+        modules = [
+          disko.nixosModules.disko
+          { disko.devices.disk.disk1.device = "/dev/vda"; }
+        ];
+      };
      ildkule = stableNixosConfig "ildkule" { };
      #ildkule-unstable = unstableNixosConfig "ildkule" { };
      shark = stableNixosConfig "shark" { };
@@ -164,6 +167,7 @@
        ];
        modules = [
          inputs.nix-gitea-themes.nixosModules.default
+          self.nixosModules.robots-txt
        ];
      };

@@ -243,50 +247,14 @@
        bluemap = pkgs.callPackage ./packages/bluemap.nix { };

        out-of-your-element = pkgs.callPackage ./packages/out-of-your-element.nix { };
-      }
-      //
-      # Mediawiki extensions
+      } //
      (lib.pipe null [
        (_: pkgs.callPackage ./packages/mediawiki-extensions { })
        (lib.flip builtins.removeAttrs ["override" "overrideDerivation"])
        (lib.mapAttrs' (name: lib.nameValuePair "mediawiki-${name}"))
      ])
-      //
-      # Machines
-      lib.genAttrs allMachines
-        (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel)
-      //
-      # Nix-topology
-      (let
-        topology' = import inputs.nix-topology {
-          pkgs = import nixpkgs {
-            system = "x86_64-linux";
-            overlays = [ inputs.nix-topology.overlays.default ];
-          };
-
-          modules = [
-            ./topology
-            {
-              nixosConfigurations = lib.mapAttrs (_name: nixosCfg: nixosCfg.extendModules {
-                modules = [
-                  inputs.nix-topology.nixosModules.default
-                  ./topology/service-extractors/greg-ng.nix
-                ];
-              }) self.nixosConfigurations;
-            }
-          ];
-        };
-      in {
-        topology = topology'.config.output;
-        topology-png = pkgs.runCommand "pvv-config-topology-png" {
-          nativeBuildInputs = [ pkgs.writableTmpDirAsHomeHook ];
-        } ''
-          mkdir -p "$out"
-          for file in '${topology'.config.output}'/*.svg; do
-            ${lib.getExe pkgs.imagemagick} -density 300 -background none "$file" "$out"/"$(basename "''${file%.svg}.png")"
-          done
-        '';
-      });
+      // lib.genAttrs allMachines
+        (machine: self.nixosConfigurations.${machine}.config.system.build.toplevel);
    };
  };
 }
--- a/hosts/kommode/services/gitea/default.nix
+++ b/hosts/kommode/services/gitea/default.nix
@@ -193,6 +193,109 @@ in {
    };
  };

+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
+
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
+
+          Sourced from:
+          - https://www.vg.no/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = [
+          "AI2Bot"
+          "Ai2Bot-Dolma"
+          "Amazonbot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "Claude-Web"
+          "ClaudeBot"
+          "Crawlspace"
+          "Diffbot"
+          "FacebookBot"
+          "FriendlyCrawler"
+          "GPTBot"
+          "Google-Extended"
+          "ICC-Crawler"
+          "ImagesiftBot"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "OAI-SearchBot"
+          "Omgili"
+          "Omgilibot"
+          "PanguBot"
+          "PerplexityBot"
+          "PetalBot"
+          "Scrapy"
+          "SemrushBot-OCOB"
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+      }
+      {
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
+  };
+
  networking.firewall.allowedTCPPorts = [ sshPort ];

  systemd.services.gitea-dump = {
--- a/topology/default.nix
+++ b/topology/default.nix
@@ -1,149 +0,0 @@
-{ lib, config, ... }:
-let
-  inherit
-    (config.lib.topology)
-    mkInternet
-    mkRouter
-    mkSwitch
-    mkDevice
-    mkConnection
-    mkConnectionRev;
-  values = import ../values.nix;
-in {
-  ### Networks
-
-  networks.pvv = {
-    name = "PVV Network";
-    cidrv4 = values.ipv4-space;
-    cidrv6 = values.ipv6-space;
-  };
-
-  networks.site-vpn = {
-    name = "OpenVPN Site to Site";
-    style = {
-      primaryColor = "#9dd68d";
-      secondaryColor = null;
-      pattern = "dashed";
-    };
-  };
-
-  networks.ntnu = {
-    name = "NTNU";
-  };
-
-  nodes.internet = mkInternet {
-    connections = mkConnection "ntnu" "wan1";
-  };
-
-  nodes.ntnu = mkRouter "NTNU" {
-    interfaceGroups = [ ["wan1"] ["eth1" "eth2" "eth3"] ];
-    connections.eth1 = mkConnection "ntnu-pvv-router" "wan1";
-    connections.eth2 = mkConnection "ntnu-veggen" "wan1";
-    connections.eth3 = mkConnection "stackit" "*";
-    interfaces.eth1.network = "ntnu";
-  };
-
-  ### Brus
-
-  nodes.ntnu-pvv-router = mkRouter "NTNU PVV Gateway" {
-    interfaceGroups = [ ["wan1"] ["eth1"] ];
-    connections.eth1 = mkConnection "brus-switch" "eth1";
-    interfaces.eth1.network = "pvv";
-  };
-
-  nodes.brus-switch = mkSwitch "Brus Switch" {
-    interfaceGroups =  [ (lib.genList (i: "eth${toString i}") 16) ];
-
-    connections.eth2 = mkConnection "bekkalokk" "enp2s0";
-    # connections.eth3 = mkConnection "bicep" "enp6s0f0";
-    connections.eth4 = mkConnection "buskerud" "eth1";
-    connections.eth5 = mkConnection "knutsen" "eth1";
-    connections.eth6 = mkConnection "powerpuff-cluster" "eth1";
-
-    connections.eth8 = mkConnection "lupine-1" "enp0s31f6";
-    connections.eth9 = mkConnection "lupine-2" "enp0s31f6";
-    connections.eth11 = mkConnection "lupine-3" "enp0s31f6";
-    connections.eth10 = mkConnection "lupine-4" "enp0s31f6";
-    connections.eth12 = mkConnection "lupine-5" "enp0s31f6";
-  };
-
-  nodes.knutsen = mkRouter "knutsen" {
-    interfaceGroups = [ ["eth1"] ["eth2"] ["vpn1"] ];
-    connections.eth2 = mkConnectionRev "brus-switch" "eth6";
-    # connections.vpn1 = mkConnection "ludvigsen" "vpn1";
-    interfaces.vpn1.network = "site-vpn";
-    interfaces.vpn1.virtual = true;
-  };
-
-  nodes.buskerud = mkDevice "buskerud" {
-    interfaceGroups = [ ["eth1"] ];
-  };
-
-  nodes.shark = {
-    guestType = "proxmox";
-    parent = config.nodes.buskerud.id;
-  };
-
-  ### Powerpuff
-
-  nodes.powerpuff-cluster = mkDevice "powerpuff-cluster" {
-    interfaceGroups = [ ["eth1"] ];
-  };
-
-  nodes.kommode = {
-    guestType = "proxmox";
-    parent = config.nodes.powerpuff-cluster.id;
-  };
-
-  nodes.bicep = {
-    guestType = "proxmox";
-    parent = config.nodes.powerpuff-cluster.id;
-  };
-
-  nodes.ustetind = {
-    guestType = "proxmox";
-    parent = config.nodes.powerpuff-cluster.id;
-  };
-
-  ### PVV
-
-  nodes.ntnu-veggen = mkRouter "NTNU-Veggen" {
-    interfaceGroups = [ ["wan1"] ["eth1"] ];
-    connections.eth1 = mkConnection "ludvigsen" "eth1";
-  };
-
-  nodes.ludvigsen = mkRouter "ludvigsen" {
-    interfaceGroups = [ ["eth1"] ["eth2"] ["vpn1"] ];
-    connections.eth2 = mkConnection "pvv-switch" "eth1";
-    interfaces.vpn1.network = "site-vpn";
-    interfaces.vpn1.virtual = true;
-    interfaces.eth1.network = "ntnu";
-    interfaces.eth2.network = "pvv";
-  };
-
-  nodes.pvv-switch = mkSwitch "PVV Switch (Terminalrommet)" {
-    interfaceGroups = [ ["eth1" "eth2" "eth3"] ];
-    connections.eth2 = mkConnection "brzeczyszczykiewicz" "eno1";
-    connections.eth3 = mkConnection "georg" "eno1";
-  };
-
-
-  ### Openstack
-
-  nodes.stackit = mkDevice "stackit" {
-    interfaceGroups = [ ["*"] ];
-  };
-
-  nodes.ildkule = {
-    guestType = "openstack";
-    parent = config.nodes.stackit.id;
-  };
-  nodes.wenche = {
-    guestType = "openstack";
-    parent = config.nodes.stackit.id;
-  };
-  nodes.bakke = {
-    guestType = "openstack";
-    parent = config.nodes.stackit.id;
-  };
-}
--- a/topology/icons/greg-ng.png
+++ b/topology/icons/greg-ng.png
--- a/topology/service-extractors/greg-ng.nix
+++ b/topology/service-extractors/greg-ng.nix
@@ -1,11 +0,0 @@
-{ config, lib, ... }:
-let
-  cfg = config.services.greg-ng or { enable = false; };
-in
-{
-  config.topology.self.services.greg-ng = lib.mkIf cfg.enable {
-    name = "Greg-ng";
-    icon = ../icons/greg-ng.png;
-    details.listen = { text = "${cfg.settings.host}:${toString cfg.settings.port}"; };
-  };
-}