kommode/gitea: add robots.txt

flake.nix

@@ -229,20 +229,7 @@
           modules = [
             inputs.nix-gitea-themes.nixosModules.default
             inputs.disko.nixosModules.disko
-          ];
+            self.nixosModules.robots-txt
-        };
-
-        grevling = stableNixosConfig "grevling" {
-          modules = [
-            ./hosts/grevling/configuration.nix
-            sops-nix.nixosModules.sops
-          ];
-        };
-
-        tuba = stableNixosConfig "grevling" {
-          modules = [
-            ./hosts/tuba/configuration.nix
-            sops-nix.nixosModules.sops
           ];
         };
 

hosts/grevling/configuration.nix

@@ -1,36 +0,0 @@
-{ config, pkgs, values, ... }:
-{
-  imports = [
-      # Include the results of the hardware scan.
-      ./hardware-configuration.nix
-      ../../base.nix
-      ../../misc/metrics-exporters.nix
-
-      ./services/openvpn
-    ];
-
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  networking.hostName = "grevling";
-
-  # systemd.network.networks."30-eno1" = values.defaultNetworkConfig // {
-  #   matchConfig.Name = "eno1";
-  #   address = with values.hosts.georg; [ (ipv4 + "/25") (ipv6 + "/64") ];
-  # };
-
-  # List packages installed in system profile
-  environment.systemPackages = with pkgs; [
-  ];
-
-  # List services that you want to enable:
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "23.05"; # Did you read the comment?
-
-}

hosts/grevling/hardware-configuration.nix

@@ -1,40 +0,0 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
-    ];
-
-  boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-intel" ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/33825f0d-5a63-40fc-83db-bfa1ebb72ba0";
-      fsType = "ext4";
-    };
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/145E-7362";
-      fsType = "vfat";
-    };
-
-  swapDevices =
-    [ { device = "/dev/disk/by-uuid/7ed27e21-3247-44cd-8bcc-5d4a2efebf57"; }
-    ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
-  # networking.interfaces.enp2s2.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
-}

hosts/grevling/services/openvpn/default.nix

@@ -1,77 +0,0 @@
-{ pkgs, lib, values, ... }:
-{
-  services.openvpn.servers."ov-tunnel" = {
-    config = let
-      conf = {
-        # TODO: use aliases
-        local = "129.241.210.191";
-        port = 1194;
-        proto = "udp";
-        dev = "tap";
-
-        # TODO: set up
-        ca = "";
-        cert = "";
-        key = "";
-        dh = "";
-
-        # Maintain a record of client <-> virtual IP address
-        # associations in this file.  If OpenVPN goes down or
-        # is restarted, reconnecting clients can be assigned
-        # the same virtual IP address from the pool that was
-        # previously assigned.
-        ifconfig-pool-persist = ./ipp.txt;
-
-        server-bridge = builtins.concatStringsSep " " [
-          "129.241.210.129"
-          "255.255.255.128"
-          "129.241.210.253"
-          "129.241.210.254"
-        ];
-
-        keepalive = "10 120";
-        cipher = "none";
-
-        user = "nobody";
-        group = "nobody";
-
-        status = "/var/log/openvpn-status.log";
-
-        client-config-dir = pkgs.writeTextDir "tuba" ''
-          # Sett IP-adr. for tap0 til tubas PVV-adr.
-          ifconfig-push ${values.services.tuba-tap} 255.255.255.128
-          # Hvordan skal man faa dette til aa funke, tro?
-          #ifconfig-ipv6-push 2001:700:300:1900::xxx/64
-          
-          # La tuba bruke std. PVV-gateway til all trafikk (unntatt
-          # VPN-tunnellen).
-          push "redirect-gateway"
-        '';
-
-        persist-key = true;
-        persist-tun = true;
-
-        verb = 5;
-
-        explicit-exit-notify = 1;
-      };
-    in lib.pipe conf [
-      (lib.filterAttrs (_: value: !(builtins.isNull value || value == false)))
-      (builtins.mapAttrs (_: value:
-        if builtins.isList value then builtins.concatStringsSep " " (map toString value)
-        else if value == true then value
-        else if builtins.any (f: f value) [
-          builtins.isString
-          builtins.isInt
-          builtins.isFloat
-          lib.isPath
-          lib.isDerivation
-        ] then toString value
-        else throw "Unknown value in grevling openvpn config, deading now\n${value}"
-      ))
-      (lib.mapAttrsToList (name: value: if value == true then name else "${name} ${value}"))
-      (builtins.concatStringsSep "\n")
-      (x: x + "\n\n")
-    ];
-  };
-}

hosts/kommode/services/gitea/default.nix

@@ -193,6 +193,109 @@ in {
     };
   };
 
+  environment.robots-txt."gitea" = {
+    virtualHost = domain;
+    rules = [
+      {
+        pre_comment = ''
+          Gitea internals
+
+          See these for more information:
+          - https://gitea.com/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = "*";
+        Disallow = [
+          "/api/*"
+          "/avatars"
+          "/*/*/src/commit/*"
+          "/*/*/commit/*"
+          "/*/*/*/refs/*"
+          "/*/*/*/star"
+          "/*/*/*/watch"
+          "/*/*/labels"
+          "/*/*/activity/*"
+          "/vendor/*"
+          "/swagger.*.json"
+          "/repo/create"
+          "/repo/migrate"
+          "/org/create"
+          "/*/*/fork"
+          "/*/*/watchers"
+          "/*/*/stargazers"
+          "/*/*/forks"
+          "*/.git/"
+          "/*.git"
+          "/*.atom"
+          "/*.rss"
+        ];
+      }
+      {
+        pre_comment = "Language Spam";
+        Disallow = "/*?lang=";
+      }
+      {
+        pre_comment = ''
+          AI bots
+
+          Sourced from:
+          - https://www.vg.no/robots.txt
+          - https://codeberg.org/robots.txt
+        '';
+        User-agent = [
+          "AI2Bot"
+          "Ai2Bot-Dolma"
+          "Amazonbot"
+          "Applebot-Extended"
+          "Bytespider"
+          "CCBot"
+          "ChatGPT-User"
+          "Claude-Web"
+          "ClaudeBot"
+          "Crawlspace"
+          "Diffbot"
+          "FacebookBot"
+          "FriendlyCrawler"
+          "GPTBot"
+          "Google-Extended"
+          "ICC-Crawler"
+          "ImagesiftBot"
+          "Kangaroo Bot"
+          "Meta-ExternalAgent"
+          "OAI-SearchBot"
+          "Omgili"
+          "Omgilibot"
+          "PanguBot"
+          "PerplexityBot"
+          "PetalBot"
+          "Scrapy"
+          "SemrushBot-OCOB"
+          "Sidetrade indexer bot"
+          "Timpibot"
+          "VelenPublicWebCrawler"
+          "Webzio-Extended"
+          "YouBot"
+          "anthropic-ai"
+          "cohere-ai"
+          "cohere-training-data-crawler"
+          "facebookexternalhit"
+          "iaskspider/2.0"
+          "img2dataset"
+          "meta-externalagent"
+          "omgili"
+          "omgilibot"
+        ];
+        Disallow = "/";
+      }
+      {
+        Crawl-delay = "2";
+      }
+      {
+        Sitemap = "https://${domain}/sitemap.xml";
+      }
+    ];
+  };
+
   networking.firewall.allowedTCPPorts = [ sshPort ];
 
   services.rsync-pull-targets = {

hosts/tuba/configuration.nix

@@ -1,36 +0,0 @@
-{ config, pkgs, values, ... }:
-{
-  imports = [
-      # Include the results of the hardware scan.
-      ./hardware-configuration.nix
-      ../../base.nix
-      ../../misc/metrics-exporters.nix
-
-      ./services/openvpn
-    ];
-
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.efi.canTouchEfiVariables = true;
-
-  networking.hostName = "tuba";
-
-  # systemd.network.networks."30-eno1" = values.defaultNetworkConfig // {
-  #   matchConfig.Name = "eno1";
-  #   address = with values.hosts.georg; [ (ipv4 + "/25") (ipv6 + "/64") ];
-  # };
-
-  # List packages installed in system profile
-  environment.systemPackages = with pkgs; [
-  ];
-
-  # List services that you want to enable:
-
-  # This value determines the NixOS release from which the default
-  # settings for stateful data, like file locations and database versions
-  # on your system were taken. It‘s perfectly fine and recommended to leave
-  # this value at the release version of the first install of this system.
-  # Before changing this value read the documentation for this option
-  # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-  system.stateVersion = "23.05"; # Did you read the comment?
-
-}

hosts/tuba/hardware-configuration.nix

@@ -1,40 +0,0 @@
-# Do not modify this file!  It was generated by ‘nixos-generate-config’
-# and may be overwritten by future invocations.  Please make changes
-# to /etc/nixos/configuration.nix instead.
-{ config, lib, pkgs, modulesPath, ... }:
-
-{
-  imports =
-    [ (modulesPath + "/installer/scan/not-detected.nix")
-    ];
-
-  boot.initrd.availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ "kvm-intel" ];
-  boot.extraModulePackages = [ ];
-
-  fileSystems."/" =
-    { device = "/dev/disk/by-uuid/33825f0d-5a63-40fc-83db-bfa1ebb72ba0";
-      fsType = "ext4";
-    };
-
-  fileSystems."/boot" =
-    { device = "/dev/disk/by-uuid/145E-7362";
-      fsType = "vfat";
-    };
-
-  swapDevices =
-    [ { device = "/dev/disk/by-uuid/7ed27e21-3247-44cd-8bcc-5d4a2efebf57"; }
-    ];
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
-  networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
-  # networking.interfaces.enp2s2.useDHCP = lib.mkDefault true;
-
-  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
-}

hosts/tuba/services/openvpn/default.nix

@@ -1,54 +0,0 @@
-{ lib, values, ... }:
-{
-  services.openvpn.servers."ov-tunnel" = {
-    config = let
-      conf = {
-        # TODO: use aliases
-        client = true;
-        dev = "tap";
-        proto = "udp";
-        remote = "129.241.210.191 1194";
-
-        resolv-retry = "infinite";
-        nobind = true;
-
-        # # TODO: set up
-        ca = "";
-        cert = "";
-        key = "";
-        remote-cert-tls = "server";
-        cipher = "none";
-
-        user = "nobody";
-        group = "nobody";
-
-        status = "/var/log/openvpn-status.log";
-
-        persist-key = true;
-        persist-tun = true;
-
-        verb = 5;
-
-        # script-security = 2;
-        # up = "systemctl restart rwhod";
-      };
-    in lib.pipe conf [
-      (lib.filterAttrs (_: value: !(builtins.isNull value || value == false)))
-      (builtins.mapAttrs (_: value:
-        if builtins.isList value then builtins.concatStringsSep " " (map toString value)
-        else if value == true then value
-        else if builtins.any (f: f value) [
-          builtins.isString
-          builtins.isInt
-          builtins.isFloat
-          lib.isPath
-          lib.isDerivation
-        ] then toString value
-        else throw "Unknown value in tuba openvpn config, deading now\n${value}"
-      ))
-      (lib.mapAttrsToList (name: value: if value == true then name else "${name} ${value}"))
-      (builtins.concatStringsSep "\n")
-      (x: x + "\n\n")
-    ];
-  };
-}

values.nix

@@ -26,12 +26,6 @@ in rec {
       ipv4 = pvv-ipv4 213;
       ipv6 = pvv-ipv6 213;
     };
-    grevling-tap = {
-      ipv4 = pvv-ipv4 251;
-    };
-    tuba-tap = {
-      ipv4 = pvv-ipv4 252;
-    };
   };
 
   hosts = {
@@ -117,14 +111,6 @@ in rec {
       ipv4 = pvv-ipv4 228;
       ipv6 = pvv-ipv6 228;
     };
-    grevling = {
-      ipv4 = pvv-ipv4 198;
-      ipv6 = pvv-ipv6 198;
-    };
-    tuba = {
-      ipv4 = pvv-ipv4 199;
-      ipv6 = pvv-ipv6 199;
-    };
   };
 
   defaultNetworkConfig = {