pvv-nixos-config/modules/robots-txt.nix

{
  config,
  pkgs,
  lib,
  ...
}:
let
  cfg = config.environment.robots-txt;

  robots-txt-format = {
    type =
      let
        coercedStrToNonEmptyListOfStr = lib.types.coercedTo lib.types.str lib.singleton (
          lib.types.nonEmptyListOf lib.types.str
        );
      in
      lib.types.listOf (
        lib.types.submodule {
          freeformType = lib.types.attrsOf coercedStrToNonEmptyListOfStr;
          options = {
            pre_comment = lib.mkOption {
              description = "Comment to add before the rule";
              type = lib.types.lines;
              default = "";
            };
            post_comment = lib.mkOption {
              description = "Comment to add after the rule";
              type = lib.types.lines;
              default = "";
            };
          };
        }
      );

    generate =
      name: value:
      let
        makeComment =
          comment:
          lib.pipe comment [
            (lib.splitString "\n")
            (lib.map (line: if line == "" then "#" else "# ${line}"))
            (lib.concatStringsSep "\n")
          ];

        ruleToString =
          rule:
          let
            user_agent = rule.User-agent or [ ];
            pre_comment = rule.pre_comment;
            post_comment = rule.post_comment;
            rest = builtins.removeAttrs rule [
              "User-agent"
              "pre_comment"
              "post_comment"
            ];
          in
          lib.concatStringsSep "\n" (
            lib.filter (x: x != null) [
              (if (pre_comment != "") then makeComment pre_comment else null)
              (
                let
                  user-agents = lib.concatMapStringsSep "\n" (value: "User-agent: ${value}") user_agent;
                in
                if user_agent == [ ] then null else user-agents
              )
              (lib.pipe rest [
                (lib.mapAttrsToList (ruleName: map (value: "${ruleName}: ${value}")))
                lib.concatLists
                (lib.concatStringsSep "\n")
              ])
              (if (post_comment != "") then makeComment post_comment else null)
            ]
          );

        content = lib.concatMapStringsSep "\n\n" ruleToString value;
      in
      pkgs.writeText name content;
  };
in
{
  options.environment.robots-txt = lib.mkOption {
    default = { };
    description = ''
      Different instances of robots.txt to use with web services.
    '';
    type = lib.types.attrsOf (
      lib.types.submodule (
        { name, ... }:
        {
          options = {
            enable = lib.mkEnableOption "this instance of robots.txt" // {
              default = true;
            };

            path = lib.mkOption {
              description = "The resulting path of the dir containing the robots.txt file";
              type = lib.types.path;
              readOnly = true;
              default = "/etc/robots-txt/${name}";
            };

            rules = lib.mkOption {
              description = "Rules to include in robots.txt";
              default = [ ];
              example = [
                {
                  User-agent = "Googlebot";
                  Disallow = "/no-googlebot";
                }
                {
                  User-agent = "Bingbot";
                  Disallow = [
                    "/no-bingbot"
                    "/no-bingbot2"
                  ];
                }
              ];
              type = robots-txt-format.type;
            };

            virtualHost = lib.mkOption {
              description = "An nginx virtual host to add the robots.txt to";
              type = lib.types.nullOr lib.types.str;
              default = null;
            };
          };
        }
      )
    );
  };

  config = {
    environment.etc = lib.mapAttrs' (name: value: {
      name = "robots-txt/${name}/robots.txt";
      value.source = robots-txt-format.generate name value.rules;
    }) cfg;

    services.nginx.virtualHosts = lib.pipe cfg [
      (lib.filterAttrs (_: value: value.virtualHost != null))
      (lib.mapAttrs' (
        name: value: {
          name = value.virtualHost;
          value = {
            locations = {
              "= /robots.txt" = {
                extraConfig = ''
                  add_header Content-Type text/plain;
                '';
                root = cfg.${name}.path;
              };
            };
          };
        }
      ))
    ];
  };
}