diff --git a/flake.nix b/flake.nix index 3d12130..b385dcd 100644 --- a/flake.nix +++ b/flake.nix @@ -91,6 +91,7 @@ modules = [ inputs.matrix-next.nixosModules.default inputs.pvv-calendar-bot.nixosModules.default + self.nixosModules.gickup ]; overlays = [ inputs.pvv-calendar-bot.overlays.x86_64-linux.default @@ -164,6 +165,7 @@ snakeoil-certs = ./modules/snakeoil-certs.nix; snappymail = ./modules/snappymail.nix; robots-txt = ./modules/robots-txt.nix; + gickup = ./modules/gickup; }; devShells = forAllSystems (system: { diff --git a/hosts/bicep/configuration.nix b/hosts/bicep/configuration.nix index 6bf7556..30b143c 100644 --- a/hosts/bicep/configuration.nix +++ b/hosts/bicep/configuration.nix @@ -8,6 +8,7 @@ ./services/nginx ./services/calendar-bot.nix + ./services/git-mirrors ./services/mysql.nix ./services/postgres.nix diff --git a/hosts/bicep/services/git-mirrors/default.nix b/hosts/bicep/services/git-mirrors/default.nix new file mode 100644 index 0000000..f7d0165 --- /dev/null +++ b/hosts/bicep/services/git-mirrors/default.nix @@ -0,0 +1,100 @@ +{ config, pkgs, lib, fp, ... }: +let + cfg = config.services.gickup; +in +{ + sops.secrets."gickup/github-token" = { + owner = "gickup"; + }; + + services.gickup = { + enable = true; + + dataDir = "/data/gickup"; + + destinationSettings = { + structured = true; + zip = false; + keep = 10; + bare = true; + lfs = true; + }; + + instances = let + defaultGithubConfig = { + settings.token_file = config.sops.secrets."gickup/github-token".path; + }; + defaultGitlabConfig = { + # settings.token_file = ... + }; + in { + "github:Git-Mediawiki/Git-Mediawiki" = defaultGithubConfig; + "github:NixOS/nixpkgs" = defaultGithubConfig; + "github:go-gitea/gitea" = defaultGithubConfig; + "github:heimdal/heimdal" = defaultGithubConfig; + "github:saltstack/salt" = defaultGithubConfig; + "github:typst/typst" = defaultGithubConfig; + "github:unmojang/FjordLauncher" = defaultGithubConfig; + "github:unmojang/drasl" = defaultGithubConfig; + "github:yushijinhun/authlib-injector" = defaultGithubConfig; + + "gitlab:mx-puppet/discord/better-discord.js" = defaultGitlabConfig; + "gitlab:mx-puppet/discord/discord-markdown" = defaultGitlabConfig; + "gitlab:mx-puppet/discord/matrix-discord-parser" = defaultGitlabConfig; + "gitlab:mx-puppet/discord/mx-puppet-discord" = defaultGitlabConfig; + "gitlab:mx-puppet/mx-puppet-bridge" = defaultGitlabConfig; + + "any:glibc" = { + settings.url = "https://sourceware.org/git/glibc.git"; + }; + + "any:out-of-your-element" = { + settings.url = "https://gitdab.com/cadence/out-of-your-element.git"; + }; + + "any:out-of-your-element-module" = { + settings.url = "https://cgit.rory.gay/nix/OOYE-module.git"; + }; + }; + }; + + services.cgit = let + domain = "bicep.pvv.ntnu.no"; + in { + ${domain} = { + enable = true; + package = pkgs.callPackage (fp /packages/cgit.nix) { }; + group = "gickup"; + scanPath = "${cfg.dataDir}/linktree"; + settings = { + enable-commit-graph = true; + enable-follow-links = true; + enable-http-clone = true; + enable-remote-branches = true; + clone-url = "https://${domain}/$CGIT_REPO_URL"; + remove-suffix = true; + root-title = "PVVSPPP"; + root-desc = "PVV Speiler Praktisk og Prominent Programvare"; + snapshots = "all"; + logo = "/PVV-logo.png"; + }; + }; + }; + + services.nginx.virtualHosts."bicep.pvv.ntnu.no" = { + forceSSL = true; + enableACME = true; + + locations."= /PVV-logo.png".alias = let + small-pvv-logo = pkgs.runCommandLocal "pvv-logo-96x96" { + nativeBuildInputs = [ pkgs.imagemagick ]; + } '' + magick '${fp /assets/logo_blue_regular.svg}' -resize 96x96 PNG:"$out" + ''; + in toString small-pvv-logo; + }; + + systemd.services."fcgiwrap-cgit-bicep.pvv.ntnu.no" = { + serviceConfig.BindReadOnlyPaths = [ cfg.dataDir ]; + }; +} diff --git a/modules/gickup/default.nix b/modules/gickup/default.nix new file mode 100644 index 0000000..f3018f4 --- /dev/null +++ b/modules/gickup/default.nix @@ -0,0 +1,310 @@ +{ config, pkgs, lib, utils, ... }: +let + cfg = config.services.gickup; + format = pkgs.formats.yaml { }; +in +{ + imports = [ + ./set-description.nix + ./hardlink-files.nix + ./import-from-toml.nix + ./update-linktree.nix + ]; + + options.services.gickup = { + enable = lib.mkEnableOption "gickup, a git repository mirroring service"; + + package = lib.mkPackageOption pkgs "gickup" { }; + gitPackage = lib.mkPackageOption pkgs "git" { }; + gitLfsPackage = lib.mkPackageOption pkgs "git-lfs" { }; + + dataDir = lib.mkOption { + type = lib.types.path; + description = "The directory to mirror repositories to."; + default = "/var/lib/gickup"; + example = "/data/gickup"; + }; + + destinationSettings = lib.mkOption { + description = '' + Settings for destination local, see gickup configuration file + + Note that `path` will be set automatically to `/var/lib/gickup` + ''; + type = lib.types.submodule { + freeformType = format.type; + }; + default = { }; + example = { + structured = true; + zip = false; + keep = 10; + bare = true; + lfs = true; + }; + }; + + instances = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule (submoduleInputs@{ name, ... }: let + submoduleName = name; + + nameParts = rec { + repoType = builtins.head (lib.splitString ":" submoduleName); + + owner = if repoType == "any" + then null + else lib.pipe submoduleName [ + (lib.removePrefix "${repoType}:") + (lib.splitString "/") + builtins.head + ]; + + repo = if repoType == "any" + then null + else lib.pipe submoduleName [ + (lib.removePrefix "${repoType}:") + (lib.splitString "/") + lib.last + ]; + + slug = if repoType == "any" + then lib.toLower (builtins.replaceStrings [ ":" "/" ] [ "-" "-" ] submoduleName) + else "${lib.toLower repoType}-${lib.toLower owner}-${lib.toLower repo}"; + }; + in { + options = { + interval = lib.mkOption { + type = lib.types.str; + default = "daily"; + example = "weekly"; + description = '' + Specification (in the format described by {manpage}`systemd.time(7)`) of the time + interval at which to run the service. + ''; + }; + + type = lib.mkOption { + type = lib.types.enum [ + "github" + "gitlab" + "gitea" + "gogs" + "bitbucket" + "onedev" + "sourcehut" + "any" + ]; + example = "github"; + default = nameParts.repoType; + description = '' + The type of the repository to mirror. + ''; + }; + + owner = lib.mkOption { + type = with lib.types; nullOr str; + example = "go-gitea"; + default = nameParts.owner; + description = '' + The owner of the repository to mirror (if applicable) + ''; + }; + + repo = lib.mkOption { + type = with lib.types; nullOr str; + example = "gitea"; + default = nameParts.repo; + description = '' + The name of the repository to mirror (if applicable) + ''; + }; + + slug = lib.mkOption { + type = lib.types.str; + default = nameParts.slug; + example = "github-go-gitea-gitea"; + description = '' + The slug of the repository to mirror. + ''; + }; + + description = lib.mkOption { + type = with lib.types; nullOr str; + example = "A project which does this and that"; + description = '' + A description of the project. This isn't used directly by gickup for anything, + but can be useful if gickup is used together with cgit or similar. + ''; + }; + + settings = lib.mkOption { + description = "Instance specific settings, see gickup configuration file"; + type = lib.types.submodule { + freeformType = format.type; + }; + default = { }; + example = { + username = "gickup"; + password = "hunter2"; + wiki = true; + issues = true; + }; + }; + }; + })); + }; + }; + + config = lib.mkIf cfg.enable { + users.users.gickup = { + isSystemUser = true; + group = "gickup"; + home = "/var/lib/gickup"; + }; + + users.groups.gickup = { }; + + services.gickup.destinationSettings.path = "/var/lib/gickup/raw"; + + systemd.tmpfiles.settings."10-gickup" = lib.mkIf (cfg.dataDir != "/var/lib/gickup") { + ${cfg.dataDir}.d = { + user = "gickup"; + group = "gickup"; + mode = "0755"; + }; + }; + + systemd.slices."system-gickup" = { + description = "Gickup git repository mirroring service"; + after = [ "network.target" ]; + }; + + systemd.targets.gickup = { + description = "Gickup git repository mirroring service"; + wants = map ({ slug, ... }: "gickup@${slug}.service") (lib.attrValues cfg.instances); + }; + + systemd.timers = { + "gickup@" = { + description = "Gickup git repository mirroring service for %i"; + + timerConfig = { + OnCalendar = "daily"; + RandomizedDelaySec = "1h"; + Persistent = true; + AccuracySec = "1s"; + }; + }; + } + // + # Overrides for mirrors which are not "daily" + (lib.pipe cfg.instances [ + builtins.attrValues + (builtins.filter (instance: instance.interval != "daily")) + (map ({ slug, interval, ... }: { + name = "gickup@${slug}"; + value = { + overrideStrategy = "asDropin"; + timerConfig.OnCalendar = interval; + }; + })) + builtins.listToAttrs + ]); + + systemd.targets.timers.wants = map ({ slug, ... }: "gickup@${slug}.timer") (lib.attrValues cfg.instances); + + systemd.services = { + "gickup@" = let + configDir = lib.pipe cfg.instances [ + (lib.mapAttrsToList (name: instance: { + name = "${instance.slug}.yml"; + path = format.generate "gickup-configuration-${name}.yml" { + destination.local = [ cfg.destinationSettings ]; + source.${instance.type} = [ + ( + (lib.optionalAttrs (instance.type != "any") { + user = instance.owner; + includeorgs = [ instance.owner ]; + include = [ instance.repo ]; + }) + // + instance.settings + ) + ]; + }; + })) + (pkgs.linkFarm "gickup-configuration-files") + ]; + in { + description = "Gickup git repository mirroring service for %i"; + after = [ "network.target" ]; + + path = [ + cfg.gitPackage + cfg.gitLfsPackage + ]; + + restartIfChanged = false; + + serviceConfig = { + Type = "oneshot"; + ExecStart = "'${pkgs.gickup}/bin/gickup' '${configDir}/%i.yml'"; + ExecStartPost = ""; + + User = "gickup"; + Group = "gickup"; + + BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [ + "${cfg.dataDir}:/var/lib/gickup" + ]; + + Slice = "system-gickup.slice"; + + SyslogIdentifier = "gickup-%i"; + StateDirectory = "gickup"; + # WorkingDirectory = "gickup"; + # RuntimeDirectory = "gickup"; + # RuntimeDirectoryMode = "0700"; + + # https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431 + RemainAfterExit = true; + + # Hardening options + AmbientCapabilities = []; + LockPersonality = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + PrivateTmp = true; + PrivateUsers = true; + ProcSubset = "pid"; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + # ProtectProc = "invisible"; + # ProtectSystem = "strict"; + RemoveIPC = true; + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + # SystemCallFilter = [ + # "@system-service" + # "~@resources" + # "~@privileged" + # ]; + UMask = "0002"; + CapabilityBoundingSet = []; + }; + }; + }; + }; +} diff --git a/modules/gickup/hardlink-files.nix b/modules/gickup/hardlink-files.nix new file mode 100644 index 0000000..c16abf7 --- /dev/null +++ b/modules/gickup/hardlink-files.nix @@ -0,0 +1,42 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.services.gickup; +in +{ + config = lib.mkIf cfg.enable { + # TODO: add a service that will look at the backed up files and hardlink + # the ones that have a matching hash together to save space. This can + # either run routinely (i.e. trigger by systemd-timer), or be activated + # whenever a gickup@.service finishes. The latter is probably better. + + # systemd.services."gickup-hardlink" = { + # serviceConfig = { + # Type = "oneshot"; + # ExecStart = let + # script = pkgs.writeShellApplication { + # name = "gickup-hardlink-files.sh"; + # runtimeInputs = [ pkgs.coreutils pkgs.jdupes ]; + # text = '' + + # ''; + # }; + # in lib.getExe script; + + # User = "gickup"; + # Group = "gickup"; + + # BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [ + # "${cfg.dataDir}:/var/lib/gickup" + # ]; + + # Slice = "system-gickup.slice"; + + # StateDirectory = "gickup"; + + # # Hardening options + # # TODO: + # PrivateNetwork = true; + # }; + # }; + }; +} diff --git a/modules/gickup/import-from-toml.nix b/modules/gickup/import-from-toml.nix new file mode 100644 index 0000000..26b09ca --- /dev/null +++ b/modules/gickup/import-from-toml.nix @@ -0,0 +1,11 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.services.gickup; +in +{ + config = lib.mkIf cfg.enable { + # TODO: import cfg.instances from a toml file to make it easier for non-nix users + # to add repositories to mirror + }; +} diff --git a/modules/gickup/set-description.nix b/modules/gickup/set-description.nix new file mode 100644 index 0000000..745769b --- /dev/null +++ b/modules/gickup/set-description.nix @@ -0,0 +1,9 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.services.gickup; +in +{ + config = lib.mkIf cfg.enable { + # TODO: create .git/description files for each repo where cfg.instances..description is set + }; +} diff --git a/modules/gickup/update-linktree.nix b/modules/gickup/update-linktree.nix new file mode 100644 index 0000000..1805f59 --- /dev/null +++ b/modules/gickup/update-linktree.nix @@ -0,0 +1,84 @@ +{ config, lib, pkgs, ... }: +let + cfg = config.services.gickup; +in +{ + config = lib.mkIf cfg.enable { + # TODO: run upon completion of cloning a repository + systemd.timers."gickup-linktree" = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnCalendar = "daily"; + Persistent = true; + Unit = "gickup-linktree.service"; + }; + }; + + # TODO: update symlink for one repo at a time (e.g. gickup-linktree@.service) + systemd.services."gickup-linktree" = { + serviceConfig = { + Type = "oneshot"; + ExecStart = let + script = pkgs.writeShellApplication { + name = "gickup-update-symlink-tree.sh"; + runtimeInputs = [ + pkgs.coreutils + pkgs.findutils + ]; + text = '' + shopt -s nullglob + + for repository in ./*/*/*; do + REPOSITORY_RELATIVE_DIRS=''${repository#"./"} + + echo "Checking $REPOSITORY_RELATIVE_DIRS" + + declare -a REVISIONS + readarray -t REVISIONS < <(find "$repository" -mindepth 1 -maxdepth 1 -printf "%f\n" | sort --numeric-sort --reverse) + + if [[ "''${#REVISIONS[@]}" == 0 ]]; then + echo "Found no revisions for $repository, continuing" + continue + fi + + LAST_REVISION="''${REVISIONS[0]}" + SYMLINK_PATH="../linktree/''${REPOSITORY_RELATIVE_DIRS}" + + mkdir -p "$(dirname "$SYMLINK_PATH")" + + EXPECTED_SYMLINK_TARGET=$(realpath "''${repository}/''${LAST_REVISION}") + EXISTING_SYMLINK_TARGET=$(realpath "$SYMLINK_PATH" || echo "") + + if [[ "$EXISTING_SYMLINK_TARGET" != "$EXPECTED_SYMLINK_TARGET" ]]; then + echo "Updating symlink for $REPOSITORY_RELATIVE_DIRS" + rm "$SYMLINK_PATH" ||: + ln -rs "$EXPECTED_SYMLINK_TARGET" "$SYMLINK_PATH" + else + echo "Symlink already up to date, continuing..." + fi + + echo "---" + done + ''; + }; + in lib.getExe script; + + User = "gickup"; + Group = "gickup"; + + BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [ + "${cfg.dataDir}:/var/lib/gickup" + ]; + + Slice = "system-gickup.slice"; + + StateDirectory = "gickup"; + WorkingDirectory = "/var/lib/gickup/raw"; + + # Hardening options + # TODO: + PrivateNetwork = true; + }; + }; + }; +} diff --git a/packages/cgit.nix b/packages/cgit.nix new file mode 100644 index 0000000..c006ae6 --- /dev/null +++ b/packages/cgit.nix @@ -0,0 +1,21 @@ +{ cgit, fetchurl, ... }: +let + pname = cgit.pname; + commit = "09d24d7cd0b7e85633f2f43808b12871bb209d69"; +in +cgit.overrideAttrs (_: { + version = "1.2.3-unstable-2024.07.16"; + + src = fetchurl { + url = "https://git.zx2c4.com/cgit/snapshot/${pname}-${commit}.tar.xz"; + hash = "sha256-gfgjAXnWRqVCP+4cmYOVdB/3OFOLJl2WBOc3bFVDsjw="; + }; + + # cgit is tightly coupled with git and needs a git source tree to build. + # IMPORTANT: Remember to check which git version cgit needs on every version + # bump (look for "GIT_VER" in the top-level Makefile). + gitSrc = fetchurl { + url = "mirror://kernel/software/scm/git/git-2.46.0.tar.xz"; + hash = "sha256-fxI0YqKLfKPr4mB0hfcWhVTCsQ38FVx+xGMAZmrCf5U="; + }; +}) diff --git a/secrets/bicep/bicep.yaml b/secrets/bicep/bicep.yaml index e3fe480..82da3b2 100644 --- a/secrets/bicep/bicep.yaml +++ b/secrets/bicep/bicep.yaml @@ -3,6 +3,8 @@ calendar-bot: mysql_password: ENC[AES256_GCM,data:Gqag8yOgPH3ntoT5TmaqJWv1j+si2qIyz5Ryfw5E2A==,iv:kQDcxnPfwJQcFovI4f87UDt18F8ah3z5xeY86KmdCyY=,tag:A1sCSNXJziAmtUWohqwJgg==,type:str] mysql: password: ENC[AES256_GCM,data:KqEe0TVdeMIzPKsmFg9x0X9xWijnOk306ycyXTm2Tpqo/O0F,iv:Y+hlQ8n1ZIP9ncXBzd2kCSs/DWVTWhiEluFVwZFKRCA=,tag:xlaUk0Wftk62LpYE5pKNQw==,type:str] +gickup: + github-token: ENC[AES256_GCM,data:H/yBDLIvEXunmaUha3c2vUWKLRIbl9QrC0t13AQDRCTnrvhabeiUFLNxZ/F+4B6sZ2aPSgZoB69WwnHvh1wLdiFp1qLWKW/jQPvzZOxE4n+jXrnSOutUWktbPzVj,iv:KFW4jRru93JIl9doVFtcNkJDWp89NlzWjPDflHxcL/U=,tag:YtgyRxkoZO9MkuP3DJh7zA==,type:str] sops: kms: [] gcp_kms: [] @@ -63,8 +65,8 @@ sops: cTh5bnJ3WW90aXRCSUp6NHFYeU1tZ0kK4afdtJwGNu6wLRI0fuu+mBVeqVeB0rgX 0q5hwyzjiRnHnyjF38CmcGgydSfDRmF6P+WIMbCwXC6LwfRhAmBGPg== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-08-15T21:18:33Z" - mac: ENC[AES256_GCM,data:uR5HgeDAYqoqB9kk1V6p0T30+v6WpQJi4+qIeCDRnoUPnQKUVR10hvBhICck+E+Uh8p+tGhM6Uf3YrAJAV0ZCUiNJjtwDJQQLUDT53vdOAXN4xADCQqNuhgVwVMaruoTheEiwOswRuhFeEwy0gBj3Ze2pu47lueHYclmEzumLeQ=,iv:t0UyXN2YaR2m7M/pV2wTLJG5wVfqTIUs7wSQMmyeTVw=,tag:O7dIffzrDAXz3kGx5uazhw==,type:str] + lastmodified: "2025-05-07T21:34:48Z" + mac: ENC[AES256_GCM,data:n6GHD+nQmZL17WvUZiMCBLRHbtpoKU6U8o/Oraj0VSRi/pQ74QWGVEcIX87kFjBvR2C+UPd3KwXzjQHhjUfHpz9EjIGi6tXLTTo8K3ptd2wCL8MW418TVO4KV+BFmHGT4kwlbdoqaJ2SA7HcfXNaC68e/2CTXhtkLpIwGXtYWJA=,iv:iC5QX/JMwno4mBljPdorNmcQSD2wy/wOYvGrUoC2yzg=,tag:GuFNQ6+d6o9DYC6Do/IEqQ==,type:str] pgp: - created_at: "2024-08-04T00:03:40Z" enc: |- @@ -87,4 +89,4 @@ sops: -----END PGP MESSAGE----- fp: F7D37890228A907440E1FD4846B9228E814A2AAC unencrypted_suffix: _unencrypted - version: 3.9.0 + version: 3.9.4