felixalb/pvv-nixos-config
1
0
Fork 0
mirror of https://git.pvv.ntnu.no/Drift/pvv-nixos-config.git synced 2025-05-12 16:33:31 +02:00
Code Issues Projects Releases Wiki Activity

bicep: add git-mirroring service

Browse Source
This commit is contained in:
h7x4 2025-03-30 17:18:28 +02:00
parent 20e3f89b79
commit 48c59201e1
No known key found for this signature in database
GPG Key ID: 9F2F7D8250F35146
10 changed files with 571 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
flake.nix
View File

@ -91,6 +91,7 @@
modules = [
inputs.matrix-next.nixosModules.default
inputs.pvv-calendar-bot.nixosModules.default
self.nixosModules.gickup
];
overlays = [
inputs.pvv-calendar-bot.overlays.x86_64-linux.default
@ -164,6 +165,7 @@
snakeoil-certs = ./modules/snakeoil-certs.nix;
snappymail = ./modules/snappymail.nix;
robots-txt = ./modules/robots-txt.nix;
gickup = ./modules/gickup;
};
devShells = forAllSystems (system: {

1
hosts/bicep/configuration.nix
View File

@ -8,6 +8,7 @@
./services/nginx
./services/calendar-bot.nix
./services/git-mirrors
./services/mysql.nix
./services/postgres.nix

92
hosts/bicep/services/git-mirrors/default.nix Normal file
View File

@ -0,0 +1,92 @@
{ config, pkgs, lib, fp, ... }:
let
cfg = config.services.gickup;
in
{
sops.secrets."gickup/github-token" = {
owner = "gickup";
};
services.gickup = {
enable = true;
dataDir = "/data/gickup";
destinationSettings = {
structured = true;
zip = false;
keep = 10;
bare = true;
lfs = true;
};
instances = let
defaultGithubConfig = {
settings.token_file = config.sops.secrets."gickup/github-token".path;
};
defaultGitlabConfig = {
# settings.token_file = ...
};
in {
"github:Git-Mediawiki/Git-Mediawiki" = defaultGithubConfig;
"github:NixOS/nixpkgs" = defaultGithubConfig;
"github:go-gitea/gitea" = defaultGithubConfig;
"github:heimdal/heimdal" = defaultGithubConfig;
"github:saltstack/salt" = defaultGithubConfig;
"github:typst/typst" = defaultGithubConfig;
"github:unmojang/FjordLauncher" = defaultGithubConfig;
"github:unmojang/drasl" = defaultGithubConfig;
"github:yushijinhun/authlib-injector" = defaultGithubConfig;
"gitlab:mx-puppet/discord/better-discord.js" = defaultGitlabConfig;
"gitlab:mx-puppet/discord/discord-markdown" = defaultGitlabConfig;
"gitlab:mx-puppet/discord/matrix-discord-parser" = defaultGitlabConfig;
"gitlab:mx-puppet/discord/mx-puppet-discord" = defaultGitlabConfig;
"gitlab:mx-puppet/mx-puppet-bridge" = defaultGitlabConfig;
"any:glibc" = {
settings.url = "https://sourceware.org/git/glibc.git";
};
};
};
services.cgit = let
domain = "bicep.pvv.ntnu.no";
in {
${domain} = {
enable = true;
package = pkgs.callPackage (fp /packages/cgit.nix) { };
group = "gickup";
scanPath = "${cfg.dataDir}/linktree";
settings = {
enable-commit-graph = true;
enable-follow-links = true;
enable-http-clone = true;
enable-remote-branches = true;
clone-url = "https://${domain}/$CGIT_REPO_URL";
remove-suffix = true;
root-title = "PVVSPPP";
root-desc = "PVV Speiler Praktisk og Prominent Programvare";
snapshots = "all";
logo = "/PVV-logo.png";
};
};
};
services.nginx.virtualHosts."bicep.pvv.ntnu.no" = {
forceSSL = true;
enableACME = true;
locations."= /PVV-logo.png".alias = let
small-pvv-logo = pkgs.runCommandLocal "pvv-logo-96x96" {
nativeBuildInputs = [ pkgs.imagemagick ];
} ''
magick '${fp /assets/logo_blue_regular.svg}' -resize 96x96 PNG:"$out"
'';
in toString small-pvv-logo;
};
systemd.services."fcgiwrap-cgit-bicep.pvv.ntnu.no" = {
serviceConfig.BindReadOnlyPaths = [ cfg.dataDir ];
};
}

312
modules/gickup/default.nix Normal file
View File

@ -0,0 +1,312 @@
{ config, pkgs, lib, utils, ... }:
let
cfg = config.services.gickup;
format = pkgs.formats.yaml { };
in
{
imports = [
./set-description.nix
./hardlink-files.nix
./import-from-toml.nix
./update-linktree.nix
];
options.services.gickup = {
enable = lib.mkEnableOption "gickup, a git repository mirroring service";
package = lib.mkPackageOption pkgs "gickup" { };
gitPackage = lib.mkPackageOption pkgs "git" { };
gitLfsPackage = lib.mkPackageOption pkgs "git-lfs" { };
dataDir = lib.mkOption {
type = lib.types.path;
description = "The directory to mirror repositories to.";
default = "/var/lib/gickup";
example = "/data/gickup";
};
destinationSettings = lib.mkOption {
description = ''
Settings for destination local, see gickup configuration file
Note that `path` will be set automatically to `/var/lib/gickup`
'';
type = lib.types.submodule {
freeformType = format.type;
};
default = { };
example = {
structured = true;
zip = false;
keep = 10;
bare = true;
lfs = true;
};
};
instances = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule (submoduleInputs@{ name, ... }: let
submoduleName = name;
nameParts = rec {
repoType = builtins.head (lib.splitString ":" submoduleName);
owner = if repoType == "any"
then null
else lib.pipe submoduleName [
(lib.removePrefix "${repoType}:")
(lib.splitString "/")
builtins.head
lib.toLower
];
repo = if repoType == "any"
then null
else lib.pipe submoduleName [
(lib.removePrefix "${repoType}:")
(lib.splitString "/")
lib.last
lib.toLower
];
slug = if repoType == "any"
then builtins.replaceStrings [ ":" "/" ] [ "-" "-" ] submoduleName
else "${repoType}-${owner}-${repo}";
};
in {
options = {
interval = lib.mkOption {
type = lib.types.str;
default = "daily";
example = "weekly";
description = ''
Specification (in the format described by {manpage}`systemd.time(7)`) of the time
interval at which to run the service.
'';
};
type = lib.mkOption {
type = lib.types.enum [
"github"
"gitlab"
"gitea"
"gogs"
"bitbucket"
"onedev"
"sourcehut"
"any"
];
example = "github";
default = nameParts.repoType;
description = ''
The type of the repository to mirror.
'';
};
owner = lib.mkOption {
type = with lib.types; nullOr str;
example = "go-gitea";
default = nameParts.owner;
description = ''
The owner of the repository to mirror (if applicable)
'';
};
repo = lib.mkOption {
type = with lib.types; nullOr str;
example = "gitea";
default = nameParts.repo;
description = ''
The name of the repository to mirror (if applicable)
'';
};
slug = lib.mkOption {
type = lib.types.str;
default = nameParts.slug;
example = "github-go-gitea-gitea";
description = ''
The slug of the repository to mirror.
'';
};
description = lib.mkOption {
type = with lib.types; nullOr str;
example = "A project which does this and that";
description = ''
A description of the project. This isn't used directly by gickup for anything,
but can be useful if gickup is used together with cgit or similar.
'';
};
settings = lib.mkOption {
description = "Instance specific settings, see gickup configuration file";
type = lib.types.submodule {
freeformType = format.type;
};
default = { };
example = {
username = "gickup";
password = "hunter2";
wiki = true;
issues = true;
};
};
};
}));
};
};
config = lib.mkIf cfg.enable {
users.users.gickup = {
isSystemUser = true;
group = "gickup";
home = "/var/lib/gickup";
};
users.groups.gickup = { };
services.gickup.destinationSettings.path = "/var/lib/gickup/raw";
systemd.tmpfiles.settings."10-gickup" = lib.mkIf (cfg.dataDir != "/var/lib/gickup") {
${cfg.dataDir}.d = {
user = "gickup";
group = "gickup";
mode = "0755";
};
};
systemd.slices."system-gickup" = {
description = "Gickup git repository mirroring service";
after = [ "network.target" ];
};
systemd.targets.gickup = {
description = "Gickup git repository mirroring service";
wants = map ({ slug, ... }: "gickup@${slug}.service") (lib.attrValues cfg.instances);
};
systemd.timers = {
"gickup@" = {
description = "Gickup git repository mirroring service for %i";
timerConfig = {
OnCalendar = "daily";
RandomizedDelaySec = "1h";
Persistent = true;
AccuracySec = "1s";
};
};
}
//
# Overrides for mirrors which are not "daily"
(lib.pipe cfg.instances [
builtins.attrValues
(builtins.filter (instance: instance.interval != "daily"))
(map ({ slug, interval, ... }: {
name = "gickup@${slug}";
value = {
overrideStrategy = "asDropin";
timerConfig.OnCalendar = interval;
};
}))
builtins.listToAttrs
]);
systemd.targets.timers.wants = map ({ slug, ... }: "gickup@${slug}.timer") (lib.attrValues cfg.instances);
systemd.services = {
"gickup@" = let
configDir = lib.pipe cfg.instances [
(lib.mapAttrsToList (name: instance: {
name = "${instance.slug}.yml";
path = format.generate "gickup-configuration-${name}.yml" {
destination.local = [ cfg.destinationSettings ];
source.${instance.type} = [
(
(lib.optionalAttrs (instance.type != "any") {
user = instance.owner;
includeorgs = [ instance.owner ];
include = [ instance.repo ];
})
//
instance.settings
)
];
};
}))
(pkgs.linkFarm "gickup-configuration-files")
];
in {
description = "Gickup git repository mirroring service for %i";
after = [ "network.target" ];
path = [
cfg.gitPackage
cfg.gitLfsPackage
];
restartIfChanged = false;
serviceConfig = {
Type = "oneshot";
ExecStart = "'${pkgs.gickup}/bin/gickup' '${configDir}/%i.yml'";
ExecStartPost = "";
User = "gickup";
Group = "gickup";
BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [
"${cfg.dataDir}:/var/lib/gickup"
];
Slice = "system-gickup.slice";
SyslogIdentifier = "gickup-%i";
StateDirectory = "gickup";
# WorkingDirectory = "gickup";
# RuntimeDirectory = "gickup";
# RuntimeDirectoryMode = "0700";
# https://discourse.nixos.org/t/how-to-prevent-custom-systemd-service-from-restarting-on-nixos-rebuild-switch/43431
RemainAfterExit = true;
# Hardening options
AmbientCapabilities = [];
LockPersonality = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateMounts = true;
PrivateTmp = true;
PrivateUsers = true;
ProcSubset = "pid";
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectHostname = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
# ProtectProc = "invisible";
# ProtectSystem = "strict";
RemoveIPC = true;
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
# SystemCallFilter = [
# "@system-service"
# "~@resources"
# "~@privileged"
# ];
UMask = "0002";
CapabilityBoundingSet = [];
};
};
};
};
}

42
modules/gickup/hardlink-files.nix Normal file
View File

@ -0,0 +1,42 @@
{ config, lib, pkgs, ... }:
let
cfg = config.services.gickup;
in
{
config = lib.mkIf cfg.enable {
# TODO: add a service that will look at the backed up files and hardlink
# the ones that have a matching hash together to save space. This can
# either run routinely (i.e. trigger by systemd-timer), or be activated
# whenever a gickup@<slug>.service finishes. The latter is probably better.
# systemd.services."gickup-hardlink" = {
# serviceConfig = {
# Type = "oneshot";
# ExecStart = let
# script = pkgs.writeShellApplication {
# name = "gickup-hardlink-files.sh";
# runtimeInputs = [ pkgs.coreutils pkgs.jdupes ];
# text = ''
# '';
# };
# in lib.getExe script;
# User = "gickup";
# Group = "gickup";
# BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [
# "${cfg.dataDir}:/var/lib/gickup"
# ];
# Slice = "system-gickup.slice";
# StateDirectory = "gickup";
# # Hardening options
# # TODO:
# PrivateNetwork = true;
# };
# };
};
}

11
modules/gickup/import-from-toml.nix Normal file
View File

@ -0,0 +1,11 @@
{ config, lib, pkgs, ... }:
let
cfg = config.services.gickup;
in
{
config = lib.mkIf cfg.enable {
# TODO: import cfg.instances from a toml file to make it easier for non-nix users
# to add repositories to mirror
};
}

9
modules/gickup/set-description.nix Normal file
View File

@ -0,0 +1,9 @@
{ config, lib, pkgs, ... }:
let
cfg = config.services.gickup;
in
{
config = lib.mkIf cfg.enable {
# TODO: create .git/description files for each repo where cfg.instances.<instance>.description is set
};
}

76
modules/gickup/update-linktree.nix Normal file
View File

@ -0,0 +1,76 @@
{ config, lib, pkgs, ... }:
let
cfg = config.services.gickup;
in
{
config = lib.mkIf cfg.enable {
# TODO: run upon completion of cloning a repository
# TODO: update symlink for one repo at a time (e.g. gickup-linktree@<instance>.service)
systemd.services."gickup-linktree" = {
serviceConfig = {
Type = "oneshot";
ExecStart = let
script = pkgs.writeShellApplication {
name = "gickup-update-symlink-tree.sh";
runtimeInputs = [
pkgs.coreutils
pkgs.findutils
];
text = ''
shopt -s nullglob
for repository in ./*/*/*; do
REPOSITORY_RELATIVE_DIRS=''${repository#"./"}
echo "Checking $REPOSITORY_RELATIVE_DIRS"
declare -a REVISIONS
readarray -t REVISIONS < <(find "$repository" -mindepth 1 -maxdepth 1 -printf "%f\n" | sort --numeric-sort --reverse)
if [[ "''${#REVISIONS[@]}" == 0 ]]; then
echo "Found no revisions for $repository, continuing"
continue
fi
LAST_REVISION="''${REVISIONS[0]}"
SYMLINK_PATH="../linktree/''${REPOSITORY_RELATIVE_DIRS}"
mkdir -p "$(dirname "$SYMLINK_PATH")"
EXPECTED_SYMLINK_TARGET=$(realpath "''${repository}/''${LAST_REVISION}")
EXISTING_SYMLINK_TARGET=$(realpath "$SYMLINK_PATH" || echo "<none>")
if [[ "$EXISTING_SYMLINK_TARGET" != "$EXPECTED_SYMLINK_TARGET" ]]; then
echo "Updating symlink for $REPOSITORY_RELATIVE_DIRS"
rm "$SYMLINK_PATH" ||:
ln -rs "$EXPECTED_SYMLINK_TARGET" "$SYMLINK_PATH"
else
echo "Symlink already up to date, continuing..."
fi
echo "---"
done
'';
};
in lib.getExe script;
User = "gickup";
Group = "gickup";
BindPaths = lib.optionals (cfg.dataDir != "/var/lib/gickup") [
"${cfg.dataDir}:/var/lib/gickup"
];
Slice = "system-gickup.slice";
StateDirectory = "gickup";
WorkingDirectory = "/var/lib/gickup/raw";
# Hardening options
# TODO:
PrivateNetwork = true;
};
};
};
}

21
packages/cgit.nix Normal file
View File

@ -0,0 +1,21 @@
{ cgit, fetchurl, ... }:
let
pname = cgit.pname;
commit = "09d24d7cd0b7e85633f2f43808b12871bb209d69";
in
cgit.overrideAttrs (_: {
version = "1.2.3-unstable-2024.07.16";
src = fetchurl {
url = "https://git.zx2c4.com/cgit/snapshot/${pname}-${commit}.tar.xz";
hash = "sha256-gfgjAXnWRqVCP+4cmYOVdB/3OFOLJl2WBOc3bFVDsjw=";
};
# cgit is tightly coupled with git and needs a git source tree to build.
# IMPORTANT: Remember to check which git version cgit needs on every version
# bump (look for "GIT_VER" in the top-level Makefile).
gitSrc = fetchurl {
url = "mirror://kernel/software/scm/git/git-2.46.0.tar.xz";
hash = "sha256-fxI0YqKLfKPr4mB0hfcWhVTCsQ38FVx+xGMAZmrCf5U=";
};
})

8
secrets/bicep/bicep.yaml
View File

@ -3,6 +3,8 @@ calendar-bot:
mysql_password: ENC[AES256_GCM,data:Gqag8yOgPH3ntoT5TmaqJWv1j+si2qIyz5Ryfw5E2A==,iv:kQDcxnPfwJQcFovI4f87UDt18F8ah3z5xeY86KmdCyY=,tag:A1sCSNXJziAmtUWohqwJgg==,type:str]
mysql:
password: ENC[AES256_GCM,data:KqEe0TVdeMIzPKsmFg9x0X9xWijnOk306ycyXTm2Tpqo/O0F,iv:Y+hlQ8n1ZIP9ncXBzd2kCSs/DWVTWhiEluFVwZFKRCA=,tag:xlaUk0Wftk62LpYE5pKNQw==,type:str]
gickup:
github-token: ENC[AES256_GCM,data:H/yBDLIvEXunmaUha3c2vUWKLRIbl9QrC0t13AQDRCTnrvhabeiUFLNxZ/F+4B6sZ2aPSgZoB69WwnHvh1wLdiFp1qLWKW/jQPvzZOxE4n+jXrnSOutUWktbPzVj,iv:KFW4jRru93JIl9doVFtcNkJDWp89NlzWjPDflHxcL/U=,tag:YtgyRxkoZO9MkuP3DJh7zA==,type:str]
sops:
kms: []
gcp_kms: []
@ -63,8 +65,8 @@ sops:
cTh5bnJ3WW90aXRCSUp6NHFYeU1tZ0kK4afdtJwGNu6wLRI0fuu+mBVeqVeB0rgX
0q5hwyzjiRnHnyjF38CmcGgydSfDRmF6P+WIMbCwXC6LwfRhAmBGPg==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2024-08-15T21:18:33Z"
mac: ENC[AES256_GCM,data:uR5HgeDAYqoqB9kk1V6p0T30+v6WpQJi4+qIeCDRnoUPnQKUVR10hvBhICck+E+Uh8p+tGhM6Uf3YrAJAV0ZCUiNJjtwDJQQLUDT53vdOAXN4xADCQqNuhgVwVMaruoTheEiwOswRuhFeEwy0gBj3Ze2pu47lueHYclmEzumLeQ=,iv:t0UyXN2YaR2m7M/pV2wTLJG5wVfqTIUs7wSQMmyeTVw=,tag:O7dIffzrDAXz3kGx5uazhw==,type:str]
lastmodified: "2025-05-07T21:34:48Z"
mac: ENC[AES256_GCM,data:n6GHD+nQmZL17WvUZiMCBLRHbtpoKU6U8o/Oraj0VSRi/pQ74QWGVEcIX87kFjBvR2C+UPd3KwXzjQHhjUfHpz9EjIGi6tXLTTo8K3ptd2wCL8MW418TVO4KV+BFmHGT4kwlbdoqaJ2SA7HcfXNaC68e/2CTXhtkLpIwGXtYWJA=,iv:iC5QX/JMwno4mBljPdorNmcQSD2wy/wOYvGrUoC2yzg=,tag:GuFNQ6+d6o9DYC6Do/IEqQ==,type:str]
pgp:
- created_at: "2024-08-04T00:03:40Z"
enc: |-
@ -87,4 +89,4 @@ sops:
-----END PGP MESSAGE-----
fp: F7D37890228A907440E1FD4846B9228E814A2AAC
unencrypted_suffix: _unencrypted
version: 3.9.0
version: 3.9.4