Commit actual files, not symlinks :(

prometheus.nix

@@ -1 +0,0 @@
-/etc/nixos/services/grafana.nix

prometheus.nix

@@ -0,0 +1,125 @@
+{ config, pkgs, ... }: 
+
+{
+  services.grafana = {
+    enable = true;
+    domain = "grafana.home.feal.no";
+    port = 2342;
+    addr = "127.0.0.1";
+  };
+
+  services.prometheus = {
+    # Server config
+    enable = true;
+    port = 9001;
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [{
+          targets = [ 
+            "chapel.home.feal.no:${toString config.services.prometheus.exporters.node.port}" 
+            "sulu.home.feal.no:9100"
+            "mccoy.home.feal.no:9100"
+            "borg.home.feal.no:9100"
+            "dlink-feal.home.feal.no:9100"
+          ];
+        }];
+      }
+      {
+        job_name = "openwrt";
+        static_configs = [{ targets = [ "dlink-feal.home.feal.no:9100" ]; } ];
+      }
+      {
+        job_name = "snmp";
+
+        static_configs = [{
+          targets = [
+            "feal-syn1.home.feal.no"
+            "feal-syn2.home.feal.no"
+          ];
+        }];
+        metrics_path = "/snmp";
+        params.module = ["synology"];
+        relabel_configs = [
+          {
+            source_labels = ["__address__"];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = ["__param_target"];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9116";
+          }
+        ];
+      }
+    ];
+
+    # Client config
+    exporters.node = {
+      enable = true;
+      enabledCollectors = [ "systemd" ];
+      port = 9002;
+    };
+  };
+
+  services.loki = {
+    enable = true;
+    configFile = /var/prometheus/loki-local-config.yaml;
+  };
+	
+	systemd.services.promtail = {
+    description = "Promtail service for Loki";
+    wantedBy = [ "multi-user.target" ];
+
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.grafana-loki}/bin/promtail --config.file /var/prometheus/promtail.yaml
+      '';
+    };
+  };
+
+  services.nginx.virtualHosts.${config.services.grafana.domain} = {
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+      proxyWebsockets = true;
+      extraConfig = ''
+        proxy_buffers 8 1024k;  
+        proxy_buffer_size 1024k;
+      '';
+    };
+  };
+
+  systemd.services.grafana-matrix-forwarder = {
+    enable = true;
+    description = "Forward Grafana Alerts to a matrix room";
+    unitConfig = {
+      Type = "simple";
+    };
+    serviceConfig = {
+      EnvironmentFile = "/var/prometheus/GRAFANASECRETS";
+      ExecStart = "/var/prometheus/grafana-matrix-forwarder --user $MATRIX_USER --password $MATRIX_PASSWORD --homeserver $MATRIX_HOMESERVER --host 127.0.0.1";
+    };
+    wantedBy = [ "multi-user.target" ];
+  };
+
+  environment.systemPackages = [
+    pkgs.prometheus-snmp-exporter
+  ];
+
+  systemd.services.prometheus-snmp-exporter = {
+    enable = true;
+    description = "Gather data from SNMP (Synology)";
+    unitConfig = {
+      Type = "simple";
+    };
+    serviceConfig = {
+      ExecStart = "${pkgs.prometheus-snmp-exporter}/bin/snmp_exporter --config.file='/var/prometheus/snmp.yml'";
+    };
+    wantedBy = [ "multi-user.target" ];
+  };
+
+}

var_prometheus/grafana-matrix-forwarder

@@ -1 +0,0 @@
-/var/prometheus/grafana-matrix-forwarder

var_prometheus/loki-local-config.yaml

@@ -1 +0,0 @@
-/var/prometheus/loki-local-config.yaml

var_prometheus/loki-local-config.yaml

@@ -0,0 +1,68 @@
+auth_enabled: false 
+ 
+server: 
+  http_listen_port: 3100 
+  grpc_listen_port: 9096 
+ 
+ingester: 
+  wal: 
+    enabled: true 
+    dir: /var/lib/loki/wal
+  lifecycler: 
+    address: 127.0.0.1 
+    ring: 
+      kvstore: 
+        store: inmemory 
+      replication_factor: 1 
+    final_sleep: 0s 
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed 
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h 
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first 
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) 
+  max_transfer_retries: 0     # Chunk transfers disabled 
+ 
+schema_config: 
+  configs: 
+    - from: 2020-10-24 
+      store: boltdb-shipper 
+      object_store: filesystem 
+      schema: v11 
+      index: 
+        prefix: index_ 
+        period: 24h 
+ 
+storage_config: 
+  boltdb_shipper: 
+    active_index_directory: /var/lib/loki/boltdb-shipper-active
+    cache_location: /var/lib/loki/boltdb-shipper-cache
+    cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space 
+    shared_store: filesystem 
+  filesystem: 
+    directory: /var/lib/loki/chunks
+ 
+compactor: 
+  working_directory: /var/lib/loki/boltdb-shipper-compactor 
+  shared_store: filesystem 
+ 
+limits_config: 
+  reject_old_samples: true 
+  reject_old_samples_max_age: 168h 
+ 
+chunk_store_config: 
+  max_look_back_period: 0s 
+ 
+table_manager: 
+  retention_deletes_enabled: false 
+  retention_period: 0s 
+ 
+ruler: 
+  storage: 
+    type: local 
+    local: 
+      directory: /var/lib/loki/rules 
+  rule_path: /var/lib/loki/rules-temp 
+  alertmanager_url: http://localhost:9093 
+  ring: 
+    kvstore: 
+      store: inmemory 
+  enable_api: true

var_prometheus/promtail.yaml

@@ -1 +0,0 @@
-/var/prometheus/promtail.yaml

var_prometheus/promtail.yaml

@@ -0,0 +1,20 @@
+server:
+  http_listen_port: 28183
+  grpc_listen_port: 0
+
+positions:
+  filename: /var/lib/loki/promtailpositions.yaml
+
+clients:
+  - url: http://127.0.0.1:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: journal
+    journal:
+      max_age: 12h
+      labels:
+        job: systemd-journal
+        host: chapel.home.feal.no
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'

var_prometheus/snmp.yml

@@ -1 +0,0 @@
-/var/prometheus/snmp.yml