alertmanager matrix receiver

k8s/operators/kube-prometheus/alertmanager-main-secret-patch.yaml

@@ -0,0 +1,5 @@
+$patch: delete
+apiVersion: v1
+kind: Secret
+metadata:
+  name: alertmanager-main

k8s/operators/kube-prometheus/apiregistration-patch.yaml

@@ -0,0 +1,5 @@
+$patch: delete
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1beta1.metrics.k8s.io

k8s/operators/kube-prometheus/kustomization.yaml

@@ -59,6 +59,15 @@ patches:
     target:
       kind: Deployment
       name: kube-state-metrics
+  - path: alertmanager-main-secret-patch.yaml
+    target:
+      kind: Secret
+      name: alertmanager-main
+  - path: node-exporter-prometheus-rule-patches.yaml
+    target:
+      kind: PrometheusRule
+      name: node-exporter-rules
+  - path: apiregistration-patch.yaml
 secretGenerator:
   - name: grafana-config
     namespace: monitoring

k8s/operators/kube-prometheus/node-exporter-prometheus-rule-patches.yaml

@@ -0,0 +1,77 @@
+# Set the dashboard for all node-exporter alerts to the Node Exporter Full dashboard
+- op: add
+  path: /spec/groups/0/rules/0/annotations/dashboard # NodeFilesystemSpaceFillingUp
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/1/annotations/dashboard # NodeFilesystemSpaceFillingUp
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/2/annotations/dashboard # NodeFilesystemAlmostOutOfSpace
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/3/annotations/dashboard # NodeFilesystemAlmostOutOfSpace
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/4/annotations/dashboard # NodeFilesystemFilesFillingUp
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/5/annotations/dashboard # NodeFilesystemFilesFillingUp
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/6/annotations/dashboard # NodeFilesystemAlmostOutOfFiles
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/7/annotations/dashboard # NodeFilesystemAlmostOutOfFiles
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/8/annotations/dashboard # NodeNetworkReceiveErrs
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/9/annotations/dashboard # NodeNetworkTransmitErrs
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/10/annotations/dashboard # NodeHighNumberConntrackEntriesUsed
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/11/annotations/dashboard # NodeTextFileCollectorScrapeError
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/12/annotations/dashboard # NodeClockSkewDetected
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/13/annotations/dashboard # NodeClockNotSynchronising
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/14/annotations/dashboard # NodeRAIDDegraded
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/15/annotations/dashboard # NodeRAIDDiskFailure
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/16/annotations/dashboard # NodeFileDescriptorLimit
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/17/annotations/dashboard # NodeFileDescriptorLimit
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/18/annotations/dashboard # NodeCPUHighUsage
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/19/annotations/dashboard # NodeSystemSaturation
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/20/annotations/dashboard # NodeMemoryMajorPagesFaults
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/21/annotations/dashboard # NodeMemoryHighUtilization
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/22/annotations/dashboard # NodeDiskIOSaturation
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+- op: add
+  path: /spec/groups/0/rules/23/annotations/dashboard # NodeSystemdServiceFailed
+  value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}
+# unclear why this one doesn't want to patch, leaving it out for now
+# - op: add
+#   path: /spec/groups/0/rules/24/annotations/dashboard # NodeBondingDegraded
+#   value: https://grafana.home.finn.io/d/rYdddlPWk/node-exporter-full?var-node={{ $labels.instance }}