JankySolutions/infra
2
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages 12 Wiki Activity Actions

more dashboard stuff

Browse source
This commit is contained in:
Finn 2024-07-27 16:54:45 -07:00
parent e342f1ade7
commit d5e276cc02
11 changed files with 24686 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

595
k8s/monitoring/grafana-dashboards/loki-and-promtail.json Normal file
View file

@ -0,0 +1,595 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"description": "Loki and Promtail metrics.",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 10880,
"graphTooltip": 0,
"id": 44,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 0
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {},
"editorMode": "code",
"expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{route=~\"api_prom_push|loki_api_v1_push\"}[$interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))",
"legendFormat": "{{status}}",
"range": true,
"refId": "A"
}
],
"title": "pushes/second",
"type": "timeseries"
},
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.0.3",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "000000001"
},
"editorMode": "code",
"expr": "sum(rate (loki_distributor_ingester_appends_total[$interval]))",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "Average batch appends sent to ingesters (Loki)",
"type": "timeseries"
},
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 0
},
"id": 8,
"interval": "",
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.0.3",
"targets": [
{
"datasource": {},
"editorMode": "code",
"expr": "rate(loki_ingester_chunk_entries_count[$interval])",
"range": true,
"refId": "A"
}
],
"title": "chunks ingested/second",
"type": "timeseries"
},
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "binBps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"mean",
"min",
"sum"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.0.3",
"targets": [
{
"datasource": {},
"editorMode": "code",
"exemplar": true,
"expr": "sum(rate(promtail_sent_bytes_total[$interval])) by (instance)",
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "sent by total by promtail",
"type": "timeseries"
},
{
"datasource": {},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"links": [],
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "binBps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "10.0.3",
"targets": [
{
"datasource": {},
"editorMode": "code",
"expr": "sum(rate(promtail_encoded_bytes_total[$interval])) by (instance)",
"format": "time_series",
"instant": false,
"legendFormat": "{{ pod }}",
"refId": "A"
}
],
"title": "encoded by promtail",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": [
{
"auto": false,
"auto_count": 30,
"auto_min": "10s",
"current": {
"selected": false,
"text": "10m",
"value": "10m"
},
"hide": 0,
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": true,
"text": "10m",
"value": "10m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
},
{
"selected": false,
"text": "6h",
"value": "6h"
},
{
"selected": false,
"text": "12h",
"value": "12h"
},
{
"selected": false,
"text": "1d",
"value": "1d"
},
{
"selected": false,
"text": "7d",
"value": "7d"
},
{
"selected": false,
"text": "14d",
"value": "14d"
},
{
"selected": false,
"text": "30d",
"value": "30d"
}
],
"query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d",
"queryValue": "",
"refresh": 2,
"skipUrlSync": false,
"type": "interval"
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "",
"title": "Loki & Promtail",
"uid": "MQHVDmtWk",
"version": 5,
"weekStart": ""
}

23899
k8s/monitoring/grafana-dashboards/node-exporter-full.json Normal file
View file

File diff suppressed because it is too large Load diff

40
k8s/monitoring/ingresses.yaml
View file

@ -4,7 +4,7 @@ metadata:
name: grafana
spec:
rules:
- host: grafana.monitoring.k8s
- host: grafana.home.finn.io
http:
paths:
- path: /
@ -18,7 +18,7 @@ spec:
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: prometheus
name: prometheus-internal
spec:
rules:
- host: prometheus.monitoring.k8s
@ -31,3 +31,39 @@ spec:
name: prometheus-k8s
port:
number: 9090
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: prometheus
annotations:
traefik.ingress.kubernetes.io/router.middlewares: monitoring-oauth2-proxy-prometheus-errors@kubernetescrd, monitoring-oauth2-proxy-prometheus@kubernetescrd
spec:
rules:
- host: prometheus.k8s.home.finn.io
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: prometheus-k8s
port:
number: 9090
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: oauth2-proxy-prometheus
spec:
rules:
- host: prometheus.k8s.home.finn.io
http:
paths:
- pathType: Prefix
path: /oauth2
backend:
service:
name: oauth2-proxy-prometheus
port:
number: 4180

10
k8s/monitoring/kustomization.yaml
View file

@ -6,9 +6,17 @@ resources:
- promtail.yaml
- ingresses.yaml
- secrets.yaml
- oauth2-proxy.yaml
secretGenerator:
- name: additional-scrape-configs
options:
disableNameSuffixHash: true
files:
- scrape-configs.yaml
- prom-scrape-configs/scrape-configs.yaml
configMapGenerator:
- name: grafana-dashboards-custom
options:
disableNameSuffixHash: true
files:
- grafana-dashboards/loki-and-promtail.json
- grafana-dashboards/node-exporter-full.json

87
k8s/monitoring/oauth2-proxy.yaml Normal file
View file

@ -0,0 +1,87 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: oauth2-proxy-prometheus
labels:
app: oauth2-proxy
instance: prometheus
spec:
selector:
matchLabels:
app: oauth2-proxy
instance: prometheus
template:
metadata:
labels:
app: oauth2-proxy
instance: prometheus
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "44180"
spec:
containers:
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:latest
args:
- --http-address=0.0.0.0:4180
- --metrics-address=0.0.0.0:44180
- --real-client-ip-header=x-forwarded-for
envFrom:
- configMapRef:
name: oauth2-proxy
- secretRef:
name: oauth2-proxy-prometheus
env:
- name: OAUTH2_PROXY_CLIENT_ID
value: prometheus
resources:
limits:
memory: "128Mi"
cpu: "500m"
ports:
- containerPort: 4180
---
apiVersion: v1
kind: Service
metadata:
name: oauth2-proxy-prometheus
spec:
selector:
app: oauth2-proxy
instance: prometheus
ports:
- name: http
port: 4180
---
apiVersion: v1
kind: ConfigMap
metadata:
name: oauth2-proxy
data:
OAUTH2_PROXY_PROVIDER: keycloak-oidc
OAUTH2_PROXY_OIDC_ISSUER_URL: https://auth.janky.solutions/realms/janky.solutions
OAUTH2_PROXY_PROVIDER_DISPLAY_NAME: Janky Solutions
OAUTH2_PROXY_EMAIL_DOMAINS: "*"
OAUTH2_PROXY_CODE_CHALLENGE_METHOD: S256
---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: oauth2-proxy-prometheus
spec:
forwardAuth:
address: http://oauth2-proxy-prometheus.monitoring.svc.cluster.local:4180/oauth2/auth
trustForwardHeader: true
---
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: oauth2-proxy-prometheus-errors
spec:
errors:
status:
- "401-403"
service:
name: oauth2-proxy-prometheus
port: 4180
query: "/oauth2/sign_in?rd={url}"

11
k8s/monitoring/scrape-configs.yaml → k8s/monitoring/prom-scrape-configs/scrape-configs.yaml
View file

@ -41,6 +41,13 @@
- signald:9595 # signald on signald
- http:8405 # haproxy on http
- media-ingest:8192
- monitoring-0:3100 # loki on legacy monitoring
- monitoring-0:9374 # smokeping_prober on legacy monitoring
- monitoring-0:3000 # grafana on legacy monitoring
- monitoring-0:9093 # alertmanager on legacy monitoring
- monitoring-0:9090 # prometheus on legacy monitoring
- monitoring-0:9595 # signald on legacy monitoring
- monitoring-0:9099 # rdap monitor on legacy monitoring
- job_name: promtail
static_configs:
- targets:
@ -60,7 +67,8 @@
- forgejo-runner-2:9080
- forgejo-runner-3:9080
- forgejo-runner-4:9080
- job_name: node
- monitoring-0:9080
- job_name: node-exporter
static_configs:
- targets:
- openwrt:9100
@ -85,6 +93,7 @@
- hypervisor-d:9100
- livingroom-tv:9100
- mobile-proxy:9100
- monitoring-0:9100
- forgejo-runner-0:9100
- forgejo-runner-1:9100
- forgejo-runner-2:9100

14
k8s/operators/kube-prometheus/grafana-deployment-patch.yaml
View file

@ -3,3 +3,17 @@
value:
- secretRef:
name: grafana-secrets
- op: add
path: /spec/template/spec/containers/0/volumeMounts/-
value:
mountPath: /grafana-dashboard-definitions/1
name: grafana-dashboards-custom
- op: add
path: /spec/template/spec/volumes/-
value:
name: grafana-dashboards-custom
configMap:
name: grafana-dashboards-custom
# - op: replace
# path: /spec/template/spec/automountServiceAccountToken
# value: true

25
k8s/operators/kube-prometheus/grafana/dashboards.yaml Normal file
View file

@ -0,0 +1,25 @@
{
"apiVersion": 1,
"providers": [
{
"folder": "Default",
"folderUid": "",
"name": "0",
"options": {
"path": "/grafana-dashboard-definitions/0"
},
"orgId": 1,
"type": "file"
},
{
"folder": "Janky Solutions",
"folderUid": "",
"name": "1",
"options": {
"path": "/grafana-dashboard-definitions/1"
},
"orgId": 1,
"type": "file"
}
]
}

0
k8s/operators/kube-prometheus/grafana.ini → k8s/operators/kube-prometheus/grafana/grafana.ini
View file

11
k8s/operators/kube-prometheus/kustomization.yaml
View file

@ -30,7 +30,7 @@ patches:
kind: Prometheus
name: k8s
- path: cluster-role-patch.yaml
- path: prometheus-cluster-role-patch.yaml
target:
kind: ClusterRole
name: prometheus-k8s
@ -45,4 +45,11 @@ secretGenerator:
namespace: monitoring
behavior: replace
files:
- grafana.ini
- grafana/grafana.ini
configMapGenerator:
- name: grafana-dashboards
namespace: monitoring
behavior: replace
files:
- grafana/dashboards.yaml

0
k8s/operators/kube-prometheus/cluster-role-patch.yaml → k8s/operators/kube-prometheus/prometheus-cluster-role-patch.yaml
View file