From fd41ce8cc21d95c016fa897e421770a8c521fc6c Mon Sep 17 00:00:00 2001 From: Finn Date: Wed, 10 Jul 2024 15:53:01 -0700 Subject: [PATCH] Add additional scrape configs for prometheus --- k8s/monitoring/ingresses.yaml | 33 ++++++ k8s/monitoring/kustomization.yaml | 8 ++ k8s/monitoring/namespace.yaml | 3 + k8s/monitoring/scrape-configs.yaml | 168 +++++++++++++++++++++++++++++ 4 files changed, 212 insertions(+) create mode 100644 k8s/monitoring/ingresses.yaml create mode 100644 k8s/monitoring/scrape-configs.yaml diff --git a/k8s/monitoring/ingresses.yaml b/k8s/monitoring/ingresses.yaml new file mode 100644 index 0000000..8d2f974 --- /dev/null +++ b/k8s/monitoring/ingresses.yaml @@ -0,0 +1,33 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana +spec: + rules: + - host: grafana.monitoring.k8s + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana + port: + number: 3000 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus +spec: + rules: + - host: prometheus.monitoring.k8s + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: prometheus-k8s + port: + number: 9090 diff --git a/k8s/monitoring/kustomization.yaml b/k8s/monitoring/kustomization.yaml index 7e6c453..8ce3934 100644 --- a/k8s/monitoring/kustomization.yaml +++ b/k8s/monitoring/kustomization.yaml @@ -4,3 +4,11 @@ namespace: monitoring resources: - namespace.yaml - promtail.yaml + - ingresses.yaml + - secrets.yaml +secretGenerator: + - name: additional-scrape-configs + options: + disableNameSuffixHash: true + files: + - scrape-configs.yaml diff --git a/k8s/monitoring/namespace.yaml b/k8s/monitoring/namespace.yaml index d325236..d78cb4f 100644 --- a/k8s/monitoring/namespace.yaml +++ b/k8s/monitoring/namespace.yaml @@ -2,3 +2,6 @@ apiVersion: v1 kind: Namespace metadata: name: monitoring + labels: + pod-security.kubernetes.io/warn: privileged + pod-security.kubernetes.io/warn-version: latest diff --git a/k8s/monitoring/scrape-configs.yaml b/k8s/monitoring/scrape-configs.yaml new file mode 100644 index 0000000..ed557a4 --- /dev/null +++ b/k8s/monitoring/scrape-configs.yaml @@ -0,0 +1,168 @@ +- job_name: 'k8s-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name +- job_name: static_http_targets + static_configs: + - targets: + - 'ubnt:9001' # mongod-exporter + - 'ubnt:9130' # unifi-exporter + - 'rpi4-build:8080' + - 'ci-runner-0:8080' + - 'ci-runner-1:8080' + - 'ci-runner-2:8080' + - 'ci-runner-3:8080' + - 'docker:9170' # docker hub prometheus exporter + - 'jellyfin:8096' # jellyfin + - 'signald:9595' # signald on signald + - 172.18.102.10:9595 # signald on finn-dev-0/entanglement + - 'http:8405' # haproxy on http + - media-ingest:8192 +- job_name: promtail + static_configs: + - targets: + - 'mx1.janky.email:9080' # promtail on mx1.janky.email + - k8s-node-1:9080 + - k8s-node-2:9080 + - k8s-node-3:9080 + - k8s-node-usb-0:9080 + - hypervisor-d:9080 + - http:9080 + - media-ingest:9080 + - matrix:9080 + - minio:9080 + - signald:9080 + - forgejo-runner-0:9080 + - forgejo-runner-1:9080 + - forgejo-runner-2:9080 + - forgejo-runner-3:9080 + - forgejo-runner-4:9080 +- job_name: node + static_configs: + - targets: + - 'openwrt:9100' + - 'octopi:9100' + - 'home-assistant:9100' + - 'rpi4-build:9100' + - 'docker:9100' + - 'jellyfin:9100' + - 'minio:9100' + - 'signald:9100' + - 'ci-runner-0:9100' + - 'ci-runner-1:9100' + - 'ci-runner-2:9100' + - 'ci-runner-3:9100' + - 'media-ingest:9100' + - mc:9100 + - http:9100 + - ubnt:9100 + - mx1.janky.email:9100 + - matrix:9100 + - dns:9100 + - hypervisor-d:9100 + - livingroom-tv:9100 + - mobile-proxy:9100 + - forgejo-runner-0:9100 + - forgejo-runner-1:9100 + - forgejo-runner-2:9100 + - forgejo-runner-3:9100 + - forgejo-runner-4:9100 +- job_name: minio + authorization: + credentials_file: /etc/prometheus/secrets/scrape-secrets/minio.token + metrics_path: /minio/v2/metrics/cluster + static_configs: + - targets: ['minio:9000'] +- job_name: 'home-assistant' + metrics_path: /api/prometheus + authorization: + credentials_file: /etc/prometheus/secrets/scrape-secrets/home-assistant.token + static_configs: + - targets: + - home-assistant:8123 +- job_name: forgejo + authorization: + credentials_file: /etc/prometheus/secrets/scrape-secrets/forgejo.token + scheme: https + static_configs: + - targets: [git.janky.solutions] +- job_name: 'blackbox-icmp' + metrics_path: /probe + params: + module: [icmp] # Send ICMP pings + static_configs: + - targets: + - openwrt + - 10.5.2.4 # zoe via wireguard + - 10.5.2.5 # plur-police via wireguard + - 8.8.8.8 + - 8.8.4.4 + - ns380201.janky.solutions + - ns328891.janky.solutions + - dan.home.callpipe.com + - 18.236.0.0 # aws us-west-2 test IP (http://ec2-reachability.amazonaws.com/) + - docker-1.janky.cloud + - taint + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 +- job_name: 'blackbox-http' + metrics_path: /probe + params: + module: [http_2xx] # Look for a HTTP 200 response + static_configs: + - targets: + - https://finn.io + - https://janky.solutions + - https://social.seattle.wa.us + - https://media.social.seattle.wa.us/liveness + - https://ha.herzfeld.casa + - https://grocy.herzfeld.casa + - https://janky.cloud + - https://bw.janky.cloud + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 127.0.0.1:9115 +- job_name: 'mdns' + file_sd_configs: + - files: + - '/tmp/mdns-sd.json' + refresh_interval: 5m +- job_name: speedtest + scrape_interval: 30m + scrape_timeout: 1m + static_configs: + - targets: + - docker:9798 +- job_name: synapse + metrics_path: /_synapse/metrics + static_configs: + - targets: [matrix:8009]