handle effects of surprise traefik upgrade

I have WIP stuff that goes uncommitted frequently enough that I think this is better than keeping the list accurate. Probably wouldn't feel that was in a multi-user environment

.forgejo/workflows/build-keycloak.yaml

@@ -20,3 +20,16 @@ jobs:
           tags: git.janky.solutions/jankysolutions/infra/keycloak:latest
           platforms: linux/amd64
           push: ${{ github.ref == 'refs/heads/main' }}
+  rollout:
+    name: roll out update
+    runs-on: ubuntu-latest
+    container:
+      image: git.janky.solutions/jankysolutions/infra/deployer:latest
+    needs: [build-keycloak]
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: roll out update
+        run: |
+          set -euo pipefail
+          echo "${{ secrets.KUBERNETES_CLIENT_CONFIG }}" > ~/.kube/config
+          kubectl -n keycloak rollout restart deployment keycloak

.forgejo/workflows/build-traefik-forward-auth.yaml

@@ -20,3 +20,16 @@ jobs:
           tags: git.janky.solutions/jankysolutions/infra/traefik-forward-auth:latest
           platforms: linux/amd64
           push: ${{ github.ref == 'refs/heads/main' }}
+  rollout:
+    name: roll out update
+    runs-on: ubuntu-latest
+    container:
+      image: git.janky.solutions/jankysolutions/infra/deployer:latest
+    needs: [build-traefik-forward-auth]
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - name: roll out update
+        run: |
+          set -euo pipefail
+          echo "${{ secrets.KUBERNETES_CLIENT_CONFIG }}" > ~/.kube/config
+          kubectl -n kube-system rollout restart deployment -l app=traefik-forward-auth

.forgejo/workflows/validate-helm-render.yaml

@@ -11,5 +11,11 @@ jobs:
     steps:
       - run: apk add --no-cache nodejs git helm kubectl bash
       - uses: actions/checkout@v4
+      - name: Deploy ~/.kube/config
+        run: |
+          set -euo pipefail
+          mkdir -p ~/.kube
+          echo "${{ secrets.KUBERNETES_CLIENT_CONFIG }}" > ~/.kube/config
+          chmod 600 ~/.kube/config
       - run: ./helm/render-all.sh
       - run: git diff --exit-code

containers/keycloak/Containerfile

@@ -1,4 +1,4 @@
-FROM quay.io/keycloak/keycloak:26.0 as builder
+FROM quay.io/keycloak/keycloak:26.1.4 as builder
 
 # Enable health and metrics support
 ENV KC_HEALTH_ENABLED=true
@@ -10,7 +10,7 @@ ENV KC_DB=postgres
 WORKDIR /opt/keycloak
 RUN /opt/keycloak/bin/kc.sh build
 
-FROM quay.io/keycloak/keycloak:26.0
+FROM quay.io/keycloak/keycloak:26.1
 COPY --from=builder /opt/keycloak/ /opt/keycloak/
 ADD themes/jankysolutions /opt/keycloak/themes/jankysolutions
 

containers/openbao/Containerfile

@@ -1,4 +1,4 @@
-FROM node:23 AS web-build
+FROM node:22 AS web-build
 RUN git clone -b v2.1.0 https://github.com/openbao/openbao /usr/src/openbao
 WORKDIR /usr/src/openbao
 RUN make ember-dist

containers/synapse/Containerfile

@@ -1,4 +1,4 @@
-FROM matrixdotorg/synapse:v1.123.0
+FROM ghcr.io/element-hq/synapse:v1.127.1
 RUN pip install boto3 humanize tqdm
 # there is probably a better way to figure out where the site packages are
 # this used to be hard coded to /usr/local/lib/python3.11/site-packages but then synapse updated it's minor python version and it broke

containers/traefik-forward-auth/Containerfile

@@ -1,4 +1,4 @@
-FROM library/golang:1.23 as builder
+FROM library/golang:1.24 as builder
 RUN git clone -b v2.3.0 https://github.com/thomseddon/traefik-forward-auth /go/traefik-forward-auth
 WORKDIR /go/traefik-forward-auth
 RUN CGO_ENABLED=0 go build -a -installsuffix nocgo -o /traefik-forward-auth ./cmd

helm/external-secrets/kustomization.yaml

@@ -7,5 +7,5 @@ helmCharts:
       enabled: false # default, bitwarden-sdk-server doesn't work with vaultwarden (https://github.com/external-secrets/bitwarden-sdk-server/issues/18)
   namespace: external-secrets
   releaseName: external-secrets
-  version: 0.13.0
+  version: 0.15.0
   repo: https://charts.external-secrets.io

helm/metrics-server/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: monitoring
+helmCharts:
+- name: metrics-server
+  releaseName: metrics-server
+  valuesInline:
+    metrics:
+      enabled: true
+    serviceMonitor:
+      enabled: true
+  version: 3.12.2
+  repo: https://kubernetes-sigs.github.io/metrics-server/

helm/render-all.sh

@@ -7,6 +7,12 @@ header="# DO NOT EDIT: This file has been automatically generated by the script
 render_helm() {
     target="${1}"
     component="${2}"
+
+    if [ ! -d "${component}" ]; then
+        echo "skipping non-existant component ${component}"
+        return
+    fi
+
     mkdir -p "${target}/${component}"
     echo "${header}" > "${target}/${component}/bundle.yaml"
     rm -rf "${component}/charts" # it doesn't seem to update them otherwise
@@ -14,11 +20,11 @@ render_helm() {
 }
 
 # main k8s cluster operators
-for component in openbao external-secrets secrets-store-csi-driver cert-manager-webhook-pdns; do
+for component in openbao external-secrets secrets-store-csi-driver ceph-csi-cephfs cert-manager-webhook-pdns; do
     render_helm ../k8s/operators "${component}"
 done
 
 # cisco k8s cluster operators
-for component in rook cert-manager-webhook-pdns traefik; do
+for component in rook cert-manager-webhook-pdns traefik metrics-server; do
     render_helm ../talos/k8s/operators "${component}"
 done

helm/rook/kustomization.yaml

@@ -4,5 +4,5 @@ helmCharts:
 - name: rook-ceph
   namespace: rook-ceph
   releaseName: rook-ceph
-  version: v1.16.2
+  version: v1.16.3
   repo: https://charts.rook.io/release

helm/secrets-store-csi-driver/kustomization.yaml

@@ -10,5 +10,5 @@ helmCharts:
       registrar:
         logVerbosity: 1
   releaseName: secrets-store-csi-driver
-  version: v1.4.5
+  version: 1.4.8
   repo: https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts

helm/traefik/kustomization.yaml

@@ -4,11 +4,13 @@ helmCharts:
 - name: traefik
   namespace: traefik
   releaseName: traefik
-  version: 34.2.0
+  version: 34.4.1
   valuesInline:
     deployment:
-      replicas: 2
+      replicas: 1
     ports:
+      web:
+        hostPort: 80
       websecure:
         hostPort: 443
         proxyProtocol:
@@ -17,4 +19,12 @@ helmCharts:
     providers:
       kubernetesCRD:
         allowCrossNamespace: true
+    logs:
+      access:
+        enabled: false
+        format: json
+        fields:
+          headers:
+            names:
+              User-Agent: keep
   repo: https://traefik.github.io/charts

k8s/forgejo-dev/kustomization.yaml

@@ -6,5 +6,5 @@ resources:
  - config.yaml
  - ingress.yaml
  - services.yaml
- - statefulset.yaml
+#  - statefulset.yaml
  - secrets.yaml

k8s/forgejo/config.yaml

@@ -1,44 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: forgejo
-data:
-  FORGEJO__repository__DEFAULT_REPO_UNITS: repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects,repo.packages,repo.actions # this is the default for 1.22, should be safe to remove when we get there
-  FORGEJO__repository__PREFERRED_LICENSES: AGPL-3.0-or-later,LGPL-3.0-or-later,GPL-3.0-or-later,Apache-2.0,MIT
-  FORGEJO__repository__ENABLE_PUSH_CREATE_USER: "true"
-  FORGEJO__metrics__ENABLED: "true"
-  FORGEJO__email.incoming__USERNAME: git@janky.solutions
-  FORGEJO__email.incoming__USE_TLS: "true"
-  FORGEJO__email.incoming__PORT: "993"
-  FORGEJO__email.incoming__HOST: mx1.janky.email
-  FORGEJO__email.incoming__REPLY_TO_ADDRESS: git+%{token}@janky.solutions
-  FORGEJO__email.incoming__ENABLED: "true"
-  FORGEJO__mailer__FROM: git@janky.solutions
-  FORGEJO__mailer__USER: git@janky.solutions
-  FORGEJO__mailer__SMTP_ADDR: mx1.janky.email
-  FORGEJO__mailer__PROTOCOL: smtps
-  FORGEJO__mailer__ENABLED: "true"
-  FORGEJO__service__NO_REPLY_ADDRESS: noreply.git.janky.solutions
-  FORGEJO__service__ALLOW_ONLY_EXTERNAL_REGISTRATION: "true"
-  FORGEJO__service__SHOW_REGISTRATION_BUTTON: "false"
-  FORGEJO__service__DEFAULT_KEEP_EMAIL_PRIVATE: "true"
-  FORGEJO__service__ENABLE_NOTIFY_MAIL: "true"
-  FORGEJO__oauth2_client__ENABLE_AUTO_REGISTRATION: "true"
-  FORGEJO__oauth2_client__REGISTER_EMAIL_CONFIRM: "false"
-  FORGEJO__openid__ENABLE_OPENID_SIGNUP: "false"
-  FORGEJO__server__ROOT_URL: https://git.janky.solutions/
-  FORGEJO__server__DOMAIN: git.janky.solutions
-  FORGEJO__DEFAULT__APP_NAME: Janky Solutions
-  DEFAULT_MERGE_STYLE: rebase
-  # FORGEJO__storage__STORAGE_TYPE: minio
-  # FORGEJO__storage__MINIO_ENDPOINT: storage.home.finn.io
-  # FORGEJO__storage__MINIO_USE_SSL: "true"
-  # FORGEJO__storage__MINIO_LOCATION: us-sea-1
-  # FORGEJO__storage__MINIO_ACCESS_KEY_ID: aQ0zCsTpCSJ8eKLtGZ3C
-  # FORGEJO__storage__MINIO_BUCKET: forgejo
-  # FORGEJO__attachment__STORAGE_TYPE: minio
-  # FORGEJO__attachment__MINIO_ENDPOINT: storage.home.finn.io
-  # FORGEJO__attachment__MINIO_USE_SSL: "true"
-  # FORGEJO__attachment__MINIO_LOCATION: us-sea-1
-  # FORGEJO__attachment__MINIO_ACCESS_KEY_ID: aQ0zCsTpCSJ8eKLtGZ3C
-  # FORGEJO__attachment__MINIO_BUCKET: forgejo

k8s/forgejo/kustomization.yaml

@@ -3,7 +3,7 @@ kind: Kustomization
 namespace: forgejo
 resources:
   - namespace.yaml
-  - config.yaml
+  # - config.yaml
   - ingress.yaml
   - forgejo-secret-sync.yaml
   - services.yaml
@@ -21,3 +21,37 @@ configMapGenerator:
   - name: forgejo-secret-sync
     files:
       - forgejo-secret-sync/forgejo-secret-sync.py
+  - name: forgejo
+    literals:
+      - FORGEJO__repository__DEFAULT_REPO_UNITS=repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects,repo.packages,repo.actions # this is the default for 1.22, should be safe to remove when we get there
+      - FORGEJO__repository__PREFERRED_LICENSES=AGPL-3.0-or-later,LGPL-3.0-or-later,GPL-3.0-or-later,Apache-2.0,MIT
+      - FORGEJO__repository__ENABLE_PUSH_CREATE_USER="true"
+      - FORGEJO__metrics__ENABLED=true
+      - FORGEJO__email.incoming__USERNAME=git@janky.solutions
+      - FORGEJO__email.incoming__USE_TLS=true
+      - FORGEJO__email.incoming__PORT=993
+      - FORGEJO__email.incoming__HOST=mx1.janky.email
+      - FORGEJO__email.incoming__REPLY_TO_ADDRESS=git+%{token}@janky.solutions
+      - FORGEJO__email.incoming__ENABLED=true
+      - FORGEJO__mailer__FROM=git@janky.solutions
+      - FORGEJO__mailer__USER=git@janky.solutions
+      - FORGEJO__mailer__SMTP_ADDR=mx1.janky.email
+      - FORGEJO__mailer__PROTOCOL=smtps
+      - FORGEJO__mailer__ENABLED=true
+      - FORGEJO__service__NO_REPLY_ADDRESS=noreply.git.janky.solutions
+      - FORGEJO__service__ALLOW_ONLY_EXTERNAL_REGISTRATION=true
+      - FORGEJO__service__SHOW_REGISTRATION_BUTTON=false
+      - FORGEJO__service__DEFAULT_KEEP_EMAIL_PRIVATE=true
+      - FORGEJO__service__ENABLE_NOTIFY_MAIL=true
+      - FORGEJO__oauth2_client__ENABLE_AUTO_REGISTRATION=true
+      - FORGEJO__oauth2_client__REGISTER_EMAIL_CONFIRM=false
+      - FORGEJO__openid__ENABLE_OPENID_SIGNUP=false
+      - FORGEJO__server__ROOT_URL=https://git.janky.solutions/
+      - FORGEJO__server__DOMAIN=git.janky.solutions
+      - FORGEJO__DEFAULT__APP_NAME=Janky Solutions
+      - DEFAULT_MERGE_STYLE=rebase
+      - FORGEJO__storage__STORAGE_TYPE=minio
+      - FORGEJO__storage__MINIO_ENDPOINT=s3.janky.solutions
+      - FORGEJO__storage__MINIO_USE_SSL=true
+      - FORGEJO__storage__MINIO_BUCKET=forgejo
+      - FORGEJO__storage__SERVE_DIRECT=false

k8s/forgejo/statefulset.yaml

@@ -14,7 +14,7 @@ spec:
         app: forgejo
     spec:
       containers:
-      - image: codeberg.org/forgejo/forgejo:10.0.0
+      - image: codeberg.org/forgejo/forgejo:10.0.3
         imagePullPolicy: Always
         name: forgejo
         resources: {}

k8s/http-ingress/external-services.yaml

@@ -0,0 +1,29 @@
+# ceph-dashboard.home.finn.io
+apiVersion: traefik.io/v1alpha1
+kind: IngressRoute
+metadata:
+  name: ceph-dashboard.home.finn.io
+spec:
+  entryPoints:
+    - websecure
+  routes:
+    - match: Host(`ceph-dashboard.k8s.home.finn.io`) && PathPrefix(`/`)
+      kind: Rule
+      services:
+        - name: ceph-dashboard-home-finn-io
+          kind: Service
+          port: 80
+      middlewares:
+        - name: kube-system-traefik-forward-auth@kubernetescrd
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ceph-dashboard-home-finn-io
+spec:
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80
+  type: ExternalName
+  externalName: 10.5.1.219

k8s/http-ingress/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: http-ingress
+resources:
+  - namespace.yaml
+  - external-services.yaml

k8s/http-ingress/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: http-ingress

k8s/keycloak/deployment.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: keycloak
-          image: git.janky.solutions/jankysolutions/infra/keycloak:25.0
+          image: git.janky.solutions/jankysolutions/infra/keycloak:latest
           imagePullPolicy: Always
           resources: {}
           volumeMounts:

k8s/keycloak/ingress.yaml

@@ -1,4 +1,4 @@
-apiVersion: traefik.containo.us/v1alpha1
+apiVersion: traefik.io/v1alpha1
 kind: ServersTransport
 metadata:
   name: keycloak-frontend

k8s/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
   - external-account-rbac
   - forgejo
   - generic-device-plugin
+  - http-ingress
   - invoiceninja
   - keycloak
   - matrix

k8s/matrix/bridge-facebook.yaml

@@ -60,7 +60,7 @@ spec:
             - secretRef:
                 name: bridge-facebook
       containers:
-      - image: dock.mau.dev/mautrix/meta:v0.4.3
+      - image: dock.mau.dev/mautrix/meta:v0.4.5
         name: bridge-facebook
         resources: {}
         command: ["/usr/bin/mautrix-meta", "-c", "/data/config.yaml", "--no-update"]

k8s/matrix/bridge-signal.yaml

@@ -57,7 +57,7 @@ spec:
             - secretRef:
                 name: bridge-signal
       containers:
-        - image: dock.mau.dev/mautrix/signal:v0.7.5
+        - image: dock.mau.dev/mautrix/signal:v0.8.1
           name: bridge-signal
           resources: {}
           ports:

k8s/matrix/janky.bot-homeserver.yaml

@@ -37,7 +37,7 @@ spec:
             - secretRef:
                 name: synapse-janky-bot
       containers:
-      - image: matrixdotorg/synapse:v1.123.0
+      - image: matrixdotorg/synapse:v1.127.1
         name: synapse
         resources: {}
         volumeMounts:

k8s/matrix/janky.solutions-homeserver.yaml

@@ -195,19 +195,7 @@ spec:
       name: homeserver-janky-solutions
       port: 8008
   - kind: Rule
-    match: Host(`matrix.janky.solutions`) && PathPrefix(`/_matrix/client/unstable/org.matrix.msc3575/sync`)
+    match: Host(`matrix.janky.solutions`) && PathRegexp(`^/_matrix/client/.*/(login|logout|refresh)$`)
-    services:
-    - kind: Service
-      name: sliding-sync-janky-solutions
-      port: 8008
-  - kind: Rule
-    match: Host(`matrix.janky.solutions`) && PathPrefix(`/client`)
-    services:
-    - kind: Service
-      name: sliding-sync-janky-solutions
-      port: 8008
-  - kind: Rule
-    match: Host(`matrix.janky.solutions`) && PathPrefix(`/_matrix/client/{version:.*}/{endpoint:(login|logout|refresh)}`)
     services:
     - kind: Service
       name: mas-janky-solutions
@@ -242,7 +230,7 @@ spec:
               name: mas-janky-solutions
       containers:
       - name: mas-janky-solutions
-        image: ghcr.io/element-hq/matrix-authentication-service:0.12
+        image: ghcr.io/element-hq/matrix-authentication-service:0.14
         args: ["server", "-c", "/data/config.yaml"]
         env:
         - name: PGPASSWORD
@@ -315,7 +303,7 @@ spec:
   teamId: matrix
   volume:
     size: 50Gi
-  numberOfInstances: 1
+  numberOfInstances: 2
   users:
     superuser:
     - superuser

k8s/miniflux/miniflux.yaml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: miniflux
-          image: docker.io/miniflux/miniflux:2.2.5
+          image: docker.io/miniflux/miniflux:2.2.6
           imagePullPolicy: Always
           resources: {}
           envFrom:

k8s/mobilizon/kustomization.yaml

@@ -4,7 +4,7 @@ namespace: mobilizon
 resources:
   - namespace.yaml
   - database.yaml
-  - mobilizon.yaml
+  # - mobilizon.yaml
   - ingress.yaml
 configMapGenerator:
   - name: mobilizon

k8s/mobilizon/mobilizon.yaml

@@ -18,7 +18,7 @@ spec:
         fsGroupChangePolicy: "OnRootMismatch"
       containers:
         - name: mobilizon
-          image: framasoft/mobilizon:5.1.1
+          image: framasoft/mobilizon:5.1.2
           ports:
             - containerPort: 4000
               name: web

k8s/monitoring/alerts-postgres.yaml

@@ -0,0 +1,20 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    prometheus: k8s
+    role: alert-rules
+  name: postgres-operator-alerts
+  namespace: monitoring
+spec:
+  groups:
+  - name: postgres-operator
+    rules:
+    - alert: PatroniHasNoLeader
+      expr: (max by (scope) (patroni_master) < 1) and (max by (scope) (patroni_standby_leader) < 1)
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: Patroni has no Leader (instance {{ $labels.instance }})
+        description: "A leader node (neither primary nor standby) cannot be found inside the cluster {{ $labels.scope }}\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

k8s/monitoring/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: monitoring
 resources:
+  - alerts-postgres.yaml
   - promtail.yaml
   - ingresses.yaml
   - secrets.yaml

k8s/monitoring/matrix-alertmanager-receiver.yaml

@@ -31,7 +31,7 @@ spec:
                 name: matrix-alertmanager-receiver
       containers:
       - name: matrix-alertmanager-receiver
-        image: docker.io/metio/matrix-alertmanager-receiver:2024.12.18
+        image: docker.io/metio/matrix-alertmanager-receiver:2025.3.26
         args: ["--config-path", "/config/config.yaml"]
         resources:
           limits:

k8s/monitoring/prom-scrape-configs/scrape-configs.yaml

@@ -29,7 +29,6 @@
 - job_name: static_http_targets
   static_configs:
     - targets:
-      - ubnt:9001 # mongod-exporter
       - rpi4-build:8080
       - docker:9170 # docker hub prometheus exporter
       - jellyfin:8096 # jellyfin

k8s/monitoring/thanos.yaml

@@ -48,6 +48,7 @@ spec:
   ports:
   - port: 9090
     targetPort: 9090
+    name: http
 ---
 apiVersion: v1
 kind: Service

k8s/netbox/netbox.yaml

@@ -13,7 +13,7 @@ spec:
     spec:
       containers:
       - name: netbox
-        image: ghcr.io/netbox-community/netbox:v4.2.0-3.0.2
+        image: ghcr.io/netbox-community/netbox:v4.2.6
         envFrom:
           - secretRef:
               name: netbox

k8s/operators/external-secrets/bitwarden.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
         - name: bitwarden-cli
-          image: ghcr.io/charlesthomas/bitwarden-cli:2025.1.2
+          image: ghcr.io/charlesthomas/bitwarden-cli:2025.2.0
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:

k8s/operators/kube-prometheus/kustomization.yaml

@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - https://github.com/prometheus-operator/kube-prometheus?ref=74f4e0cda3f3c2a4e8a1ab7d9bdbee019a47c851
+  - https://github.com/prometheus-operator/kube-prometheus?ref=v0.14.0
 patches:
   - path: node-exporter-patch.yaml
     target:

k8s/operators/secrets-store-csi-driver/bundle.yaml

@@ -7,8 +7,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver
   namespace: secrets-store-csi-driver
 ---
@@ -24,8 +24,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-keep-crds
   namespace: secrets-store-csi-driver
 ---
@@ -41,8 +41,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-upgrade-crds
   namespace: secrets-store-csi-driver
 ---
@@ -55,8 +55,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
     rbac.authorization.k8s.io/aggregate-to-admin: "true"
     rbac.authorization.k8s.io/aggregate-to-edit: "true"
   name: secretproviderclasses-admin-role
@@ -82,8 +82,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secretproviderclasses-role
 rules:
 - apiGroups:
@@ -149,8 +149,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
     rbac.authorization.k8s.io/aggregate-to-view: "true"
   name: secretproviderclasses-viewer-role
 rules:
@@ -172,8 +172,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
     rbac.authorization.k8s.io/aggregate-to-view: "true"
   name: secretproviderclasspodstatuses-viewer-role
 rules:
@@ -194,8 +194,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secretprovidersyncing-role
 rules:
 - apiGroups:
@@ -223,8 +223,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-keep-crds
 rules:
 - apiGroups:
@@ -247,8 +247,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-upgrade-crds
 rules:
 - apiGroups:
@@ -269,8 +269,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secretproviderclasses-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -289,8 +289,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secretprovidersyncing-rolebinding
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -313,8 +313,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-keep-crds
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -337,8 +337,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-upgrade-crds
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -357,8 +357,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver
   namespace: secrets-store-csi-driver
 spec:
@@ -374,8 +374,8 @@ spec:
         app.kubernetes.io/instance: secrets-store-csi-driver
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: secrets-store-csi-driver
-        app.kubernetes.io/version: 1.4.5
+        app.kubernetes.io/version: 1.4.8
-        helm.sh/chart: secrets-store-csi-driver-1.4.5
+        helm.sh/chart: secrets-store-csi-driver-1.4.8
     spec:
       affinity:
         nodeAffinity:
@@ -422,7 +422,7 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: spec.nodeName
-        image: registry.k8s.io/csi-secrets-store/driver:v1.4.5
+        image: registry.k8s.io/csi-secrets-store/driver:v1.4.8
         imagePullPolicy: IfNotPresent
         livenessProbe:
           failureThreshold: 5
@@ -520,8 +520,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-keep-crds
   namespace: secrets-store-csi-driver
 spec:
@@ -538,7 +538,7 @@ spec:
         - secretproviderclasspodstatuses.secrets-store.csi.x-k8s.io
         - -p
         - '{"metadata":{"annotations": {"helm.sh/resource-policy": "keep"}}}'
-        image: registry.k8s.io/csi-secrets-store/driver-crds:v1.4.5
+        image: registry.k8s.io/csi-secrets-store/driver-crds:v1.4.8
         imagePullPolicy: IfNotPresent
         name: crds-keep
       nodeSelector:
@@ -560,8 +560,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store-csi-driver-upgrade-crds
   namespace: secrets-store-csi-driver
 spec:
@@ -575,7 +575,7 @@ spec:
         - apply
         - -f
         - crds/
-        image: registry.k8s.io/csi-secrets-store/driver-crds:v1.4.5
+        image: registry.k8s.io/csi-secrets-store/driver-crds:v1.4.8
         imagePullPolicy: IfNotPresent
         name: crds-upgrade
       nodeSelector:
@@ -593,8 +593,8 @@ metadata:
     app.kubernetes.io/instance: secrets-store-csi-driver
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: secrets-store-csi-driver
-    app.kubernetes.io/version: 1.4.5
+    app.kubernetes.io/version: 1.4.8
-    helm.sh/chart: secrets-store-csi-driver-1.4.5
+    helm.sh/chart: secrets-store-csi-driver-1.4.8
   name: secrets-store.csi.k8s.io
 spec:
   attachRequired: false

k8s/s3staticsites/ingress.yaml

@@ -79,3 +79,27 @@ spec:
                 name:  s3staticsites
                 port:
                   name: http
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: janky-solutions
+  labels:
+    name: janky-solutions
+  annotations:
+     cert-manager.io/cluster-issuer: letsencrypt
+spec:
+  tls:
+    - hosts: [janky.solutions]
+      secretName: janky.solutions
+  rules:
+    - host: janky.solutions
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name:  s3staticsites
+                port:
+                  name: http

k8s/shlink/shlink.yaml

@@ -18,7 +18,7 @@ spec:
         fsGroup: 1001
         fsGroupChangePolicy: "OnRootMismatch"
       containers:
-      - image: ghcr.io/shlinkio/shlink:4.4.1
+      - image: ghcr.io/shlinkio/shlink:4.4.6
         name: shlink
         resources: {}
         ports:

k8s/snipeit/kustomization.yaml

@@ -16,7 +16,7 @@ configMapGenerator:
       - DB_HOST=mysql
       - APP_URL=https://snipe.herzfeld.casa
       - APP_FORCE_TLS=true
-      - APP_TIMEZONE=US/Pacific
+      - APP_TIMEZONE=America/Los_Angeles
       - APP_LOCALE=en-US
       - MAIL_MAILER=smtp
       - MAIL_HOST=mx1.janky.email

k8s/snipeit/statefulset.yaml

@@ -18,7 +18,7 @@ spec:
         fsGroupChangePolicy: "OnRootMismatch"
       containers:
       - name: snipe
-        image: snipe/snipe-it:v7.1.16
+        image: snipe/snipe-it:v8.0.4
         ports:
         - containerPort: 80
           name: web

k8s/spoolman/kustomization.yaml

@@ -3,6 +3,6 @@ kind: Kustomization
 namespace: spoolman
 resources:
  - namespace.yaml
- - database.yaml
+#  - database.yaml
- - deployment.yaml
+#  - deployment.yaml
  - ingress.yaml

k8s/system/blocked-user-agents.yaml

@@ -43,7 +43,7 @@ spec:
   entryPoints:
     - websecure
   routes:
-  - match: HeadersRegexp(`User-Agent`, `.*(meta-externalagent|ClaudeBot|Amazonbot|SemrushBot|Googlebot|YandexBot|Bytespider).*`)
+  - match: HeadersRegexp(`User-Agent`, `.*(meta-externalagent|ClaudeBot|Amazonbot|SemrushBot|Googlebot|YandexBot|Bytespider|GPTBot|Chrome/114.0.0.0|Chrome/56.0.8415.1887|Android 5.0|Android 6.0|Android 7.0|Android 8.0).*`)
     kind: Rule
     priority: 1000
     services:

k8s/system/traefik-dashboard.yaml

@@ -1,4 +1,4 @@
-apiVersion: traefik.containo.us/v1alpha1
+apiVersion: traefik.io/v1alpha1
 kind: IngressRoute
 metadata:
   name: traefik-dashboard

k8s/system/traefik-forward-auth-herzfeld-casa.yaml

@@ -4,18 +4,21 @@ metadata:
   name: traefik-forward-auth-herzfeld-casa
   namespace: kube-system
   labels:
-    app: traefik-forward-auth-herzfeld-casa
+    app: traefik-forward-auth
+    instance: herzfeld-casa
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: traefik-forward-auth-herzfeld-casa
+      app: traefik-forward-auth
+      instance: herzfeld-casa
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
-        app: traefik-forward-auth-herzfeld-casa
+        app: traefik-forward-auth
+        instance: herzfeld-casa
     spec:
       terminationGracePeriodSeconds: 60
       containers:
@@ -41,12 +44,13 @@ metadata:
   namespace: kube-system
 spec:
   selector:
-    app: traefik-forward-auth-herzfeld-casa
+    app: traefik-forward-auth
+    instance: herzfeld-casa
   ports:
   - name: auth-http
     port: 4181
 ---
-apiVersion: traefik.containo.us/v1alpha1
+apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
   name: traefik-forward-auth-herzfeld-casa

k8s/system/traefik-forward-auth.yaml

@@ -5,17 +5,20 @@ metadata:
   namespace: kube-system
   labels:
     app: traefik-forward-auth
+    instance: default
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: traefik-forward-auth
+      instance: default
   strategy:
     type: Recreate
   template:
     metadata:
       labels:
         app: traefik-forward-auth
+        instance: default
     spec:
       terminationGracePeriodSeconds: 60
       containers:
@@ -42,11 +45,12 @@ metadata:
 spec:
   selector:
     app: traefik-forward-auth
+    instance: default
   ports:
   - name: auth-http
     port: 4181
 ---
-apiVersion: traefik.containo.us/v1alpha1
+apiVersion: traefik.io/v1alpha1
 kind: Middleware
 metadata:
   name: traefik-forward-auth

k8s/system/traefik-helm-chart.yaml

@@ -33,9 +33,12 @@ spec:
     providers:
       kubernetesCRD:
         allowCrossNamespace: true
+        allowExternalNameServices: true
+      kubernetesIngress:
+        allowExternalNameServices: true
     logs:
       access:
-        enabled: false
+        enabled: true
         format: json
         fields:
           headers:

k8s/system/traefik/external-services.yaml

@@ -3,6 +3,7 @@
     (list "minio-console" "minio-console.home.finn.io" "http://minio:9001")
     (list "minio" "storage.home.finn.io" "http://minio:9000")
     (list "jellyfin" "jellyfin.janky.solutions" "http://jellyfin:8096")
+    (list "jellyseerr" "jellyfin-requests.janky.solutions" "http://media-ingest:5055")
     (list "dns" "dns.janky.solutions" "http://dns:9191")
     (list "dns443" "dns.janky.solutions:443" "http://dns:9191")
     (list "legacy-monitoring" "monitoring.home.finn.io" "http://monitoring-0:3000")

k8s/wordpress/hannah.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: wordpress
-        image: library/wordpress:6.7.1
+        image: library/wordpress:6.7.2
         env:
           - name: WORDPRESS_DB_HOST
             value: hannah-db
@@ -128,6 +128,7 @@ kind: ExternalSecret
 metadata:
   name: hannah-db
 spec:
+  refreshInterval: "0"
   target:
     name: hannah-db
   dataFrom:
@@ -153,3 +154,35 @@ spec:
                 name:  hannah
                 port:
                   name: web
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: seattledoulaservices.com
+  annotations:
+     cert-manager.io/cluster-issuer: letsencrypt
+spec:
+  tls:
+    - hosts: [seattledoulaservices.com, www.seattledoulaservices.com]
+      secretName: seattledoulaservices.com
+  rules:
+    - host: seattledoulaservices.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name:  hannah
+                port:
+                  name: web
+    - host: www.seattledoulaservices.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name:  hannah
+                port:
+                  name: web

talos/config_patch.yaml

@@ -12,3 +12,5 @@ machine:
                   - bind
                   - rshared
                   - rw
+        extraArgs:
+            rotate-server-certificates: true

talos/k8s/kubelet-serving-cert-approver/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # This will raise some warnings, issue filed here: https://github.com/alex1989hu/kubelet-serving-cert-approver/issues/255
+  - https://github.com/alex1989hu/kubelet-serving-cert-approver//deploy/standalone?ref=v0.9.0

talos/k8s/kustomization.yaml

@@ -2,5 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - operators
+  - kubelet-serving-cert-approver
   - monitoring
   - rook

talos/k8s/monitoring/kube-state-metrics.yaml

@@ -178,7 +178,7 @@ spec:
         - --port=8081
         - --telemetry-host=127.0.0.1
         - --telemetry-port=8082
-        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0
+        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.15.0
         name: kube-state-metrics
         resources:
           limits:
@@ -202,7 +202,7 @@ spec:
         - --secure-listen-address=:8443
         - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
         - --upstream=http://127.0.0.1:8081/
-        image: quay.io/brancz/kube-rbac-proxy:v0.18.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.19.0
         name: kube-rbac-proxy-main
         ports:
         - containerPort: 8443
@@ -229,7 +229,7 @@ spec:
         - --secure-listen-address=:9443
         - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
         - --upstream=http://127.0.0.1:8082/
-        image: quay.io/brancz/kube-rbac-proxy:v0.18.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.19.0
         name: kube-rbac-proxy-self
         ports:
         - containerPort: 9443

talos/k8s/monitoring/kubernetesControlPlane-prometheusRule.yaml

@@ -0,0 +1,301 @@
+# based on https://github.com/prometheus-operator/kube-prometheus/blob/v0.14.0/manifests/kubernetesControlPlane-prometheusRule.yaml
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    app.kubernetes.io/component: prometheus
+    app.kubernetes.io/instance: k8s
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/part-of: kube-prometheus
+    app.kubernetes.io/version: 2.54.1
+    prometheus: k8s
+    role: alert-rules
+  name: prometheus-k8s-prometheus-rules
+  namespace: monitoring
+spec:
+  groups:
+  - name: prometheus
+    rules:
+    - alert: PrometheusBadConfig
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusbadconfig
+        summary: Failed Prometheus configuration reload.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        max_over_time(prometheus_config_last_reload_successful{job="prometheus-k8s",namespace="monitoring"}[5m]) == 0
+      for: 10m
+      labels:
+        severity: critical
+    - alert: PrometheusSDRefreshFailure
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to refresh SD with mechanism {{$labels.mechanism}}.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheussdrefreshfailure
+        summary: Failed Prometheus SD refresh.
+      expr: |
+        increase(prometheus_sd_refresh_failures_total{job="prometheus-k8s",namespace="monitoring"}[10m]) > 0
+      for: 20m
+      labels:
+        severity: warning
+    - alert: PrometheusKubernetesListWatchFailures
+      annotations:
+        description: Kubernetes service discovery of Prometheus {{$labels.namespace}}/{{$labels.pod}} is experiencing {{ printf "%.0f" $value }} failures with LIST/WATCH requests to the Kubernetes API in the last 5 minutes.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuskuberneteslistwatchfailures
+        summary: Requests in Kubernetes SD are failing.
+      expr: |
+        increase(prometheus_sd_kubernetes_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusNotificationQueueRunningFull
+      annotations:
+        description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotificationqueuerunningfull
+        summary: Prometheus alert notification queue predicted to run full in less than 30m.
+      expr: |
+        # Without min_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        (
+          predict_linear(prometheus_notifications_queue_length{job="prometheus-k8s",namespace="monitoring"}[5m], 60 * 30)
+        >
+          min_over_time(prometheus_notifications_queue_capacity{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
+      annotations:
+        description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstosomealertmanagers
+        summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
+      expr: |
+        (
+          rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        /
+          rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+        * 100
+        > 1
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusNotConnectedToAlertmanagers
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotconnectedtoalertmanagers
+        summary: Prometheus is not connected to any Alertmanagers.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        max_over_time(prometheus_notifications_alertmanagers_discovered{job="prometheus-k8s",namespace="monitoring"}[5m]) < 1
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusTSDBReloadsFailing
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbreloadsfailing
+        summary: Prometheus has issues reloading blocks from disk.
+      expr: |
+        increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
+      for: 4h
+      labels:
+        severity: warning
+    - alert: PrometheusTSDBCompactionsFailing
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustsdbcompactionsfailing
+        summary: Prometheus has issues compacting blocks.
+      expr: |
+        increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
+      for: 4h
+      labels:
+        severity: warning
+    - alert: PrometheusNotIngestingSamples
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusnotingestingsamples
+        summary: Prometheus is not ingesting samples.
+      expr: |
+        (
+          sum without(type) (rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m])) <= 0
+        and
+          (
+            sum without(scrape_job) (prometheus_target_metadata_cache_entries{job="prometheus-k8s",namespace="monitoring"}) > 0
+          or
+            sum without(rule_group) (prometheus_rule_group_rules{job="prometheus-k8s",namespace="monitoring"}) > 0
+          )
+        )
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusDuplicateTimestamps
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with different values but duplicated timestamp.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusduplicatetimestamps
+        summary: Prometheus is dropping samples with duplicate timestamps.
+      expr: |
+        rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusOutOfOrderTimestamps
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value  }} samples/s with timestamps arriving out of order.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusoutofordertimestamps
+        summary: Prometheus drops samples with out-of-order timestamps.
+      expr: |
+        rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 10m
+      labels:
+        severity: warning
+    - alert: PrometheusRemoteStorageFailures
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotestoragefailures
+        summary: Prometheus fails to send samples to remote storage.
+      expr: |
+        (
+          (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring"}[5m]))
+        /
+          (
+            (rate(prometheus_remote_storage_failed_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) or rate(prometheus_remote_storage_samples_failed_total{job="prometheus-k8s",namespace="monitoring"}[5m]))
+          +
+            (rate(prometheus_remote_storage_succeeded_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]) or rate(prometheus_remote_storage_samples_total{job="prometheus-k8s",namespace="monitoring"}[5m]))
+          )
+        )
+        * 100
+        > 1
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusRemoteWriteBehind
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritebehind
+        summary: Prometheus remote write is behind.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        (
+          max_over_time(prometheus_remote_storage_highest_timestamp_in_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
+        - ignoring(remote_name, url) group_right
+          max_over_time(prometheus_remote_storage_queue_highest_sent_timestamp_seconds{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+        > 120
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusRemoteWriteDesiredShards
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-k8s",namespace="monitoring"}` $labels.instance | query | first | value }}.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusremotewritedesiredshards
+        summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
+      expr: |
+        # Without max_over_time, failed scrapes could create false negatives, see
+        # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
+        (
+          max_over_time(prometheus_remote_storage_shards_desired{job="prometheus-k8s",namespace="monitoring"}[5m])
+        >
+          max_over_time(prometheus_remote_storage_shards_max{job="prometheus-k8s",namespace="monitoring"}[5m])
+        )
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusRuleFailures
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusrulefailures
+        summary: Prometheus is failing rule evaluations.
+      expr: |
+        increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: critical
+    - alert: PrometheusMissingRuleEvaluations
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusmissingruleevaluations
+        summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
+      expr: |
+        increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusTargetLimitHit
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because the number of targets exceeded the configured target_limit.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetlimithit
+        summary: Prometheus has dropped targets because some scrape configs have exceeded the targets limit.
+      expr: |
+        increase(prometheus_target_scrape_pool_exceeded_target_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusLabelLimitHit
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has dropped {{ printf "%.0f" $value }} targets because some samples exceeded the configured label_limit, label_name_length_limit or label_value_length_limit.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuslabellimithit
+        summary: Prometheus has dropped targets because some scrape configs have exceeded the labels limit.
+      expr: |
+        increase(prometheus_target_scrape_pool_exceeded_label_limits_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusScrapeBodySizeLimitHit
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured body_size_limit.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapebodysizelimithit
+        summary: Prometheus has dropped some targets that exceeded body size limit.
+      expr: |
+        increase(prometheus_target_scrapes_exceeded_body_size_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusScrapeSampleLimitHit
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed {{ printf "%.0f" $value }} scrapes in the last 5m because some targets exceeded the configured sample_limit.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheusscrapesamplelimithit
+        summary: Prometheus has failed scrapes that have exceeded the configured sample limit.
+      expr: |
+        increase(prometheus_target_scrapes_exceeded_sample_limit_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusTargetSyncFailure
+      annotations:
+        description: '{{ printf "%.0f" $value }} targets in Prometheus {{$labels.namespace}}/{{$labels.pod}} have failed to sync because invalid configuration was supplied.'
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheustargetsyncfailure
+        summary: Prometheus has failed to sync targets.
+      expr: |
+        increase(prometheus_target_sync_failed_total{job="prometheus-k8s",namespace="monitoring"}[30m]) > 0
+      for: 5m
+      labels:
+        severity: critical
+    - alert: PrometheusHighQueryLoad
+      annotations:
+        description: Prometheus {{$labels.namespace}}/{{$labels.pod}} query API has less than 20% available capacity in its query engine for the last 15 minutes.
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheushighqueryload
+        summary: Prometheus is reaching its maximum capacity serving concurrent requests.
+      expr: |
+        avg_over_time(prometheus_engine_queries{job="prometheus-k8s",namespace="monitoring"}[5m]) / max_over_time(prometheus_engine_queries_concurrent_max{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0.8
+      for: 15m
+      labels:
+        severity: warning
+    - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
+      annotations:
+        description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
+        runbook_url: https://runbooks.prometheus-operator.dev/runbooks/prometheus/prometheuserrorsendingalertstoanyalertmanager
+        summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
+      expr: |
+        min without (alertmanager) (
+          rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring",alertmanager!~``}[5m])
+        /
+          rate(prometheus_notifications_sent_total{job="prometheus-k8s",namespace="monitoring",alertmanager!~``}[5m])
+        )
+        * 100
+        > 3
+      for: 15m
+      labels:
+        severity: critical

talos/k8s/monitoring/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 namespace: monitoring
 resources:
   - kube-state-metrics.yaml
+  - kubernetesControlPlane-prometheusRule.yaml
   - node-exporter.yaml
   - prom.yaml
   - servicemonitors.yaml

talos/k8s/monitoring/node-exporter.yaml

@@ -32,7 +32,7 @@ spec:
         - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run/k3s/containerd/.+|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/)
         - --collector.netclass.ignored-devices=^(veth.*|[a-f0-9]{15})$
         - --collector.netdev.device-exclude=^(veth.*|[a-f0-9]{15})$
-        image: quay.io/prometheus/node-exporter:v1.8.2
+        image: quay.io/prometheus/node-exporter:v1.9.0
         imagePullPolicy: IfNotPresent
         name: node-exporter
         ports:
@@ -76,7 +76,7 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: status.podIP
-        image: quay.io/brancz/kube-rbac-proxy:v0.18.2
+        image: quay.io/brancz/kube-rbac-proxy:v0.19.0
         imagePullPolicy: IfNotPresent
         name: kube-rbac-proxy
         ports:

talos/k8s/monitoring/servicemonitors.yaml

@@ -1,3 +1,4 @@
+# this file is heavily based on https://github.com/prometheus-operator/kube-prometheus/blob/v0.14.0/manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -264,6 +265,19 @@ spec:
     scheme: https
     tlsConfig:
       insecureSkipVerify: true
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    honorLabels: true
+    interval: 30s
+    path: /metrics/resource
+    port: https-metrics
+    relabelings:
+    - action: replace
+      sourceLabels:
+      - __metrics_path__
+      targetLabel: metrics_path
+    scheme: https
+    tlsConfig:
+      insecureSkipVerify: true
   jobLabel: app.kubernetes.io/name
   namespaceSelector:
     matchNames:

talos/k8s/operators/kustomization.yaml

@@ -4,6 +4,7 @@ resources:
   - cert-manager
   - cert-manager-webhook-pdns
   - local-path-provisioner
+  - metrics-server
   - prometheus-operator
   - rook
   - traefik

talos/k8s/operators/metrics-server/bundle.yaml

@@ -0,0 +1,273 @@
+# DO NOT EDIT: This file has been automatically generated by the script in helm/render-all.sh, edits may get overwritten
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: metrics-server
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: system:metrics-server
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes/metrics
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - namespaces
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: system:metrics-server-aggregated-reader
+rules:
+- apiGroups:
+  - metrics.k8s.io
+  resources:
+  - pods
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: metrics-server-auth-reader
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: metrics-server:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: monitoring
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: system:metrics-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:metrics-server
+subjects:
+- kind: ServiceAccount
+  name: metrics-server
+  namespace: monitoring
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: metrics-server
+  namespace: monitoring
+spec:
+  ports:
+  - appProtocol: https
+    name: https
+    port: 443
+    protocol: TCP
+    targetPort: https
+  selector:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/name: metrics-server
+  type: ClusterIP
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: metrics-server
+  namespace: monitoring
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: metrics-server
+      app.kubernetes.io/name: metrics-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: metrics-server
+        app.kubernetes.io/name: metrics-server
+    spec:
+      containers:
+      - args:
+        - --secure-port=10250
+        - --cert-dir=/tmp
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        - --kubelet-use-node-status-port
+        - --metric-resolution=15s
+        - --authorization-always-allow-paths=/metrics
+        image: registry.k8s.io/metrics-server/metrics-server:v0.7.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /livez
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 0
+          periodSeconds: 10
+        name: metrics-server
+        ports:
+        - containerPort: 10250
+          name: https
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /readyz
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 20
+          periodSeconds: 10
+        resources:
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 1000
+          seccompProfile:
+            type: RuntimeDefault
+        volumeMounts:
+        - mountPath: /tmp
+          name: tmp
+      priorityClassName: system-cluster-critical
+      serviceAccountName: metrics-server
+      volumes:
+      - emptyDir: {}
+        name: tmp
+---
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: v1beta1.metrics.k8s.io
+spec:
+  group: metrics.k8s.io
+  groupPriorityMinimum: 100
+  insecureSkipTLSVerify: true
+  service:
+    name: metrics-server
+    namespace: monitoring
+    port: 443
+  version: v1beta1
+  versionPriority: 100
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    app.kubernetes.io/instance: metrics-server
+    app.kubernetes.io/managed-by: Helm
+    app.kubernetes.io/name: metrics-server
+    app.kubernetes.io/version: 0.7.2
+    helm.sh/chart: metrics-server-3.12.2
+  name: metrics-server
+  namespace: monitoring
+spec:
+  endpoints:
+  - interval: 1m
+    path: /metrics
+    port: https
+    scheme: https
+    scrapeTimeout: 10s
+    tlsConfig:
+      insecureSkipVerify: true
+  jobLabel: app.kubernetes.io/instance
+  namespaceSelector:
+    matchNames:
+    - default
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: metrics-server
+      app.kubernetes.io/name: metrics-server

talos/k8s/operators/metrics-server/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - bundle.yaml

talos/k8s/operators/metrics-server/namespace.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: rook-ceph
+  name: monitoring
   labels:
     pod-security.kubernetes.io/enforce: privileged

talos/k8s/operators/prometheus-operator/kustomization.yaml

@@ -2,5 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: monitoring
 resources:
-  - namespace.yaml
+  - https://github.com/prometheus-operator/prometheus-operator?ref=v0.80.0
-  - https://github.com/prometheus-operator/prometheus-operator?ref=v0.79.2

talos/k8s/operators/rook/bundle.yaml

@@ -15095,7 +15095,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-cmd-reporter
@@ -15117,7 +15117,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-mgr
@@ -15130,7 +15130,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-osd
@@ -15149,7 +15149,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-rgw
@@ -15162,7 +15162,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-system
@@ -15400,7 +15400,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-system
@@ -15716,7 +15716,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rbd-csi-nodeplugin
@@ -15948,7 +15948,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-cluster-mgmt
@@ -15981,7 +15981,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-global
@@ -16169,7 +16169,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-mgr-cluster
@@ -16225,7 +16225,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-object-bucket
@@ -16301,7 +16301,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-system
@@ -16502,7 +16502,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-system
@@ -16592,7 +16592,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-global
@@ -16651,7 +16651,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-system
@@ -16890,7 +16890,7 @@ metadata:
     app.kubernetes.io/created-by: helm
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/part-of: rook-ceph-operator
-    helm.sh/chart: rook-ceph-v1.16.2
+    helm.sh/chart: rook-ceph-v1.16.3
     operator: rook
     storage-backend: ceph
   name: rook-ceph-operator
@@ -16906,7 +16906,7 @@ spec:
     metadata:
       labels:
         app: rook-ceph-operator
-        helm.sh/chart: rook-ceph-v1.16.2
+        helm.sh/chart: rook-ceph-v1.16.3
     spec:
       containers:
       - args:
@@ -16933,7 +16933,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: docker.io/rook/ceph:v1.16.2
+        image: docker.io/rook/ceph:v1.16.3
         imagePullPolicy: IfNotPresent
         name: rook-ceph-operator
         resources:

talos/k8s/operators/rook/cluster.yaml

@@ -11,7 +11,7 @@ spec:
     # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/.
     # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v19.2.0-20240927
     # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities
-    image: quay.io/ceph/ceph:v19.2.0
+    image: quay.io/ceph/ceph:v19.2.1
     # Whether to allow unsupported versions of Ceph. Currently Reef and Squid are supported.
     # Future versions such as Tentacle (v20) would require this to be set to `true`.
     # Do not set to true in production.
@@ -66,6 +66,7 @@ spec:
     ssl: false
     # The url of the Prometheus instance
     # prometheusEndpoint: <protocol>://<prometheus-host>:<port>
+    prometheusEndpoint: http://prometheus-operated.monitoring.svc.cluster.local:9090
     # Whether SSL should be verified if the Prometheus server is using https
     # prometheusEndpointSSLVerify: false
   # enable prometheus alerting for cluster
@@ -221,6 +222,11 @@ spec:
   #   monitoring:
   #   crashcollector:
   resources:
+    osd-hdd:
+      limits:
+        memory: 10Gi
+      requests:
+        memory: 4Gi
   #The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory
   #   mgr:
   #     limits:

talos/k8s/operators/traefik/bundle.yaml

@@ -7,7 +7,7 @@ metadata:
     app.kubernetes.io/instance: traefik-traefik
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: traefik
-    helm.sh/chart: traefik-34.2.0
+    helm.sh/chart: traefik-34.4.1
   name: traefik
   namespace: traefik
 ---
@@ -18,7 +18,7 @@ metadata:
     app.kubernetes.io/instance: traefik-traefik
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: traefik
-    helm.sh/chart: traefik-34.2.0
+    helm.sh/chart: traefik-34.4.1
   name: traefik-traefik
 rules:
 - apiGroups:
@@ -94,7 +94,7 @@ metadata:
     app.kubernetes.io/instance: traefik-traefik
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: traefik
-    helm.sh/chart: traefik-34.2.0
+    helm.sh/chart: traefik-34.4.1
   name: traefik-traefik
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -112,7 +112,7 @@ metadata:
     app.kubernetes.io/instance: traefik-traefik
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: traefik
-    helm.sh/chart: traefik-34.2.0
+    helm.sh/chart: traefik-34.4.1
   name: traefik
   namespace: traefik
 spec:
@@ -137,12 +137,12 @@ metadata:
     app.kubernetes.io/instance: traefik-traefik
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: traefik
-    helm.sh/chart: traefik-34.2.0
+    helm.sh/chart: traefik-34.4.1
   name: traefik
   namespace: traefik
 spec:
   minReadySeconds: 0
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app.kubernetes.io/instance: traefik-traefik
@@ -162,7 +162,7 @@ spec:
         app.kubernetes.io/instance: traefik-traefik
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: traefik
-        helm.sh/chart: traefik-34.2.0
+        helm.sh/chart: traefik-34.4.1
     spec:
       automountServiceAccountToken: true
       containers:
@@ -195,7 +195,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: docker.io/traefik:v3.3.2
+        image: docker.io/traefik:v3.3.4
         imagePullPolicy: IfNotPresent
         lifecycle: null
         livenessProbe:
@@ -217,6 +217,7 @@ spec:
           name: traefik
           protocol: TCP
         - containerPort: 8000
+          hostPort: 80
           name: web
           protocol: TCP
         - containerPort: 8443
@@ -267,7 +268,7 @@ metadata:
     app.kubernetes.io/instance: traefik-traefik
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: traefik
-    helm.sh/chart: traefik-34.2.0
+    helm.sh/chart: traefik-34.4.1
   name: traefik
 spec:
   controller: traefik.io/ingress-controller

talos/k8s/rook/buckets.yaml

@@ -37,3 +37,11 @@ metadata:
 spec:
   bucketName: loki
   storageClassName: rook-ceph-bucket
+---
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: forgejo
+spec:
+  bucketName: forgejo
+  storageClassName: rook-ceph-bucket

talos/k8s/rook/dashboard.yaml

@@ -0,0 +1,20 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ceph-dashboard.k8s.home.finn.io
+  labels:
+    name: ceph-dashboard.k8s.home.finn.io
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+spec:
+  rules:
+    - host: ceph-dashboard.k8s.home.finn.io
+      http:
+        paths:
+          - pathType: Prefix
+            path: /
+            backend:
+              service:
+                name: rook-ceph-mgr-dashboard
+                port:
+                  number: 7000

talos/k8s/rook/kustomization.yaml

@@ -3,5 +3,7 @@ kind: Kustomization
 namespace: rook-ceph
 resources:
   - buckets.yaml
+  - dashboard.yaml
+  - filesystems.yaml
   - s3-pool.yaml
   - servicemonitor.yaml

tf/bao.tf

@@ -3,8 +3,8 @@ resource "vault_auth_backend" "kubernetes" {
 }
 
 resource "vault_kubernetes_auth_backend_config" "example" {
-  backend                = vault_auth_backend.kubernetes.path
+  backend         = vault_auth_backend.kubernetes.path
-  kubernetes_host        = "https://kubernetes.default.svc.cluster.local:443"
+  kubernetes_host = "https://kubernetes.default.svc.cluster.local:443"
 }
 
 resource "vault_kubernetes_auth_backend_role" "k8s-default" {
@@ -13,7 +13,7 @@ resource "vault_kubernetes_auth_backend_role" "k8s-default" {
   bound_service_account_names      = ["default"]
   bound_service_account_namespaces = ["*"]
   token_ttl                        = 3600
-  token_policies                   = [
+  token_policies = [
     vault_policy.k8s_default.name
   ]
 }
@@ -30,6 +30,6 @@ resource "vault_policy" "k8s_default" {
 
   policy = templatefile("bao-policies/k8s-default.hcl", {
     k8s_auth_backend_accessor = vault_auth_backend.kubernetes.accessor,
-    k8s_secrets_path = vault_mount.static_secrets.path,
+    k8s_secrets_path          = vault_mount.static_secrets.path,
   })
 }

tf/keycloak-clients.tf

@@ -2,16 +2,16 @@
 module "keycloak_client_tofu" {
   source = "./keycloak-client"
 
-  realm = keycloak_realm.dev.id
+  realm       = keycloak_realm.dev.id
   vault_mount = vault_mount.static_secrets.path
 
-  client_id = "tofu"
+  client_id                = "tofu"
   service_accounts_enabled = true
 }
 
 data "keycloak_openid_client" "realm_management" {
-    realm_id = keycloak_realm.dev.id
+  realm_id  = keycloak_realm.dev.id
-    client_id = "realm-management"
+  client_id = "realm-management"
 }
 
 resource "keycloak_openid_client_service_account_role" "client_service_account_role" {

tf/keycloak-normal-flow.tf

@@ -1,6 +1,6 @@
 resource "keycloak_authentication_flow" "webauthn_browser" {
-  realm_id = keycloak_realm.dev.id
+  realm_id    = keycloak_realm.dev.id
-  alias    = "webauthn_browser"
+  alias       = "webauthn_browser"
   description = "browser based authentication"
 }
 
@@ -18,20 +18,20 @@ resource "keycloak_authentication_subflow" "webauthn_flow" {
   parent_flow_alias = keycloak_authentication_flow.webauthn_browser.alias
   provider_id       = "basic-flow"
   requirement       = "ALTERNATIVE"
-  depends_on = [ keycloak_authentication_execution.auth_cookie ]
+  depends_on        = [keycloak_authentication_execution.auth_cookie]
 }
 
 resource "keycloak_authentication_execution" "user_pass" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.webauthn_flow.alias
-  authenticator = "auth-username-password-form"
+  authenticator     = "auth-username-password-form"
-  requirement = "REQUIRED"
+  requirement       = "REQUIRED"
 }
 
 resource "keycloak_authentication_execution" "webauthn" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.webauthn_flow.alias
-  authenticator = "webauthn-authenticator"
+  authenticator     = "webauthn-authenticator"
-  requirement = "REQUIRED"
+  requirement       = "REQUIRED"
 }
 

tf/keycloak-passkey-flow.tf

@@ -1,6 +1,6 @@
 resource "keycloak_authentication_flow" "passkey" {
-  realm_id = keycloak_realm.dev.id
+  realm_id    = keycloak_realm.dev.id
-  alias    = "passkey"
+  alias       = "passkey"
   description = "browser based authentication"
 }
 
@@ -17,14 +17,14 @@ resource "keycloak_authentication_subflow" "passkey_forms" {
   parent_flow_alias = keycloak_authentication_flow.passkey.alias
   provider_id       = "basic-flow"
   requirement       = "ALTERNATIVE"
-  depends_on = [ keycloak_authentication_execution.auth_cookie ]
+  depends_on        = [keycloak_authentication_execution.auth_cookie]
 }
 
 resource "keycloak_authentication_execution" "passkey_username" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.passkey_forms.alias
-  authenticator = "auth-username-form"
+  authenticator     = "auth-username-form"
-  requirement = "REQUIRED"
+  requirement       = "REQUIRED"
 }
 
 resource "keycloak_authentication_subflow" "passkey_passwordless_or_2fa" {
@@ -33,15 +33,15 @@ resource "keycloak_authentication_subflow" "passkey_passwordless_or_2fa" {
   parent_flow_alias = keycloak_authentication_subflow.passkey_forms.alias
   provider_id       = "basic-flow"
   requirement       = "REQUIRED"
-  depends_on = [ keycloak_authentication_execution.passkey_username ]
+  depends_on        = [keycloak_authentication_execution.passkey_username]
 }
 
 resource "keycloak_authentication_execution" "passkey_webauthn_passwordless" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.passkey_passwordless_or_2fa.alias
-  authenticator = "webauthn-authenticator-passwordless"
+  authenticator     = "webauthn-authenticator-passwordless"
-  requirement = "ALTERNATIVE"
+  requirement       = "ALTERNATIVE"
-  depends_on = [ keycloak_authentication_execution.passkey_username ]
+  depends_on        = [keycloak_authentication_execution.passkey_username]
 }
 
 resource "keycloak_authentication_subflow" "passkey_password_and_second_factor" {
@@ -53,10 +53,10 @@ resource "keycloak_authentication_subflow" "passkey_password_and_second_factor"
 }
 
 resource "keycloak_authentication_execution" "passkey_password" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.passkey_password_and_second_factor.alias
-  authenticator = "auth-password-form"
+  authenticator     = "auth-password-form"
-  requirement = "REQUIRED"
+  requirement       = "REQUIRED"
 }
 
 resource "keycloak_authentication_subflow" "passkey_second_factor" {
@@ -68,22 +68,22 @@ resource "keycloak_authentication_subflow" "passkey_second_factor" {
 }
 
 resource "keycloak_authentication_execution" "passkey_user_configured_condition" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.passkey_second_factor.alias
-  authenticator = "conditional-user-configured"
+  authenticator     = "conditional-user-configured"
-  requirement = "REQUIRED"
+  requirement       = "REQUIRED"
 }
 
 resource "keycloak_authentication_execution" "passkey_webauthn" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.passkey_second_factor.alias
-  authenticator = "webauthn-authenticator"
+  authenticator     = "webauthn-authenticator"
-  requirement = "ALTERNATIVE"
+  requirement       = "ALTERNATIVE"
 }
 
 resource "keycloak_authentication_execution" "passkey_otp" {
-  realm_id = keycloak_realm.dev.id
+  realm_id          = keycloak_realm.dev.id
   parent_flow_alias = keycloak_authentication_subflow.passkey_second_factor.alias
-  authenticator = "auth-otp-form"
+  authenticator     = "auth-otp-form"
-  requirement = "ALTERNATIVE"
+  requirement       = "ALTERNATIVE"
 }

tf/keycloak.tf

@@ -1,11 +1,11 @@
 resource "keycloak_realm" "dev" {
-  realm   = "dev.janky.solutions"
+  realm                       = "dev.janky.solutions"
-  enabled = true
+  enabled                     = true
-  display_name = "Janky Solutions (dev)"
+  display_name                = "Janky Solutions (dev)"
   default_signature_algorithm = "RS256"
 }
 
 resource "keycloak_authentication_bindings" "browser_authentication_binding" {
-  realm_id        = keycloak_realm.dev.id
+  realm_id     = keycloak_realm.dev.id
-  browser_flow  = keycloak_authentication_flow.passkey.alias
+  browser_flow = keycloak_authentication_flow.passkey.alias
 }

tf/providers.tf

@@ -1,9 +1,9 @@
 data "terraform_remote_state" "kube" {
   backend = "kubernetes"
   config = {
-    secret_suffix    = "state"
+    secret_suffix = "state"
-    namespace        = "tofu"
+    namespace     = "tofu"
-    config_path      = "../.kubeconfig"
+    config_path   = "../.kubeconfig"
   }
 }
 
@@ -12,13 +12,13 @@ provider "vault" {}
 terraform {
   required_providers {
     keycloak = {
-      source = "mrparkers/keycloak"
+      source  = "mrparkers/keycloak"
       version = ">= 4.0.0"
     }
   }
 }
 
 provider "keycloak" {
-    realm         = "dev.janky.solutions"
+  realm = "dev.janky.solutions"
-    url           = "https://auth.janky.solutions"
+  url   = "https://auth.janky.solutions"
 }