From eced9bae679bc2411a3af60c3691788f8d0cc450 Mon Sep 17 00:00:00 2001 From: Finn Date: Thu, 8 Aug 2024 12:52:32 -0700 Subject: [PATCH] Move janky.solutions matrix homeserver into kubernetes --- k8s/kustomization.yaml | 4 +- .../appservices-janky.solutions/facebook.yaml | 14 ++ .../appservices-janky.solutions/signal.yaml | 14 ++ .../appservices-janky.solutions/telegram.yaml | 17 ++ ...{bridge-meta.yaml => bridge-facebook.yaml} | 6 +- k8s/matrix/bridge-init/initialize-bridge.py | 14 -- k8s/matrix/bridge-init/initialize-bridge.sh | 7 - k8s/matrix/bridge-signal.yaml | 9 +- k8s/matrix/bridge-telegram.yaml | 9 +- k8s/matrix/bridge.yaml | 21 -- .../config-janky.solutions/homeserver.yaml | 82 ++++++++ k8s/matrix/config-janky.solutions/log.yaml | 21 ++ k8s/matrix/janky.solutions-homeserver.yaml | 192 ++++++++++++++++++ k8s/matrix/kustomization.yaml | 6 +- k8s/monitoring/kustomization.yaml | 1 - k8s/monitoring/namespace.yaml | 7 - .../prom-scrape-configs/scrape-configs.yaml | 1 + .../postgres-operator/kustomization.yaml | 1 + k8s/system/traefik/external-services.yaml | 1 - 19 files changed, 354 insertions(+), 73 deletions(-) create mode 100644 k8s/matrix/appservices-janky.solutions/facebook.yaml create mode 100644 k8s/matrix/appservices-janky.solutions/signal.yaml create mode 100644 k8s/matrix/appservices-janky.solutions/telegram.yaml rename k8s/matrix/{bridge-meta.yaml => bridge-facebook.yaml} (99%) delete mode 100755 k8s/matrix/bridge-init/initialize-bridge.py delete mode 100755 k8s/matrix/bridge-init/initialize-bridge.sh delete mode 100644 k8s/matrix/bridge.yaml create mode 100644 k8s/matrix/config-janky.solutions/homeserver.yaml create mode 100644 k8s/matrix/config-janky.solutions/log.yaml create mode 100644 k8s/matrix/janky.solutions-homeserver.yaml delete mode 100644 k8s/monitoring/namespace.yaml diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 69377c8..6516278 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization resources: - operators - system - - adsb + # - adsb - forgejo - generic-device-plugin - invoiceninja @@ -11,6 +11,6 @@ resources: - monitoring - s3staticsites - shlink - - system-upgrade + - system-upgrade-controller - keycloak - miniflux diff --git a/k8s/matrix/appservices-janky.solutions/facebook.yaml b/k8s/matrix/appservices-janky.solutions/facebook.yaml new file mode 100644 index 0000000..aa59f7f --- /dev/null +++ b/k8s/matrix/appservices-janky.solutions/facebook.yaml @@ -0,0 +1,14 @@ +id: facebook +url: http://bridge-facebook:29319 +as_token: SECRET_FACEBOOK_AS_TOKEN +hs_token: SECRET_FACEBOOK_HS_TOKEN +sender_localpart: SECRET_FACEBOOK_LOCALPART +rate_limited: false +namespaces: + users: + - regex: ^@facebookbot:janky\.solutions$ + exclusive: true + - regex: ^@facebook_.*:janky\.solutions$ + exclusive: true +de.sorunome.msc2409.push_ephemeral: true +push_ephemeral: true diff --git a/k8s/matrix/appservices-janky.solutions/signal.yaml b/k8s/matrix/appservices-janky.solutions/signal.yaml new file mode 100644 index 0000000..21f0346 --- /dev/null +++ b/k8s/matrix/appservices-janky.solutions/signal.yaml @@ -0,0 +1,14 @@ +id: signal +url: http://bridge-signal:29328 +as_token: SECRET_SIGNAL_AS_TOKEN +hs_token: SECRET_SIGNAL_HS_TOKEN +sender_localpart: SECRET_SIGNAL_LOCALPART +rate_limited: false +namespaces: + users: + - regex: ^@signalbot:janky\.solutions$ + exclusive: true + - regex: ^@signal_.*:janky\.solutions$ + exclusive: true +de.sorunome.msc2409.push_ephemeral: true +push_ephemeral: true diff --git a/k8s/matrix/appservices-janky.solutions/telegram.yaml b/k8s/matrix/appservices-janky.solutions/telegram.yaml new file mode 100644 index 0000000..a6d4fba --- /dev/null +++ b/k8s/matrix/appservices-janky.solutions/telegram.yaml @@ -0,0 +1,17 @@ +id: telegram +as_token: SECRET_TELEGRAM_AS_TOKEN +hs_token: SECRET_TELEGRAM_HS_TOKEN +namespaces: + users: + - exclusive: true + regex: '@telegram_.*:janky\.solutions' + - exclusive: true + regex: '@telegrambot:janky\.solutions' + aliases: + - exclusive: true + regex: \#telegram_.*:janky\.solutions +url: http://bridge-telegram:29317 +sender_localpart: SECRET_TELEGRAM_SENDER_LOCALPART +rate_limited: false +de.sorunome.msc2409.push_ephemeral: true +push_ephemeral: true diff --git a/k8s/matrix/bridge-meta.yaml b/k8s/matrix/bridge-facebook.yaml similarity index 99% rename from k8s/matrix/bridge-meta.yaml rename to k8s/matrix/bridge-facebook.yaml index 9328b78..e5fe55f 100644 --- a/k8s/matrix/bridge-meta.yaml +++ b/k8s/matrix/bridge-facebook.yaml @@ -46,9 +46,9 @@ spec: app: bridge-facebook spec: initContainers: - - name: template-config + - name: initialize-secrets image: docker.io/library/python - command: ["python", "/init/initialize-bridge.py"] + command: ["python", "/init/initialize-secrets.py", "config.yaml"] volumeMounts: - name: init mountPath: /init @@ -78,7 +78,7 @@ spec: name: bridge-facebook - name: init configMap: - name: bridge-init + name: secrets-init volumeClaimTemplates: - metadata: name: storage diff --git a/k8s/matrix/bridge-init/initialize-bridge.py b/k8s/matrix/bridge-init/initialize-bridge.py deleted file mode 100755 index 519fd66..0000000 --- a/k8s/matrix/bridge-init/initialize-bridge.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python -import os - -with open("/config/config.yaml") as r: - c = r.read() - -for i in ["AS_TOKEN", "HS_TOKEN", "TG_API_ID", "TG_API_HASH"]: - value = os.getenv(i) - if value is not None: - c = c.replace(i, value) - print("replaced", i) - -with open("/data/config.yaml", 'w') as w: - w.write(c) diff --git a/k8s/matrix/bridge-init/initialize-bridge.sh b/k8s/matrix/bridge-init/initialize-bridge.sh deleted file mode 100755 index 7b6b3a2..0000000 --- a/k8s/matrix/bridge-init/initialize-bridge.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -euo pipefail -cp /config/config.yaml /data/config.yaml -sed -i "s#AS_TOKEN#${AS_TOKEN}#g" /data/config.yaml -sed -i "s#HS_TOKEN#${HS_TOKEN}#g" /data/config.yaml -sed -i "s#TG_API_ID#${TG_API_ID}#g" /data/config.yaml -sed -i "s#TG_API_HASH#${TG_API_HASH}#g" /data/config.yaml diff --git a/k8s/matrix/bridge-signal.yaml b/k8s/matrix/bridge-signal.yaml index 476198d..c6f777a 100644 --- a/k8s/matrix/bridge-signal.yaml +++ b/k8s/matrix/bridge-signal.yaml @@ -41,14 +41,11 @@ spec: metadata: labels: app: bridge-signal - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8000" spec: initContainers: - - name: template-config + - name: initialize-secrets image: docker.io/library/python - command: ["python", "/init/initialize-bridge.py"] + command: ["python", "/init/initialize-secrets.py", "config.yaml"] volumeMounts: - name: init mountPath: /init @@ -77,7 +74,7 @@ spec: name: bridge-signal - name: init configMap: - name: bridge-init + name: secrets-init volumeClaimTemplates: - metadata: name: storage diff --git a/k8s/matrix/bridge-telegram.yaml b/k8s/matrix/bridge-telegram.yaml index 51a3090..432ac98 100644 --- a/k8s/matrix/bridge-telegram.yaml +++ b/k8s/matrix/bridge-telegram.yaml @@ -62,14 +62,11 @@ spec: metadata: labels: app: bridge-telegram - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8000" spec: initContainers: - - name: template-config + - name: initialize-secrets image: docker.io/library/python - command: ["python", "/init/initialize-bridge.py"] + command: ["python", "/init/initialize-secrets.py", "config.yaml"] volumeMounts: - name: init mountPath: /init @@ -99,7 +96,7 @@ spec: name: bridge-telegram - name: init configMap: - name: bridge-init + name: secrets-init volumeClaimTemplates: - metadata: name: storage diff --git a/k8s/matrix/bridge.yaml b/k8s/matrix/bridge.yaml deleted file mode 100644 index d02e708..0000000 --- a/k8s/matrix/bridge.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: bridge-init - namespace: matrix -data: - initialize-bridge.py: | # TODO: migrate this and it's users to secrets-init/secrets-init.py which is similar but more generic - #!/usr/bin/python - import os - - with open("/config/config.yaml") as r: - c = r.read() - - for i in ["AS_TOKEN", "HS_TOKEN", "TG_API_ID", "TG_API_HASH"]: - value = os.getenv(i) - if value is not None: - c = c.replace(i, value) - print("replaced", i) - - with open("/data/config.yaml", 'w') as w: - w.write(c) diff --git a/k8s/matrix/config-janky.solutions/homeserver.yaml b/k8s/matrix/config-janky.solutions/homeserver.yaml new file mode 100644 index 0000000..13e630e --- /dev/null +++ b/k8s/matrix/config-janky.solutions/homeserver.yaml @@ -0,0 +1,82 @@ +server_name: "janky.solutions" +pid_file: /data/homeserver.pid +listeners: + - port: 8008 + tls: false + type: http + x_forwarded: true + resources: + - names: [client, federation] + compress: false + - port: 8000 + type: http + bind_addresses: ['0.0.0.0'] + resources: + - names: [metrics] +database: + name: psycopg2 + args: + host: janky-solutions-homeserver-database.matrix.svc.cluster.local + dbname: matrix +log_config: "/config/log.yaml" +media_store_path: /data/media_store +registration_shared_secret: "SECRET_registration_shared_secret" +report_stats: false +macaroon_secret_key: "SECRET_macaroon_secret_key" +form_secret: "SECRET_form_secret" +signing_key_path: "/secrets/janky.solutions.signing.key" +trusted_key_servers: + - server_name: "matrix.org" +public_baseurl: https://matrix.janky.solutions +ip_range_whitelist: [10.5.1.245,10.5.1.1] +oidc_providers: + - idp_id: keycloak + idp_name: "Janky Solutions Auth" + issuer: "https://auth.janky.solutions/realms/janky.solutions" + client_id: "synapse" + client_secret: SECRET_oidc_secret + scopes: ["openid", "profile"] + user_mapping_provider: + config: + localpart_template: "{{ user.preferred_username }}" + display_name_template: "{{ user.name }}" + backchannel_logout_enabled: true # Optional +password_config: + enabled: false +app_service_config_files: + - /appservices/facebook.yaml + - /appservices/telegram.yaml + - /appservices/signal.yaml +media_storage_providers: +- module: s3_storage_provider.S3StorageProviderBackend + store_local: True + store_remote: True + store_synchronous: True + config: + bucket: matrix-media + # All of the below options are optional, for use with non-AWS S3-like + # services, or to specify access tokens here instead of some external method. + region_name: sea-01 + endpoint_url: https://storage.home.finn.io + access_key_id: SECRET_AWS_ACCESS_KEY_ID + secret_access_key: SECRET_AWS_SECRET_ACCESS_KEY + + # Server Side Encryption for Customer-provided keys + #sse_customer_key: + # Your SSE-C algorithm is very likely AES256 + # Default is AES256. + #sse_customer_algo: + + # The object storage class used when uploading files to the bucket. + # Default is STANDARD. + #storage_class: "STANDARD_IA" + + # Prefix for all media in bucket, can't be changed once media has been uploaded + # Useful if sharing the bucket between Synapses + # Blank if not provided + prefix: "janky.solutions/" + + # The maximum number of concurrent threads which will be used to connect + # to S3. Each thread manages a single connection. Default is 40. + # + #threadpool_size: 20 diff --git a/k8s/matrix/config-janky.solutions/log.yaml b/k8s/matrix/config-janky.solutions/log.yaml new file mode 100644 index 0000000..bf690a3 --- /dev/null +++ b/k8s/matrix/config-janky.solutions/log.yaml @@ -0,0 +1,21 @@ +version: 1 +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' +handlers: + console: + class: logging.StreamHandler + formatter: precise +loggers: + # This is just here so we can leave `loggers` in the config regardless of whether + # we configure other loggers below (avoid empty yaml dict error). + _placeholder: + level: "INFO" + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO +root: + level: INFO + handlers: [console] +disable_existing_loggers: false diff --git a/k8s/matrix/janky.solutions-homeserver.yaml b/k8s/matrix/janky.solutions-homeserver.yaml new file mode 100644 index 0000000..eca9a5b --- /dev/null +++ b/k8s/matrix/janky.solutions-homeserver.yaml @@ -0,0 +1,192 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: homeserver-janky-solutions +spec: + selector: + matchLabels: + app: synapse + instance: janky.solutions + serviceName: homeserver-janky-solutions + replicas: 1 + template: + metadata: + labels: + app: synapse + instance: janky.solutions + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8000" + prometheus.io/path: /_synapse/metrics + spec: + securityContext: + fsGroup: 991 + fsGroupChangePolicy: "OnRootMismatch" + initContainers: + - name: initialize-homeserver-secrets + image: library/python + command: ["python", "/init/initialize-secrets.py", "homeserver.yaml"] + volumeMounts: + - name: init + mountPath: /init + - name: data + mountPath: /data + - name: config + mountPath: /config + envFrom: + - secretRef: + name: synapse-janky-solutions + - name: initialize-bridge-secrets + image: library/python + command: ["python", "/init/initialize-secrets.py", "facebook.yaml", "telegram.yaml", "signal.yaml"] + volumeMounts: + - name: init + mountPath: /init + - name: data + mountPath: /data + - name: appservices + mountPath: /config + envFrom: + - secretRef: + name: synapse-janky-solutions + containers: + - image: git.janky.solutions/jankysolutions/infra/synapse:latest + name: synapse + resources: {} + volumeMounts: + - name: data + mountPath: /data + - name: config + mountPath: /config + - name: secrets + mountPath: /secrets + - name: appservices + mountPath: /appservices + env: + - name: SYNAPSE_SERVER_NAME + value: matrix.janky.solutions + - name: SYNAPSE_REPORT_STATS + value: "no" + - name: TZ + value: America/Los_Angeles + - name: PGUSER + valueFrom: + secretKeyRef: + name: matrix.janky-solutions-homeserver-database.credentials.postgresql.acid.zalan.do + key: username + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: matrix.janky-solutions-homeserver-database.credentials.postgresql.acid.zalan.do + key: password + ports: + - name: http + containerPort: 8008 + - name: metrics + containerPort: 8000 + volumes: + - name: config + configMap: + name: synapse-janky-solutions + - name: appservices + configMap: + name: appservices-janky-solutions + - name: init + configMap: + name: secrets-init + - name: secrets + secret: + secretName: synapse-janky-solutions + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: homeserver-janky-solutions +spec: + ports: + - name: http + port: 8008 + selector: + app: synapse + instance: janky.solutions +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: homeserver-janky-solutions + annotations: + cert-manager.io/cluster-issuer: letsencrypt +spec: + rules: + - host: matrix.janky.solutions + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: homeserver-janky-solutions + port: + name: http +--- +apiVersion: "acid.zalan.do/v1" +kind: postgresql +metadata: + name: janky-solutions-homeserver-database +spec: + teamId: matrix + volume: + size: 10Gi + numberOfInstances: 2 + users: + superuser: + - superuser + - createdb + matrix: [] + databases: + matrix: matrix + patroni: + initdb: + encoding: UTF8 + locale: C + pg_hba: + - local all all trust + - hostssl all +zalandos 127.0.0.1/32 pam + - host all all 127.0.0.1/32 md5 + - hostssl all +zalandos ::1/128 pam + - host all all ::1/128 md5 + - local replication standby trust + - hostssl replication standby all md5 + - hostnossl all all all reject + - hostssl all +zalandos all pam + - hostssl all all all md5 + - host all all all md5 + resources: + limits: + cpu: "4" + memory: 4Gi + postgresql: + version: "16" +--- +apiVersion: v1 +kind: Service +metadata: + name: janky-solutions-homeserver-database-nodeport + namespace: matrix +spec: + type: NodePort + ports: + - name: postgresql + port: 5432 + nodePort: 30003 + selector: + k8s.home.finn.io: janky-solutions-homeserver-database + spilo-role: master diff --git a/k8s/matrix/kustomization.yaml b/k8s/matrix/kustomization.yaml index 86182ad..deb12be 100644 --- a/k8s/matrix/kustomization.yaml +++ b/k8s/matrix/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization namespace: matrix resources: - namespace.yaml - - bridge-meta.yaml + - bridge-facebook.yaml - bridge-signal.yaml - bridge-telegram.yaml - janky.bot-homeserver.yaml @@ -11,10 +11,6 @@ resources: - secrets.yaml - secrets-init.yaml configMapGenerator: - - name: bridge-init - files: - - bridge-init/initialize-bridge.py - - bridge-init/initialize-bridge.sh - name: synapse-janky-bot files: - config-janky.bot/homeserver.yaml diff --git a/k8s/monitoring/kustomization.yaml b/k8s/monitoring/kustomization.yaml index 4a65b3f..1cc816f 100644 --- a/k8s/monitoring/kustomization.yaml +++ b/k8s/monitoring/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: monitoring resources: - - namespace.yaml - promtail.yaml - ingresses.yaml - secrets.yaml diff --git a/k8s/monitoring/namespace.yaml b/k8s/monitoring/namespace.yaml deleted file mode 100644 index d78cb4f..0000000 --- a/k8s/monitoring/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: monitoring - labels: - pod-security.kubernetes.io/warn: privileged - pod-security.kubernetes.io/warn-version: latest diff --git a/k8s/monitoring/prom-scrape-configs/scrape-configs.yaml b/k8s/monitoring/prom-scrape-configs/scrape-configs.yaml index 6e2199c..6f69cc8 100644 --- a/k8s/monitoring/prom-scrape-configs/scrape-configs.yaml +++ b/k8s/monitoring/prom-scrape-configs/scrape-configs.yaml @@ -49,6 +49,7 @@ - monitoring-0:9595 # signald on legacy monitoring - monitoring-0:9099 # rdap monitor on legacy monitoring - hypervisor-d:9162 # apcupsd_exporter on hypervisor-d + - hypervisor-d:9633 # smartctl_exporter on hypervisor-d - job_name: promtail static_configs: - targets: diff --git a/k8s/operators/postgres-operator/kustomization.yaml b/k8s/operators/postgres-operator/kustomization.yaml index 458eeec..3588c50 100644 --- a/k8s/operators/postgres-operator/kustomization.yaml +++ b/k8s/operators/postgres-operator/kustomization.yaml @@ -16,3 +16,4 @@ configMapGenerator: - logical_backup_s3_bucket=k8s-db-backups - logical_backup_s3_endpoint=storage.home.finn.io - logical_backup_s3_region=us-sea-1 + - custom_pod_annotations=prometheus.io/scrape:true,prometheus.io/port:8008 diff --git a/k8s/system/traefik/external-services.yaml b/k8s/system/traefik/external-services.yaml index 886cfc7..3a3024c 100644 --- a/k8s/system/traefik/external-services.yaml +++ b/k8s/system/traefik/external-services.yaml @@ -5,7 +5,6 @@ (list "jellyfin" "jellyfin.janky.solutions" "http://jellyfin:8096") (list "dns" "dns.janky.solutions" "http://dns:9191") (list "dns443" "dns.janky.solutions:443" "http://dns:9191") - (list "matrix" "matrix.janky.solutions" "http://matrix:8008") }} http: routers: