add system-upgrade controller

This commit is contained in:
Finn 2024-06-17 18:35:42 -07:00
parent d5a33f8c2f
commit 2ab5523f3a
6 changed files with 959 additions and 4 deletions

View file

@ -10,3 +10,4 @@ resources:
- monitoring - monitoring
- s3staticsites - s3staticsites
- shlink - shlink
- system-upgrade

View file

@ -8,7 +8,7 @@ data:
# additional_secret_mount: "some-secret-name" # additional_secret_mount: "some-secret-name"
# additional_secret_mount_path: "/some/dir" # additional_secret_mount_path: "/some/dir"
api_port: "8080" api_port: "8080"
aws_region: eu-central-1 # aws_region: eu-central-1
cluster_domain: cluster.local cluster_domain: cluster.local
cluster_history_entries: "1000" cluster_history_entries: "1000"
cluster_labels: application:spilo cluster_labels: application:spilo
@ -47,7 +47,7 @@ data:
# enable_lazy_spilo_upgrade: "false" # enable_lazy_spilo_upgrade: "false"
enable_master_load_balancer: "false" enable_master_load_balancer: "false"
enable_master_pooler_load_balancer: "false" enable_master_pooler_load_balancer: "false"
enable_password_rotation: "false" enable_password_rotation: "true"
enable_patroni_failsafe_mode: "false" enable_patroni_failsafe_mode: "false"
enable_secrets_deletion: "true" enable_secrets_deletion: "true"
enable_persistent_volume_claim_deletion: "true" enable_persistent_volume_claim_deletion: "true"
@ -97,7 +97,7 @@ data:
# logical_backup_s3_access_key_id: "" # logical_backup_s3_access_key_id: ""
logical_backup_s3_sse: "AES256" logical_backup_s3_sse: "AES256"
# logical_backup_s3_retention_time: "" # logical_backup_s3_retention_time: ""
logical_backup_schedule: "5/* * * * *" # set to agressively frequent to test, used to be 30 00 * * * logical_backup_schedule: "*/5 * * * *" # set to agressively frequent to test, used to be 30 00 * * *
logical_backup_cronjob_environment_secret: backups-secret logical_backup_cronjob_environment_secret: backups-secret
major_version_upgrade_mode: "manual" major_version_upgrade_mode: "manual"
# major_version_upgrade_team_allow_list: "" # major_version_upgrade_team_allow_list: ""
@ -157,7 +157,7 @@ data:
spilo_allow_privilege_escalation: "true" spilo_allow_privilege_escalation: "true"
# spilo_runasuser: 101 # spilo_runasuser: 101
# spilo_runasgroup: 103 # spilo_runasgroup: 103
# spilo_fsgroup: 103 spilo_fsgroup: "103"
spilo_privileged: "false" spilo_privileged: "false"
storage_resize_mode: "pvc" storage_resize_mode: "pvc"
super_username: postgres super_username: postgres

View file

@ -0,0 +1,642 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: plans.upgrade.cattle.io
spec:
group: upgrade.cattle.io
names:
categories:
- upgrade
kind: Plan
plural: plans
singular: plan
preserveUnknownFields: false
scope: Namespaced
versions:
- additionalPrinterColumns:
- jsonPath: .spec.upgrade.image
name: Image
type: string
- jsonPath: .spec.channel
name: Channel
type: string
- jsonPath: .spec.version
name: Version
type: string
name: v1
schema:
openAPIV3Schema:
properties:
spec:
properties:
channel:
nullable: true
type: string
concurrency:
type: integer
cordon:
type: boolean
drain:
nullable: true
properties:
deleteEmptydirData:
nullable: true
type: boolean
deleteLocalData:
nullable: true
type: boolean
disableEviction:
type: boolean
force:
type: boolean
gracePeriod:
nullable: true
type: integer
ignoreDaemonSets:
nullable: true
type: boolean
podSelector:
nullable: true
properties:
matchExpressions:
items:
properties:
key:
nullable: true
type: string
operator:
nullable: true
type: string
values:
items:
nullable: true
type: string
nullable: true
type: array
type: object
nullable: true
type: array
matchLabels:
additionalProperties:
nullable: true
type: string
nullable: true
type: object
type: object
skipWaitForDeleteTimeout:
type: integer
timeout:
nullable: true
type: integer
type: object
exclusive:
type: boolean
imagePullSecrets:
items:
properties:
name:
nullable: true
type: string
type: object
nullable: true
type: array
jobActiveDeadlineSecs:
type: integer
nodeSelector:
nullable: true
properties:
matchExpressions:
items:
properties:
key:
nullable: true
type: string
operator:
nullable: true
type: string
values:
items:
nullable: true
type: string
nullable: true
type: array
type: object
nullable: true
type: array
matchLabels:
additionalProperties:
nullable: true
type: string
nullable: true
type: object
type: object
prepare:
nullable: true
properties:
args:
items:
nullable: true
type: string
nullable: true
type: array
command:
items:
nullable: true
type: string
nullable: true
type: array
envFrom:
items:
properties:
configMapRef:
nullable: true
properties:
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
prefix:
nullable: true
type: string
secretRef:
nullable: true
properties:
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
type: object
nullable: true
type: array
envs:
items:
properties:
name:
nullable: true
type: string
value:
nullable: true
type: string
valueFrom:
nullable: true
properties:
configMapKeyRef:
nullable: true
properties:
key:
nullable: true
type: string
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
fieldRef:
nullable: true
properties:
apiVersion:
nullable: true
type: string
fieldPath:
nullable: true
type: string
type: object
resourceFieldRef:
nullable: true
properties:
containerName:
nullable: true
type: string
divisor:
nullable: true
type: string
resource:
nullable: true
type: string
type: object
secretKeyRef:
nullable: true
properties:
key:
nullable: true
type: string
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
type: object
type: object
nullable: true
type: array
image:
nullable: true
type: string
securityContext:
nullable: true
properties:
allowPrivilegeEscalation:
nullable: true
type: boolean
capabilities:
nullable: true
properties:
add:
items:
nullable: true
type: string
nullable: true
type: array
drop:
items:
nullable: true
type: string
nullable: true
type: array
type: object
privileged:
nullable: true
type: boolean
procMount:
nullable: true
type: string
readOnlyRootFilesystem:
nullable: true
type: boolean
runAsGroup:
nullable: true
type: integer
runAsNonRoot:
nullable: true
type: boolean
runAsUser:
nullable: true
type: integer
seLinuxOptions:
nullable: true
properties:
level:
nullable: true
type: string
role:
nullable: true
type: string
type:
nullable: true
type: string
user:
nullable: true
type: string
type: object
seccompProfile:
nullable: true
properties:
localhostProfile:
nullable: true
type: string
type:
nullable: true
type: string
type: object
windowsOptions:
nullable: true
properties:
gmsaCredentialSpec:
nullable: true
type: string
gmsaCredentialSpecName:
nullable: true
type: string
hostProcess:
nullable: true
type: boolean
runAsUserName:
nullable: true
type: string
type: object
type: object
volumes:
items:
properties:
destination:
nullable: true
type: string
name:
nullable: true
type: string
source:
nullable: true
type: string
type: object
nullable: true
type: array
type: object
secrets:
items:
properties:
ignoreUpdates:
type: boolean
name:
nullable: true
type: string
path:
nullable: true
type: string
type: object
nullable: true
type: array
serviceAccountName:
nullable: true
type: string
tolerations:
items:
properties:
effect:
nullable: true
type: string
key:
nullable: true
type: string
operator:
nullable: true
type: string
tolerationSeconds:
nullable: true
type: integer
value:
nullable: true
type: string
type: object
nullable: true
type: array
upgrade:
nullable: true
properties:
args:
items:
nullable: true
type: string
nullable: true
type: array
command:
items:
nullable: true
type: string
nullable: true
type: array
envFrom:
items:
properties:
configMapRef:
nullable: true
properties:
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
prefix:
nullable: true
type: string
secretRef:
nullable: true
properties:
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
type: object
nullable: true
type: array
envs:
items:
properties:
name:
nullable: true
type: string
value:
nullable: true
type: string
valueFrom:
nullable: true
properties:
configMapKeyRef:
nullable: true
properties:
key:
nullable: true
type: string
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
fieldRef:
nullable: true
properties:
apiVersion:
nullable: true
type: string
fieldPath:
nullable: true
type: string
type: object
resourceFieldRef:
nullable: true
properties:
containerName:
nullable: true
type: string
divisor:
nullable: true
type: string
resource:
nullable: true
type: string
type: object
secretKeyRef:
nullable: true
properties:
key:
nullable: true
type: string
name:
nullable: true
type: string
optional:
nullable: true
type: boolean
type: object
type: object
type: object
nullable: true
type: array
image:
nullable: true
type: string
securityContext:
nullable: true
properties:
allowPrivilegeEscalation:
nullable: true
type: boolean
capabilities:
nullable: true
properties:
add:
items:
nullable: true
type: string
nullable: true
type: array
drop:
items:
nullable: true
type: string
nullable: true
type: array
type: object
privileged:
nullable: true
type: boolean
procMount:
nullable: true
type: string
readOnlyRootFilesystem:
nullable: true
type: boolean
runAsGroup:
nullable: true
type: integer
runAsNonRoot:
nullable: true
type: boolean
runAsUser:
nullable: true
type: integer
seLinuxOptions:
nullable: true
properties:
level:
nullable: true
type: string
role:
nullable: true
type: string
type:
nullable: true
type: string
user:
nullable: true
type: string
type: object
seccompProfile:
nullable: true
properties:
localhostProfile:
nullable: true
type: string
type:
nullable: true
type: string
type: object
windowsOptions:
nullable: true
properties:
gmsaCredentialSpec:
nullable: true
type: string
gmsaCredentialSpecName:
nullable: true
type: string
hostProcess:
nullable: true
type: boolean
runAsUserName:
nullable: true
type: string
type: object
type: object
volumes:
items:
properties:
destination:
nullable: true
type: string
name:
nullable: true
type: string
source:
nullable: true
type: string
type: object
nullable: true
type: array
type: object
version:
nullable: true
type: string
required:
- upgrade
type: object
status:
properties:
applying:
items:
nullable: true
type: string
nullable: true
type: array
conditions:
items:
properties:
lastTransitionTime:
nullable: true
type: string
lastUpdateTime:
nullable: true
type: string
message:
nullable: true
type: string
reason:
nullable: true
type: string
status:
nullable: true
type: string
type:
nullable: true
type: string
type: object
nullable: true
type: array
latestHash:
nullable: true
type: string
latestVersion:
nullable: true
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}

View file

@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- system-upgrade-controller.yaml
- crd.yaml
- plan.yaml

View file

@ -0,0 +1,42 @@
# Server plan
apiVersion: upgrade.cattle.io/v1
kind: Plan
metadata:
name: server-plan
namespace: system-upgrade
spec:
concurrency: 1
cordon: true
nodeSelector:
matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: In
values:
- "true"
serviceAccountName: system-upgrade
upgrade:
image: rancher/k3s-upgrade
channel: https://update.k3s.io/v1-release/channels/stable
---
# Agent plan
apiVersion: upgrade.cattle.io/v1
kind: Plan
metadata:
name: agent-plan
namespace: system-upgrade
spec:
concurrency: 1
cordon: true
nodeSelector:
matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: DoesNotExist
prepare:
args:
- prepare
- server-plan
image: rancher/k3s-upgrade
serviceAccountName: system-upgrade
upgrade:
image: rancher/k3s-upgrade
channel: https://update.k3s.io/v1-release/channels/stable

View file

@ -0,0 +1,264 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system-upgrade-controller
rules:
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- namespaces
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- update
- apiGroups:
- upgrade.cattle.io
resources:
- plans
- plans/status
verbs:
- get
- list
- watch
- create
- patch
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: system-upgrade-controller
namespace: system-upgrade
rules:
- apiGroups:
- batch
resources:
- jobs
verbs:
- create
- delete
- deletecollection
- patch
- update
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system-upgrade-controller-drainer
rules:
- apiGroups:
- ""
resources:
- pods/eviction
verbs:
- create
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- patch
- apiGroups:
- apps
resources:
- statefulsets
- daemonsets
- replicasets
verbs:
- get
- list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system-upgrade-drainer
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system-upgrade-controller-drainer
subjects:
- kind: ServiceAccount
name: system-upgrade
namespace: system-upgrade
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system-upgrade
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system-upgrade-controller
subjects:
- kind: ServiceAccount
name: system-upgrade
namespace: system-upgrade
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: system-upgrade
namespace: system-upgrade
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: system-upgrade-controller
subjects:
- kind: ServiceAccount
name: system-upgrade
namespace: system-upgrade
---
apiVersion: v1
kind: Namespace
metadata:
labels:
pod-security.kubernetes.io/enforce: privileged
name: system-upgrade
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: system-upgrade
namespace: system-upgrade
---
apiVersion: v1
data:
SYSTEM_UPGRADE_CONTROLLER_DEBUG: "false"
SYSTEM_UPGRADE_CONTROLLER_THREADS: "2"
SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900"
SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "99"
SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: Always
SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: rancher/kubectl:v1.25.4
SYSTEM_UPGRADE_JOB_PRIVILEGED: "true"
SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: "900"
SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m
kind: ConfigMap
metadata:
name: default-controller-env
namespace: system-upgrade
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: system-upgrade-controller
namespace: system-upgrade
spec:
selector:
matchLabels:
upgrade.cattle.io/controller: system-upgrade-controller
template:
metadata:
labels:
upgrade.cattle.io/controller: system-upgrade-controller
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
containers:
- env:
- name: SYSTEM_UPGRADE_CONTROLLER_NAME
valueFrom:
fieldRef:
fieldPath: metadata.labels['upgrade.cattle.io/controller']
- name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
envFrom:
- configMapRef:
name: default-controller-env
image: rancher/system-upgrade-controller:v0.13.4
imagePullPolicy: IfNotPresent
name: system-upgrade-controller
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
runAsGroup: 65534
runAsNonRoot: true
runAsUser: 65534
seccompProfile:
type: RuntimeDefault
volumeMounts:
- mountPath: /etc/ssl
name: etc-ssl
readOnly: true
- mountPath: /etc/pki
name: etc-pki
readOnly: true
- mountPath: /etc/ca-certificates
name: etc-ca-certificates
readOnly: true
- mountPath: /tmp
name: tmp
serviceAccountName: system-upgrade
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/controlplane
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
- effect: NoExecute
key: node-role.kubernetes.io/etcd
operator: Exists
volumes:
- hostPath:
path: /etc/ssl
type: DirectoryOrCreate
name: etc-ssl
- hostPath:
path: /etc/pki
type: DirectoryOrCreate
name: etc-pki
- hostPath:
path: /etc/ca-certificates
type: DirectoryOrCreate
name: etc-ca-certificates
- emptyDir: {}
name: tmp