From 2ab5523f3a53c522f843407beaf92ef87a8f74ba Mon Sep 17 00:00:00 2001 From: Finn Date: Mon, 17 Jun 2024 18:35:42 -0700 Subject: [PATCH] add system-upgrade controller --- k8s/kustomization.yaml | 1 + .../postgres-operator/configmap.yaml | 8 +- k8s/system-upgrade-controller/crd.yaml | 642 ++++++++++++++++++ .../kustomization.yaml | 6 + k8s/system-upgrade-controller/plan.yaml | 42 ++ .../system-upgrade-controller.yaml | 264 +++++++ 6 files changed, 959 insertions(+), 4 deletions(-) create mode 100644 k8s/system-upgrade-controller/crd.yaml create mode 100644 k8s/system-upgrade-controller/kustomization.yaml create mode 100644 k8s/system-upgrade-controller/plan.yaml create mode 100644 k8s/system-upgrade-controller/system-upgrade-controller.yaml diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index aa64723..a5fa499 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -10,3 +10,4 @@ resources: - monitoring - s3staticsites - shlink + - system-upgrade diff --git a/k8s/operators/postgres-operator/configmap.yaml b/k8s/operators/postgres-operator/configmap.yaml index a941540..db67750 100644 --- a/k8s/operators/postgres-operator/configmap.yaml +++ b/k8s/operators/postgres-operator/configmap.yaml @@ -8,7 +8,7 @@ data: # additional_secret_mount: "some-secret-name" # additional_secret_mount_path: "/some/dir" api_port: "8080" - aws_region: eu-central-1 + # aws_region: eu-central-1 cluster_domain: cluster.local cluster_history_entries: "1000" cluster_labels: application:spilo @@ -47,7 +47,7 @@ data: # enable_lazy_spilo_upgrade: "false" enable_master_load_balancer: "false" enable_master_pooler_load_balancer: "false" - enable_password_rotation: "false" + enable_password_rotation: "true" enable_patroni_failsafe_mode: "false" enable_secrets_deletion: "true" enable_persistent_volume_claim_deletion: "true" @@ -97,7 +97,7 @@ data: # logical_backup_s3_access_key_id: "" logical_backup_s3_sse: "AES256" # logical_backup_s3_retention_time: "" - logical_backup_schedule: "5/* * * * *" # set to agressively frequent to test, used to be 30 00 * * * + logical_backup_schedule: "*/5 * * * *" # set to agressively frequent to test, used to be 30 00 * * * logical_backup_cronjob_environment_secret: backups-secret major_version_upgrade_mode: "manual" # major_version_upgrade_team_allow_list: "" @@ -157,7 +157,7 @@ data: spilo_allow_privilege_escalation: "true" # spilo_runasuser: 101 # spilo_runasgroup: 103 - # spilo_fsgroup: 103 + spilo_fsgroup: "103" spilo_privileged: "false" storage_resize_mode: "pvc" super_username: postgres diff --git a/k8s/system-upgrade-controller/crd.yaml b/k8s/system-upgrade-controller/crd.yaml new file mode 100644 index 0000000..6fb5343 --- /dev/null +++ b/k8s/system-upgrade-controller/crd.yaml @@ -0,0 +1,642 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: plans.upgrade.cattle.io +spec: + group: upgrade.cattle.io + names: + categories: + - upgrade + kind: Plan + plural: plans + singular: plan + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.upgrade.image + name: Image + type: string + - jsonPath: .spec.channel + name: Channel + type: string + - jsonPath: .spec.version + name: Version + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + spec: + properties: + channel: + nullable: true + type: string + concurrency: + type: integer + cordon: + type: boolean + drain: + nullable: true + properties: + deleteEmptydirData: + nullable: true + type: boolean + deleteLocalData: + nullable: true + type: boolean + disableEviction: + type: boolean + force: + type: boolean + gracePeriod: + nullable: true + type: integer + ignoreDaemonSets: + nullable: true + type: boolean + podSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + skipWaitForDeleteTimeout: + type: integer + timeout: + nullable: true + type: integer + type: object + exclusive: + type: boolean + imagePullSecrets: + items: + properties: + name: + nullable: true + type: string + type: object + nullable: true + type: array + jobActiveDeadlineSecs: + type: integer + nodeSelector: + nullable: true + properties: + matchExpressions: + items: + properties: + key: + nullable: true + type: string + operator: + nullable: true + type: string + values: + items: + nullable: true + type: string + nullable: true + type: array + type: object + nullable: true + type: array + matchLabels: + additionalProperties: + nullable: true + type: string + nullable: true + type: object + type: object + prepare: + nullable: true + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + envs: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + volumes: + items: + properties: + destination: + nullable: true + type: string + name: + nullable: true + type: string + source: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + secrets: + items: + properties: + ignoreUpdates: + type: boolean + name: + nullable: true + type: string + path: + nullable: true + type: string + type: object + nullable: true + type: array + serviceAccountName: + nullable: true + type: string + tolerations: + items: + properties: + effect: + nullable: true + type: string + key: + nullable: true + type: string + operator: + nullable: true + type: string + tolerationSeconds: + nullable: true + type: integer + value: + nullable: true + type: string + type: object + nullable: true + type: array + upgrade: + nullable: true + properties: + args: + items: + nullable: true + type: string + nullable: true + type: array + command: + items: + nullable: true + type: string + nullable: true + type: array + envFrom: + items: + properties: + configMapRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + prefix: + nullable: true + type: string + secretRef: + nullable: true + properties: + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + nullable: true + type: array + envs: + items: + properties: + name: + nullable: true + type: string + value: + nullable: true + type: string + valueFrom: + nullable: true + properties: + configMapKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + fieldRef: + nullable: true + properties: + apiVersion: + nullable: true + type: string + fieldPath: + nullable: true + type: string + type: object + resourceFieldRef: + nullable: true + properties: + containerName: + nullable: true + type: string + divisor: + nullable: true + type: string + resource: + nullable: true + type: string + type: object + secretKeyRef: + nullable: true + properties: + key: + nullable: true + type: string + name: + nullable: true + type: string + optional: + nullable: true + type: boolean + type: object + type: object + type: object + nullable: true + type: array + image: + nullable: true + type: string + securityContext: + nullable: true + properties: + allowPrivilegeEscalation: + nullable: true + type: boolean + capabilities: + nullable: true + properties: + add: + items: + nullable: true + type: string + nullable: true + type: array + drop: + items: + nullable: true + type: string + nullable: true + type: array + type: object + privileged: + nullable: true + type: boolean + procMount: + nullable: true + type: string + readOnlyRootFilesystem: + nullable: true + type: boolean + runAsGroup: + nullable: true + type: integer + runAsNonRoot: + nullable: true + type: boolean + runAsUser: + nullable: true + type: integer + seLinuxOptions: + nullable: true + properties: + level: + nullable: true + type: string + role: + nullable: true + type: string + type: + nullable: true + type: string + user: + nullable: true + type: string + type: object + seccompProfile: + nullable: true + properties: + localhostProfile: + nullable: true + type: string + type: + nullable: true + type: string + type: object + windowsOptions: + nullable: true + properties: + gmsaCredentialSpec: + nullable: true + type: string + gmsaCredentialSpecName: + nullable: true + type: string + hostProcess: + nullable: true + type: boolean + runAsUserName: + nullable: true + type: string + type: object + type: object + volumes: + items: + properties: + destination: + nullable: true + type: string + name: + nullable: true + type: string + source: + nullable: true + type: string + type: object + nullable: true + type: array + type: object + version: + nullable: true + type: string + required: + - upgrade + type: object + status: + properties: + applying: + items: + nullable: true + type: string + nullable: true + type: array + conditions: + items: + properties: + lastTransitionTime: + nullable: true + type: string + lastUpdateTime: + nullable: true + type: string + message: + nullable: true + type: string + reason: + nullable: true + type: string + status: + nullable: true + type: string + type: + nullable: true + type: string + type: object + nullable: true + type: array + latestHash: + nullable: true + type: string + latestVersion: + nullable: true + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/k8s/system-upgrade-controller/kustomization.yaml b/k8s/system-upgrade-controller/kustomization.yaml new file mode 100644 index 0000000..b1cf29b --- /dev/null +++ b/k8s/system-upgrade-controller/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - system-upgrade-controller.yaml + - crd.yaml + - plan.yaml diff --git a/k8s/system-upgrade-controller/plan.yaml b/k8s/system-upgrade-controller/plan.yaml new file mode 100644 index 0000000..cbf64b5 --- /dev/null +++ b/k8s/system-upgrade-controller/plan.yaml @@ -0,0 +1,42 @@ +# Server plan +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: server-plan + namespace: system-upgrade +spec: + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "true" + serviceAccountName: system-upgrade + upgrade: + image: rancher/k3s-upgrade + channel: https://update.k3s.io/v1-release/channels/stable +--- +# Agent plan +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: agent-plan + namespace: system-upgrade +spec: + concurrency: 1 + cordon: true + nodeSelector: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: DoesNotExist + prepare: + args: + - prepare + - server-plan + image: rancher/k3s-upgrade + serviceAccountName: system-upgrade + upgrade: + image: rancher/k3s-upgrade + channel: https://update.k3s.io/v1-release/channels/stable diff --git a/k8s/system-upgrade-controller/system-upgrade-controller.yaml b/k8s/system-upgrade-controller/system-upgrade-controller.yaml new file mode 100644 index 0000000..0b0f9ee --- /dev/null +++ b/k8s/system-upgrade-controller/system-upgrade-controller.yaml @@ -0,0 +1,264 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-upgrade-controller +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - nodes + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - update +- apiGroups: + - upgrade.cattle.io + resources: + - plans + - plans/status + verbs: + - get + - list + - watch + - create + - patch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system-upgrade-controller + namespace: system-upgrade +rules: +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - deletecollection + - patch + - update + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system-upgrade-controller-drainer +rules: +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch +- apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - replicasets + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-upgrade-drainer +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system-upgrade-controller-drainer +subjects: +- kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system-upgrade-controller +subjects: +- kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system-upgrade + namespace: system-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: system-upgrade-controller +subjects: +- kind: ServiceAccount + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/enforce: privileged + name: system-upgrade +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: system-upgrade + namespace: system-upgrade +--- +apiVersion: v1 +data: + SYSTEM_UPGRADE_CONTROLLER_DEBUG: "false" + SYSTEM_UPGRADE_CONTROLLER_THREADS: "2" + SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: "900" + SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: "99" + SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: Always + SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: rancher/kubectl:v1.25.4 + SYSTEM_UPGRADE_JOB_PRIVILEGED: "true" + SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: "900" + SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m +kind: ConfigMap +metadata: + name: default-controller-env + namespace: system-upgrade +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: system-upgrade-controller + namespace: system-upgrade +spec: + selector: + matchLabels: + upgrade.cattle.io/controller: system-upgrade-controller + template: + metadata: + labels: + upgrade.cattle.io/controller: system-upgrade-controller + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - env: + - name: SYSTEM_UPGRADE_CONTROLLER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['upgrade.cattle.io/controller'] + - name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + envFrom: + - configMapRef: + name: default-controller-env + image: rancher/system-upgrade-controller:v0.13.4 + imagePullPolicy: IfNotPresent + name: system-upgrade-controller + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /etc/ssl + name: etc-ssl + readOnly: true + - mountPath: /etc/pki + name: etc-pki + readOnly: true + - mountPath: /etc/ca-certificates + name: etc-ca-certificates + readOnly: true + - mountPath: /tmp + name: tmp + serviceAccountName: system-upgrade + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/controlplane + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoExecute + key: node-role.kubernetes.io/etcd + operator: Exists + volumes: + - hostPath: + path: /etc/ssl + type: DirectoryOrCreate + name: etc-ssl + - hostPath: + path: /etc/pki + type: DirectoryOrCreate + name: etc-pki + - hostPath: + path: /etc/ca-certificates + type: DirectoryOrCreate + name: etc-ca-certificates + - emptyDir: {} + name: tmp