From 05f8489d3d1aed6640d908b5472024a51e4b0516 Mon Sep 17 00:00:00 2001 From: Masaki Yatsu Date: Mon, 1 Dec 2025 16:45:37 +0900 Subject: [PATCH] chore(clickhouse): set pod security standards and k8s resources --- clickhouse/.gitignore | 3 + clickhouse/README.md | 18 ++++++ ...khouse-installation-template.gomplate.yaml | 60 +++++++++++++++++++ .../clickhouse-operator-values.gomplate.yaml | 17 ++++++ ...ickhouse.yaml => clickhouse.gomplate.yaml} | 5 +- clickhouse/justfile | 53 ++++++++++++++-- clickhouse/zookeeper.yaml | 20 ++++--- 7 files changed, 161 insertions(+), 15 deletions(-) create mode 100644 clickhouse/clickhouse-installation-template.gomplate.yaml create mode 100644 clickhouse/clickhouse-operator-values.gomplate.yaml rename clickhouse/{clickhouse.yaml => clickhouse.gomplate.yaml} (79%) diff --git a/clickhouse/.gitignore b/clickhouse/.gitignore index 2c7b9cc..56a5c08 100644 --- a/clickhouse/.gitignore +++ b/clickhouse/.gitignore @@ -1,2 +1,5 @@ clickhouse-credentials-external-secret.yaml clickhouse-ingress.yaml +clickhouse-installation-template.yaml +clickhouse-operator-values.yaml +clickhouse.yaml diff --git a/clickhouse/README.md b/clickhouse/README.md index 3921caf..735ac2b 100644 --- a/clickhouse/README.md +++ b/clickhouse/README.md @@ -26,3 +26,21 @@ An optional web-based query interface for ClickHouse is available: ```bash just ch-ui::install ``` + +## Pod Security Standards + +The ClickHouse namespace is configured with **baseline** enforcement: + +- `pod-security.kubernetes.io/enforce=baseline` +- `pod-security.kubernetes.io/warn=baseline` + +### Optional Capabilities + +ClickHouse can use the following Linux capabilities for enhanced performance, but they are **not required** for normal operation: + +| Capability | Purpose | Impact if disabled | +|------------|--------------------------------------------------|-----------------------------------------------| +| `IPC_LOCK` | `mlock` to prevent binary from being paged out | Slightly slower startup under memory pressure | +| `SYS_NICE` | Thread priority control via `os_thread_priority` | Setting has no effect | + +These capabilities are disabled by default to comply with baseline Pod Security Standards. To enable them, the namespace must allow privileged pods, and you need to uncomment the `add` line in `clickhouse-installation-template.yaml`. diff --git a/clickhouse/clickhouse-installation-template.gomplate.yaml b/clickhouse/clickhouse-installation-template.gomplate.yaml new file mode 100644 index 0000000..52777c4 --- /dev/null +++ b/clickhouse/clickhouse-installation-template.gomplate.yaml @@ -0,0 +1,60 @@ +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallationTemplate +metadata: + name: clickhouse-security-template +spec: + defaults: + templates: + podTemplate: clickhouse-secure-pod + configuration: + settings: + max_server_memory_usage: {{ .Env.CLICKHOUSE_MAX_SERVER_MEMORY }} + templates: + podTemplates: + - name: clickhouse-secure-pod + spec: + securityContext: + runAsUser: 101 + runAsGroup: 101 + fsGroup: 101 + containers: + - name: clickhouse + image: {{ .Env.CLICKHOUSE_IMAGE }} + resources: + requests: + cpu: {{ .Env.CLICKHOUSE_CPU_REQUEST }} + memory: {{ .Env.CLICKHOUSE_MEMORY_REQUEST }} + limits: + cpu: "{{ .Env.CLICKHOUSE_CPU_LIMIT }}" + memory: {{ .Env.CLICKHOUSE_MEMORY_LIMIT }} + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: [ALL] + # IPC_LOCK: mlock to prevent binary from being paged out (optional, for stability under high I/O) + # SYS_NICE: os_thread_priority setting (optional, for thread priority control) + # Uncomment below if namespace allows privileged pods: + # add: [IPC_LOCK, SYS_NICE] + seccompProfile: + type: RuntimeDefault + command: + - clickhouse-server + - --config-file=/etc/clickhouse-server/config.xml + - name: clickhouse-log + image: registry.access.redhat.com/ubi8/ubi-minimal:latest + resources: + requests: + cpu: {{ .Env.CLICKHOUSE_LOG_CPU_REQUEST }} + memory: {{ .Env.CLICKHOUSE_LOG_MEMORY_REQUEST }} + limits: + cpu: {{ .Env.CLICKHOUSE_LOG_CPU_LIMIT }} + memory: {{ .Env.CLICKHOUSE_LOG_MEMORY_LIMIT }} + command: + - /bin/sh + - -c + - -- + args: + - while true; do sleep 30; done; + templating: + policy: auto diff --git a/clickhouse/clickhouse-operator-values.gomplate.yaml b/clickhouse/clickhouse-operator-values.gomplate.yaml new file mode 100644 index 0000000..4b6f829 --- /dev/null +++ b/clickhouse/clickhouse-operator-values.gomplate.yaml @@ -0,0 +1,17 @@ +operator: + resources: + requests: + cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_REQUEST }} + memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_REQUEST }} + limits: + cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_LIMIT }} + memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_LIMIT }} + +metrics: + resources: + requests: + cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_REQUEST }} + memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_REQUEST }} + limits: + cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_LIMIT }} + memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_LIMIT }} diff --git a/clickhouse/clickhouse.yaml b/clickhouse/clickhouse.gomplate.yaml similarity index 79% rename from clickhouse/clickhouse.yaml rename to clickhouse/clickhouse.gomplate.yaml index 023dd5f..411eb19 100644 --- a/clickhouse/clickhouse.yaml +++ b/clickhouse/clickhouse.gomplate.yaml @@ -25,8 +25,9 @@ spec: default/password: "disabled" default/networks/ip: "127.0.0.1" profiles: - default/max_memory_usage: 4000000000 # 4GB - default/max_bytes_before_external_group_by: 2000000000 # 2GB + default/max_memory_usage: {{ .Env.CLICKHOUSE_MAX_MEMORY_USAGE }} + default/max_bytes_before_external_group_by: {{ .Env.CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY }} + default/max_bytes_before_external_sort: {{ .Env.CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT }} default/add_http_cors_header: 1 templates: volumeClaimTemplates: diff --git a/clickhouse/justfile b/clickhouse/justfile index c1a7ffc..58154e2 100644 --- a/clickhouse/justfile +++ b/clickhouse/justfile @@ -2,9 +2,38 @@ set fallback := true export CLICKHOUSE_NAMESPACE := env("CLICKHOUSE_NAMESPACE", "clickhouse") export CLICKHOUSE_HOST := env("CLICKHOUSE_HOST", "") -export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.3") +export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.5") +export CLICKHOUSE_IMAGE := env("CLICKHOUSE_IMAGE", "clickhouse/clickhouse-server:25.10") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") +# ClickHouse resource settings +export CLICKHOUSE_MEMORY_REQUEST := env("CLICKHOUSE_MEMORY_REQUEST", "1Gi") +export CLICKHOUSE_MEMORY_LIMIT := env("CLICKHOUSE_MEMORY_LIMIT", "8Gi") +export CLICKHOUSE_CPU_REQUEST := env("CLICKHOUSE_CPU_REQUEST", "200m") +export CLICKHOUSE_CPU_LIMIT := env("CLICKHOUSE_CPU_LIMIT", "2") + +# ClickHouse memory settings (bytes) +# max_server_memory_usage: Server-wide limit, should be ~75% of MEMORY_LIMIT (default: 0 = auto 90% of RAM) +export CLICKHOUSE_MAX_SERVER_MEMORY := env("CLICKHOUSE_MAX_SERVER_MEMORY", "6442450944") +# max_memory_usage: Per-query limit (default: 10GB) +export CLICKHOUSE_MAX_MEMORY_USAGE := env("CLICKHOUSE_MAX_MEMORY_USAGE", "4294967296") +# max_bytes_before_external_group_by: Spill to disk threshold for GROUP BY (default: 0 = disabled) +export CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY := env("CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY", "2147483648") +# max_bytes_before_external_sort: Spill to disk threshold for ORDER BY (default: 0 = disabled) +export CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT := env("CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT", "2147483648") + +# ClickHouse log sidecar resource settings +export CLICKHOUSE_LOG_MEMORY_REQUEST := env("CLICKHOUSE_LOG_MEMORY_REQUEST", "64Mi") +export CLICKHOUSE_LOG_MEMORY_LIMIT := env("CLICKHOUSE_LOG_MEMORY_LIMIT", "128Mi") +export CLICKHOUSE_LOG_CPU_REQUEST := env("CLICKHOUSE_LOG_CPU_REQUEST", "10m") +export CLICKHOUSE_LOG_CPU_LIMIT := env("CLICKHOUSE_LOG_CPU_LIMIT", "100m") + +# ClickHouse Operator resource settings +export CLICKHOUSE_OPERATOR_MEMORY_REQUEST := env("CLICKHOUSE_OPERATOR_MEMORY_REQUEST", "64Mi") +export CLICKHOUSE_OPERATOR_MEMORY_LIMIT := env("CLICKHOUSE_OPERATOR_MEMORY_LIMIT", "256Mi") +export CLICKHOUSE_OPERATOR_CPU_REQUEST := env("CLICKHOUSE_OPERATOR_CPU_REQUEST", "50m") +export CLICKHOUSE_OPERATOR_CPU_LIMIT := env("CLICKHOUSE_OPERATOR_CPU_LIMIT", "500m") + [private] default: @just --list --unsorted --list-submodules @@ -20,8 +49,17 @@ remove-helm-repo: # Create ClickHouse namespace create-namespace: - @kubectl get namespace ${CLICKHOUSE_NAMESPACE} &>/dev/null || \ + #!/bin/bash + set -euo pipefail + if ! kubectl get namespace ${CLICKHOUSE_NAMESPACE} &>/dev/null; then kubectl create namespace ${CLICKHOUSE_NAMESPACE} + fi + kubectl label namespace ${CLICKHOUSE_NAMESPACE} \ + pod-security.kubernetes.io/enforce=baseline \ + pod-security.kubernetes.io/enforce-version=latest \ + pod-security.kubernetes.io/warn=baseline \ + pod-security.kubernetes.io/warn-version=latest \ + --overwrite # Delete ClickHouse namespace delete-namespace: @@ -74,8 +112,13 @@ install: just install-zookeeper just create-credentials just add-helm-repo + gomplate -f clickhouse-operator-values.gomplate.yaml -o clickhouse-operator-values.yaml helm upgrade --install clickhouse-operator clickhouse-operator/altinity-clickhouse-operator \ - --version ${CLICKHOUSE_CHART_VERSION} -n ${CLICKHOUSE_NAMESPACE} --wait + --version ${CLICKHOUSE_CHART_VERSION} -n ${CLICKHOUSE_NAMESPACE} \ + -f clickhouse-operator-values.yaml --wait + gomplate -f clickhouse-installation-template.gomplate.yaml -o clickhouse-installation-template.yaml + gomplate -f clickhouse.gomplate.yaml -o clickhouse.yaml + kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./clickhouse-installation-template.yaml kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./clickhouse.yaml echo "Waiting for ClickHouse installation to be ready..." kubectl wait --for=jsonpath='{.status.status}'=Completed \ @@ -103,7 +146,7 @@ uninstall: -n ${CLICKHOUSE_NAMESPACE} &>/dev/null; then echo "Deleting ClickHouseInstallation resources..." kubectl delete clickhouseinstallations.clickhouse.altinity.com --all \ - -n ${CLICKHOUSE_NAMESPACE} --timeout=30s --ignore-not-found || { + -n ${CLICKHOUSE_NAMESPACE} --timeout=60s --ignore-not-found || { echo "Graceful deletion timed out, forcing finalizer removal..." for chi in $(kubectl get clickhouseinstallations.clickhouse.altinity.com \ -n ${CLICKHOUSE_NAMESPACE} -o name 2>/dev/null); do @@ -480,7 +523,7 @@ install-zookeeper: # Uninstall ZooKeeper uninstall-zookeeper: - kubectl delete -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml + kubectl delete -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml --ignore-not-found # Clean up ClickHouse resources cleanup: diff --git a/clickhouse/zookeeper.yaml b/clickhouse/zookeeper.yaml index b42cabd..66574da 100644 --- a/clickhouse/zookeeper.yaml +++ b/clickhouse/zookeeper.yaml @@ -115,11 +115,17 @@ spec: env: - name: SERVERS value: "1" - -# See those links for proper startup settings: -# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper -# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper -# https://github.com/ClickHouse/ClickHouse/issues/11781 + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: [ALL] + seccompProfile: + type: RuntimeDefault + # See those links for proper startup settings: + # https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper + # https://clickhouse.yandex/docs/en/operations/tips/#zookeeper + # https://github.com/ClickHouse/ClickHouse/issues/11781 command: - bash - -x @@ -174,8 +180,6 @@ spec: fi && mkdir -pv ${ZOO_DATA_DIR} && mkdir -pv ${ZOO_DATA_LOG_DIR} && - whoami && - chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" && export MY_ID=$((ORD+1)) && echo $MY_ID > $ZOO_DATA_DIR/myid && for (( i=1; i<=$SERVERS; i++ )); do @@ -246,9 +250,9 @@ spec: volumeMounts: - name: datadir-volume mountPath: /var/lib/zookeeper - # Run as a non-privileged user securityContext: runAsUser: 1000 + runAsGroup: 1000 fsGroup: 1000 volumeClaimTemplates: - metadata: