chore(clickhouse): set pod security standards and k8s resources

2025-12-01 16:45:37 +09:00
parent ea328fe517
commit 05f8489d3d
7 changed files with 161 additions and 15 deletions
--- a/clickhouse/.gitignore
+++ b/clickhouse/.gitignore
@@ -1,2 +1,5 @@
 clickhouse-credentials-external-secret.yaml
 clickhouse-ingress.yaml
+clickhouse-installation-template.yaml
+clickhouse-operator-values.yaml
+clickhouse.yaml
--- a/clickhouse/README.md
+++ b/clickhouse/README.md
@@ -26,3 +26,21 @@ An optional web-based query interface for ClickHouse is available:
 ```bash
 just ch-ui::install
 ```
+
+## Pod Security Standards
+
+The ClickHouse namespace is configured with **baseline** enforcement:
+
+- `pod-security.kubernetes.io/enforce=baseline`
+- `pod-security.kubernetes.io/warn=baseline`
+
+### Optional Capabilities
+
+ClickHouse can use the following Linux capabilities for enhanced performance, but they are **not required** for normal operation:
+
+| Capability | Purpose                                          | Impact if disabled                            |
+|------------|--------------------------------------------------|-----------------------------------------------|
+| `IPC_LOCK` | `mlock` to prevent binary from being paged out   | Slightly slower startup under memory pressure |
+| `SYS_NICE` | Thread priority control via `os_thread_priority` | Setting has no effect                         |
+
+These capabilities are disabled by default to comply with baseline Pod Security Standards. To enable them, the namespace must allow privileged pods, and you need to uncomment the `add` line in `clickhouse-installation-template.yaml`.
--- a/clickhouse/clickhouse-installation-template.gomplate.yaml
+++ b/clickhouse/clickhouse-installation-template.gomplate.yaml
@@ -0,0 +1,60 @@
+apiVersion: clickhouse.altinity.com/v1
+kind: ClickHouseInstallationTemplate
+metadata:
+  name: clickhouse-security-template
+spec:
+  defaults:
+    templates:
+      podTemplate: clickhouse-secure-pod
+  configuration:
+    settings:
+      max_server_memory_usage: {{ .Env.CLICKHOUSE_MAX_SERVER_MEMORY }}
+  templates:
+    podTemplates:
+      - name: clickhouse-secure-pod
+        spec:
+          securityContext:
+            runAsUser: 101
+            runAsGroup: 101
+            fsGroup: 101
+          containers:
+            - name: clickhouse
+              image: {{ .Env.CLICKHOUSE_IMAGE }}
+              resources:
+                requests:
+                  cpu: {{ .Env.CLICKHOUSE_CPU_REQUEST }}
+                  memory: {{ .Env.CLICKHOUSE_MEMORY_REQUEST }}
+                limits:
+                  cpu: "{{ .Env.CLICKHOUSE_CPU_LIMIT }}"
+                  memory: {{ .Env.CLICKHOUSE_MEMORY_LIMIT }}
+              securityContext:
+                allowPrivilegeEscalation: false
+                runAsNonRoot: true
+                capabilities:
+                  drop: [ALL]
+                  # IPC_LOCK: mlock to prevent binary from being paged out (optional, for stability under high I/O)
+                  # SYS_NICE: os_thread_priority setting (optional, for thread priority control)
+                  # Uncomment below if namespace allows privileged pods:
+                  # add: [IPC_LOCK, SYS_NICE]
+                seccompProfile:
+                  type: RuntimeDefault
+              command:
+                - clickhouse-server
+                - --config-file=/etc/clickhouse-server/config.xml
+            - name: clickhouse-log
+              image: registry.access.redhat.com/ubi8/ubi-minimal:latest
+              resources:
+                requests:
+                  cpu: {{ .Env.CLICKHOUSE_LOG_CPU_REQUEST }}
+                  memory: {{ .Env.CLICKHOUSE_LOG_MEMORY_REQUEST }}
+                limits:
+                  cpu: {{ .Env.CLICKHOUSE_LOG_CPU_LIMIT }}
+                  memory: {{ .Env.CLICKHOUSE_LOG_MEMORY_LIMIT }}
+              command:
+                - /bin/sh
+                - -c
+                - --
+              args:
+                - while true; do sleep 30; done;
+  templating:
+    policy: auto
--- a/clickhouse/clickhouse-operator-values.gomplate.yaml
+++ b/clickhouse/clickhouse-operator-values.gomplate.yaml
@@ -0,0 +1,17 @@
+operator:
+  resources:
+    requests:
+      cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_REQUEST }}
+      memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_REQUEST }}
+    limits:
+      cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_LIMIT }}
+      memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_LIMIT }}
+
+metrics:
+  resources:
+    requests:
+      cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_REQUEST }}
+      memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_REQUEST }}
+    limits:
+      cpu: {{ .Env.CLICKHOUSE_OPERATOR_CPU_LIMIT }}
+      memory: {{ .Env.CLICKHOUSE_OPERATOR_MEMORY_LIMIT }}
--- a/clickhouse/clickhouse.gomplate.yaml
+++ b/clickhouse/clickhouse.gomplate.yaml
@@ -25,8 +25,9 @@ spec:
      default/password: "disabled"
      default/networks/ip: "127.0.0.1"
    profiles:
-      default/max_memory_usage: 4000000000 # 4GB
-      default/max_bytes_before_external_group_by: 2000000000 # 2GB
+      default/max_memory_usage: {{ .Env.CLICKHOUSE_MAX_MEMORY_USAGE }}
+      default/max_bytes_before_external_group_by: {{ .Env.CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY }}
+      default/max_bytes_before_external_sort: {{ .Env.CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT }}
      default/add_http_cors_header: 1
  templates:
    volumeClaimTemplates:
--- a/clickhouse/justfile
+++ b/clickhouse/justfile
@@ -2,9 +2,38 @@ set fallback := true

 export CLICKHOUSE_NAMESPACE := env("CLICKHOUSE_NAMESPACE", "clickhouse")
 export CLICKHOUSE_HOST := env("CLICKHOUSE_HOST", "")
-export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.3")
+export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.5")
+export CLICKHOUSE_IMAGE := env("CLICKHOUSE_IMAGE", "clickhouse/clickhouse-server:25.10")
 export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets")

+# ClickHouse resource settings
+export CLICKHOUSE_MEMORY_REQUEST := env("CLICKHOUSE_MEMORY_REQUEST", "1Gi")
+export CLICKHOUSE_MEMORY_LIMIT := env("CLICKHOUSE_MEMORY_LIMIT", "8Gi")
+export CLICKHOUSE_CPU_REQUEST := env("CLICKHOUSE_CPU_REQUEST", "200m")
+export CLICKHOUSE_CPU_LIMIT := env("CLICKHOUSE_CPU_LIMIT", "2")
+
+# ClickHouse memory settings (bytes)
+# max_server_memory_usage: Server-wide limit, should be ~75% of MEMORY_LIMIT (default: 0 = auto 90% of RAM)
+export CLICKHOUSE_MAX_SERVER_MEMORY := env("CLICKHOUSE_MAX_SERVER_MEMORY", "6442450944")
+# max_memory_usage: Per-query limit (default: 10GB)
+export CLICKHOUSE_MAX_MEMORY_USAGE := env("CLICKHOUSE_MAX_MEMORY_USAGE", "4294967296")
+# max_bytes_before_external_group_by: Spill to disk threshold for GROUP BY (default: 0 = disabled)
+export CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY := env("CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY", "2147483648")
+# max_bytes_before_external_sort: Spill to disk threshold for ORDER BY (default: 0 = disabled)
+export CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT := env("CLICKHOUSE_MAX_BYTES_BEFORE_EXTERNAL_SORT", "2147483648")
+
+# ClickHouse log sidecar resource settings
+export CLICKHOUSE_LOG_MEMORY_REQUEST := env("CLICKHOUSE_LOG_MEMORY_REQUEST", "64Mi")
+export CLICKHOUSE_LOG_MEMORY_LIMIT := env("CLICKHOUSE_LOG_MEMORY_LIMIT", "128Mi")
+export CLICKHOUSE_LOG_CPU_REQUEST := env("CLICKHOUSE_LOG_CPU_REQUEST", "10m")
+export CLICKHOUSE_LOG_CPU_LIMIT := env("CLICKHOUSE_LOG_CPU_LIMIT", "100m")
+
+# ClickHouse Operator resource settings
+export CLICKHOUSE_OPERATOR_MEMORY_REQUEST := env("CLICKHOUSE_OPERATOR_MEMORY_REQUEST", "64Mi")
+export CLICKHOUSE_OPERATOR_MEMORY_LIMIT := env("CLICKHOUSE_OPERATOR_MEMORY_LIMIT", "256Mi")
+export CLICKHOUSE_OPERATOR_CPU_REQUEST := env("CLICKHOUSE_OPERATOR_CPU_REQUEST", "50m")
+export CLICKHOUSE_OPERATOR_CPU_LIMIT := env("CLICKHOUSE_OPERATOR_CPU_LIMIT", "500m")
+
 [private]
 default:
    @just --list --unsorted --list-submodules
@@ -20,8 +49,17 @@ remove-helm-repo:

 # Create ClickHouse namespace
 create-namespace:
-    @kubectl get namespace ${CLICKHOUSE_NAMESPACE} &>/dev/null || \
+    #!/bin/bash
+    set -euo pipefail
+    if ! kubectl get namespace ${CLICKHOUSE_NAMESPACE} &>/dev/null; then
        kubectl create namespace ${CLICKHOUSE_NAMESPACE}
+    fi
+    kubectl label namespace ${CLICKHOUSE_NAMESPACE} \
+        pod-security.kubernetes.io/enforce=baseline \
+        pod-security.kubernetes.io/enforce-version=latest \
+        pod-security.kubernetes.io/warn=baseline \
+        pod-security.kubernetes.io/warn-version=latest \
+        --overwrite

 # Delete ClickHouse namespace
 delete-namespace:
@@ -74,8 +112,13 @@ install:
    just install-zookeeper
    just create-credentials
    just add-helm-repo
+    gomplate -f clickhouse-operator-values.gomplate.yaml -o clickhouse-operator-values.yaml
    helm upgrade --install clickhouse-operator clickhouse-operator/altinity-clickhouse-operator \
-        --version ${CLICKHOUSE_CHART_VERSION} -n ${CLICKHOUSE_NAMESPACE} --wait
+        --version ${CLICKHOUSE_CHART_VERSION} -n ${CLICKHOUSE_NAMESPACE} \
+        -f clickhouse-operator-values.yaml --wait
+    gomplate -f clickhouse-installation-template.gomplate.yaml -o clickhouse-installation-template.yaml
+    gomplate -f clickhouse.gomplate.yaml -o clickhouse.yaml
+    kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./clickhouse-installation-template.yaml
    kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./clickhouse.yaml
    echo "Waiting for ClickHouse installation to be ready..."
    kubectl wait --for=jsonpath='{.status.status}'=Completed \
@@ -103,7 +146,7 @@ uninstall:
        -n ${CLICKHOUSE_NAMESPACE} &>/dev/null; then
        echo "Deleting ClickHouseInstallation resources..."
        kubectl delete clickhouseinstallations.clickhouse.altinity.com --all \
-            -n ${CLICKHOUSE_NAMESPACE} --timeout=30s --ignore-not-found || {
+            -n ${CLICKHOUSE_NAMESPACE} --timeout=60s --ignore-not-found || {
            echo "Graceful deletion timed out, forcing finalizer removal..."
            for chi in $(kubectl get clickhouseinstallations.clickhouse.altinity.com \
                -n ${CLICKHOUSE_NAMESPACE} -o name 2>/dev/null); do
@@ -480,7 +523,7 @@ install-zookeeper:

 # Uninstall ZooKeeper
 uninstall-zookeeper:
-    kubectl delete -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml
+    kubectl delete -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml --ignore-not-found

 # Clean up ClickHouse resources
 cleanup:
--- a/clickhouse/zookeeper.yaml
+++ b/clickhouse/zookeeper.yaml
@@ -115,7 +115,13 @@ spec:
          env:
            - name: SERVERS
              value: "1"
-
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            capabilities:
+              drop: [ALL]
+            seccompProfile:
+              type: RuntimeDefault
          # See those links for proper startup settings:
          # https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper
          # https://clickhouse.yandex/docs/en/operations/tips/#zookeeper
@@ -174,8 +180,6 @@ spec:
              fi &&
              mkdir -pv ${ZOO_DATA_DIR} &&
              mkdir -pv ${ZOO_DATA_LOG_DIR} &&
-              whoami &&
-              chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" &&
              export MY_ID=$((ORD+1)) &&
              echo $MY_ID > $ZOO_DATA_DIR/myid &&
              for (( i=1; i<=$SERVERS; i++ )); do
@@ -246,9 +250,9 @@ spec:
          volumeMounts:
            - name: datadir-volume
              mountPath: /var/lib/zookeeper
-      # Run as a non-privileged user
      securityContext:
        runAsUser: 1000
+        runAsGroup: 1000
        fsGroup: 1000
  volumeClaimTemplates:
    - metadata: