From d9ee90c32cd08cba0371271abeb9af1f98495a19 Mon Sep 17 00:00:00 2001 From: Masaki Yatsu Date: Thu, 4 Dec 2025 11:34:22 +0900 Subject: [PATCH] feat(clickhouse): enable Prometheus monitoring --- clickhouse/.gitignore | 1 + clickhouse/README.md | 28 ++++++++ ...khouse-installation-template.gomplate.yaml | 4 ++ .../clickhouse-servicemonitor.gomplate.yaml | 67 +++++++++++++++++++ clickhouse/clickhouse.gomplate.yaml | 11 +++ clickhouse/justfile | 37 ++++++++++ 6 files changed, 148 insertions(+) create mode 100644 clickhouse/clickhouse-servicemonitor.gomplate.yaml diff --git a/clickhouse/.gitignore b/clickhouse/.gitignore index 56a5c08..ecc9b99 100644 --- a/clickhouse/.gitignore +++ b/clickhouse/.gitignore @@ -2,4 +2,5 @@ clickhouse-credentials-external-secret.yaml clickhouse-ingress.yaml clickhouse-installation-template.yaml clickhouse-operator-values.yaml +clickhouse-servicemonitor.yaml clickhouse.yaml diff --git a/clickhouse/README.md b/clickhouse/README.md index 735ac2b..7abea01 100644 --- a/clickhouse/README.md +++ b/clickhouse/README.md @@ -44,3 +44,31 @@ ClickHouse can use the following Linux capabilities for enhanced performance, bu | `SYS_NICE` | Thread priority control via `os_thread_priority` | Setting has no effect | These capabilities are disabled by default to comply with baseline Pod Security Standards. To enable them, the namespace must allow privileged pods, and you need to uncomment the `add` line in `clickhouse-installation-template.yaml`. + +## Monitoring + +ClickHouse exposes Prometheus metrics on port 9363. When Prometheus (kube-prometheus-stack) is installed, monitoring can be enabled during installation or manually. + +### Enable Monitoring + +```bash +just clickhouse::setup-monitoring +``` + +This creates a ServiceMonitor and a metrics Service for Prometheus to scrape. + +### Grafana Dashboard + +Import the ClickHouse dashboard from Grafana.com: + +1. Open Grafana → **Dashboards** → **New** → **Import** +2. Enter Dashboard ID: `14192` +3. Click **Load**, select **Prometheus** data source, then **Import** + +The dashboard includes panels for memory, connections, queries, I/O, replication, merge operations, cache, and ZooKeeper metrics. + +### Remove Monitoring + +```bash +just clickhouse::remove-monitoring +``` diff --git a/clickhouse/clickhouse-installation-template.gomplate.yaml b/clickhouse/clickhouse-installation-template.gomplate.yaml index 52777c4..9d8dc54 100644 --- a/clickhouse/clickhouse-installation-template.gomplate.yaml +++ b/clickhouse/clickhouse-installation-template.gomplate.yaml @@ -20,6 +20,10 @@ spec: containers: - name: clickhouse image: {{ .Env.CLICKHOUSE_IMAGE }} + ports: + - name: prometheus + containerPort: 9363 + protocol: TCP resources: requests: cpu: {{ .Env.CLICKHOUSE_CPU_REQUEST }} diff --git a/clickhouse/clickhouse-servicemonitor.gomplate.yaml b/clickhouse/clickhouse-servicemonitor.gomplate.yaml new file mode 100644 index 0000000..a1b5778 --- /dev/null +++ b/clickhouse/clickhouse-servicemonitor.gomplate.yaml @@ -0,0 +1,67 @@ +{{- if .Env.MONITORING_ENABLED }} +--- +apiVersion: v1 +kind: Service +metadata: + name: clickhouse-metrics + namespace: {{ .Env.CLICKHOUSE_NAMESPACE }} + labels: + app: clickhouse + clickhouse.altinity.com/chi: clickhouse +spec: + type: ClusterIP + ports: + - name: prometheus + port: 9363 + targetPort: 9363 + protocol: TCP + selector: + clickhouse.altinity.com/chi: clickhouse +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: clickhouse + namespace: {{ .Env.CLICKHOUSE_NAMESPACE }} + labels: + app: clickhouse + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: clickhouse + clickhouse.altinity.com/chi: clickhouse + namespaceSelector: + matchNames: + - {{ .Env.CLICKHOUSE_NAMESPACE }} + endpoints: + - port: prometheus + path: /metrics + interval: 30s + scrapeTimeout: 10s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: clickhouse-operator + namespace: {{ .Env.CLICKHOUSE_NAMESPACE }} + labels: + app: clickhouse-operator + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: altinity-clickhouse-operator + namespaceSelector: + matchNames: + - {{ .Env.CLICKHOUSE_NAMESPACE }} + endpoints: + - port: ch-metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s + - port: op-metrics + path: /metrics + interval: 30s + scrapeTimeout: 10s +{{- end }} diff --git a/clickhouse/clickhouse.gomplate.yaml b/clickhouse/clickhouse.gomplate.yaml index c7f5df4..42589d6 100644 --- a/clickhouse/clickhouse.gomplate.yaml +++ b/clickhouse/clickhouse.gomplate.yaml @@ -29,6 +29,17 @@ spec: transposed + # Enable Prometheus metrics endpoint + prometheus.xml: | + + + /metrics + 9363 + true + true + true + + users: admin/k8s_secret_password: clickhouse-credentials/admin admin/networks/ip: "::/0" diff --git a/clickhouse/justfile b/clickhouse/justfile index 116cafb..e0da5e0 100644 --- a/clickhouse/justfile +++ b/clickhouse/justfile @@ -5,6 +5,8 @@ export CLICKHOUSE_HOST := env("CLICKHOUSE_HOST", "") export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.5") export CLICKHOUSE_IMAGE := env("CLICKHOUSE_IMAGE", "clickhouse/clickhouse-server:25.10") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") +export PROMETHEUS_NAMESPACE := env("PROMETHEUS_NAMESPACE", "monitoring") +export MONITORING_ENABLED := env("MONITORING_ENABLED", "") # ClickHouse resource settings export CLICKHOUSE_MEMORY_REQUEST := env("CLICKHOUSE_MEMORY_REQUEST", "1Gi") @@ -107,6 +109,16 @@ install: --placeholder="e.g., clickhouse.example.com" ) done + # Check if Prometheus is available and ask about monitoring + if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then + if [ -z "${MONITORING_ENABLED}" ]; then + if gum confirm "Enable Prometheus monitoring?"; then + MONITORING_ENABLED="true" + else + MONITORING_ENABLED="false" + fi + fi + fi echo "Installing ClickHouse..." just create-namespace just install-zookeeper @@ -124,6 +136,10 @@ install: kubectl wait --for=jsonpath='{.status.status}'=Completed \ clickhouseinstallation/clickhouse -n ${CLICKHOUSE_NAMESPACE} --timeout=600s just setup-ingress ${CLICKHOUSE_HOST} + # Setup monitoring if enabled + if [ "${MONITORING_ENABLED}" = "true" ]; then + just setup-monitoring + fi echo "ClickHouse installation completed successfully" echo "ClickHouse API at: https://${CLICKHOUSE_HOST}" @@ -137,6 +153,27 @@ setup-ingress host: kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f clickhouse-ingress.yaml echo "ClickHouse Ingress configured successfully" +# Setup Prometheus monitoring for ClickHouse +setup-monitoring: + #!/bin/bash + set -euo pipefail + echo "Setting up Prometheus monitoring for ClickHouse..." + kubectl label namespace ${CLICKHOUSE_NAMESPACE} buun.channel/enable-monitoring=true --overwrite + MONITORING_ENABLED="true" gomplate -f clickhouse-servicemonitor.gomplate.yaml \ + -o clickhouse-servicemonitor.yaml + kubectl apply -f clickhouse-servicemonitor.yaml + echo "Prometheus monitoring configured successfully" + +# Remove Prometheus monitoring for ClickHouse +remove-monitoring: + #!/bin/bash + set -euo pipefail + echo "Removing Prometheus monitoring for ClickHouse..." + kubectl delete servicemonitor clickhouse clickhouse-operator -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found + kubectl delete service clickhouse-metrics -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found + kubectl label namespace ${CLICKHOUSE_NAMESPACE} buun.channel/enable-monitoring- --ignore-not-found + echo "Prometheus monitoring removed" + # Uninstall ClickHouse (delete_volumes='false' to preserve PVCs and namespace) uninstall delete-volumes='true': #!/bin/bash