diff --git a/clickhouse/.gitignore b/clickhouse/.gitignore new file mode 100644 index 0000000..34b9e4c --- /dev/null +++ b/clickhouse/.gitignore @@ -0,0 +1 @@ +clickhouse-credentials-external-secret.yaml diff --git a/clickhouse/clickhouse-credentials-external-secret.gomplate.yaml b/clickhouse/clickhouse-credentials-external-secret.gomplate.yaml new file mode 100644 index 0000000..38a2c47 --- /dev/null +++ b/clickhouse/clickhouse-credentials-external-secret.gomplate.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: clickhouse-credentials-external-secret + namespace: {{ .Env.CLICKHOUSE_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: clickhouse-credentials + creationPolicy: Owner + template: + type: Opaque + data: + admin: "{{ `{{ .admin }}` }}" + data: + - secretKey: admin + remoteRef: + key: clickhouse/credentials + property: admin \ No newline at end of file diff --git a/clickhouse/clickhouse.yaml b/clickhouse/clickhouse.yaml new file mode 100644 index 0000000..b2307eb --- /dev/null +++ b/clickhouse/clickhouse.yaml @@ -0,0 +1,42 @@ +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + name: clickhouse +spec: + defaults: + templates: + dataVolumeClaimTemplate: data-volume-template + logVolumeClaimTemplate: log-volume-template + configuration: + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 1 + zookeeper: + nodes: + - host: zookeeper + port: 2181 + users: + admin/k8s_secret_password: clickhouse-credentials/admin + admin/networks/ip: "::/0" + admin/access_management: 1 + profiles: + default/max_memory_usage: 4000000000 # 4GB + default/max_bytes_before_external_group_by: 2000000000 # 2GB + templates: + volumeClaimTemplates: + - name: data-volume-template + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + - name: log-volume-template + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi diff --git a/clickhouse/justfile b/clickhouse/justfile new file mode 100644 index 0000000..e58d912 --- /dev/null +++ b/clickhouse/justfile @@ -0,0 +1,403 @@ +set fallback := true + +export CLICKHOUSE_NAMESPACE := env("CLICKHOUSE_NAMESPACE", "clickhouse") +export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.3") +export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") + +[private] +default: + @just --list --unsorted --list-submodules + +# Add Helm repository +add-helm-repo: + helm repo add clickhouse-operator https://docs.altinity.com/clickhouse-operator/ + helm repo update + +# Remove Helm repository +remove-helm-repo: + helm repo remove clickhouse-operator + +# Create ClickHouse namespace +create-namespace: + @kubectl get namespace ${CLICKHOUSE_NAMESPACE} &>/dev/null || \ + kubectl create namespace ${CLICKHOUSE_NAMESPACE} + +# Delete ClickHouse namespace +delete-namespace: + @kubectl delete namespace ${CLICKHOUSE_NAMESPACE} --ignore-not-found + +# Create ClickHouse credentials secret +create-credentials: + #!/bin/bash + set -euo pipefail + echo "Setting up ClickHouse credentials..." + + # Generate admin password + ADMIN_PASSWORD=$(just utils::random-password) + + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "External Secrets available. Storing credentials in Vault and creating ExternalSecret..." + just vault::put clickhouse/credentials admin="$ADMIN_PASSWORD" + gomplate -f clickhouse-credentials-external-secret.gomplate.yaml -o clickhouse-credentials-external-secret.yaml + kubectl apply -f clickhouse-credentials-external-secret.yaml + echo "Waiting for credentials secret to be ready..." + kubectl wait --for=condition=Ready externalsecret/clickhouse-credentials-external-secret \ + -n ${CLICKHOUSE_NAMESPACE} --timeout=60s + else + echo "External Secrets not available. Creating Kubernetes Secret directly..." + kubectl delete secret clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found + kubectl create secret generic clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} \ + --from-literal=admin="$ADMIN_PASSWORD" + echo "Credentials secret created directly in Kubernetes" + fi + echo "ClickHouse credentials setup completed" + +# Delete ClickHouse credentials secret +delete-credentials-secret: + @kubectl delete secret clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found + @kubectl delete externalsecret clickhouse-credentials-external-secret -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found + +# Install ClickHouse +install: + just create-namespace + just install-zookeeper + just create-credentials + just add-helm-repo + helm upgrade --install clickhouse-operator clickhouse-operator/altinity-clickhouse-operator \ + --version ${CLICKHOUSE_CHART_VERSION} -n ${CLICKHOUSE_NAMESPACE} --wait + kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./clickhouse.yaml + echo "Waiting for ClickHouse installation to be ready..." + kubectl wait --for=jsonpath='{.status.status}'=Completed \ + clickhouseinstallation/clickhouse -n ${CLICKHOUSE_NAMESPACE} --timeout=600s + echo "ClickHouse installation completed successfully" + +# Uninstall ClickHouse +uninstall: + #!/bin/bash + set -euo pipefail + echo "Uninstalling ClickHouse..." + if kubectl get clickhouseinstallations.clickhouse.altinity.com \ + -n ${CLICKHOUSE_NAMESPACE} &>/dev/null; then + echo "Deleting ClickHouseInstallation resources..." + kubectl delete clickhouseinstallations.clickhouse.altinity.com --all \ + -n ${CLICKHOUSE_NAMESPACE} --timeout=30s --ignore-not-found || { + echo "Graceful deletion timed out, forcing finalizer removal..." + for chi in $(kubectl get clickhouseinstallations.clickhouse.altinity.com \ + -n ${CLICKHOUSE_NAMESPACE} -o name 2>/dev/null); do + kubectl patch "$chi" -n ${CLICKHOUSE_NAMESPACE} \ + --type='merge' -p='{"metadata":{"finalizers":null}}' || true + done + } + fi + helm uninstall clickhouse-operator -n ${CLICKHOUSE_NAMESPACE} --wait --ignore-not-found + just uninstall-zookeeper + just delete-credentials-secret + just delete-namespace + echo "ClickHouse uninstalled successfully" + +# Print ClickHouse admin password +admin-password: + #!/bin/bash + set -euo pipefail + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "Getting password from Vault..." + just vault::get clickhouse/credentials admin + else + echo "Getting password from Kubernetes Secret..." + kubectl get secret clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} \ + -o jsonpath='{.data.admin}' | base64 -d + echo + fi + +# Connect to ClickHouse as admin +connect-admin: check-env + @just utils::check-connection clickhouse-clickhouse.clickhouse 9000 + @clickhouse client --host clickhouse-clickhouse.clickhouse --port 9000 \ + --user admin --password $(just clickhouse::admin-password) + +# Connect to ClickHouse +connect user: check-env + @just utils::check-connection clickhouse-clickhouse.clickhouse 9000 + @clickhouse client --host clickhouse-clickhouse.clickhouse --port 9000 \ + --user {{ user }} --ask-password + +# Create ClickHouse user +create-user username='' password='': + #!/bin/bash + set -euo pipefail + USERNAME=${USERNAME:-"{{ username }}"} + PASSWORD=${PASSWORD:-"{{ password }}"} + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username: " --width=100) + done + if just user-exists ${USERNAME} &>/dev/null; then + echo "User ${USERNAME} already exists" >&2 + exit + fi + if [ -z "${PASSWORD}" ]; then + PASSWORD=$(gum input --prompt="Password: " --password --width=100 \ + --placeholder="Empty to generate a random password") + fi + if [ -z "${PASSWORD}" ]; then + PASSWORD=$(just utils::random-password) + echo "Generated random password: ${PASSWORD}" + fi + echo "Creating ClickHouse user '${USERNAME}'..." + just connect-admin </dev/null; then + echo "User ${USERNAME} does not exist." >&2 + exit + fi + echo "Deleting ClickHouse user '${USERNAME}'..." + just connect-admin <&2 + exit 1 + fi + if ! just user-exists ${USERNAME}; then + echo "User ${USERNAME} does not exist." >&2 + exit 1 + fi + echo "Granting all privileges on '${DB_NAME}' to ClickHouse user '${USERNAME}'..." + just connect-admin <&2 + exit 1 + fi + if ! just user-exists ${USERNAME}; then + echo "User ${USERNAME} does not exist." >&2 + exit 1 + fi + echo "Revoking all privileges on '${DB_NAME}' from ClickHouse user '${USERNAME}'..." + just connect-admin </dev/null; then + if just user-exists ${USERNAME} &>/dev/null; then + just revoke "${DB_NAME}" "${USERNAME}" + else + echo "User ${USERNAME} does not exist, skipping revoke." + fi + just delete-db "${DB_NAME}" + else + echo "Database ${DB_NAME} does not exist, skipping database deletion." + fi + if just user-exists ${USERNAME} &>/dev/null; then + just delete-user "${USERNAME}" + else + echo "User ${USERNAME} does not exist, skipping user deletion." + fi + echo "Cleanup completed." + +# List all users in ClickHouse +list-users: + #!/bin/bash + set -euo pipefail + just connect-admin </dev/null; then + kubectl create namespace zookeeper + fi + kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml + +# Uninstall ZooKeeper +uninstall-zookeeper: + kubectl delete -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml + +# Clean up ClickHouse resources +cleanup: + #!/bin/bash + set -euo pipefail + echo "This will delete all ClickHouse resources and secrets." + if gum confirm "Are you sure you want to proceed?"; then + echo "Cleaning up ClickHouse resources..." + just vault::delete clickhouse/credentials || true + echo "Cleanup completed" + else + echo "Cleanup cancelled" + fi + +# Check the environment +[private] +check-env: + #!/bin/bash + set -euo pipefail + if ! command -v clickhouse &>/dev/null; then + echo "clickhouse CLI is not installed. Please install it first." + exit 1 + fi diff --git a/clickhouse/zookeeper.yaml b/clickhouse/zookeeper.yaml new file mode 100644 index 0000000..b42cabd --- /dev/null +++ b/clickhouse/zookeeper.yaml @@ -0,0 +1,261 @@ +# https://raw.githubusercontent.com/Altinity/clickhouse-operator/refs/heads/master/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node.yaml +# Setup Service to provide access to Zookeeper for clients +apiVersion: v1 +kind: Service +metadata: + # DNS would be like zookeeper.zoons + name: zookeeper + labels: + app: zookeeper +spec: + ports: + - port: 2181 + name: client + - port: 7000 + name: prometheus + selector: + app: zookeeper + what: node +--- +# Setup Headless Service for StatefulSet +apiVersion: v1 +kind: Service +metadata: + # DNS would be like zookeeper-0.zookeepers.etc + name: zookeepers + labels: + app: zookeeper +spec: + ports: + - port: 2888 + name: server + - port: 3888 + name: leader-election + clusterIP: None + selector: + app: zookeeper + what: node +--- +# Setup max number of unavailable pods in StatefulSet +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: zookeeper-pod-disruption-budget +spec: + selector: + matchLabels: + app: zookeeper + maxUnavailable: 1 +--- +# Setup Zookeeper StatefulSet +# Possible params: +# 1. replicas +# 2. memory +# 3. cpu +# 4. storage +# 5. storageClassName +# 6. user to run app +apiVersion: apps/v1 +kind: StatefulSet +metadata: + # nodes would be named as zookeeper-0, zookeeper-1, zookeeper-2 + name: zookeeper + labels: + app: zookeeper +spec: + selector: + matchLabels: + app: zookeeper + serviceName: zookeepers + replicas: 1 + updateStrategy: + type: RollingUpdate + podManagementPolicy: OrderedReady + template: + metadata: + labels: + app: zookeeper + what: node + annotations: + prometheus.io/port: '7000' + prometheus.io/scrape: 'true' + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - zookeeper + # TODO think about multi-AZ EKS + # topologyKey: topology.kubernetes.io/zone + topologyKey: "kubernetes.io/hostname" + containers: + - name: kubernetes-zookeeper + imagePullPolicy: IfNotPresent + image: "docker.io/zookeeper:3.8.4" + resources: + requests: + memory: "512M" + cpu: "1" + limits: + memory: "4Gi" + cpu: "2" + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + - containerPort: 7000 + name: prometheus + env: + - name: SERVERS + value: "1" + +# See those links for proper startup settings: +# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper +# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper +# https://github.com/ClickHouse/ClickHouse/issues/11781 + command: + - bash + - -x + - -c + - | + HOST=`hostname -s` && + DOMAIN=`hostname -d` && + CLIENT_PORT=2181 && + SERVER_PORT=2888 && + ELECTION_PORT=3888 && + PROMETHEUS_PORT=7000 && + ZOO_DATA_DIR=/var/lib/zookeeper/data && + ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog && + { + echo "clientPort=${CLIENT_PORT}" + echo 'tickTime=2000' + echo 'initLimit=300' + echo 'syncLimit=10' + echo 'maxClientCnxns=2000' + echo 'maxTimeToWaitForEpoch=2000' + echo 'maxSessionTimeout=60000000' + echo "dataDir=${ZOO_DATA_DIR}" + echo "dataLogDir=${ZOO_DATA_LOG_DIR}" + echo 'autopurge.snapRetainCount=10' + echo 'autopurge.purgeInterval=1' + echo 'preAllocSize=131072' + echo 'snapCount=3000000' + echo 'leaderServes=yes' + echo 'standaloneEnabled=false' + echo '4lw.commands.whitelist=*' + echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider' + echo "metricsProvider.httpPort=${PROMETHEUS_PORT}" + echo "skipACL=true" + echo "fastleader.maxNotificationInterval=10000" + } > /conf/zoo.cfg && + { + echo "zookeeper.root.logger=CONSOLE" + echo "zookeeper.console.threshold=INFO" + echo "log4j.rootLogger=\${zookeeper.root.logger}" + echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" + echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" + echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" + echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n" + } > /conf/log4j.properties && + echo 'JVMFLAGS="-Xms128M -Xmx4G -XX:ActiveProcessorCount=8 -XX:+AlwaysPreTouch -Djute.maxbuffer=8388608 -XX:MaxGCPauseMillis=50"' > /conf/java.env && + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} && + ORD=${BASH_REMATCH[2]}; + else + echo "Failed to parse name and ordinal of Pod" && + exit 1; + fi && + mkdir -pv ${ZOO_DATA_DIR} && + mkdir -pv ${ZOO_DATA_LOG_DIR} && + whoami && + chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" && + export MY_ID=$((ORD+1)) && + echo $MY_ID > $ZOO_DATA_DIR/myid && + for (( i=1; i<=$SERVERS; i++ )); do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg; + done && + if [[ $SERVERS -eq 1 ]]; then + echo "group.1=1" >> /conf/zoo.cfg; + else + echo "group.1=1:2:3" >> /conf/zoo.cfg; + fi && + for (( i=1; i<=$SERVERS; i++ )); do + WEIGHT=1 + if [[ $i == 1 ]]; then + WEIGHT=10 + fi + echo "weight.$i=$WEIGHT" >> /conf/zoo.cfg; + done && + zkServer.sh start-foreground + readinessProbe: + exec: + command: + - bash + - -c + - ' + IFS=; + MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;); + while [[ "$MNTR" == "This ZooKeeper instance is not currently serving requests" ]]; + do + echo "wait mntr works"; + sleep 1; + MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;); + done; + STATE=$(echo -e $MNTR | grep zk_server_state | cut -d " " -f 2); + if [[ "$STATE" =~ "leader" ]]; then + echo "check leader state"; + SYNCED_FOLLOWERS=$(echo -e $MNTR | grep zk_synced_followers | awk -F"[[:space:]]+" "{print \$2}" | cut -d "." -f 1); + if [[ "$SYNCED_FOLLOWERS" != "0" ]]; then + ./bin/zkCli.sh ls /; + exit $?; + else + exit 0; + fi; + elif [[ "$STATE" =~ "follower" ]]; then + echo "check follower state"; + PEER_STATE=$(echo -e $MNTR | grep zk_peer_state); + if [[ "$PEER_STATE" =~ "following - broadcast" ]]; then + ./bin/zkCli.sh ls /; + exit $?; + else + exit 1; + fi; + else + exit 1; + fi + ' + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 60 + livenessProbe: + exec: + command: + - bash + - -xc + - 'date && OK=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi' + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + volumeMounts: + - name: datadir-volume + mountPath: /var/lib/zookeeper + # Run as a non-privileged user + securityContext: + runAsUser: 1000 + fsGroup: 1000 + volumeClaimTemplates: + - metadata: + name: datadir-volume + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 25Gi diff --git a/justfile b/justfile index f63725f..a0ae84c 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,7 @@ export PATH := "./node_modules/.bin:" + env_var('PATH') default: @just --list --unsorted --list-submodules +mod clickhouse mod datahub mod env mod external-secrets