feat(clickhouse): Add ClickHouse
This commit is contained in:
Masaki Yatsu
1
clickhouse/.gitignore
vendored
Normal file
1
clickhouse/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
clickhouse-credentials-external-secret.yaml
|
||||
@@ -0,0 +1,22 @@
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: clickhouse-credentials-external-secret
|
||||
namespace: {{ .Env.CLICKHOUSE_NAMESPACE }}
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
name: vault-secret-store
|
||||
kind: ClusterSecretStore
|
||||
target:
|
||||
name: clickhouse-credentials
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
type: Opaque
|
||||
data:
|
||||
admin: "{{ `{{ .admin }}` }}"
|
||||
data:
|
||||
- secretKey: admin
|
||||
remoteRef:
|
||||
key: clickhouse/credentials
|
||||
property: admin
|
||||
42
clickhouse/clickhouse.yaml
Normal file
42
clickhouse/clickhouse.yaml
Normal file
@@ -0,0 +1,42 @@
|
||||
apiVersion: clickhouse.altinity.com/v1
|
||||
kind: ClickHouseInstallation
|
||||
metadata:
|
||||
name: clickhouse
|
||||
spec:
|
||||
defaults:
|
||||
templates:
|
||||
dataVolumeClaimTemplate: data-volume-template
|
||||
logVolumeClaimTemplate: log-volume-template
|
||||
configuration:
|
||||
clusters:
|
||||
- name: default
|
||||
layout:
|
||||
shardsCount: 1
|
||||
replicasCount: 1
|
||||
zookeeper:
|
||||
nodes:
|
||||
- host: zookeeper
|
||||
port: 2181
|
||||
users:
|
||||
admin/k8s_secret_password: clickhouse-credentials/admin
|
||||
admin/networks/ip: "::/0"
|
||||
admin/access_management: 1
|
||||
profiles:
|
||||
default/max_memory_usage: 4000000000 # 4GB
|
||||
default/max_bytes_before_external_group_by: 2000000000 # 2GB
|
||||
templates:
|
||||
volumeClaimTemplates:
|
||||
- name: data-volume-template
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
- name: log-volume-template
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 100Mi
|
||||
403
clickhouse/justfile
Normal file
403
clickhouse/justfile
Normal file
@@ -0,0 +1,403 @@
|
||||
set fallback := true
|
||||
|
||||
export CLICKHOUSE_NAMESPACE := env("CLICKHOUSE_NAMESPACE", "clickhouse")
|
||||
export CLICKHOUSE_CHART_VERSION := env("CLICKHOUSE_CHART_VERSION", "0.25.3")
|
||||
export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets")
|
||||
|
||||
[private]
|
||||
default:
|
||||
@just --list --unsorted --list-submodules
|
||||
|
||||
# Add Helm repository
|
||||
add-helm-repo:
|
||||
helm repo add clickhouse-operator https://docs.altinity.com/clickhouse-operator/
|
||||
helm repo update
|
||||
|
||||
# Remove Helm repository
|
||||
remove-helm-repo:
|
||||
helm repo remove clickhouse-operator
|
||||
|
||||
# Create ClickHouse namespace
|
||||
create-namespace:
|
||||
@kubectl get namespace ${CLICKHOUSE_NAMESPACE} &>/dev/null || \
|
||||
kubectl create namespace ${CLICKHOUSE_NAMESPACE}
|
||||
|
||||
# Delete ClickHouse namespace
|
||||
delete-namespace:
|
||||
@kubectl delete namespace ${CLICKHOUSE_NAMESPACE} --ignore-not-found
|
||||
|
||||
# Create ClickHouse credentials secret
|
||||
create-credentials:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
echo "Setting up ClickHouse credentials..."
|
||||
|
||||
# Generate admin password
|
||||
ADMIN_PASSWORD=$(just utils::random-password)
|
||||
|
||||
if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then
|
||||
echo "External Secrets available. Storing credentials in Vault and creating ExternalSecret..."
|
||||
just vault::put clickhouse/credentials admin="$ADMIN_PASSWORD"
|
||||
gomplate -f clickhouse-credentials-external-secret.gomplate.yaml -o clickhouse-credentials-external-secret.yaml
|
||||
kubectl apply -f clickhouse-credentials-external-secret.yaml
|
||||
echo "Waiting for credentials secret to be ready..."
|
||||
kubectl wait --for=condition=Ready externalsecret/clickhouse-credentials-external-secret \
|
||||
-n ${CLICKHOUSE_NAMESPACE} --timeout=60s
|
||||
else
|
||||
echo "External Secrets not available. Creating Kubernetes Secret directly..."
|
||||
kubectl delete secret clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found
|
||||
kubectl create secret generic clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} \
|
||||
--from-literal=admin="$ADMIN_PASSWORD"
|
||||
echo "Credentials secret created directly in Kubernetes"
|
||||
fi
|
||||
echo "ClickHouse credentials setup completed"
|
||||
|
||||
# Delete ClickHouse credentials secret
|
||||
delete-credentials-secret:
|
||||
@kubectl delete secret clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found
|
||||
@kubectl delete externalsecret clickhouse-credentials-external-secret -n ${CLICKHOUSE_NAMESPACE} --ignore-not-found
|
||||
|
||||
# Install ClickHouse
|
||||
install:
|
||||
just create-namespace
|
||||
just install-zookeeper
|
||||
just create-credentials
|
||||
just add-helm-repo
|
||||
helm upgrade --install clickhouse-operator clickhouse-operator/altinity-clickhouse-operator \
|
||||
--version ${CLICKHOUSE_CHART_VERSION} -n ${CLICKHOUSE_NAMESPACE} --wait
|
||||
kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./clickhouse.yaml
|
||||
echo "Waiting for ClickHouse installation to be ready..."
|
||||
kubectl wait --for=jsonpath='{.status.status}'=Completed \
|
||||
clickhouseinstallation/clickhouse -n ${CLICKHOUSE_NAMESPACE} --timeout=600s
|
||||
echo "ClickHouse installation completed successfully"
|
||||
|
||||
# Uninstall ClickHouse
|
||||
uninstall:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
echo "Uninstalling ClickHouse..."
|
||||
if kubectl get clickhouseinstallations.clickhouse.altinity.com \
|
||||
-n ${CLICKHOUSE_NAMESPACE} &>/dev/null; then
|
||||
echo "Deleting ClickHouseInstallation resources..."
|
||||
kubectl delete clickhouseinstallations.clickhouse.altinity.com --all \
|
||||
-n ${CLICKHOUSE_NAMESPACE} --timeout=30s --ignore-not-found || {
|
||||
echo "Graceful deletion timed out, forcing finalizer removal..."
|
||||
for chi in $(kubectl get clickhouseinstallations.clickhouse.altinity.com \
|
||||
-n ${CLICKHOUSE_NAMESPACE} -o name 2>/dev/null); do
|
||||
kubectl patch "$chi" -n ${CLICKHOUSE_NAMESPACE} \
|
||||
--type='merge' -p='{"metadata":{"finalizers":null}}' || true
|
||||
done
|
||||
}
|
||||
fi
|
||||
helm uninstall clickhouse-operator -n ${CLICKHOUSE_NAMESPACE} --wait --ignore-not-found
|
||||
just uninstall-zookeeper
|
||||
just delete-credentials-secret
|
||||
just delete-namespace
|
||||
echo "ClickHouse uninstalled successfully"
|
||||
|
||||
# Print ClickHouse admin password
|
||||
admin-password:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then
|
||||
echo "Getting password from Vault..."
|
||||
just vault::get clickhouse/credentials admin
|
||||
else
|
||||
echo "Getting password from Kubernetes Secret..."
|
||||
kubectl get secret clickhouse-credentials -n ${CLICKHOUSE_NAMESPACE} \
|
||||
-o jsonpath='{.data.admin}' | base64 -d
|
||||
echo
|
||||
fi
|
||||
|
||||
# Connect to ClickHouse as admin
|
||||
connect-admin: check-env
|
||||
@just utils::check-connection clickhouse-clickhouse.clickhouse 9000
|
||||
@clickhouse client --host clickhouse-clickhouse.clickhouse --port 9000 \
|
||||
--user admin --password $(just clickhouse::admin-password)
|
||||
|
||||
# Connect to ClickHouse
|
||||
connect user: check-env
|
||||
@just utils::check-connection clickhouse-clickhouse.clickhouse 9000
|
||||
@clickhouse client --host clickhouse-clickhouse.clickhouse --port 9000 \
|
||||
--user {{ user }} --ask-password
|
||||
|
||||
# Create ClickHouse user
|
||||
create-user username='' password='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
USERNAME=${USERNAME:-"{{ username }}"}
|
||||
PASSWORD=${PASSWORD:-"{{ password }}"}
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
if just user-exists ${USERNAME} &>/dev/null; then
|
||||
echo "User ${USERNAME} already exists" >&2
|
||||
exit
|
||||
fi
|
||||
if [ -z "${PASSWORD}" ]; then
|
||||
PASSWORD=$(gum input --prompt="Password: " --password --width=100 \
|
||||
--placeholder="Empty to generate a random password")
|
||||
fi
|
||||
if [ -z "${PASSWORD}" ]; then
|
||||
PASSWORD=$(just utils::random-password)
|
||||
echo "Generated random password: ${PASSWORD}"
|
||||
fi
|
||||
echo "Creating ClickHouse user '${USERNAME}'..."
|
||||
just connect-admin <<EOF
|
||||
CREATE USER '${USERNAME}' IDENTIFIED BY '${PASSWORD}';
|
||||
EOF
|
||||
echo "User ${USERNAME} created."
|
||||
|
||||
# Create new user for ClickHouse (deprecated - use create-user instead)
|
||||
create-user-old username='' password='' database='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
USERNAME="{{ username }}"
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
PASSWORD="{{ password }}"
|
||||
while [ -z "${PASSWORD}" ]; do
|
||||
PASSWORD=$(gum input --prompt="Password: " --password --width=100)
|
||||
done
|
||||
DATABASE="{{ database }}"
|
||||
while [ -z "${DATABASE}" ]; do
|
||||
DATABASE=$(gum input --prompt="Database: " --width=100)
|
||||
done
|
||||
if [ "${DATABASE}" != "default" ]; then
|
||||
if ! just db-exists ${DATABASE}; then
|
||||
echo "Database '${DATABASE}' does not exist"
|
||||
just create-db ${DATABASE}
|
||||
fi
|
||||
fi
|
||||
echo "Creating ClickHouse user '${USERNAME}'..."
|
||||
just connect-admin <<EOF
|
||||
CREATE USER '${USERNAME}' IDENTIFIED BY '${PASSWORD}' DEFAULT DATABASE '${DATABASE}';
|
||||
EOF
|
||||
|
||||
# Delete ClickHouse user
|
||||
delete-user username='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
USERNAME=${USERNAME:-"{{ username }}"}
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
if ! just user-exists ${USERNAME} &>/dev/null; then
|
||||
echo "User ${USERNAME} does not exist." >&2
|
||||
exit
|
||||
fi
|
||||
echo "Deleting ClickHouse user '${USERNAME}'..."
|
||||
just connect-admin <<EOF
|
||||
DROP USER '${USERNAME}';
|
||||
EOF
|
||||
echo "User ${USERNAME} deleted."
|
||||
|
||||
# Grant all privileges on database to user
|
||||
grant db_name='' username='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
DB_NAME=${DB_NAME:-"{{ db_name }}"}
|
||||
USERNAME=${USERNAME:-"{{ username }}"}
|
||||
while [ -z "${DB_NAME}" ]; do
|
||||
DB_NAME=$(gum input --prompt="Database name: " --width=100)
|
||||
done
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
if ! just db-exists ${DB_NAME}; then
|
||||
echo "Database ${DB_NAME} does not exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! just user-exists ${USERNAME}; then
|
||||
echo "User ${USERNAME} does not exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Granting all privileges on '${DB_NAME}' to ClickHouse user '${USERNAME}'..."
|
||||
just connect-admin <<EOF
|
||||
GRANT ALL ON ${DB_NAME}.* TO '${USERNAME}';
|
||||
EOF
|
||||
echo "Privileges granted."
|
||||
|
||||
# Grant user access to ClickHouse database (deprecated - use grant instead)
|
||||
grant-user username='' database='default':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
USERNAME="{{ username }}"
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
DATABASE="{{ database }}"
|
||||
while [ -z "${DATABASE}" ]; do
|
||||
DATABASE=$(gum input --prompt="Database: " --width=100)
|
||||
done
|
||||
echo "Granting SELECT permission on '${DATABASE}' to ClickHouse user '${USERNAME}'..."
|
||||
just connect-admin <<EOF
|
||||
GRANT ALL ON ${DATABASE}.* TO '${USERNAME}';
|
||||
EOF
|
||||
|
||||
# Revoke all privileges on database from user
|
||||
revoke db_name='' username='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
DB_NAME=${DB_NAME:-"{{ db_name }}"}
|
||||
USERNAME=${USERNAME:-"{{ username }}"}
|
||||
while [ -z "${DB_NAME}" ]; do
|
||||
DB_NAME=$(gum input --prompt="Database name: " --width=100)
|
||||
done
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
if ! just db-exists ${DB_NAME}; then
|
||||
echo "Database ${DB_NAME} does not exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! just user-exists ${USERNAME}; then
|
||||
echo "User ${USERNAME} does not exist." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Revoking all privileges on '${DB_NAME}' from ClickHouse user '${USERNAME}'..."
|
||||
just connect-admin <<EOF
|
||||
REVOKE ALL ON ${DB_NAME}.* FROM '${USERNAME}';
|
||||
EOF
|
||||
echo "Privileges revoked."
|
||||
|
||||
# Create ClickHouse database and user
|
||||
create-user-and-db username='' db_name='' password='':
|
||||
just create-db "{{ db_name }}"
|
||||
just create-user "{{ username }}" "{{ password }}"
|
||||
just grant "{{ db_name }}" "{{ username }}"
|
||||
|
||||
# Delete ClickHouse database and user
|
||||
delete-user-and-db username='' db_name='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
DB_NAME=${DB_NAME:-"{{ db_name }}"}
|
||||
USERNAME=${USERNAME:-"{{ username }}"}
|
||||
if just db-exists ${DB_NAME} &>/dev/null; then
|
||||
if just user-exists ${USERNAME} &>/dev/null; then
|
||||
just revoke "${DB_NAME}" "${USERNAME}"
|
||||
else
|
||||
echo "User ${USERNAME} does not exist, skipping revoke."
|
||||
fi
|
||||
just delete-db "${DB_NAME}"
|
||||
else
|
||||
echo "Database ${DB_NAME} does not exist, skipping database deletion."
|
||||
fi
|
||||
if just user-exists ${USERNAME} &>/dev/null; then
|
||||
just delete-user "${USERNAME}"
|
||||
else
|
||||
echo "User ${USERNAME} does not exist, skipping user deletion."
|
||||
fi
|
||||
echo "Cleanup completed."
|
||||
|
||||
# List all users in ClickHouse
|
||||
list-users:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
just connect-admin <<EOF
|
||||
SHOW USERS;
|
||||
EOF
|
||||
|
||||
# Check if user exists in ClickHouse
|
||||
[no-exit-message]
|
||||
user-exists username='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
USERNAME="{{ username }}"
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
result=$(just connect-admin <<EOF
|
||||
SELECT name FROM system.users WHERE name = '${USERNAME}';
|
||||
EOF
|
||||
)
|
||||
if [[ -n "$result" && "$result" == *"$USERNAME"* ]]; then
|
||||
echo "User '$USERNAME' exists."
|
||||
exit 0
|
||||
else
|
||||
echo "User '$USERNAME' does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create DB in ClickHouse
|
||||
create-db database='default':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
DATABASE="{{ database }}"
|
||||
while [ -z "${DATABASE}" ]; do
|
||||
DATABASE=$(gum input --prompt="Database: " --width=100)
|
||||
done
|
||||
echo "Creating ClickHouse database '${DATABASE}'..."
|
||||
just connect-admin <<EOF
|
||||
CREATE DATABASE IF NOT EXISTS ${DATABASE};
|
||||
EOF
|
||||
|
||||
# Check if DB exists in ClickHouse
|
||||
[no-exit-message]
|
||||
db-exists database='default':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
DATABASE="{{ database }}"
|
||||
while [ -z "${DATABASE}" ]; do
|
||||
DATABASE=$(gum input --prompt="Database: " --width=100)
|
||||
done
|
||||
result=$(just connect-admin <<EOF
|
||||
SHOW DATABASES LIKE '${DATABASE}';
|
||||
EOF
|
||||
)
|
||||
if [[ -n "$result" && "$result" == *"$DATABASE"* ]]; then
|
||||
echo "Database '$DATABASE' exists."
|
||||
exit 0
|
||||
else
|
||||
echo "Database '$DATABASE' does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Print user privilege
|
||||
user-privilege username='':
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
USERNAME="{{ username }}"
|
||||
while [ -z "${USERNAME}" ]; do
|
||||
USERNAME=$(gum input --prompt="Username: " --width=100)
|
||||
done
|
||||
just connect-admin <<EOF
|
||||
SHOW GRANTS FOR '${USERNAME}';
|
||||
EOF
|
||||
|
||||
# Install ZooKeeper
|
||||
install-zookeeper:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
if ! kubectl get namespace zookeeper &>/dev/null; then
|
||||
kubectl create namespace zookeeper
|
||||
fi
|
||||
kubectl apply -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml
|
||||
|
||||
# Uninstall ZooKeeper
|
||||
uninstall-zookeeper:
|
||||
kubectl delete -n ${CLICKHOUSE_NAMESPACE} -f ./zookeeper.yaml
|
||||
|
||||
# Clean up ClickHouse resources
|
||||
cleanup:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
echo "This will delete all ClickHouse resources and secrets."
|
||||
if gum confirm "Are you sure you want to proceed?"; then
|
||||
echo "Cleaning up ClickHouse resources..."
|
||||
just vault::delete clickhouse/credentials || true
|
||||
echo "Cleanup completed"
|
||||
else
|
||||
echo "Cleanup cancelled"
|
||||
fi
|
||||
|
||||
# Check the environment
|
||||
[private]
|
||||
check-env:
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
if ! command -v clickhouse &>/dev/null; then
|
||||
echo "clickhouse CLI is not installed. Please install it first."
|
||||
exit 1
|
||||
fi
|
||||
261
clickhouse/zookeeper.yaml
Normal file
261
clickhouse/zookeeper.yaml
Normal file
@@ -0,0 +1,261 @@
|
||||
# https://raw.githubusercontent.com/Altinity/clickhouse-operator/refs/heads/master/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node.yaml
|
||||
# Setup Service to provide access to Zookeeper for clients
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
# DNS would be like zookeeper.zoons
|
||||
name: zookeeper
|
||||
labels:
|
||||
app: zookeeper
|
||||
spec:
|
||||
ports:
|
||||
- port: 2181
|
||||
name: client
|
||||
- port: 7000
|
||||
name: prometheus
|
||||
selector:
|
||||
app: zookeeper
|
||||
what: node
|
||||
---
|
||||
# Setup Headless Service for StatefulSet
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
# DNS would be like zookeeper-0.zookeepers.etc
|
||||
name: zookeepers
|
||||
labels:
|
||||
app: zookeeper
|
||||
spec:
|
||||
ports:
|
||||
- port: 2888
|
||||
name: server
|
||||
- port: 3888
|
||||
name: leader-election
|
||||
clusterIP: None
|
||||
selector:
|
||||
app: zookeeper
|
||||
what: node
|
||||
---
|
||||
# Setup max number of unavailable pods in StatefulSet
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: zookeeper-pod-disruption-budget
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: zookeeper
|
||||
maxUnavailable: 1
|
||||
---
|
||||
# Setup Zookeeper StatefulSet
|
||||
# Possible params:
|
||||
# 1. replicas
|
||||
# 2. memory
|
||||
# 3. cpu
|
||||
# 4. storage
|
||||
# 5. storageClassName
|
||||
# 6. user to run app
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
# nodes would be named as zookeeper-0, zookeeper-1, zookeeper-2
|
||||
name: zookeeper
|
||||
labels:
|
||||
app: zookeeper
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: zookeeper
|
||||
serviceName: zookeepers
|
||||
replicas: 1
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
podManagementPolicy: OrderedReady
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: zookeeper
|
||||
what: node
|
||||
annotations:
|
||||
prometheus.io/port: '7000'
|
||||
prometheus.io/scrape: 'true'
|
||||
spec:
|
||||
affinity:
|
||||
podAntiAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchExpressions:
|
||||
- key: "app"
|
||||
operator: In
|
||||
values:
|
||||
- zookeeper
|
||||
# TODO think about multi-AZ EKS
|
||||
# topologyKey: topology.kubernetes.io/zone
|
||||
topologyKey: "kubernetes.io/hostname"
|
||||
containers:
|
||||
- name: kubernetes-zookeeper
|
||||
imagePullPolicy: IfNotPresent
|
||||
image: "docker.io/zookeeper:3.8.4"
|
||||
resources:
|
||||
requests:
|
||||
memory: "512M"
|
||||
cpu: "1"
|
||||
limits:
|
||||
memory: "4Gi"
|
||||
cpu: "2"
|
||||
ports:
|
||||
- containerPort: 2181
|
||||
name: client
|
||||
- containerPort: 2888
|
||||
name: server
|
||||
- containerPort: 3888
|
||||
name: leader-election
|
||||
- containerPort: 7000
|
||||
name: prometheus
|
||||
env:
|
||||
- name: SERVERS
|
||||
value: "1"
|
||||
|
||||
# See those links for proper startup settings:
|
||||
# https://github.com/kow3ns/kubernetes-zookeeper/blob/master/docker/scripts/start-zookeeper
|
||||
# https://clickhouse.yandex/docs/en/operations/tips/#zookeeper
|
||||
# https://github.com/ClickHouse/ClickHouse/issues/11781
|
||||
command:
|
||||
- bash
|
||||
- -x
|
||||
- -c
|
||||
- |
|
||||
HOST=`hostname -s` &&
|
||||
DOMAIN=`hostname -d` &&
|
||||
CLIENT_PORT=2181 &&
|
||||
SERVER_PORT=2888 &&
|
||||
ELECTION_PORT=3888 &&
|
||||
PROMETHEUS_PORT=7000 &&
|
||||
ZOO_DATA_DIR=/var/lib/zookeeper/data &&
|
||||
ZOO_DATA_LOG_DIR=/var/lib/zookeeper/datalog &&
|
||||
{
|
||||
echo "clientPort=${CLIENT_PORT}"
|
||||
echo 'tickTime=2000'
|
||||
echo 'initLimit=300'
|
||||
echo 'syncLimit=10'
|
||||
echo 'maxClientCnxns=2000'
|
||||
echo 'maxTimeToWaitForEpoch=2000'
|
||||
echo 'maxSessionTimeout=60000000'
|
||||
echo "dataDir=${ZOO_DATA_DIR}"
|
||||
echo "dataLogDir=${ZOO_DATA_LOG_DIR}"
|
||||
echo 'autopurge.snapRetainCount=10'
|
||||
echo 'autopurge.purgeInterval=1'
|
||||
echo 'preAllocSize=131072'
|
||||
echo 'snapCount=3000000'
|
||||
echo 'leaderServes=yes'
|
||||
echo 'standaloneEnabled=false'
|
||||
echo '4lw.commands.whitelist=*'
|
||||
echo 'metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider'
|
||||
echo "metricsProvider.httpPort=${PROMETHEUS_PORT}"
|
||||
echo "skipACL=true"
|
||||
echo "fastleader.maxNotificationInterval=10000"
|
||||
} > /conf/zoo.cfg &&
|
||||
{
|
||||
echo "zookeeper.root.logger=CONSOLE"
|
||||
echo "zookeeper.console.threshold=INFO"
|
||||
echo "log4j.rootLogger=\${zookeeper.root.logger}"
|
||||
echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender"
|
||||
echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}"
|
||||
echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout"
|
||||
echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"
|
||||
} > /conf/log4j.properties &&
|
||||
echo 'JVMFLAGS="-Xms128M -Xmx4G -XX:ActiveProcessorCount=8 -XX:+AlwaysPreTouch -Djute.maxbuffer=8388608 -XX:MaxGCPauseMillis=50"' > /conf/java.env &&
|
||||
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
|
||||
NAME=${BASH_REMATCH[1]} &&
|
||||
ORD=${BASH_REMATCH[2]};
|
||||
else
|
||||
echo "Failed to parse name and ordinal of Pod" &&
|
||||
exit 1;
|
||||
fi &&
|
||||
mkdir -pv ${ZOO_DATA_DIR} &&
|
||||
mkdir -pv ${ZOO_DATA_LOG_DIR} &&
|
||||
whoami &&
|
||||
chown -Rv zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" &&
|
||||
export MY_ID=$((ORD+1)) &&
|
||||
echo $MY_ID > $ZOO_DATA_DIR/myid &&
|
||||
for (( i=1; i<=$SERVERS; i++ )); do
|
||||
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$SERVER_PORT:$ELECTION_PORT" >> /conf/zoo.cfg;
|
||||
done &&
|
||||
if [[ $SERVERS -eq 1 ]]; then
|
||||
echo "group.1=1" >> /conf/zoo.cfg;
|
||||
else
|
||||
echo "group.1=1:2:3" >> /conf/zoo.cfg;
|
||||
fi &&
|
||||
for (( i=1; i<=$SERVERS; i++ )); do
|
||||
WEIGHT=1
|
||||
if [[ $i == 1 ]]; then
|
||||
WEIGHT=10
|
||||
fi
|
||||
echo "weight.$i=$WEIGHT" >> /conf/zoo.cfg;
|
||||
done &&
|
||||
zkServer.sh start-foreground
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- bash
|
||||
- -c
|
||||
- '
|
||||
IFS=;
|
||||
MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;);
|
||||
while [[ "$MNTR" == "This ZooKeeper instance is not currently serving requests" ]];
|
||||
do
|
||||
echo "wait mntr works";
|
||||
sleep 1;
|
||||
MNTR=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "mntr" >&3 ; tee <&3; exec 3<&- ;);
|
||||
done;
|
||||
STATE=$(echo -e $MNTR | grep zk_server_state | cut -d " " -f 2);
|
||||
if [[ "$STATE" =~ "leader" ]]; then
|
||||
echo "check leader state";
|
||||
SYNCED_FOLLOWERS=$(echo -e $MNTR | grep zk_synced_followers | awk -F"[[:space:]]+" "{print \$2}" | cut -d "." -f 1);
|
||||
if [[ "$SYNCED_FOLLOWERS" != "0" ]]; then
|
||||
./bin/zkCli.sh ls /;
|
||||
exit $?;
|
||||
else
|
||||
exit 0;
|
||||
fi;
|
||||
elif [[ "$STATE" =~ "follower" ]]; then
|
||||
echo "check follower state";
|
||||
PEER_STATE=$(echo -e $MNTR | grep zk_peer_state);
|
||||
if [[ "$PEER_STATE" =~ "following - broadcast" ]]; then
|
||||
./bin/zkCli.sh ls /;
|
||||
exit $?;
|
||||
else
|
||||
exit 1;
|
||||
fi;
|
||||
else
|
||||
exit 1;
|
||||
fi
|
||||
'
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 60
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- bash
|
||||
- -xc
|
||||
- 'date && OK=$(exec 3<>/dev/tcp/127.0.0.1/2181 ; printf "ruok" >&3 ; IFS=; tee <&3; exec 3<&- ;); if [[ "$OK" == "imok" ]]; then exit 0; else exit 1; fi'
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 5
|
||||
volumeMounts:
|
||||
- name: datadir-volume
|
||||
mountPath: /var/lib/zookeeper
|
||||
# Run as a non-privileged user
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
fsGroup: 1000
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: datadir-volume
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 25Gi
|
||||
Reference in New Issue
Block a user