diff --git a/10_Postgres/cnpg-values.yaml b/10_Postgres/cnpg-values.yaml new file mode 100644 index 0000000..5fd887a --- /dev/null +++ b/10_Postgres/cnpg-values.yaml @@ -0,0 +1,27 @@ +# Pod Security Context for restricted Pod Security Standards +#podSecurityContext: +# runAsNonRoot: true +# seccompProfile: +# type: RuntimeDefault +# fsGroup: 10001 +# +## Container Security Context for restricted Pod Security Standards +#containerSecurityContext: +# allowPrivilegeEscalation: false +# readOnlyRootFilesystem: true +# runAsUser: 10001 +# runAsGroup: 10001 +# seccompProfile: +# type: RuntimeDefault +# capabilities: +# drop: +# - ALL +# +resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 100m + memory: 256Mi + diff --git a/10_Postgres/justfile b/10_Postgres/justfile new file mode 100644 index 0000000..7cbd8b6 --- /dev/null +++ b/10_Postgres/justfile @@ -0,0 +1,647 @@ +set fallback := true + +export CNPG_NAMESPACE := env("CNPG_NAMESPACE", "postgres") +export CNPG_CHART_VERSION := env("CNPG_CHART_VERSION", "0.26.1") +export CNPG_CLUSTER_CHART_VERSION := env("CNPG_CLUSTER_CHART_VERSION", "0.3.1") +export POSTGRES_STORAGE_SIZE := env("POSTGRES_STORAGE_SIZE", "20Gi") +export POSTGRES_MAX_CONNECTIONS := env("POSTGRES_MAX_CONNECTIONS", "200") +export K8S_VAULT_NAMESPACE := env("K8S_VAULT_NAMESPACE", "vault") +export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") + +[private] +default: + @just --list --unsorted --list-submodules + +# Add Helm repository +add-helm-repo: + @helm repo add cnpg https://cloudnative-pg.github.io/charts + @helm repo update + +# Remove Helm repository +remove-helm-repo: + @helm repo remove cnpg + +# Install CloudNativePG and create a cluster +install: + @just install-cnpg + @just create-cluster + +# Uninstall CloudNativePG and delete the cluster +uninstall: + @just delete-cluster + @just uninstall-cnpg + +# Install CloudNativePG +install-cnpg: + @just add-helm-repo + @helm upgrade --cleanup-on-fail --install cnpg cnpg/cloudnative-pg \ + --version ${CNPG_CHART_VERSION} \ + -n ${CNPG_NAMESPACE} --create-namespace --wait \ + -f cnpg-values.yaml + + @kubectl label namespace ${CNPG_NAMESPACE} \ + pod-security.kubernetes.io/enforce=restricted --overwrite + +# Uninstall CloudNativePG +uninstall-cnpg: + @helm uninstall cnpg -n ${CNPG_NAMESPACE} --wait + @kubectl delete namespace ${CNPG_NAMESPACE} --ignore-not-found + +# Create Postgres cluster +create-cluster: + #!/bin/bash + set -euo pipefail + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "External Secrets Operator detected. Creating admin credentials via ExternalSecret..." + password=$(just utils::random-password) + just vault::put-root postgres/admin username=postgres password="${password}" + + kubectl delete externalsecret postgres-cluster-superuser -n ${CNPG_NAMESPACE} --ignore-not-found + gomplate -f postgres-superuser-external-secret.gomplate.yaml | kubectl apply -f - + + echo "Waiting for ExternalSecret to sync..." + kubectl wait --for=condition=Ready externalsecret/postgres-cluster-superuser \ + -n ${CNPG_NAMESPACE} --timeout=60s + else + echo "External Secrets Operator not found. Creating superuser secret directly..." + password=$(just utils::random-password) + kubectl delete secret postgres-cluster-superuser -n ${CNPG_NAMESPACE} --ignore-not-found + kubectl create secret generic postgres-cluster-superuser -n ${CNPG_NAMESPACE} \ + --from-literal=username=postgres \ + --from-literal=password="${password}" + + if helm status vault -n ${K8S_VAULT_NAMESPACE} &>/dev/null; then + just vault::put-root postgres/admin username=postgres password="${password}" + fi + fi + + gomplate -f postgres-cluster-values.gomplate.yaml -o postgres-cluster-values.yaml + helm upgrade --install postgres-cluster cnpg/cluster \ + --version ${CNPG_CLUSTER_CHART_VERSION} \ + -n ${CNPG_NAMESPACE} --wait -f postgres-cluster-values.yaml + + echo "Waiting for PostgreSQL cluster to be ready..." + kubectl wait --for=condition=Ready clusters.postgresql.cnpg.io/postgres-cluster \ + -n ${CNPG_NAMESPACE} --timeout=300s + +# Delete Postgres cluster +delete-cluster: + @helm uninstall postgres-cluster -n ${CNPG_NAMESPACE} --ignore-not-found --wait + @kubectl delete externalsecret postgres-cluster-superuser -n ${CNPG_NAMESPACE} --ignore-not-found + @kubectl delete secret postgres-cluster-superuser -n ${CNPG_NAMESPACE} --ignore-not-found + +# Print Postgres username +admin-username: + @echo "postgres" + +# Print Postgres password +admin-password: + @kubectl get -n ${CNPG_NAMESPACE} secret postgres-cluster-superuser \ + -o jsonpath="{.data.password}" | base64 --decode + @echo + +# Create Postgres database +create-db db_name='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-{{ db_name }}} + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + if just db-exists ${DB_NAME} &>/dev/null; then + echo "Database ${DB_NAME} already exists" >&2 + exit + fi + echo "Creating database ${DB_NAME}..." + just psql -c "\"CREATE DATABASE ${DB_NAME};\"" + echo "Database ${DB_NAME} created." + +# Delete Postgres database +delete-db db_name='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-{{ db_name }}} + if ! just db-exists ${DB_NAME} &>/dev/null; then + echo "Database ${DB_NAME} does not exist." >&2 + exit + fi + # Terminate all connections to the database + just psql -c "\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = '${DB_NAME}' AND pid <> pg_backend_pid();\"" + # Force disconnect if needed + just psql -c "\"UPDATE pg_database SET datallowconn = false WHERE datname = '${DB_NAME}';\"" + just psql -c "\"SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = '${DB_NAME}';\"" + just psql -c "\"DROP DATABASE ${DB_NAME};\"" + echo "Database ${DB_NAME} deleted." + +# Check if database exists +[no-exit-message] +db-exists db_name='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-{{ db_name }}} + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + if echo '\l' | just postgres::psql | grep -E "^ *${DB_NAME} *\|" &>/dev/null; then + echo "Database ${DB_NAME} exists." + else + echo "Database ${DB_NAME} does not exist." >&2 + exit 1 + fi + +# Create Postgres user +create-user username='' password='': + #!/bin/bash + set -euo pipefail + USERNAME=${USERNAME:-"{{ username }}"} + PASSWORD=${PASSWORD:-"{{ password }}"} + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username: " --width=100) + done + if just user-exists ${USERNAME} &>/dev/null; then + echo "User ${USERNAME} already exists" >&2 + exit + fi + if [ -z "${PASSWORD}" ]; then + PASSWORD=$(gum input --prompt="Password: " --password --width=100 \ + --placeholder="Empty to generate a random password") + fi + if [ -z "${PASSWORD}" ]; then + PASSWORD=$(just random-password) + echo "Generated random password: ${PASSWORD}" + fi + just psql -c "\"CREATE USER ${USERNAME} WITH LOGIN PASSWORD '${PASSWORD}';\"" + echo "User ${USERNAME} created." + +# Delete Postgres user +delete-user username='': + #!/bin/bash + set -euo pipefail + USERNAME=${USERNAME:-"{{ username }}"} + if ! just user-exists ${USERNAME} &>/dev/null; then + echo "User ${USERNAME} does not exist." >&2 + exit + fi + just psql -c "\"ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public REVOKE ALL ON TABLES FROM ${USERNAME};\"" + just psql -c "\"ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public REVOKE ALL ON SEQUENCES FROM ${USERNAME};\"" + just psql -c "\"ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public REVOKE ALL ON FUNCTIONS FROM ${USERNAME};\"" + just psql -c "\"ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public REVOKE ALL ON TYPES FROM ${USERNAME};\"" + just psql -c "\"ALTER SCHEMA public OWNER TO postgres;\"" + just psql -c "\"DROP USER ${USERNAME};\"" + echo "User ${USERNAME} deleted." + +# Check if user exists +[no-exit-message] +user-exists username='': + #!/bin/bash + set -euo pipefail + USERNAME=${USERNAME:-"{{ username }}"} + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username: " --width=100) + done + if echo '\du' | just postgres::psql | grep -E "^ *${USERNAME} *\|" &>/dev/null; then + echo "User ${USERNAME} exists." + else + echo "User ${USERNAME} does not exist." >&2 + exit 1 + fi + +# Change user password +change-password username='' password='': + #!/bin/bash + set -euo pipefail + USERNAME=${USERNAME:-"{{ username }}"} + PASSWORD=${PASSWORD:-"{{ password }}"} + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username: " --width=100) + done + if ! just user-exists ${USERNAME} &>/dev/null; then + echo "User ${USERNAME} does not exist." >&2 + exit 1 + fi + if [ -z "${PASSWORD}" ]; then + PASSWORD=$(gum input --prompt="New password: " --password --width=100 \ + --placeholder="Empty to generate a random password") + fi + if [ -z "${PASSWORD}" ]; then + PASSWORD=$(just utils::random-password) + echo "Generated random password: ${PASSWORD}" + fi + just psql -c "\"ALTER USER ${USERNAME} WITH PASSWORD '${PASSWORD}';\"" + echo "Password changed for user ${USERNAME}." + +# Grant all privileges on database to user +grant db_name='' username='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-"{{ db_name }}"} + USERNAME=${USERNAME:-"{{ username }}"} + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username: " --width=100) + done + if ! just psql ${DB_NAME} -U postgres -P pager=off -c "\"SELECT 1;\""; then + echo "Database ${DB_NAME} does not exist." >&2 + exit 1 + fi + just psql -c "\"GRANT ALL PRIVILEGES ON DATABASE ${DB_NAME} TO ${USERNAME};\"" + # Grant CREATE permission on public schema (needed for PostgreSQL 15+) + just psql -d ${DB_NAME} -c "\"GRANT CREATE ON SCHEMA public TO ${USERNAME};\"" + echo "Privileges granted." + +# Revoke all privileges on database from user +revoke db_name='' username='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-"{{ db_name }}"} + USERNAME=${USERNAME:-"{{ username }}"} + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username: " --width=100) + done + if ! just psql -U postgres ${DB_NAME} -P pager=off -c "\"SELECT 1;\""; then + echo "Database ${DB_NAME} does not exist." >&2 + exit 1 + fi + just psql -c "\"REVOKE ALL PRIVILEGES ON DATABASE ${DB_NAME} FROM ${USERNAME};\"" + echo "Privileges revoked." + +# Create Postgres database and user +create-user-and-db username='' db_name='' password='': + @just create-db "{{ db_name }}" + @just create-user "{{ username }}" "{{ password }}" + @just grant "{{ db_name }}" "{{ username }}" + +# Delete Postgres database and user +delete-user-and-db username='' db_name='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-"{{ db_name }}"} + USERNAME=${USERNAME:-"{{ username }}"} + if just db-exists ${DB_NAME} &>/dev/null; then + if just user-exists ${USERNAME} &>/dev/null; then + just revoke "${DB_NAME}" "${USERNAME}" + else + echo "User ${USERNAME} does not exist, skipping revoke." + fi + just delete-db "${DB_NAME}" + else + echo "Database ${DB_NAME} does not exist, skipping database deletion." + fi + if just user-exists ${USERNAME} &>/dev/null; then + just delete-user "${USERNAME}" + else + echo "User ${USERNAME} does not exist, skipping user deletion." + fi + echo "Cleanup completed." + +# Create logical replication slot for CDC +create-replication-slot slot_name='' db_name='postgres' plugin='pgoutput': + #!/bin/bash + set -euo pipefail + SLOT_NAME=${SLOT_NAME:-"{{ slot_name }}"} + DB_NAME=${DB_NAME:-"{{ db_name }}"} + PLUGIN=${PLUGIN:-"{{ plugin }}"} + while [ -z "${SLOT_NAME}" ]; do + SLOT_NAME=$(gum input --prompt="Replication slot name: " --width=100 \ + --placeholder="e.g., airbyte_slot") + done + if kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -tAc \ + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = '${SLOT_NAME}';" | grep -q "${SLOT_NAME}"; then + echo "Replication slot '${SLOT_NAME}' already exists." + exit 0 + fi + echo "Creating replication slot '${SLOT_NAME}' with plugin '${PLUGIN}'..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "SELECT pg_create_logical_replication_slot('${SLOT_NAME}', '${PLUGIN}');" + echo "Replication slot '${SLOT_NAME}' created." + +# Delete replication slot +delete-replication-slot slot_name='' db_name='postgres': + #!/bin/bash + set -euo pipefail + SLOT_NAME=${SLOT_NAME:-"{{ slot_name }}"} + DB_NAME=${DB_NAME:-"{{ db_name }}"} + while [ -z "${SLOT_NAME}" ]; do + SLOT_NAME=$(gum input --prompt="Replication slot name to delete: " --width=100) + done + if ! kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -tAc \ + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = '${SLOT_NAME}';" | grep -q "${SLOT_NAME}"; then + echo "Replication slot '${SLOT_NAME}' does not exist." + exit 1 + fi + echo "Deleting replication slot '${SLOT_NAME}'..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "SELECT pg_drop_replication_slot('${SLOT_NAME}');" + echo "Replication slot '${SLOT_NAME}' deleted." + +# List all replication slots +list-replication-slots: + @echo "Replication slots:" + @kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d postgres -c \ + "SELECT slot_name, plugin, slot_type, database, active, restart_lsn FROM pg_replication_slots;" + +# Create publication for CDC +create-publication pub_name='' db_name='' tables='': + #!/bin/bash + set -euo pipefail + PUB_NAME=${PUB_NAME:-"{{ pub_name }}"} + DB_NAME=${DB_NAME:-"{{ db_name }}"} + TABLES="${TABLES:-{{ tables }}}" + while [ -z "${PUB_NAME}" ]; do + PUB_NAME=$(gum input --prompt="Publication name: " --width=100 \ + --placeholder="e.g., airbyte_publication") + done + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + if kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -tAc \ + "SELECT pubname FROM pg_publication WHERE pubname = '${PUB_NAME}';" | grep -q "${PUB_NAME}"; then + echo "Publication '${PUB_NAME}' already exists in database '${DB_NAME}'." + exit 0 + fi + if [ -z "${TABLES}" ]; then + echo "Select tables to include in publication:" + echo "1) All tables (ALL TABLES)" + echo "2) All user tables (exclude system/internal tables)" + echo "3) Specific tables (comma-separated list)" + CHOICE=$(gum choose "All tables" "User tables only" "Specific tables") + case "${CHOICE}" in + "All tables") + TABLES="ALL TABLES" + ;; + "User tables only") + # Get list of user tables (excluding _airbyte* and other system tables) + USER_TABLES=$(kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -tAc \ + "SELECT string_agg(tablename, ', ') FROM pg_tables + WHERE schemaname = 'public' + AND tablename NOT LIKE '\_%' + AND tablename NOT LIKE 'pg_%';") + if [ -z "${USER_TABLES}" ]; then + echo "No user tables found in database '${DB_NAME}'" + exit 1 + fi + TABLES="TABLE ${USER_TABLES}" + echo "Including tables: ${USER_TABLES}" + ;; + "Specific tables") + TABLES=$(gum input --prompt="Enter table names (comma-separated): " --width=100 \ + --placeholder="e.g., users, products, orders") + TABLES="TABLE ${TABLES}" + ;; + esac + elif [ "${TABLES}" = "ALL" ]; then + TABLES="ALL TABLES" + fi + echo "Creating publication '${PUB_NAME}' in database '${DB_NAME}'..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "CREATE PUBLICATION ${PUB_NAME} FOR ${TABLES};" + if [ "${TABLES}" != "ALL TABLES" ]; then + echo "Setting REPLICA IDENTITY for included tables..." + TABLE_LIST=$(echo "${TABLES}" | sed 's/TABLE //') + IFS=',' read -ra TABLE_ARRAY <<< "${TABLE_LIST}" + for table in "${TABLE_ARRAY[@]}"; do + table=$(echo "$table" | xargs) # trim whitespace + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "ALTER TABLE ${table} REPLICA IDENTITY FULL;" 2>/dev/null || true + done + fi + echo "Publication '${PUB_NAME}' created." + +# Delete publication +delete-publication pub_name='' db_name='': + #!/bin/bash + set -euo pipefail + PUB_NAME=${PUB_NAME:-"{{ pub_name }}"} + DB_NAME=${DB_NAME:-"{{ db_name }}"} + while [ -z "${PUB_NAME}" ]; do + PUB_NAME=$(gum input --prompt="Publication name to delete: " --width=100) + done + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + if ! kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -tAc \ + "SELECT pubname FROM pg_publication WHERE pubname = '${PUB_NAME}';" | grep -q "${PUB_NAME}"; then + echo "Publication '${PUB_NAME}' does not exist in database '${DB_NAME}'." + exit 1 + fi + echo "Deleting publication '${PUB_NAME}' from database '${DB_NAME}'..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "DROP PUBLICATION ${PUB_NAME};" + echo "Publication '${PUB_NAME}' deleted." + +# List all publications in a database +list-publications db_name='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-"{{ db_name }}"} + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + echo "Publications in database '${DB_NAME}':" + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "SELECT pubname, puballtables, pubinsert, pubupdate, pubdelete FROM pg_publication;" + +# Grant CDC privileges to user +grant-cdc-privileges username='' db_name='': + #!/bin/bash + set -euo pipefail + USERNAME=${USERNAME:-"{{ username }}"} + DB_NAME=${DB_NAME:-"{{ db_name }}"} + while [ -z "${USERNAME}" ]; do + USERNAME=$(gum input --prompt="Username to grant CDC privileges: " --width=100) + done + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name: " --width=100) + done + echo "Granting CDC privileges to user '${USERNAME}' on database '${DB_NAME}'..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c "ALTER USER ${USERNAME} WITH REPLICATION;" + echo "Granting schema and table privileges..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c \ + "GRANT USAGE ON SCHEMA public TO ${USERNAME}; + GRANT CREATE ON SCHEMA public TO ${USERNAME}; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${USERNAME}; + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${USERNAME};" + echo "Granting pg_read_all_data role..." + kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d ${DB_NAME} -c "GRANT pg_read_all_data TO ${USERNAME};" 2>/dev/null || true + echo "CDC privileges granted to user '${USERNAME}'" + +# Setup CDC (Change Data Capture) +setup-cdc db_name='' slot_name='' pub_name='' username='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-"{{ db_name }}"} + SLOT_NAME=${SLOT_NAME:-"{{ slot_name }}"} + PUB_NAME=${PUB_NAME:-"{{ pub_name }}"} + USERNAME=${USERNAME:-"{{ username }}"} + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name for CDC setup: " --width=100) + done + while [ -z "${SLOT_NAME}" ]; do + SLOT_NAME=$(gum input --prompt="Replication slot name: " --width=100 \ + --placeholder="e.g., demo_slot") + done + while [ -z "${PUB_NAME}" ]; do + PUB_NAME=$(gum input --prompt="Publication name: " --width=100 \ + --placeholder="e.g., demo_pub") + done + echo "Setting up CDC on database '${DB_NAME}'..." + WAL_LEVEL=$(kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d postgres -tAc "SHOW wal_level;") + if [ "${WAL_LEVEL}" != "logical" ]; then + echo "WARNING: wal_level is '${WAL_LEVEL}', should be 'logical' for CDC" + echo "Please ensure PostgreSQL is configured with wal_level=logical" + exit 1 + fi + just create-replication-slot "${SLOT_NAME}" "${DB_NAME}" + just create-publication "${PUB_NAME}" "${DB_NAME}" + if [ -n "${USERNAME}" ]; then + echo "" + just grant-cdc-privileges "${USERNAME}" "${DB_NAME}" + fi + echo "" + echo "CDC setup completed for database '${DB_NAME}'" + echo " Replication Method: Logical Replication (CDC)" + echo " Replication Slot: ${SLOT_NAME}" + echo " Publication: ${PUB_NAME}" + if [ -n "${USERNAME}" ]; then + echo " User with CDC privileges: ${USERNAME}" + fi + +# Cleanup CDC (removes slot and publication) +cleanup-cdc db_name='' slot_name='' pub_name='': + #!/bin/bash + set -euo pipefail + DB_NAME=${DB_NAME:-"{{ db_name }}"} + SLOT_NAME=${SLOT_NAME:-"{{ slot_name }}"} + PUB_NAME=${PUB_NAME:-"{{ pub_name }}"} + + while [ -z "${DB_NAME}" ]; do + DB_NAME=$(gum input --prompt="Database name for CDC cleanup: " --width=100) + done + while [ -z "${SLOT_NAME}" ]; do + SLOT_NAME=$(gum input --prompt="Replication slot name to delete: " --width=100 \ + --placeholder="e.g., demo_slot") + done + while [ -z "${PUB_NAME}" ]; do + PUB_NAME=$(gum input --prompt="Publication name to delete: " --width=100 \ + --placeholder="e.g., demo_pub") + done + echo "Cleaning up CDC configuration for database '${DB_NAME}'..." + + # Check if slot is active + SLOT_ACTIVE=$(kubectl exec -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- \ + psql -U postgres -d postgres -tAc \ + "SELECT active FROM pg_replication_slots WHERE slot_name = '${SLOT_NAME}';" 2>/dev/null || echo "") + if [ "${SLOT_ACTIVE}" = "t" ]; then + echo "WARNING: Replication slot '${SLOT_NAME}' is currently active!" + echo "Please stop any active replication connections first." + if ! gum confirm "Proceed with deletion anyway?"; then + echo "Cleanup cancelled" + exit 1 + fi + fi + + # Delete in correct order: Slot first, then Publication + echo "Step 1: Deleting replication slot '${SLOT_NAME}'..." + just delete-replication-slot "${SLOT_NAME}" "${DB_NAME}" || \ + echo "Replication slot '${SLOT_NAME}' not found or already deleted" + + echo "Step 2: Deleting publication '${PUB_NAME}'..." + just delete-publication "${PUB_NAME}" "${DB_NAME}" || \ + echo "Publication '${PUB_NAME}' not found or already deleted" + + echo "CDC cleanup completed for database '${DB_NAME}'" + +# Run psql +[no-exit-message] +psql *args='': + @kubectl exec -it -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- psql {{ args }} + +# Dump Postgres database by pg_dump +[no-cd] +dump db_name file exclude_tables='': + #!/bin/bash + set -euo pipefail + + DUMP_OPTIONS="-Fc" + if [ -n "{{ exclude_tables }}" ]; then + IFS=',' read -ra TABLES <<< "{{ exclude_tables }}" + for table in "${TABLES[@]}"; do + DUMP_OPTIONS="$DUMP_OPTIONS --exclude-table=$table" + done + fi + + kubectl exec -i -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- bash -c \ + "pg_dump -d postgresql://$(just postgres::admin-username):$(just postgres::admin-password)@localhost/{{ db_name }} $DUMP_OPTIONS > \ + /var/lib/postgresql/data/db.dump" + kubectl cp -n ${CNPG_NAMESPACE} -c postgres \ + postgres-cluster-1:/var/lib/postgresql/data/db.dump {{ file }} + kubectl exec -i -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- rm /var/lib/postgresql/data/db.dump + +# Restore Postgres database by pg_restore +[no-cd] +restore db_name file: + just postgres::create-db {{ db_name }} + kubectl cp {{ file }} -n ${CNPG_NAMESPACE} -c postgres \ + postgres-cluster-1:/var/lib/postgresql/data/db.dump + kubectl exec -i -n ${CNPG_NAMESPACE} postgres-cluster-1 -c postgres -- bash -c \ + "pg_restore --clean --if-exists \ + -d postgresql://$(just postgres::admin-username):$(just postgres::admin-password)@localhost/{{ db_name }} \ + /var/lib/postgresql/data/db.dump" + +# Enable Prometheus monitoring +enable-monitoring: + #!/bin/bash + set -euo pipefail + echo "Enabling Prometheus PodMonitor for PostgreSQL cluster..." + + # Label namespace to enable monitoring + kubectl label namespace ${CNPG_NAMESPACE} buun.channel/enable-monitoring=true --overwrite + + # Enable PodMonitor + kubectl patch cluster postgres-cluster -n ${CNPG_NAMESPACE} --type=merge -p '{"spec":{"monitoring":{"enablePodMonitor":true}}}' + + echo "Waiting for PodMonitor to be created..." + sleep 3 + + # Add release label to PodMonitor + kubectl label podmonitor postgres-cluster -n ${CNPG_NAMESPACE} release=kube-prometheus-stack --overwrite + + kubectl get podmonitor -n ${CNPG_NAMESPACE} -l cnpg.io/cluster=postgres-cluster + echo "✓ PostgreSQL monitoring enabled" + +# Disable Prometheus monitoring +disable-monitoring: + #!/bin/bash + set -euo pipefail + echo "Disabling Prometheus PodMonitor for PostgreSQL cluster..." + + # Disable PodMonitor + kubectl patch cluster postgres-cluster -n ${CNPG_NAMESPACE} --type=merge -p '{"spec":{"monitoring":{"enablePodMonitor":false}}}' + + # Remove namespace label + kubectl label namespace ${CNPG_NAMESPACE} buun.channel/enable-monitoring- --ignore-not-found + + echo "✓ PostgreSQL monitoring disabled" diff --git a/10_Postgres/pgdb-example.yaml b/10_Postgres/pgdb-example.yaml new file mode 100644 index 0000000..f543f6c --- /dev/null +++ b/10_Postgres/pgdb-example.yaml @@ -0,0 +1,9 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cluster-example +spec: + instances: 3 + + storage: + size: 1Gi