set fallback := true export AIRBYTE_NAMESPACE := env("AIRBYTE_NAMESPACE", "airbyte") export AIRBYTE_CHART_VERSION := env("AIRBYTE_CHART_VERSION", "1.8.3") export AIRBYTE_WEBAPP_IMAGE_TAG := env("AIRBYTE_WEBAPP_IMAGE_TAG", "1.7.4") export AIRBYTE_HOST := env("AIRBYTE_HOST", "") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") export AIRBYTE_STORAGE_TYPE := env("AIRBYTE_STORAGE_TYPE", "") export AIRBYTE_STORAGE_BUCKET := env("AIRBYTE_STORAGE_BUCKET", "airbyte-storage") [private] default: @just --list --unsorted --list-submodules # Get Salesforce refresh token for manual OAuth setup get-salesforce-refresh-token: #!/bin/bash set -euo pipefail echo "=== Salesforce Refresh Token Generator ===" echo "" # Get Client ID CLIENT_ID=$(gum input --prompt="Salesforce Client ID: " --width=100) # Get Client Secret CLIENT_SECRET=$(gum input --prompt="Salesforce Client Secret: " --width=100 --password) # Generate authorization URL AUTH_URL="https://login.salesforce.com/services/oauth2/authorize?response_type=code&client_id=${CLIENT_ID}&redirect_uri=https://login.salesforce.com/services/oauth2/success&scope=api%20refresh_token%20offline_access" echo "" echo "Step 1: Open the following URL in your browser:" echo "" echo "${AUTH_URL}" echo "" echo "Step 2: After authentication, you will be redirected to a URL like:" echo "https://login.salesforce.com/services/oauth2/success?code=XXXXX" echo "" # Get authorization code AUTH_CODE=$(gum input --prompt="Paste the 'code' parameter from the URL: " --width=100) echo "" echo "Exchanging authorization code for refresh token..." # Exchange code for tokens RESPONSE=$(curl -s -X POST https://login.salesforce.com/services/oauth2/token \ -d "grant_type=authorization_code" \ -d "client_id=${CLIENT_ID}" \ -d "client_secret=${CLIENT_SECRET}" \ -d "redirect_uri=https://login.salesforce.com/services/oauth2/success" \ -d "code=${AUTH_CODE}") # Extract refresh token REFRESH_TOKEN=$(echo "${RESPONSE}" | jq -r '.refresh_token // empty') if [ -z "${REFRESH_TOKEN}" ]; then echo "" echo "Error: Failed to get refresh token" echo "Response: ${RESPONSE}" exit 1 fi echo "" echo "=== SUCCESS ===" echo "" echo "Client ID: ${CLIENT_ID}" echo "Client Secret: ${CLIENT_SECRET}" echo "Refresh Token: ${REFRESH_TOKEN}" echo "" echo "Use these values in Airbyte:" echo "1. Click 'Authenticate your Salesforce account'" echo "2. Click 'Set up manually'" echo "3. Enter the above Client ID, Client Secret, and Refresh Token" # Add Helm repository add-helm-repo: helm repo add airbyte https://airbytehq.github.io/helm-charts helm repo update # Remove Helm repository remove-helm-repo: helm repo remove airbyte # Create Airbyte namespace create-namespace: @kubectl get namespace ${AIRBYTE_NAMESPACE} &>/dev/null || \ kubectl create namespace ${AIRBYTE_NAMESPACE} # Delete Airbyte namespace delete-namespace: @kubectl delete namespace ${AIRBYTE_NAMESPACE} --ignore-not-found # Setup database for Airbyte setup-database: #!/bin/bash set -euo pipefail echo "Setting up Airbyte databases..." # Airbyte requires multiple databases DATABASES=("airbyte_db" "airbyte_configs" "airbyte_jobs" "temporal" "temporal_visibility") for DB_NAME in "${DATABASES[@]}"; do if just postgres::db-exists "$DB_NAME" &>/dev/null; then echo "Database '$DB_NAME' already exists." else echo "Creating new database '$DB_NAME'..." just postgres::create-db "$DB_NAME" fi done # Generate password for user creation/update if just postgres::user-exists airbyte &>/dev/null; then echo "User 'airbyte' already exists." # Check if we can get existing password from Vault/Secret if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then # Try to get existing password from Vault if DB_PASSWORD=$(just vault::get airbyte/database password 2>/dev/null); then echo "Using existing password from Vault." else echo "Generating new password and updating Vault..." DB_PASSWORD=$(just utils::random-password) just postgres::change-password airbyte "$DB_PASSWORD" fi else # For direct Secret approach, generate new password echo "Generating new password for existing user..." DB_PASSWORD=$(just utils::random-password) just postgres::change-password airbyte "$DB_PASSWORD" fi else echo "Creating new user 'airbyte'..." DB_PASSWORD=$(just utils::random-password) just postgres::create-user airbyte "$DB_PASSWORD" fi echo "Ensuring database permissions..." for DB_NAME in "${DATABASES[@]}"; do just postgres::grant "$DB_NAME" airbyte done if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "External Secrets available. Storing credentials in Vault and creating ExternalSecret..." just vault::put airbyte/database username=airbyte password="$DB_PASSWORD" gomplate -f airbyte-database-external-secret.gomplate.yaml -o airbyte-database-external-secret.yaml kubectl apply -f airbyte-database-external-secret.yaml echo "Waiting for database secret to be ready..." kubectl wait --for=condition=Ready externalsecret/airbyte-database-external-secret \ -n ${AIRBYTE_NAMESPACE} --timeout=60s else echo "External Secrets not available. Creating Kubernetes Secret directly..." kubectl delete secret airbyte-database-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found kubectl create secret generic airbyte-database-secret -n ${AIRBYTE_NAMESPACE} \ --from-literal=username=airbyte \ --from-literal=password="$DB_PASSWORD" echo "Database secret created directly in Kubernetes" fi echo "Database setup completed." # Delete database secret delete-database-secret: @kubectl delete secret airbyte-database-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found @kubectl delete externalsecret airbyte-database-external-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found # Setup OAuth2 Proxy for Airbyte # Note: Airbyte OSS doesn't support direct OIDC integration, so we use OAuth2 Proxy setup-oauth2-proxy: #!/bin/bash set -euo pipefail export AIRBYTE_HOST=${AIRBYTE_HOST:-} while [ -z "${AIRBYTE_HOST}" ]; do AIRBYTE_HOST=$( gum input --prompt="Airbyte host (FQDN): " --width=100 \ --placeholder="e.g., airbyte.example.com" ) done echo "Setting up OAuth2 Proxy for Airbyte..." just oauth2-proxy::setup-for-app \ airbyte \ "${AIRBYTE_HOST}" \ "${AIRBYTE_NAMESPACE}" \ "airbyte-airbyte-webapp-svc:80" echo "Disabling Airbyte webapp Ingress to prevent authentication bypass..." helm upgrade airbyte airbyte/airbyte \ --namespace ${AIRBYTE_NAMESPACE} \ --reuse-values \ --set webapp.ingress.enabled=false echo "OAuth2 Proxy setup completed" echo "Access Airbyte at: https://${AIRBYTE_HOST}" # Remove OAuth2 Proxy from Airbyte remove-oauth2-proxy: @echo "Removing OAuth2 Proxy for Airbyte..." @just oauth2-proxy::remove-for-app airbyte "${AIRBYTE_NAMESPACE}" @echo "Re-enabling Airbyte webapp Ingress..." @helm upgrade airbyte airbyte/airbyte \ --namespace ${AIRBYTE_NAMESPACE} \ --reuse-values \ --set webapp.ingress.enabled=true # Setup MinIO storage for Airbyte # Note: This creates airbyte user/bucket in MinIO and stores credentials in Vault. # The credentials are then synced to Kubernetes via ExternalSecret. setup-minio-storage: #!/bin/bash set -euo pipefail echo "Setting up MinIO storage for Airbyte..." # Check if MinIO is available if ! kubectl get service minio -n minio &>/dev/null; then echo "Error: MinIO is not installed. Please install MinIO first with 'just minio::install'" exit 1 fi # Create MinIO user and bucket for Airbyte just minio::create-user airbyte "${AIRBYTE_STORAGE_BUCKET}" if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "Creating ExternalSecret for MinIO credentials..." # This ExternalSecret will merge MinIO access_key/secret_key into # the Helm-managed airbyte-airbyte-secrets secret gomplate -f airbyte-minio-external-secret.gomplate.yaml -o airbyte-minio-external-secret.yaml kubectl apply -f airbyte-minio-external-secret.yaml echo "Waiting for MinIO secret to be ready..." kubectl wait --for=condition=Ready externalsecret/airbyte-minio-external-secret \ -n ${AIRBYTE_NAMESPACE} --timeout=60s else echo "External Secrets not available. Creating Kubernetes Secret directly..." # Get credentials from Vault (stored by minio::create-user) ACCESS_KEY=airbyte SECRET_KEY=$(just vault::get airbyte/minio secret_key 2>/dev/null || echo "") if [ -z "$SECRET_KEY" ]; then echo "Error: Could not retrieve MinIO credentials. Please check Vault." exit 1 fi kubectl delete secret airbyte-minio-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found kubectl create secret generic airbyte-minio-secret -n ${AIRBYTE_NAMESPACE} \ --from-literal=access_key="$ACCESS_KEY" \ --from-literal=secret_key="$SECRET_KEY" \ --from-literal=bucket="${AIRBYTE_STORAGE_BUCKET}" \ --from-literal=endpoint="http://minio.minio.svc.cluster.local:9000" echo "MinIO secret created directly in Kubernetes" fi echo "MinIO storage setup completed" # Delete MinIO storage secret # Note: This removes both the standalone MinIO secret and the ExternalSecret that # merges MinIO credentials into airbyte-airbyte-secrets (Helm-managed secret). delete-minio-secret: @kubectl delete secret airbyte-minio-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found @kubectl delete externalsecret airbyte-minio-external-secret -n ${AIRBYTE_NAMESPACE} \ --ignore-not-found # Setup local storage for Airbyte setup-local-storage: #!/bin/bash set -euo pipefail echo "Setting up local storage for Airbyte..." # Detect if Longhorn is available export LONGHORN_AVAILABLE="false" if kubectl get storageclass longhorn &>/dev/null && \ kubectl get pods -n longhorn &>/dev/null | grep -q longhorn-manager; then echo "Longhorn detected - using ReadWriteMany with longhorn storage class" export LONGHORN_AVAILABLE="true" else echo "Longhorn not detected - using ReadWriteOnce with default storage class" export LONGHORN_AVAILABLE="false" fi # Create PVC for local storage if it doesn't exist if ! kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} &>/dev/null; then echo "Creating PersistentVolumeClaim for Airbyte storage..." gomplate -f airbyte-storage-pvc.gomplate.yaml -o airbyte-storage-pvc.yaml kubectl apply -f airbyte-storage-pvc.yaml echo "Waiting for PVC to be bound..." # Wait for PVC to be bound (check status.phase instead of conditions) for i in {1..90}; do STATUS=$(kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") if [ "$STATUS" = "Bound" ]; then echo "PVC successfully bound" break elif [ $i -eq 90 ]; then echo "Timeout waiting for PVC to be bound after 3 minutes" kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} exit 1 fi sleep 2 done ACCESS_MODE=$( kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} \ -o jsonpath='{.spec.accessModes[0]}' ) STORAGE_CLASS=$( kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} \ -o jsonpath='{.spec.storageClassName}' ) echo "PVC created with access mode: $ACCESS_MODE, storage class: ${STORAGE_CLASS:-default}" else echo "PVC airbyte-storage-pvc already exists" fi echo "Local storage setup completed" # Delete local storage PVC delete-local-storage: @kubectl delete pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} --ignore-not-found # Install Airbyte (full setup) install: #!/bin/bash set -euo pipefail export AIRBYTE_HOST=${AIRBYTE_HOST:-} while [ -z "${AIRBYTE_HOST}" ]; do AIRBYTE_HOST=$( gum input --prompt="Airbyte host (FQDN): " --width=100 \ --placeholder="e.g., airbyte.example.com" ) done # Ask for storage type if not set if [ -z "${AIRBYTE_STORAGE_TYPE:-}" ]; then AIRBYTE_STORAGE_TYPE=$(gum choose --header="Select storage type:" "local" "minio") fi echo "Selected storage type: ${AIRBYTE_STORAGE_TYPE}" echo "Installing Airbyte..." just create-namespace just setup-database just add-helm-repo # Setup storage based on type if [ "${AIRBYTE_STORAGE_TYPE}" = "minio" ]; then just setup-minio-storage else just setup-local-storage fi # Generate values file from template gomplate -f airbyte-values.gomplate.yaml -o airbyte-values.yaml # Install Airbyte using Helm (without --wait to avoid ConfigError blocking) helm upgrade --install airbyte airbyte/airbyte \ --namespace ${AIRBYTE_NAMESPACE} \ --version ${AIRBYTE_CHART_VERSION} \ -f airbyte-values.yaml \ --timeout=15m # Post-install: Re-sync ExternalSecrets for idempotency # Problem: Helm's pre-install hook recreates airbyte-airbyte-secrets, # causing ExternalSecret to lose sync state and MinIO credentials to be missing. # Solution: Force ExternalSecret re-creation after Helm install. echo "DEBUG: AIRBYTE_STORAGE_TYPE=${AIRBYTE_STORAGE_TYPE}" if [ "${AIRBYTE_STORAGE_TYPE}" = "minio" ]; then echo "Re-syncing MinIO ExternalSecret after Helm installation..." kubectl delete externalsecret airbyte-minio-external-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found sleep 2 gomplate -f airbyte-minio-external-secret.gomplate.yaml -o airbyte-minio-external-secret.yaml kubectl apply -f airbyte-minio-external-secret.yaml echo "Waiting for MinIO ExternalSecret to sync..." kubectl wait --for=condition=Ready externalsecret/airbyte-minio-external-secret \ -n ${AIRBYTE_NAMESPACE} --timeout=60s echo "MinIO credentials synchronized to airbyte-airbyte-secrets" # Restart pods that failed due to ConfigError echo "Restarting pods to pick up MinIO credentials..." kubectl delete pod -n ${AIRBYTE_NAMESPACE} -l app.kubernetes.io/name=server --ignore-not-found kubectl delete pod -n ${AIRBYTE_NAMESPACE} -l app.kubernetes.io/name=worker --ignore-not-found kubectl delete pod -n ${AIRBYTE_NAMESPACE} -l app.kubernetes.io/name=workload-launcher --ignore-not-found fi # Wait for all deployments to be ready after secret synchronization echo "Waiting for all Airbyte deployments to be ready..." kubectl wait --for=condition=available deployment --all -n ${AIRBYTE_NAMESPACE} --timeout=10m echo "Airbyte installation completed" echo "" # Prompt for OAuth2 Proxy setup if gum confirm "Setup OAuth2 Proxy for Keycloak authentication?"; then export AIRBYTE_HOST="${AIRBYTE_HOST}" just setup-oauth2-proxy else echo "Access Airbyte at: https://${AIRBYTE_HOST}" echo "Post-installation notes:" echo " • Default credentials: airbyte / password" echo " • Run 'just setup-oauth2-proxy' later to enable Keycloak authentication" echo " • Set up connectors from the UI" fi # Uninstall Airbyte (complete removal) uninstall delete-db='true': #!/bin/bash set -euo pipefail echo "Uninstalling Airbyte..." # Remove OAuth2 Proxy if it exists if kubectl get deployment oauth2-proxy-airbyte -n ${AIRBYTE_NAMESPACE} &>/dev/null; then echo "Removing associated OAuth2 Proxy..." just remove-oauth2-proxy fi helm uninstall airbyte -n ${AIRBYTE_NAMESPACE} --ignore-not-found just delete-database-secret just delete-minio-secret just delete-local-storage just delete-namespace if [ "{{ delete-db }}" = "true" ]; then just postgres::delete-db airbyte_db || true just postgres::delete-db airbyte_configs || true just postgres::delete-db airbyte_jobs || true just postgres::delete-db temporal || true just postgres::delete-db temporal_visibility || true just postgres::delete-user airbyte || true fi echo "Airbyte uninstalled" # Clean up database and secrets cleanup: #!/bin/bash set -euo pipefail echo "This will delete the Airbyte databases and all secrets." if gum confirm "Are you sure you want to proceed?"; then echo "Cleaning up Airbyte resources..." just postgres::delete-db airbyte_db || true just postgres::delete-db airbyte_configs || true just postgres::delete-db airbyte_jobs || true just postgres::delete-db temporal || true just postgres::delete-db temporal_visibility || true just postgres::delete-user airbyte || true just vault::delete airbyte/database || true just vault::delete airbyte/minio || true just vault::delete oauth2-proxy/airbyte || true echo "Cleanup completed" else echo "Cleanup cancelled" fi