diff --git a/airbyte/.gitignore b/airbyte/.gitignore new file mode 100644 index 0000000..26b5db1 --- /dev/null +++ b/airbyte/.gitignore @@ -0,0 +1,4 @@ +airbyte-values.yaml +airbyte-database-external-secret.yaml +airbyte-minio-external-secret.yaml +airbyte-storage-pvc.yaml diff --git a/airbyte/airbyte-database-external-secret.gomplate.yaml b/airbyte/airbyte-database-external-secret.gomplate.yaml new file mode 100644 index 0000000..28a7e0e --- /dev/null +++ b/airbyte/airbyte-database-external-secret.gomplate.yaml @@ -0,0 +1,28 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: airbyte-database-external-secret + namespace: {{ .Env.AIRBYTE_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: airbyte-database-secret + creationPolicy: Owner + template: + type: Opaque + data: + username: "{{ `{{ .username }}` }}" + password: "{{ `{{ .password }}` }}" + DATABASE_PASSWORD: "{{ `{{ .password }}` }}" + data: + - secretKey: username + remoteRef: + key: airbyte/database + property: username + - secretKey: password + remoteRef: + key: airbyte/database + property: password \ No newline at end of file diff --git a/airbyte/airbyte-minio-external-secret.gomplate.yaml b/airbyte/airbyte-minio-external-secret.gomplate.yaml new file mode 100644 index 0000000..d62803d --- /dev/null +++ b/airbyte/airbyte-minio-external-secret.gomplate.yaml @@ -0,0 +1,38 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: airbyte-minio-external-secret + namespace: {{ .Env.AIRBYTE_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + # Target: airbyte-airbyte-secrets is managed by Helm's pre-install hook + # We use creationPolicy: Merge to add MinIO credentials to the existing secret + # Note: This may need re-sync after Helm reinstalls due to timing issues + name: airbyte-airbyte-secrets + creationPolicy: Merge + template: + type: Opaque + data: + access_key: "{{ `{{ .access_key }}` }}" + secret_key: "{{ `{{ .secret_key }}` }}" + data: + - secretKey: access_key + remoteRef: + key: airbyte/minio + property: access_key + - secretKey: secret_key + remoteRef: + key: airbyte/minio + property: secret_key + - secretKey: bucket + remoteRef: + key: airbyte/minio + property: bucket + - secretKey: endpoint + remoteRef: + key: airbyte/minio + property: endpoint \ No newline at end of file diff --git a/airbyte/airbyte-oauth-external-secret.gomplate.yaml b/airbyte/airbyte-oauth-external-secret.gomplate.yaml new file mode 100644 index 0000000..10e089c --- /dev/null +++ b/airbyte/airbyte-oauth-external-secret.gomplate.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: airbyte-oauth-external-secret + namespace: {{ .Env.AIRBYTE_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: airbyte-oauth-secret + creationPolicy: Owner + template: + type: Opaque + data: + client_id: "{{ `{{ .client_id }}` }}" + data: + - secretKey: client_id + remoteRef: + key: airbyte/oauth + property: client_id \ No newline at end of file diff --git a/airbyte/airbyte-storage-pvc.gomplate.yaml b/airbyte/airbyte-storage-pvc.gomplate.yaml new file mode 100644 index 0000000..fd01327 --- /dev/null +++ b/airbyte/airbyte-storage-pvc.gomplate.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: airbyte-storage-pvc + namespace: {{ .Env.AIRBYTE_NAMESPACE }} +spec: + accessModes: + {{- if .Env.LONGHORN_AVAILABLE }} + - ReadWriteMany + {{- else }} + - ReadWriteOnce + {{- end }} + resources: + requests: + storage: 10Gi + {{- if .Env.LONGHORN_AVAILABLE }} + storageClassName: longhorn + {{- end }} \ No newline at end of file diff --git a/airbyte/airbyte-values.gomplate.yaml b/airbyte/airbyte-values.gomplate.yaml new file mode 100644 index 0000000..47fe444 --- /dev/null +++ b/airbyte/airbyte-values.gomplate.yaml @@ -0,0 +1,221 @@ +# Airbyte Helm Chart Values +# Configuration for Airbyte deployment + +global: + # Temporal database configuration + database: + type: external + secretName: airbyte-database-secret + + # Storage configuration + storage: + {{- if eq (.Env.AIRBYTE_STORAGE_TYPE | default "minio") "minio" }} + type: s3 + s3: + bucket: {{ .Env.AIRBYTE_STORAGE_BUCKET }} + region: us-east-1 + authenticationType: credentials + accessKeyIdSecretKey: access_key + secretAccessKeySecretKey: secret_key + credentialsSecretName: airbyte-minio-secret + endpoint: http://minio.minio.svc.cluster.local:9000 + pathStyleAccess: true + {{- else }} + type: local + local: + root: /airbyte + {{- end }} + + # Database configuration + database: + type: postgres + host: postgres-cluster-rw.postgres.svc.cluster.local + port: 5432 + database: airbyte_db + user: airbyte + secretName: airbyte-database-secret + + # Deployment mode + deploymentMode: oss + + # Security context + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + +# Webapp configuration +webapp: + enabled: true + replicaCount: 1 + + image: + # https://hub.docker.com/r/airbyte/webapp/tags + repository: airbyte/webapp + tag: "{{ .Env.AIRBYTE_WEBAPP_IMAGE_TAG }}" + + service: + type: ClusterIP + port: 80 + + ingress: + enabled: true + className: traefik + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + hosts: + - host: {{ .Env.AIRBYTE_HOST }} + paths: + - path: / + pathType: Prefix + tls: + - hosts: + - {{ .Env.AIRBYTE_HOST }} + + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + +# Server configuration +server: + enabled: true + replicaCount: 1 + + service: + type: ClusterIP + port: 8001 + + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + + # JVM settings + env: + - name: JAVA_OPTS + value: "-Xmx1g -Xms512m" + +# Worker configuration +worker: + enabled: true + replicaCount: 2 + + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + + # JVM settings + env: + - name: JAVA_OPTS + value: "-Xmx1g -Xms512m" + +# Scheduler configuration +scheduler: + enabled: true + replicaCount: 1 + + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + +# Connector builder server +connectorBuilderServer: + enabled: true + replicaCount: 1 + + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "1Gi" + cpu: "1000m" + +# Airbyte bootloader (database initialization) +airbyte-bootloader: + enabled: true + + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "1000m" + +# Temporal configuration with external PostgreSQL +# Note: These environment variables may cause Helm warnings about duplicates, +# but they are necessary for external PostgreSQL configuration +temporal: + enabled: true + replicaCount: 1 + + extraEnv: + - name: DB + value: "postgres12" + - name: POSTGRES_SEEDS + value: "postgres-cluster-rw.postgres.svc.cluster.local" + - name: DATABASE_DB + value: "temporal" + - name: POSTGRES_USER + value: "airbyte" + - name: POSTGRES_PWD + valueFrom: + secretKeyRef: + name: airbyte-database-secret + key: DATABASE_PASSWORD + - name: SKIP_DB_CREATE + value: "true" + + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" + +# Metrics configuration +metrics: + enabled: true + + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + +# Pod labels +podLabels: + app: airbyte + +# Pod annotations +podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + +# PostgreSQL (disabled - using external) +postgresql: + enabled: false + +# MinIO (disabled - using external if configured) +minio: + enabled: false diff --git a/airbyte/justfile b/airbyte/justfile new file mode 100644 index 0000000..d8564f6 --- /dev/null +++ b/airbyte/justfile @@ -0,0 +1,376 @@ +set fallback := true + +export AIRBYTE_NAMESPACE := env("AIRBYTE_NAMESPACE", "airbyte") +export AIRBYTE_CHART_VERSION := env("AIRBYTE_CHART_VERSION", "1.8.2") +export AIRBYTE_WEBAPP_IMAGE_TAG := env("AIRBYTE_WEBAPP_IMAGE_TAG", "1.7.4") +export AIRBYTE_HOST := env("AIRBYTE_HOST", "") +export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") +export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") +export AIRBYTE_STORAGE_TYPE := env("AIRBYTE_STORAGE_TYPE", "") +export AIRBYTE_STORAGE_BUCKET := env("AIRBYTE_STORAGE_BUCKET", "airbyte-storage") + +[private] +default: + @just --list --unsorted --list-submodules + +# Add Helm repository +add-helm-repo: + helm repo add airbyte https://airbytehq.github.io/helm-charts + helm repo update + +# Remove Helm repository +remove-helm-repo: + helm repo remove airbyte + +# Create Airbyte namespace +create-namespace: + @kubectl get namespace ${AIRBYTE_NAMESPACE} &>/dev/null || \ + kubectl create namespace ${AIRBYTE_NAMESPACE} + +# Delete Airbyte namespace +delete-namespace: + @kubectl delete namespace ${AIRBYTE_NAMESPACE} --ignore-not-found + +# Setup database for Airbyte +setup-database: + #!/bin/bash + set -euo pipefail + echo "Setting up Airbyte databases..." + + # Airbyte requires multiple databases + DATABASES=("airbyte_db" "airbyte_configs" "airbyte_jobs" "temporal" "temporal_visibility") + + for DB_NAME in "${DATABASES[@]}"; do + if just postgres::db-exists "$DB_NAME" &>/dev/null; then + echo "Database '$DB_NAME' already exists." + else + echo "Creating new database '$DB_NAME'..." + just postgres::create-db "$DB_NAME" + fi + done + + # Generate password for user creation/update + if just postgres::user-exists airbyte &>/dev/null; then + echo "User 'airbyte' already exists." + # Check if we can get existing password from Vault/Secret + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + # Try to get existing password from Vault + if DB_PASSWORD=$(just vault::get airbyte/database password 2>/dev/null); then + echo "Using existing password from Vault." + else + echo "Generating new password and updating Vault..." + DB_PASSWORD=$(just utils::random-password) + just postgres::change-password airbyte "$DB_PASSWORD" + fi + else + # For direct Secret approach, generate new password + echo "Generating new password for existing user..." + DB_PASSWORD=$(just utils::random-password) + just postgres::change-password airbyte "$DB_PASSWORD" + fi + else + echo "Creating new user 'airbyte'..." + DB_PASSWORD=$(just utils::random-password) + just postgres::create-user airbyte "$DB_PASSWORD" + fi + + echo "Ensuring database permissions..." + for DB_NAME in "${DATABASES[@]}"; do + just postgres::grant "$DB_NAME" airbyte + done + + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "External Secrets available. Storing credentials in Vault and creating ExternalSecret..." + just vault::put airbyte/database username=airbyte password="$DB_PASSWORD" + gomplate -f airbyte-database-external-secret.gomplate.yaml -o airbyte-database-external-secret.yaml + kubectl apply -f airbyte-database-external-secret.yaml + echo "Waiting for database secret to be ready..." + kubectl wait --for=condition=Ready externalsecret/airbyte-database-external-secret \ + -n ${AIRBYTE_NAMESPACE} --timeout=60s + else + echo "External Secrets not available. Creating Kubernetes Secret directly..." + kubectl delete secret airbyte-database-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found + kubectl create secret generic airbyte-database-secret -n ${AIRBYTE_NAMESPACE} \ + --from-literal=username=airbyte \ + --from-literal=password="$DB_PASSWORD" + echo "Database secret created directly in Kubernetes" + fi + echo "Database setup completed." + +# Delete database secret +delete-database-secret: + @kubectl delete secret airbyte-database-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found + @kubectl delete externalsecret airbyte-database-external-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found + +# Setup OAuth2 Proxy for Airbyte + +# Note: Airbyte OSS doesn't support direct OIDC integration, so we use OAuth2 Proxy +setup-oauth2-proxy: + #!/bin/bash + set -euo pipefail + export AIRBYTE_HOST=${AIRBYTE_HOST:-} + while [ -z "${AIRBYTE_HOST}" ]; do + AIRBYTE_HOST=$( + gum input --prompt="Airbyte host (FQDN): " --width=100 \ + --placeholder="e.g., airbyte.example.com" + ) + done + echo "Setting up OAuth2 Proxy for Airbyte..." + just oauth2-proxy::setup-for-app airbyte "${AIRBYTE_HOST}" "${AIRBYTE_NAMESPACE}" "airbyte-airbyte-webapp-svc:80" + echo "Disabling Airbyte webapp Ingress to prevent authentication bypass..." + helm upgrade airbyte airbyte/airbyte \ + --namespace ${AIRBYTE_NAMESPACE} \ + --reuse-values \ + --set webapp.ingress.enabled=false + echo "OAuth2 Proxy setup completed" + echo "Access Airbyte at: https://${AIRBYTE_HOST}" + +# Remove OAuth2 Proxy from Airbyte +remove-oauth2-proxy: + @echo "Removing OAuth2 Proxy for Airbyte..." + @just oauth2-proxy::remove-for-app airbyte "${AIRBYTE_NAMESPACE}" + @echo "Re-enabling Airbyte webapp Ingress..." + @helm upgrade airbyte airbyte/airbyte \ + --namespace ${AIRBYTE_NAMESPACE} \ + --reuse-values \ + --set webapp.ingress.enabled=true + +# Setup MinIO storage for Airbyte +# Note: This creates airbyte user/bucket in MinIO and stores credentials in Vault. + +# The credentials are then synced to Kubernetes via ExternalSecret. +setup-minio-storage: + #!/bin/bash + set -euo pipefail + echo "Setting up MinIO storage for Airbyte..." + + # Check if MinIO is available + if ! kubectl get service minio -n minio &>/dev/null; then + echo "Error: MinIO is not installed. Please install MinIO first with 'just minio::install'" + exit 1 + fi + + # Create MinIO user and bucket for Airbyte + just minio::create-user airbyte "${AIRBYTE_STORAGE_BUCKET}" + + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "Creating ExternalSecret for MinIO credentials..." + # This ExternalSecret will merge MinIO access_key/secret_key into + # the Helm-managed airbyte-airbyte-secrets secret + gomplate -f airbyte-minio-external-secret.gomplate.yaml -o airbyte-minio-external-secret.yaml + kubectl apply -f airbyte-minio-external-secret.yaml + echo "Waiting for MinIO secret to be ready..." + kubectl wait --for=condition=Ready externalsecret/airbyte-minio-external-secret \ + -n ${AIRBYTE_NAMESPACE} --timeout=60s + else + echo "External Secrets not available. Creating Kubernetes Secret directly..." + # Get credentials from Vault (stored by minio::create-user) + ACCESS_KEY=airbyte + SECRET_KEY=$(just vault::get airbyte/minio secret_key 2>/dev/null || echo "") + if [ -z "$SECRET_KEY" ]; then + echo "Error: Could not retrieve MinIO credentials. Please check Vault." + exit 1 + fi + kubectl delete secret airbyte-minio-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found + kubectl create secret generic airbyte-minio-secret -n ${AIRBYTE_NAMESPACE} \ + --from-literal=access_key="$ACCESS_KEY" \ + --from-literal=secret_key="$SECRET_KEY" \ + --from-literal=bucket="${AIRBYTE_STORAGE_BUCKET}" \ + --from-literal=endpoint="http://minio.minio.svc.cluster.local:9000" + echo "MinIO secret created directly in Kubernetes" + fi + echo "MinIO storage setup completed" + +# Delete MinIO storage secret +# Note: This removes both the standalone MinIO secret and the ExternalSecret that + +# merges MinIO credentials into airbyte-airbyte-secrets (Helm-managed secret). +delete-minio-secret: + @kubectl delete secret airbyte-minio-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found + @kubectl delete externalsecret airbyte-minio-external-secret -n ${AIRBYTE_NAMESPACE} \ + --ignore-not-found + +# Setup local storage for Airbyte +setup-local-storage: + #!/bin/bash + set -euo pipefail + echo "Setting up local storage for Airbyte..." + + # Detect if Longhorn is available + export LONGHORN_AVAILABLE="false" + + if kubectl get storageclass longhorn &>/dev/null && \ + kubectl get pods -n longhorn &>/dev/null | grep -q longhorn-manager; then + echo "Longhorn detected - using ReadWriteMany with longhorn storage class" + export LONGHORN_AVAILABLE="true" + else + echo "Longhorn not detected - using ReadWriteOnce with default storage class" + export LONGHORN_AVAILABLE="false" + fi + + # Create PVC for local storage if it doesn't exist + if ! kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} &>/dev/null; then + echo "Creating PersistentVolumeClaim for Airbyte storage..." + gomplate -f airbyte-storage-pvc.gomplate.yaml -o airbyte-storage-pvc.yaml + kubectl apply -f airbyte-storage-pvc.yaml + echo "Waiting for PVC to be bound..." + # Wait for PVC to be bound (check status.phase instead of conditions) + for i in {1..90}; do + STATUS=$(kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound") + if [ "$STATUS" = "Bound" ]; then + echo "PVC successfully bound" + break + elif [ $i -eq 90 ]; then + echo "Timeout waiting for PVC to be bound after 3 minutes" + kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} + exit 1 + fi + sleep 2 + done + ACCESS_MODE=$( + kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} \ + -o jsonpath='{.spec.accessModes[0]}' + ) + STORAGE_CLASS=$( + kubectl get pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} \ + -o jsonpath='{.spec.storageClassName}' + ) + echo "PVC created with access mode: $ACCESS_MODE, storage class: ${STORAGE_CLASS:-default}" + else + echo "PVC airbyte-storage-pvc already exists" + fi + echo "Local storage setup completed" + +# Delete local storage PVC +delete-local-storage: + @kubectl delete pvc airbyte-storage-pvc -n ${AIRBYTE_NAMESPACE} --ignore-not-found + +# Install Airbyte (full setup) +install: + #!/bin/bash + set -euo pipefail + export AIRBYTE_HOST=${AIRBYTE_HOST:-} + while [ -z "${AIRBYTE_HOST}" ]; do + AIRBYTE_HOST=$( + gum input --prompt="Airbyte host (FQDN): " --width=100 \ + --placeholder="e.g., airbyte.example.com" + ) + done + + # Ask for storage type if not set + if [ -z "${AIRBYTE_STORAGE_TYPE:-}" ]; then + AIRBYTE_STORAGE_TYPE=$(gum choose --header="Select storage type:" "local" "minio") + fi + echo "Selected storage type: ${AIRBYTE_STORAGE_TYPE}" + + echo "Installing Airbyte..." + just create-namespace + just setup-database + just add-helm-repo + + # Setup storage based on type + if [ "${AIRBYTE_STORAGE_TYPE}" = "minio" ]; then + just setup-minio-storage + else + just setup-local-storage + fi + + # Generate values file from template + gomplate -f airbyte-values.gomplate.yaml -o airbyte-values.yaml + + # Install Airbyte using Helm (without --wait to avoid ConfigError blocking) + helm upgrade --install airbyte airbyte/airbyte \ + --namespace ${AIRBYTE_NAMESPACE} \ + --version ${AIRBYTE_CHART_VERSION} \ + -f airbyte-values.yaml \ + --timeout=15m + + # Post-install: Re-sync ExternalSecrets for idempotency + # Problem: Helm's pre-install hook recreates airbyte-airbyte-secrets, + # causing ExternalSecret to lose sync state and MinIO credentials to be missing. + # Solution: Force ExternalSecret re-creation after Helm install. + echo "DEBUG: AIRBYTE_STORAGE_TYPE=${AIRBYTE_STORAGE_TYPE}" + if [ "${AIRBYTE_STORAGE_TYPE}" = "minio" ]; then + echo "Re-syncing MinIO ExternalSecret after Helm installation..." + kubectl delete externalsecret airbyte-minio-external-secret -n ${AIRBYTE_NAMESPACE} --ignore-not-found + sleep 2 + gomplate -f airbyte-minio-external-secret.gomplate.yaml -o airbyte-minio-external-secret.yaml + kubectl apply -f airbyte-minio-external-secret.yaml + echo "Waiting for MinIO ExternalSecret to sync..." + kubectl wait --for=condition=Ready externalsecret/airbyte-minio-external-secret \ + -n ${AIRBYTE_NAMESPACE} --timeout=60s + echo "MinIO credentials synchronized to airbyte-airbyte-secrets" + + # Restart pods that failed due to ConfigError + echo "Restarting pods to pick up MinIO credentials..." + kubectl delete pod -n ${AIRBYTE_NAMESPACE} -l app.kubernetes.io/name=server --ignore-not-found + kubectl delete pod -n ${AIRBYTE_NAMESPACE} -l app.kubernetes.io/name=worker --ignore-not-found + kubectl delete pod -n ${AIRBYTE_NAMESPACE} -l app.kubernetes.io/name=workload-launcher --ignore-not-found + fi + + # Wait for all deployments to be ready after secret synchronization + echo "Waiting for all Airbyte deployments to be ready..." + kubectl wait --for=condition=available deployment --all -n ${AIRBYTE_NAMESPACE} --timeout=10m + + echo "Airbyte installation completed" + echo "" + + # Prompt for OAuth2 Proxy setup + if gum confirm "Setup OAuth2 Proxy for Keycloak authentication?"; then + export AIRBYTE_HOST="${AIRBYTE_HOST}" + just setup-oauth2-proxy + else + echo "Access Airbyte at: https://${AIRBYTE_HOST}" + echo "Post-installation notes:" + echo " • Default credentials: airbyte / password" + echo " • Run 'just setup-oauth2-proxy' later to enable Keycloak authentication" + echo " • Set up connectors from the UI" + fi + +# Uninstall Airbyte (complete removal) +uninstall delete-db='true': + #!/bin/bash + set -euo pipefail + echo "Uninstalling Airbyte..." + + # Remove OAuth2 Proxy if it exists + if kubectl get deployment oauth2-proxy-airbyte -n ${AIRBYTE_NAMESPACE} &>/dev/null; then + echo "Removing associated OAuth2 Proxy..." + just remove-oauth2-proxy + fi + + helm uninstall airbyte -n ${AIRBYTE_NAMESPACE} --ignore-not-found + just delete-database-secret + just delete-minio-secret + just delete-local-storage + just delete-namespace + if [ "{{ delete-db }}" = "true" ]; then + just postgres::delete-db airbyte_db || true + just postgres::delete-db airbyte_configs || true + just postgres::delete-db airbyte_jobs || true + just postgres::delete-db temporal || true + just postgres::delete-db temporal_visibility || true + just postgres::delete-user airbyte || true + fi + echo "Airbyte uninstalled" + +# Clean up database and secrets +cleanup: + #!/bin/bash + set -euo pipefail + echo "This will delete the Airbyte databases and all secrets." + if gum confirm "Are you sure you want to proceed?"; then + echo "Cleaning up Airbyte resources..." + just postgres::delete-db airbyte_db || true + just postgres::delete-db airbyte_configs || true + just postgres::delete-db airbyte_jobs || true + just postgres::delete-db temporal || true + just postgres::delete-db temporal_visibility || true + just postgres::delete-user airbyte || true + just vault::delete airbyte/database || true + just vault::delete airbyte/minio || true + just vault::delete oauth2-proxy/airbyte || true + echo "Cleanup completed" + else + echo "Cleanup cancelled" + fi diff --git a/justfile b/justfile index b9845e0..61f69ef 100644 --- a/justfile +++ b/justfile @@ -6,6 +6,7 @@ export PATH := "./node_modules/.bin:" + env_var('PATH') default: @just --list --unsorted --list-submodules +mod airbyte mod airflow mod ch-ui mod clickhouse