set fallback := true export LAKEKEEPER_NAMESPACE := env("LAKEKEEPER_NAMESPACE", "lakekeeper") export LAKEKEEPER_CHART_VERSION := env("LAKEKEEPER_CHART_VERSION", "0.7.1") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") [private] default: @just --list --unsorted --list-submodules # Add Helm repository add-helm-repo: helm repo add lakekeeper https://lakekeeper.github.io/lakekeeper-charts/ helm repo update # Remove Helm repository remove-helm-repo: helm repo remove lakekeeper # Create namespace create-namespace: @kubectl get namespace ${LAKEKEEPER_NAMESPACE} &>/dev/null || \ kubectl create namespace ${LAKEKEEPER_NAMESPACE} # Setup database for Lakekeeper setup-database: #!/bin/bash set -euo pipefail echo "Setting up Lakekeeper database..." if just postgres::db-exists lakekeeper &>/dev/null; then echo "Database 'lakekeeper' already exists." else echo "Creating new database 'lakekeeper'..." just postgres::create-db lakekeeper fi # Generate password for user creation/update if just postgres::user-exists lakekeeper &>/dev/null; then echo "User 'lakekeeper' already exists." # Check if we can get existing password from Vault/Secret if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then # Try to get existing password from Vault if DB_PASSWORD=$(just vault::get lakekeeper/database password 2>/dev/null); then echo "Using existing password from Vault." else echo "Generating new password and updating Vault..." DB_PASSWORD=$(just utils::random-password) just postgres::change-password lakekeeper "$DB_PASSWORD" fi else # For direct Secret approach, generate new password echo "Generating new password for existing user..." DB_PASSWORD=$(just utils::random-password) just postgres::change-password lakekeeper "$DB_PASSWORD" fi else echo "Creating new user 'lakekeeper'..." DB_PASSWORD=$(just utils::random-password) just postgres::create-user lakekeeper "$DB_PASSWORD" fi echo "Ensuring database permissions..." just postgres::grant lakekeeper lakekeeper if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "External Secrets available. Storing credentials in Vault and creating ExternalSecret..." just vault::put lakekeeper/database \ username=lakekeeper \ password="$DB_PASSWORD" \ host=postgres-cluster-rw.postgres \ port=5432 \ database=lakekeeper gomplate -f lakekeeper-database-external-secret.gomplate.yaml -o lakekeeper-database-external-secret.yaml kubectl apply -f lakekeeper-database-external-secret.yaml echo "Waiting for database secret to be ready..." kubectl wait --for=condition=Ready externalsecret/lakekeeper-database-external-secret \ -n ${LAKEKEEPER_NAMESPACE} --timeout=60s else echo "External Secrets not available. Creating Kubernetes Secret directly..." kubectl delete secret lakekeeper-database-secret -n ${LAKEKEEPER_NAMESPACE} --ignore-not-found kubectl create secret generic lakekeeper-database-secret -n ${LAKEKEEPER_NAMESPACE} \ --from-literal=username=lakekeeper \ --from-literal=password="$DB_PASSWORD" \ --from-literal=host=postgres-cluster-rw.postgres \ --from-literal=port=5432 \ --from-literal=database=lakekeeper echo "Database secret created directly in Kubernetes" fi echo "Database setup completed." # Delete database secret delete-database-secret: @kubectl delete secret lakekeeper-database-secret -n ${LAKEKEEPER_NAMESPACE} --ignore-not-found @kubectl delete externalsecret lakekeeper-database-external-secret -n ${LAKEKEEPER_NAMESPACE} --ignore-not-found # Create OIDC client in Keycloak for Lakekeeper authentication create-oidc-client: #!/bin/bash set -euo pipefail if [ -z "${LAKEKEEPER_HOST:-}" ]; then echo "Error: LAKEKEEPER_HOST environment variable is required" exit 1 fi echo "Creating Lakekeeper OAuth client in Keycloak..." # Ensure lakekeeper scope exists echo "Creating 'lakekeeper' client scope if it doesn't exist..." just keycloak::create-client-scope ${KEYCLOAK_REALM} lakekeeper "Lakekeeper API scope" # Add audience mapper to lakekeeper scope echo "Adding audience mapper to 'lakekeeper' client scope..." just keycloak::add-audience-mapper-to-scope ${KEYCLOAK_REALM} lakekeeper lakekeeper # Check if client already exists if just keycloak::client-exists ${KEYCLOAK_REALM} lakekeeper &>/dev/null; then echo "Client 'lakekeeper' already exists, skipping creation..." echo "Existing client will preserve roles and mappers" else echo "Creating new public client for PKCE flow..." # Create public client (no client secret) for PKCE flow just keycloak::create-client \ realm=${KEYCLOAK_REALM} \ client_id=lakekeeper \ redirect_url="https://${LAKEKEEPER_HOST}/ui/callback" \ post_logout_redirect_uris="https://${LAKEKEEPER_HOST}/ui/logout,https://${LAKEKEEPER_HOST}/ui/,https://${LAKEKEEPER_HOST}/" \ access_token_lifespan="43200" \ scopes="openid,profile,lakekeeper" fi # Ensure the lakekeeper scope is added to the client (for both existing and new clients) echo "Adding 'lakekeeper' scope to client..." just keycloak::add-scope-to-client ${KEYCLOAK_REALM} lakekeeper lakekeeper echo "OAuth client configured successfully for PKCE authentication" # Delete OIDC client (for cleanup purposes) delete-oidc-client: @just keycloak::delete-client ${KEYCLOAK_REALM} lakekeeper # Create OIDC API client for programmatic access (dlt, etc.) create-oidc-api-client client_name='lakekeeper-api': #!/bin/bash set -euo pipefail echo "Creating Lakekeeper OIDC API client '{{ client_name }}' in Keycloak..." # Ensure lakekeeper scope exists (should be created by create-oidc-client) echo "Ensuring 'lakekeeper' client scope exists..." just keycloak::create-client-scope ${KEYCLOAK_REALM} lakekeeper "Lakekeeper API scope" just keycloak::add-audience-mapper-to-scope ${KEYCLOAK_REALM} lakekeeper lakekeeper # Check if client already exists if just keycloak::client-exists ${KEYCLOAK_REALM} {{ client_name }} &>/dev/null; then echo "Client '{{ client_name }}' already exists." echo "To recreate, first delete it with: just lakekeeper::delete-oidc-api-client {{ client_name }}" exit 0 fi # Create confidential client with service account echo "Creating confidential client with service account..." CLIENT_SECRET=$(just utils::random-password) just keycloak::create-client \ realm=${KEYCLOAK_REALM} \ client_id={{ client_name }} \ redirect_url="http://localhost" \ client_secret="$CLIENT_SECRET" # Enable service account for client credentials flow echo "Enabling service account for client credentials flow..." just keycloak::enable-service-account ${KEYCLOAK_REALM} {{ client_name }} # Add lakekeeper scope echo "Adding 'lakekeeper' scope to client..." just keycloak::add-scope-to-client ${KEYCLOAK_REALM} {{ client_name }} lakekeeper # Store credentials in Vault if available if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "Storing credentials in Vault..." just vault::put lakekeeper/api-client/{{ client_name }} \ client_id={{ client_name }} \ client_secret="$CLIENT_SECRET" else echo "External Secrets not available. Credentials not stored in Vault." fi echo "" echo "OIDC API client '{{ client_name }}' created successfully" echo "Client ID: {{ client_name }}" echo "Client Secret: $CLIENT_SECRET" echo "" echo "Use these credentials for OAuth2 Client Credentials Flow:" echo " OIDC_CLIENT_ID={{ client_name }}" echo " OIDC_CLIENT_SECRET=$CLIENT_SECRET" echo "" if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "Credentials stored in Vault at: lakekeeper/api-client/{{ client_name }}" fi # Delete OIDC API client delete-oidc-api-client client_name='lakekeeper-api': #!/bin/bash set -euo pipefail echo "Deleting Lakekeeper OIDC API client '{{ client_name }}'..." just keycloak::delete-client ${KEYCLOAK_REALM} {{ client_name }} if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "Deleting credentials from Vault..." just vault::delete lakekeeper/api-client/{{ client_name }} || true fi echo "OIDC API client deleted" # Install Lakekeeper install: #!/bin/bash set -euo pipefail export LAKEKEEPER_HOST=${LAKEKEEPER_HOST:-} while [ -z "${LAKEKEEPER_HOST}" ]; do LAKEKEEPER_HOST=$( gum input --prompt="Lakekeeper host (FQDN): " --width=100 \ --placeholder="e.g., lakekeeper.example.com" ) done echo "Installing Lakekeeper..." just create-namespace kubectl label namespace ${LAKEKEEPER_NAMESPACE} \ pod-security.kubernetes.io/enforce=restricted --overwrite just setup-database just create-oidc-client just add-helm-repo # Helm chart will automatically create the encryption key secret gomplate -f lakekeeper-values.gomplate.yaml -o lakekeeper-values.yaml # Use --wait=false to avoid circular dependency: # Helm waits for post-install hooks (migration job) to complete, # but migration job can't start until Helm deployment finishes helm upgrade --install lakekeeper lakekeeper/lakekeeper \ --version ${LAKEKEEPER_CHART_VERSION} -n ${LAKEKEEPER_NAMESPACE} \ --timeout=10m --wait=false \ -f lakekeeper-values.yaml echo "Waiting for database migration to complete..." kubectl wait --for=condition=complete job/lakekeeper-db-migration-1 \ -n ${LAKEKEEPER_NAMESPACE} --timeout=300s echo "Waiting for Lakekeeper deployment to be ready..." kubectl wait --for=condition=available deployment/lakekeeper \ -n ${LAKEKEEPER_NAMESPACE} --timeout=300s echo "Lakekeeper installation completed" echo "Access Lakekeeper at: https://${LAKEKEEPER_HOST}" just create-oidc-api-client # Uninstall Lakekeeper uninstall delete-db='true': #!/bin/bash set -euo pipefail echo "Uninstalling Lakekeeper..." helm uninstall lakekeeper -n ${LAKEKEEPER_NAMESPACE} --ignore-not-found # Force delete stuck resources echo "Checking for stuck resources..." kubectl delete all --all -n ${LAKEKEEPER_NAMESPACE} --force --grace-period=0 2>/dev/null || true # Delete PVCs PVCS=$(kubectl get pvc -n ${LAKEKEEPER_NAMESPACE} -o name 2>/dev/null || true) if [ -n "$PVCS" ]; then echo "Deleting PersistentVolumeClaims..." kubectl delete pvc --all -n ${LAKEKEEPER_NAMESPACE} --force --grace-period=0 2>/dev/null || true fi just delete-database-secret just delete-oidc-client if [ "{{ delete-db }}" = "true" ]; then just postgres::delete-db lakekeeper fi # Clean up Keycloak client just keycloak::delete-client ${KEYCLOAK_REALM} lakekeeper || true echo "Lakekeeper uninstalled" # Clean up database and secrets cleanup: #!/bin/bash set -euo pipefail echo "This will delete the Lakekeeper database and all secrets." if gum confirm "Are you sure you want to proceed?"; then echo "Cleaning up Lakekeeper resources..." just postgres::delete-db lakekeeper || true just vault::delete lakekeeper/database || true just vault::delete lakekeeper/oauth || true just keycloak::delete-client ${KEYCLOAK_REALM} lakekeeper || true echo "Cleanup completed" else echo "Cleanup cancelled" fi # Create warehouse with vended credentials enabled (STS) create-warehouse warehouse_name='default' bucket='warehouse': #!/bin/bash set -euo pipefail echo "Creating warehouse '{{ warehouse_name }}' with vended credentials (STS) enabled..." # Get MinIO credentials MINIO_ACCESS_KEY=$(kubectl get secret -n minio minio -o jsonpath='{.data.rootUser}' | base64 -d) MINIO_SECRET_KEY=$(kubectl get secret -n minio minio -o jsonpath='{.data.rootPassword}' | base64 -d) # Create warehouse JSON configuration WAREHOUSE_CONFIG=$(cat </dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token echo "Authenticating with Keycloak..." TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Create warehouse echo "Creating warehouse..." RESPONSE=$(curl -s -w "\n%{http_code}" -X POST \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN" \ -H "Content-Type: application/json" \ -d "$WAREHOUSE_CONFIG") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Warehouse '{{ warehouse_name }}' created successfully with vended credentials enabled" echo "Response: $BODY" else echo "Error: Failed to create warehouse (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # Create Iceberg namespace in a warehouse create-warehouse-namespace warehouse_name namespace: #!/bin/bash set -euo pipefail echo "Creating namespace '{{ namespace }}' in warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token echo "Authenticating with Keycloak..." TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Get warehouse ID from warehouse name echo "Getting warehouse ID for '{{ warehouse_name }}'..." WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" echo "Available warehouses:" echo "$LIST_BODY" | jq -r '.warehouses[] | .name' 2>/dev/null || echo "Could not parse warehouse names" exit 1 fi echo "Warehouse ID: $WAREHOUSE_ID" else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" echo "Response: $LIST_BODY" exit 1 fi # Create namespace echo "Creating namespace..." NAMESPACE_CONFIG=$(cat </dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # List warehouses RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Warehouses:" echo "$BODY" | jq -r '.warehouses[] | " - \(.name) (ID: \(.id))"' else echo "Error: Failed to list warehouses (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # Get warehouse details by name get-warehouse warehouse_name: #!/bin/bash set -euo pipefail echo "Getting details for warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # List warehouses to find the ID LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" echo "Available warehouses:" echo "$LIST_BODY" | jq -r '.warehouses[] | " - \(.name)"' exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" echo "Response: $LIST_BODY" exit 1 fi # Get warehouse details RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse/${WAREHOUSE_ID}" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Warehouse '{{ warehouse_name }}' details:" echo "$BODY" | jq . echo "" echo "Storage Profile:" echo "$BODY" | jq '.["storage-profile"]' else echo "Error: Failed to get warehouse details (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # Delete namespace from a warehouse delete-warehouse-namespace warehouse_name namespace: #!/bin/bash set -euo pipefail echo "This will delete namespace '{{ namespace }}' from warehouse '{{ warehouse_name }}'." if ! gum confirm "Are you sure you want to proceed?"; then echo "Deletion cancelled" exit 0 fi echo "Deleting namespace '{{ namespace }}' from warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Get warehouse ID from warehouse name WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" echo "Available warehouses:" echo "$LIST_BODY" | jq -r '.warehouses[] | .name' 2>/dev/null || echo "Could not parse warehouse names" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" echo "Response: $LIST_BODY" exit 1 fi # Delete namespace with recursive flag to delete all tables RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}?recursive=true" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Namespace '{{ namespace }}' deleted successfully from warehouse '{{ warehouse_name }}'" elif [ "$HTTP_CODE" = "404" ]; then echo "Namespace '{{ namespace }}' not found in warehouse '{{ warehouse_name }}'" else echo "Error: Failed to delete namespace (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # List all namespaces in a warehouse list-warehouse-namespaces warehouse_name: #!/bin/bash set -euo pipefail echo "Listing namespaces in warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Get warehouse ID from warehouse name WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" echo "Available warehouses:" echo "$LIST_BODY" | jq -r '.warehouses[] | .name' 2>/dev/null || echo "Could not parse warehouse names" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" echo "Response: $LIST_BODY" exit 1 fi # List namespaces RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Namespaces in warehouse '{{ warehouse_name }}':" echo "$BODY" | jq -r '.namespaces[] | " - \(.[0])"' else echo "Error: Failed to list namespaces (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # List all tables in a namespace list-tables warehouse_name namespace: #!/bin/bash set -euo pipefail echo "Listing tables in namespace '{{ namespace }}' of warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Get warehouse ID from warehouse name WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" exit 1 fi # List tables RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/tables" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then TABLE_COUNT=$(echo "$BODY" | jq '.identifiers | length') echo "Tables in namespace '{{ namespace }}' ($TABLE_COUNT):" echo "$BODY" | jq -r '.identifiers[] | " - \(.namespace[0]).\(.name)"' else echo "Error: Failed to list tables (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # Delete a table from a namespace delete-table warehouse_name namespace table_name: #!/bin/bash set -euo pipefail echo "Deleting table '{{ table_name }}' from namespace '{{ namespace }}' in warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" exit 1 fi # Get warehouse ID WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" exit 1 fi # Delete table RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/tables/{{ table_name }}?purgeRequested=true" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Table '{{ table_name }}' deleted successfully" elif [ "$HTTP_CODE" = "404" ]; then echo "Table '{{ table_name }}' not found" else echo "Error: Failed to delete table (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # List all views in a namespace list-views warehouse_name namespace: #!/bin/bash set -euo pipefail echo "Listing views in namespace '{{ namespace }}' of warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Get warehouse ID from warehouse name WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" exit 1 fi # List views RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/views" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then VIEW_COUNT=$(echo "$BODY" | jq '.identifiers | length') echo "Views in namespace '{{ namespace }}' ($VIEW_COUNT):" echo "$BODY" | jq -r '.identifiers[] | " - \(.namespace[0]).\(.name)"' else echo "Error: Failed to list views (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # Delete a view from a namespace delete-view warehouse_name namespace view_name: #!/bin/bash set -euo pipefail echo "Deleting view '{{ view_name }}' from namespace '{{ namespace }}' in warehouse '{{ warehouse_name }}'..." # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" exit 1 fi # Get warehouse ID WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" exit 1 fi # Delete view RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/views/{{ view_name }}" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "View '{{ view_name }}' deleted successfully" elif [ "$HTTP_CODE" = "404" ]; then echo "View '{{ view_name }}' not found" else echo "Error: Failed to delete view (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi # Delete all views in a namespace (force cleanup) delete-all-views-in-namespace warehouse_name namespace: #!/bin/bash set -euo pipefail echo "Deleting all views in namespace '{{ namespace }}' of warehouse '{{ warehouse_name }}'..." # Get API client credentials CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" exit 1 fi # Get warehouse ID WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" exit 1 fi else echo "Error: Failed to list warehouses" exit 1 fi # List views VIEWS_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/views" \ -H "Authorization: Bearer $ACCESS_TOKEN") VIEWS_HTTP_CODE=$(echo "$VIEWS_RESPONSE" | tail -n1) VIEWS_BODY=$(echo "$VIEWS_RESPONSE" | sed '$d') if [ "$VIEWS_HTTP_CODE" -ge 200 ] && [ "$VIEWS_HTTP_CODE" -lt 300 ]; then VIEW_NAMES=$(echo "$VIEWS_BODY" | jq -r '.identifiers[] | .name') if [ -z "$VIEW_NAMES" ]; then echo "No views found in namespace '{{ namespace }}'" else VIEW_COUNT=$(echo "$VIEW_NAMES" | wc -l | tr -d ' ') echo "Found $VIEW_COUNT views to delete:" echo "$VIEW_NAMES" | while read -r view; do echo " - $view" done echo "$VIEW_NAMES" | while read -r view; do echo "Deleting view '$view'..." DEL_RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/views/${view}" \ -H "Authorization: Bearer $ACCESS_TOKEN") DEL_HTTP_CODE=$(echo "$DEL_RESPONSE" | tail -n1) if [ "$DEL_HTTP_CODE" -ge 200 ] && [ "$DEL_HTTP_CODE" -lt 300 ]; then echo " View '$view' deleted" else DEL_BODY=$(echo "$DEL_RESPONSE" | sed '$d') echo " Warning: Failed to delete view '$view' (HTTP $DEL_HTTP_CODE)" echo " Response: $DEL_BODY" fi done fi else echo "Error: Failed to list views (HTTP $VIEWS_HTTP_CODE)" echo "Response: $VIEWS_BODY" exit 1 fi # Delete all tables in a namespace (force cleanup) delete-all-tables-in-namespace warehouse_name namespace: #!/bin/bash set -euo pipefail echo "Deleting all tables in namespace '{{ namespace }}' of warehouse '{{ warehouse_name }}'..." # Get API client credentials CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" exit 1 fi # Get warehouse ID WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" exit 1 fi else echo "Error: Failed to list warehouses" exit 1 fi # List tables TABLES_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/tables" \ -H "Authorization: Bearer $ACCESS_TOKEN") TABLES_HTTP_CODE=$(echo "$TABLES_RESPONSE" | tail -n1) TABLES_BODY=$(echo "$TABLES_RESPONSE" | sed '$d') if [ "$TABLES_HTTP_CODE" -ge 200 ] && [ "$TABLES_HTTP_CODE" -lt 300 ]; then TABLE_NAMES=$(echo "$TABLES_BODY" | jq -r '.identifiers[] | .name') if [ -z "$TABLE_NAMES" ]; then echo "No tables found in namespace '{{ namespace }}'" else TABLE_COUNT=$(echo "$TABLE_NAMES" | wc -l | tr -d ' ') echo "Found $TABLE_COUNT tables to delete:" echo "$TABLE_NAMES" | while read -r table; do echo " - $table" done echo "$TABLE_NAMES" | while read -r table; do echo "Deleting table '$table'..." DEL_RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/{{ namespace }}/tables/${table}?purgeRequested=true" \ -H "Authorization: Bearer $ACCESS_TOKEN") DEL_HTTP_CODE=$(echo "$DEL_RESPONSE" | tail -n1) if [ "$DEL_HTTP_CODE" -ge 200 ] && [ "$DEL_HTTP_CODE" -lt 300 ]; then echo " Table '$table' deleted" else DEL_BODY=$(echo "$DEL_RESPONSE" | sed '$d') echo " Warning: Failed to delete table '$table' (HTTP $DEL_HTTP_CODE)" echo " Response: $DEL_BODY" fi done fi else echo "Error: Failed to list tables (HTTP $TABLES_HTTP_CODE)" echo "Response: $TABLES_BODY" exit 1 fi # Delete warehouse delete-warehouse warehouse_name force='false': #!/bin/bash set -euo pipefail # Get API client credentials for authentication CLIENT_SECRET=$(just vault::get lakekeeper/api-client/lakekeeper-api client_secret 2>/dev/null || echo "") if [ -z "$CLIENT_SECRET" ]; then echo "Error: Could not retrieve API client credentials" echo "Please ensure 'lakekeeper-api' client exists" exit 1 fi # Get OAuth2 token TOKEN_RESPONSE=$(curl -s -X POST "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}/protocol/openid-connect/token" \ -H "Content-Type: application/x-www-form-urlencoded" \ -d "grant_type=client_credentials" \ -d "client_id=lakekeeper-api" \ -d "client_secret=$CLIENT_SECRET" \ -d "scope=lakekeeper") ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r '.access_token') if [ "$ACCESS_TOKEN" = "null" ] || [ -z "$ACCESS_TOKEN" ]; then echo "Error: Failed to obtain access token" echo "Response: $TOKEN_RESPONSE" exit 1 fi # Get warehouse ID from warehouse name WAREHOUSE_LIST_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse" \ -H "Authorization: Bearer $ACCESS_TOKEN") LIST_HTTP_CODE=$(echo "$WAREHOUSE_LIST_RESPONSE" | tail -n1) LIST_BODY=$(echo "$WAREHOUSE_LIST_RESPONSE" | sed '$d') if [ "$LIST_HTTP_CODE" -ge 200 ] && [ "$LIST_HTTP_CODE" -lt 300 ]; then WAREHOUSE_ID=$(echo "$LIST_BODY" | jq -r '.warehouses[] | select(.name == "{{ warehouse_name }}") | .id') if [ -z "$WAREHOUSE_ID" ] || [ "$WAREHOUSE_ID" = "null" ]; then echo "Error: Warehouse '{{ warehouse_name }}' not found" echo "Available warehouses:" echo "$LIST_BODY" | jq -r '.warehouses[] | .name' 2>/dev/null || echo "Could not parse warehouse names" exit 1 fi else echo "Error: Failed to list warehouses (HTTP $LIST_HTTP_CODE)" echo "Response: $LIST_BODY" exit 1 fi # If force option is enabled, delete all namespaces first if [ "{{ force }}" = "true" ]; then echo "Force deletion enabled. Deleting all namespaces in warehouse '{{ warehouse_name }}'..." # List namespaces NAMESPACE_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces" \ -H "Authorization: Bearer $ACCESS_TOKEN") NS_HTTP_CODE=$(echo "$NAMESPACE_RESPONSE" | tail -n1) NS_BODY=$(echo "$NAMESPACE_RESPONSE" | sed '$d') if [ "$NS_HTTP_CODE" -ge 200 ] && [ "$NS_HTTP_CODE" -lt 300 ]; then # Extract namespace names and delete each one NAMESPACES=$(echo "$NS_BODY" | jq -r '.namespaces[] | .[0]') if [ -n "$NAMESPACES" ]; then echo "Found namespaces to delete:" echo "$NAMESPACES" | while read -r ns; do echo " - $ns" done echo "$NAMESPACES" | while read -r ns; do echo "Deleting namespace '$ns' (including all tables and views)..." # Delete all tables first TABLES_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/${ns}/tables" \ -H "Authorization: Bearer $ACCESS_TOKEN") TABLES_HTTP_CODE=$(echo "$TABLES_RESPONSE" | tail -n1) TABLES_BODY=$(echo "$TABLES_RESPONSE" | sed '$d') if [ "$TABLES_HTTP_CODE" -ge 200 ] && [ "$TABLES_HTTP_CODE" -lt 300 ]; then TABLE_NAMES=$(echo "$TABLES_BODY" | jq -r '.identifiers[] | .name' 2>/dev/null) if [ -n "$TABLE_NAMES" ]; then TABLE_COUNT=$(echo "$TABLE_NAMES" | wc -l | tr -d ' ') echo " Deleting $TABLE_COUNT tables..." echo "$TABLE_NAMES" | while read -r table; do curl -s -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/${ns}/tables/${table}?purgeRequested=true" \ -H "Authorization: Bearer $ACCESS_TOKEN" > /dev/null done fi fi # Delete all views VIEWS_RESPONSE=$(curl -s -w "\n%{http_code}" -X GET \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/${ns}/views" \ -H "Authorization: Bearer $ACCESS_TOKEN") VIEWS_HTTP_CODE=$(echo "$VIEWS_RESPONSE" | tail -n1) VIEWS_BODY=$(echo "$VIEWS_RESPONSE" | sed '$d') if [ "$VIEWS_HTTP_CODE" -ge 200 ] && [ "$VIEWS_HTTP_CODE" -lt 300 ]; then VIEW_NAMES=$(echo "$VIEWS_BODY" | jq -r '.identifiers[] | .name' 2>/dev/null) if [ -n "$VIEW_NAMES" ]; then VIEW_COUNT=$(echo "$VIEW_NAMES" | wc -l | tr -d ' ') echo " Deleting $VIEW_COUNT views..." echo "$VIEW_NAMES" | while read -r view; do curl -s -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/${ns}/views/${view}" \ -H "Authorization: Bearer $ACCESS_TOKEN" > /dev/null done fi fi # Delete namespace DEL_RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/catalog/v1/${WAREHOUSE_ID}/namespaces/${ns}" \ -H "Authorization: Bearer $ACCESS_TOKEN") DEL_HTTP_CODE=$(echo "$DEL_RESPONSE" | tail -n1) if [ "$DEL_HTTP_CODE" -ge 200 ] && [ "$DEL_HTTP_CODE" -lt 300 ]; then echo " Namespace '$ns' deleted" else DEL_BODY=$(echo "$DEL_RESPONSE" | sed '$d') echo " Warning: Failed to delete namespace '$ns' (HTTP $DEL_HTTP_CODE)" echo " Response: $DEL_BODY" fi done else echo "No namespaces found in warehouse '{{ warehouse_name }}'" fi fi fi echo "This will delete the warehouse '{{ warehouse_name }}' and all its data." if ! gum confirm "Are you sure you want to proceed?"; then echo "Deletion cancelled" exit 0 fi echo "Deleting warehouse '{{ warehouse_name }}'..." # Delete warehouse RESPONSE=$(curl -s -w "\n%{http_code}" -X DELETE \ "http://lakekeeper.${LAKEKEEPER_NAMESPACE}.svc.cluster.local:8181/management/v1/warehouse/${WAREHOUSE_ID}" \ -H "Authorization: Bearer $ACCESS_TOKEN") HTTP_CODE=$(echo "$RESPONSE" | tail -n1) BODY=$(echo "$RESPONSE" | sed '$d') if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then echo "Warehouse '{{ warehouse_name }}' deleted successfully" elif [ "$HTTP_CODE" = "409" ]; then echo "Error: Warehouse is not empty (HTTP 409)" echo "Response: $BODY" echo "" echo "The warehouse still contains namespaces or data." echo "To delete all namespaces automatically, use:" echo " just lakekeeper::delete-warehouse {{ warehouse_name }} true" exit 1 else echo "Error: Failed to delete warehouse (HTTP $HTTP_CODE)" echo "Response: $BODY" exit 1 fi