diff --git a/lakekeeper/README.md b/lakekeeper/README.md index 0f2882e..1449b69 100644 --- a/lakekeeper/README.md +++ b/lakekeeper/README.md @@ -23,15 +23,89 @@ The installation automatically: - Creates PostgreSQL database and user - Stores credentials in Vault (if External Secrets is available) or Kubernetes Secrets -- Creates Keycloak OIDC client with PKCE flow for secure authentication +- Creates Keycloak OIDC client with PKCE flow for Web UI authentication +- Creates API client (`lakekeeper-api`) for programmatic access with OAuth2 Client Credentials Flow - Configures audience mapper for JWT tokens - Runs database migrations - Configures Traefik ingress with TLS +**IMPORTANT**: During installation, API client credentials will be displayed. Save these for programmatic access (dlt, PyIceberg, etc.). + ## Access Access Lakekeeper at `https://lakekeeper.yourdomain.com` and authenticate via Keycloak. +## Programmatic Access + +### API Client Credentials + +During installation, a default API client `lakekeeper-api` is automatically created for programmatic access (dlt, Python scripts, etc.). + +**IMPORTANT**: The client ID and secret are displayed during installation. Save these credentials securely. + +If you need additional API clients or lost the credentials: + +```bash +# Create additional API client with custom name +just lakekeeper::create-oidc-api-client my-app + +# Recreate default client (delete first, then create) +just lakekeeper::delete-oidc-api-client lakekeeper-api +just lakekeeper::create-oidc-api-client lakekeeper-api +``` + +Each API client has: + +- **Service account enabled** for OAuth2 Client Credentials Flow +- **`lakekeeper` scope** with audience mapper (`aud: lakekeeper`) +- **Client credentials** stored in Vault (if External Secrets is available) + +### Using API Clients + +#### dlt (Data Load Tool) + +Configure dlt to use the API client credentials: + +```bash +export OIDC_CLIENT_ID=lakekeeper-api +export OIDC_CLIENT_SECRET= +export ICEBERG_CATALOG_URL=http://lakekeeper.lakekeeper.svc.cluster.local:8181/catalog +export ICEBERG_WAREHOUSE=default +``` + +The dlt Iceberg REST destination automatically uses these credentials for OAuth2 authentication. + +#### PyIceberg + +```python +from pyiceberg.catalog import load_catalog + +catalog = load_catalog( + "rest_catalog", + **{ + "uri": "http://lakekeeper.lakekeeper.svc.cluster.local:8181/catalog", + "warehouse": "default", + "credential": f"{client_id}:{client_secret}", # OAuth2 format + } +) +``` + +#### Trino Integration + +Trino uses its own OIDC client with service account. This is automatically configured by `just trino::enable-iceberg-catalog`. You don't need to create a separate API client for Trino. + +### Deleting API Clients + +```bash +# Delete default API client +just lakekeeper::delete-oidc-api-client + +# Delete custom-named client +just lakekeeper::delete-oidc-api-client my-app +``` + +This removes the Keycloak client and Vault credentials. + ## Cleanup To remove all Lakekeeper resources and secrets from Vault: diff --git a/lakekeeper/justfile b/lakekeeper/justfile index 13ab458..8bef729 100644 --- a/lakekeeper/justfile +++ b/lakekeeper/justfile @@ -139,6 +139,76 @@ create-oidc-client: delete-oidc-client: @just keycloak::delete-client ${KEYCLOAK_REALM} lakekeeper +# Create OIDC API client for programmatic access (dlt, etc.) +create-oidc-api-client client_name='lakekeeper-api': + #!/bin/bash + set -euo pipefail + echo "Creating Lakekeeper OIDC API client '{{ client_name }}' in Keycloak..." + + # Ensure lakekeeper scope exists (should be created by create-oidc-client) + echo "Ensuring 'lakekeeper' client scope exists..." + just keycloak::create-client-scope ${KEYCLOAK_REALM} lakekeeper "Lakekeeper API scope" + just keycloak::add-audience-mapper-to-scope ${KEYCLOAK_REALM} lakekeeper lakekeeper + + # Check if client already exists + if just keycloak::client-exists ${KEYCLOAK_REALM} {{ client_name }} &>/dev/null; then + echo "Client '{{ client_name }}' already exists." + echo "To recreate, first delete it with: just lakekeeper::delete-oidc-api-client {{ client_name }}" + exit 1 + fi + + # Create confidential client with service account + echo "Creating confidential client with service account..." + CLIENT_SECRET=$(just utils::random-password) + just keycloak::create-client \ + realm=${KEYCLOAK_REALM} \ + client_id={{ client_name }} \ + redirect_url="http://localhost" \ + client_secret="$CLIENT_SECRET" + + # Enable service account for client credentials flow + echo "Enabling service account for client credentials flow..." + just keycloak::enable-service-account ${KEYCLOAK_REALM} {{ client_name }} + + # Add lakekeeper scope + echo "Adding 'lakekeeper' scope to client..." + just keycloak::add-scope-to-client ${KEYCLOAK_REALM} {{ client_name }} lakekeeper + + # Store credentials in Vault if available + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "Storing credentials in Vault..." + just vault::put lakekeeper/api-client/{{ client_name }} \ + client_id={{ client_name }} \ + client_secret="$CLIENT_SECRET" + else + echo "External Secrets not available. Credentials not stored in Vault." + fi + + echo "" + echo "OIDC API client '{{ client_name }}' created successfully" + echo "Client ID: {{ client_name }}" + echo "Client Secret: $CLIENT_SECRET" + echo "" + echo "Use these credentials for OAuth2 Client Credentials Flow:" + echo " OIDC_CLIENT_ID={{ client_name }}" + echo " OIDC_CLIENT_SECRET=$CLIENT_SECRET" + echo "" + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "Credentials stored in Vault at: lakekeeper/api-client/{{ client_name }}" + fi + +# Delete OIDC API client +delete-oidc-api-client client_name='lakekeeper-api': + #!/bin/bash + set -euo pipefail + echo "Deleting Lakekeeper OIDC API client '{{ client_name }}'..." + just keycloak::delete-client ${KEYCLOAK_REALM} {{ client_name }} + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "Deleting credentials from Vault..." + just vault::delete lakekeeper/api-client/{{ client_name }} || true + fi + echo "OIDC API client deleted" + # Install Lakekeeper install: #!/bin/bash @@ -154,6 +224,7 @@ install: just create-namespace just setup-database just create-oidc-client + just create-oidc-api-client just add-helm-repo # Helm chart will automatically create the encryption key secret