baschno/buun-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

keycloak(feat): add recipes for client management

Browse Source
This commit is contained in:
Masaki Yatsu
2025-09-18 14:08:47 +09:00
parent a12601e652
commit dc30a37a42
13 changed files with 918 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
airflow/README.md
View File

@@ -27,15 +27,14 @@ This document covers Airflow installation, deployment, and debugging in the buun
just airflow::install
```
3. **Access Airflow Web UI**:
- Navigate to your Airflow instance (e.g., `https://airflow.buun.dev`)
- Login with your Keycloak credentials
4. **Assign User Roles** (if needed):
3. **Assign User Roles** (required for DAG execution):
```bash
# Add user role for DAG execution permissions
just airflow::assign-role <username> airflow_user
just airflow::assign-role <username> airflow_op
# Check user's current roles
just airflow::list-user-roles <username>
# Available roles:
# - airflow_admin: Full administrative access
@@ -44,6 +43,12 @@ This document covers Airflow installation, deployment, and debugging in the buun
# - airflow_viewer: Viewer access (read-only)
```
**Note**: New users have only Viewer access by default and cannot execute DAGs without role assignment.
4. **Access Airflow Web UI**:
- Navigate to your Airflow instance (e.g., `https://airflow.buun.dev`)
- Login with your Keycloak credentials
### Uninstalling
```bash

61
airflow/justfile
View File

@@ -140,16 +140,23 @@ create-oauth-client:
exit 1
fi
echo "Creating Airflow OAuth client in Keycloak..."
# Delete existing client to ensure fresh creation
echo "Removing existing client if present..."
just keycloak::delete-client ${KEYCLOAK_REALM} airflow || true
CLIENT_SECRET=$(just utils::random-password)
just keycloak::create-client \
${KEYCLOAK_REALM} \
airflow \
"https://${AIRFLOW_HOST}/auth/oauth-authorized/keycloak" \
"$CLIENT_SECRET"
# Check if client already exists
if just keycloak::client-exists ${KEYCLOAK_REALM} airflow &>/dev/null; then
echo "Client 'airflow' already exists, skipping creation..."
echo "Existing client will preserve roles and mappers"
# Get existing client secret for Vault/Secret synchronization
CLIENT_SECRET=$(just keycloak::get-client-secret ${KEYCLOAK_REALM} airflow | grep "Client 'airflow' secret:" | cut -d' ' -f4)
echo "Retrieved existing client secret for synchronization"
else
echo "Creating new client..."
CLIENT_SECRET=$(just utils::random-password)
just keycloak::create-client \
${KEYCLOAK_REALM} \
airflow \
"https://${AIRFLOW_HOST}/auth/oauth-authorized/keycloak" \
"$CLIENT_SECRET"
fi
if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then
echo "External Secrets available. Storing credentials in Vault and creating ExternalSecret..."
@@ -188,6 +195,14 @@ create-keycloak-roles:
just keycloak::create-client-role ${KEYCLOAK_REALM} airflow "$role" || true
done
# Create client roles mapper to include roles in JWT tokens
echo "Creating client roles mapper..."
just keycloak::add-client-roles-mapper airflow "airflow_roles" "Airflow Roles Mapper"
# Add client roles mapper to profile scope for userinfo endpoint
echo "Adding client roles mapper to profile scope..."
just keycloak::add-client-roles-to-profile-scope ${KEYCLOAK_REALM} airflow "airflow_roles"
echo "Keycloak roles created successfully"
echo "Role mappings:"
echo " - airflow_admin -> Airflow Admin (full access)"
@@ -238,6 +253,20 @@ assign-role username='' role='':
;;
esac
# List user's Airflow roles
list-user-roles username='':
#!/bin/bash
set -euo pipefail
USERNAME="{{ username }}"
# Interactive input if not provided
while [ -z "${USERNAME}" ]; do
USERNAME=$(gum input --prompt="Username: " --width=100)
done
echo "Checking Airflow roles for user '${USERNAME}'..."
just keycloak::list-user-client-roles ${KEYCLOAK_REALM} "${USERNAME}" airflow
# Remove Airflow role from user
remove-role username='' role='':
#!/bin/bash
@@ -298,7 +327,7 @@ install:
fi
KEYCLOAK_HOST=${KEYCLOAK_HOST} KEYCLOAK_REALM=${KEYCLOAK_REALM} \
gomplate -f webserver_config.py.gomplate -o webserver_config.py
gomplate -f webserver_config.gomplate.py -o webserver_config.py
kubectl delete configmap airflow-api-server-config -n ${AIRFLOW_NAMESPACE} --ignore-not-found
kubectl create configmap airflow-api-server-config -n ${AIRFLOW_NAMESPACE} \
--from-file=webserver_config.py=webserver_config.py
@@ -494,15 +523,15 @@ logs-dag-errors dag_file='':
LOG_DATE=$(date +%Y-%m-%d)
LOG_DIR="/opt/airflow/logs/dag_processor/${LOG_DATE}/dags-folder"
if [ -n "{{dag_file}}" ]; then
if [ -n "{{ dag_file }}" ]; then
# Show specific DAG file errors
LOG_FILE="${LOG_DIR}/{{dag_file}}.log"
echo "📋 Showing errors for DAG file: {{dag_file}}"
LOG_FILE="${LOG_DIR}/{{ dag_file }}.log"
echo "📋 Showing errors for DAG file: {{ dag_file }}"
echo "📂 Log file: ${LOG_FILE}"
echo ""
kubectl exec -n ${AIRFLOW_NAMESPACE} ${DAG_PROCESSOR_POD} -c dag-processor -- \
cat "${LOG_FILE}" 2>/dev/null | jq -r 'select(.level == "error") | .timestamp + " " + .event + ": " + .error_detail[0].exc_value' || \
echo "❌ No error log found for {{dag_file}} or file doesn't exist"
echo "❌ No error log found for {{ dag_file }} or file doesn't exist"
else
# List all DAG files with errors
echo "📋 Available DAG error logs:"
@@ -566,10 +595,10 @@ logs-test-import dag_file:
echo "❌ DAG processor pod not found"
exit 1
fi
echo "🧪 Testing import of DAG file: {{dag_file}}"
echo "🧪 Testing import of DAG file: {{ dag_file }}"
echo ""
kubectl exec -n ${AIRFLOW_NAMESPACE} ${DAG_PROCESSOR_POD} -c dag-processor -- \
python /opt/airflow/dags/{{dag_file}}
python /opt/airflow/dags/{{ dag_file }}
# Clean up database and secrets
cleanup:

139
airflow/webserver_config.gomplate.py Normal file
View File

@@ -0,0 +1,139 @@
import os
import logging
from flask_appbuilder.security.manager import AUTH_OAUTH
from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
log = logging.getLogger(__name__)
AUTH_TYPE = AUTH_OAUTH
AUTH_USER_REGISTRATION = True
AUTH_ROLES_SYNC_AT_LOGIN = True
AUTH_USER_REGISTRATION_ROLE = "Viewer"
# Keycloak OIDC configuration
KEYCLOAK_HOST = "{{ .Env.KEYCLOAK_HOST }}"
KEYCLOAK_REALM = "{{ .Env.KEYCLOAK_REALM }}"
OIDC_ISSUER = f"https://{KEYCLOAK_HOST}/realms/{KEYCLOAK_REALM}"
# OAuth Providers configuration
OAUTH_PROVIDERS = [{
'name': 'keycloak',
'icon': 'fa-key',
'token_key': 'access_token',
'remote_app': {
'client_id': os.environ.get('AIRFLOW_OAUTH_CLIENT_ID', ''),
'client_secret': os.environ.get('AIRFLOW_OAUTH_CLIENT_SECRET', ''),
'server_metadata_url': f'{OIDC_ISSUER}/.well-known/openid-configuration',
'api_base_url': f'{OIDC_ISSUER}/protocol/openid-connect',
'access_token_url': f'{OIDC_ISSUER}/protocol/openid-connect/token',
'authorize_url': f'{OIDC_ISSUER}/protocol/openid-connect/auth',
'request_token_url': None,
'client_kwargs': {
'scope': 'openid profile email'
}
}
}]
# Role mappings from Keycloak to Airflow
AUTH_ROLES_MAPPING = {
"airflow_admin": ["Admin"],
"airflow_op": ["Op"],
"airflow_user": ["User"],
"airflow_viewer": ["Viewer"]
}
# Use the correct claim name for client roles
AUTH_ROLE_CLAIM = "airflow_roles"
# Security Manager Override
class KeycloakSecurityManager(FabAirflowSecurityManagerOverride):
"""Custom Security Manager for Keycloak integration"""
def __init__(self, appbuilder):
super().__init__(appbuilder)
def get_oauth_user_info(self, provider, response):
"""Extract user info and roles from Keycloak token"""
if provider == "keycloak":
import jwt
import base64
import json
# Get access token
token = response.get("access_token")
if not token:
log.error("No access token found in OAuth response")
return None
try:
# Decode token without verification for debugging
# In production, you should verify the signature
parts = token.split('.')
if len(parts) == 3:
# Decode payload
payload_b64 = parts[1]
# Add padding if needed
payload_b64 += '=' * (4 - len(payload_b64) % 4)
payload = json.loads(base64.b64decode(payload_b64))
log.info(f"Decoded token payload: {payload}")
# Extract user information
userinfo = {
"username": payload.get("preferred_username"),
"email": payload.get("email"),
"first_name": payload.get("given_name"),
"last_name": payload.get("family_name"),
}
# Extract roles from different possible locations
roles = []
# Check realm access roles
realm_access = payload.get("realm_access", {})
realm_roles = realm_access.get("roles", [])
# Check resource access (client roles)
resource_access = payload.get("resource_access", {})
client_access = resource_access.get("airflow", {})
client_roles = client_access.get("roles", [])
# Check airflow_roles claim directly
direct_roles = payload.get("airflow_roles", [])
log.info(f"Realm roles: {realm_roles}")
log.info(f"Client roles: {client_roles}")
log.info(f"Direct airflow roles: {direct_roles}")
# Prefer client roles, then direct roles, then realm roles
if client_roles:
roles = client_roles
log.info(f"Using client roles: {roles}")
elif direct_roles:
roles = direct_roles
log.info(f"Using direct airflow roles: {roles}")
elif realm_roles:
# Map common realm roles to Airflow roles
role_mapping = {
'admin': 'Admin',
'user': 'User',
'viewer': 'Viewer'
}
roles = [role_mapping.get(role.lower(), 'Viewer') for role in realm_roles]
log.info(f"Using mapped realm roles: {roles}")
else:
roles = ['Viewer']
log.info("No roles found, defaulting to Viewer")
userinfo["role_keys"] = roles
log.info(f"Final userinfo: {userinfo}")
return userinfo
except Exception as e:
log.error(f"Error decoding JWT token: {e}")
# Fallback to default behavior
return super().get_oauth_user_info(provider, response)
SECURITY_MANAGER_CLASS = KeycloakSecurityManager

52
airflow/webserver_config.py.gomplate
View File

@@ -1,52 +0,0 @@
import os
import logging
from flask_appbuilder.security.manager import AUTH_OAUTH
from airflow.providers.fab.auth_manager.security_manager.override import FabAirflowSecurityManagerOverride
log = logging.getLogger(__name__)
AUTH_TYPE = AUTH_OAUTH
AUTH_USER_REGISTRATION = True
AUTH_ROLES_SYNC_AT_LOGIN = True
AUTH_USER_REGISTRATION_ROLE = "Viewer"
# Keycloak OIDC configuration
KEYCLOAK_HOST = "{{ .Env.KEYCLOAK_HOST }}"
KEYCLOAK_REALM = "{{ .Env.KEYCLOAK_REALM }}"
OIDC_ISSUER = f"https://{KEYCLOAK_HOST}/realms/{KEYCLOAK_REALM}"
# OAuth Providers configuration
OAUTH_PROVIDERS = [{
'name': 'keycloak',
'icon': 'fa-key',
'token_key': 'access_token',
'remote_app': {
'client_id': os.environ.get('AIRFLOW_OAUTH_CLIENT_ID', ''),
'client_secret': os.environ.get('AIRFLOW_OAUTH_CLIENT_SECRET', ''),
'server_metadata_url': f'{OIDC_ISSUER}/.well-known/openid-configuration',
'api_base_url': f'{OIDC_ISSUER}/protocol/openid-connect',
'access_token_url': f'{OIDC_ISSUER}/protocol/openid-connect/token',
'authorize_url': f'{OIDC_ISSUER}/protocol/openid-connect/auth',
'request_token_url': None,
'client_kwargs': {
'scope': 'openid profile email'
}
}
}]
# Role mappings from Keycloak to Airflow
AUTH_ROLES_MAPPING = {
"airflow_admin": ["Admin"],
"airflow_op": ["Op"],
"airflow_user": ["User"],
"airflow_viewer": ["Viewer"]
}
# Security Manager Override
class KeycloakSecurityManager(FabAirflowSecurityManagerOverride):
"""Custom Security Manager for Keycloak integration"""
def __init__(self, appbuilder):
super().__init__(appbuilder)
SECURITY_MANAGER_CLASS = KeycloakSecurityManager