baschno/buun-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(jupyterhub): unlimited max TTL for admin vault token

Browse Source
This commit is contained in:
Masaki Yatsu
2025-09-08 15:52:20 +09:00
parent 2bf82c7f38
commit c82c6aa22b
9 changed files with 367 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
jupyterhub/jupyterhub-admin-policy.hcl Normal file
View File

@@ -0,0 +1,50 @@
# JupyterHub Minimal Admin Policy
# Provides only necessary permissions for JupyterHub operations
# Read Keycloak credentials for OIDC authentication
path "secret/data/keycloak/admin" {
capabilities = ["read"]
}
# Full access to user secrets namespace for notebook users
path "secret/data/jupyter/*" {
capabilities = ["create", "read", "update", "delete", "list", "patch"]
}
# List secrets for user management
path "secret/metadata/jupyter/*" {
capabilities = ["list"]
}
# Token creation and management for user-specific tokens
path "auth/token/create" {
capabilities = ["create", "update"]
}
# Create orphan tokens (requires sudo for policy override)
path "auth/token/create-orphan" {
capabilities = ["create", "update", "sudo"]
}
path "auth/token/lookup-self" {
capabilities = ["read"]
}
path "auth/token/renew-self" {
capabilities = ["update"]
}
# Create user-specific policies dynamically
path "sys/policies/acl/jupyter-user-*" {
capabilities = ["create", "read", "update", "delete"]
}
# Read user policies to allow token creation with these policies
path "sys/policies/acl/*" {
capabilities = ["read", "list"]
}
# System capabilities check
path "sys/capabilities-self" {
capabilities = ["read"]
}

40
jupyterhub/jupyterhub-values.gomplate.yaml
View File

@@ -5,11 +5,8 @@ hub:
NOTEBOOK_VAULT_TOKEN_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_TTL | quote }}
NOTEBOOK_VAULT_TOKEN_MAX_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_MAX_TTL | quote }}
{{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
# Vault Agent will provide token via file
# Vault Agent provides renewable token via file (unlimited max TTL)
VAULT_TOKEN_FILE: "/vault/secrets/vault-token"
{{- else }}
# Traditional token via environment variable
JUPYTERHUB_VAULT_TOKEN: {{ .Env.JUPYTERHUB_VAULT_TOKEN | quote }}
{{- end }}
# Install packages at container startup
@@ -63,24 +60,24 @@ hub:
# Set environment variables for spawned containers
import hvac
{{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
def get_vault_token():
"""Read Vault token from file written by Vault Agent"""
"""Read Vault token from file"""
import os
token_file = os.environ.get('VAULT_TOKEN_FILE', '/vault/secrets/vault-token')
token_file = '/vault/secrets/vault-token'
try:
with open(token_file, 'r') as f:
token = f.read().strip()
if token:
return token
else:
raise Exception(f"Empty token file: {token_file}")
except FileNotFoundError:
# Fallback to environment variable for backward compatibility
return os.environ.get("JUPYTERHUB_VAULT_TOKEN")
print(f"Token file not found: {token_file}")
except Exception as e:
# Log error but attempt fallback
print(f"Error reading token file {token_file}: {e}")
return os.environ.get("JUPYTERHUB_VAULT_TOKEN")
return None
{{- end }}
async def pre_spawn_hook(spawner):
"""Set essential environment variables for spawned containers"""
@@ -94,6 +91,7 @@ hub:
# Logging configuration
spawner.environment["BUUNSTACK_LOG_LEVEL"] = "{{ .Env.JUPYTER_BUUNSTACK_LOG_LEVEL }}"
{{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
# Create user-specific Vault token directly
try:
username = spawner.user.name
@@ -105,7 +103,7 @@ hub:
spawner.log.info(f"pre_spawn_hook starting for {username}")
spawner.log.info(f"Vault address: {vault_addr}")
spawner.log.info(f"Vault token source: {'file' if os.path.exists(os.environ.get('VAULT_TOKEN_FILE', '/vault/secrets/vault-token')) else 'env'}")
spawner.log.info(f"Vault token source: {'file' if os.path.exists('/vault/secrets/vault-token') else 'env'}")
spawner.log.info(f"Vault token present: {bool(vault_token)}, length: {len(vault_token) if vault_token else 0}")
if not vault_token:
@@ -141,7 +139,7 @@ hub:
user_token_ttl = os.environ.get("NOTEBOOK_VAULT_TOKEN_TTL", "24h")
user_token_max_ttl = os.environ.get("NOTEBOOK_VAULT_TOKEN_MAX_TTL", "168h")
token_response = vault_client.auth.token.create(
token_response = vault_client.auth.token.create_orphan(
policies=[user_policy_name],
ttl=user_token_ttl,
renewable=True,
@@ -161,6 +159,7 @@ hub:
spawner.log.error("Failed to create user-specific Vault token for {}: {}".format(spawner.user.name, e))
import traceback
spawner.log.error("Full traceback: {}".format(traceback.format_exc()))
{{- end }}
c.KubeSpawner.pre_spawn_hook = pre_spawn_hook
@@ -172,12 +171,18 @@ hub:
- name: vault-config
configMap:
name: vault-agent-config
- name: vault-admin-token
secret:
secretName: jupyterhub-vault-token
extraVolumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
- name: vault-config
mountPath: /vault/config
- name: vault-admin-token
mountPath: /vault/admin-token
readOnly: true
extraContainers:
- name: vault-agent
@@ -195,8 +200,8 @@ hub:
- /bin/sh
- -c
- |
# Start Vault Agent
vault agent -config=/vault/config/agent.hcl
# Start token renewal script (handles both retrieval and renewal)
exec sh /vault/config/vault-token-renewer.sh
env:
- name: VAULT_ADDR
value: {{ .Env.VAULT_ADDR | quote }}
@@ -205,6 +210,9 @@ hub:
mountPath: /vault/secrets
- name: vault-config
mountPath: /vault/config
- name: vault-admin-token
mountPath: /vault/admin-token
readOnly: true
resources:
requests:
cpu: 50m

18
jupyterhub/jupyterhub-vault-token-external-secret.gomplate.yaml Normal file
View File

@@ -0,0 +1,18 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: jupyterhub-vault-token
namespace: {{ .Env.JUPYTERHUB_NAMESPACE }}
spec:
refreshInterval: 1h
secretStoreRef:
name: vault-secret-store
kind: ClusterSecretStore
target:
name: jupyterhub-vault-token
creationPolicy: Owner
data:
- secretKey: token
remoteRef:
key: jupyterhub/vault-token
property: token

119
jupyterhub/justfile
View File

@@ -20,7 +20,6 @@ export JUPYTER_PROFILE_TENSORFLOW_ENABLED := env("JUPYTER_PROFILE_TENSORFLOW_ENA
export JUPYTER_PROFILE_BUUN_STACK_ENABLED := env("JUPYTER_PROFILE_BUUN_STACK_ENABLED", "false")
export JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED := env("JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED", "false")
export JUPYTERHUB_VAULT_TOKEN_TTL := env("JUPYTERHUB_VAULT_TOKEN_TTL", "24h")
export JUPYTERHUB_VAULT_TOKEN_MAX_TTL := env("JUPYTERHUB_VAULT_TOKEN_MAX_TTL", "720h")
export NOTEBOOK_VAULT_TOKEN_TTL := env("NOTEBOOK_VAULT_TOKEN_TTL", "24h")
export NOTEBOOK_VAULT_TOKEN_MAX_TTL := env("NOTEBOOK_VAULT_TOKEN_MAX_TTL", "168h")
export VAULT_AGENT_LOG_LEVEL := env("VAULT_AGENT_LOG_LEVEL", "info")
@@ -54,7 +53,7 @@ delete-namespace:
kubectl delete namespace ${JUPYTERHUB_NAMESPACE} --ignore-not-found
# Install JupyterHub
install:
install root_token='':
#!/bin/bash
set -euo pipefail
export JUPYTERHUB_HOST=${JUPYTERHUB_HOST:-}
@@ -129,8 +128,16 @@ install:
if [ "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" = "true" ]; then
echo "Setting up Vault Agent for automatic token management..."
echo " Token TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
echo " Token Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
just setup-vault-integration
export VAULT_TOKEN="{{ root_token }}"
while [ -z "${VAULT_TOKEN}" ]; do
VAULT_TOKEN=$(gum input --prompt="Vault root token: " --password --width=100)
done
just setup-vault-integration ${VAULT_TOKEN}
just create-jupyterhub-vault-token ${VAULT_TOKEN}
# Create ExternalSecret for admin vault token
echo "Creating ExternalSecret for admin vault token..."
gomplate -f jupyterhub-vault-token-external-secret.gomplate.yaml | kubectl apply -f -
# Read user policy template for Vault
export USER_POLICY_HCL=$(cat user_policy.hcl)
@@ -155,6 +162,7 @@ uninstall:
helm uninstall jupyterhub -n ${JUPYTERHUB_NAMESPACE} --wait --ignore-not-found
kubectl delete pods -n ${JUPYTERHUB_NAMESPACE} -l app.kubernetes.io/component=singleuser-server
kubectl delete -n ${JUPYTERHUB_NAMESPACE} pvc jupyter-nfs-pvc --ignore-not-found
kubectl delete -n ${JUPYTERHUB_NAMESPACE} externalsecret jupyterhub-vault-token --ignore-not-found
if kubectl get pv jupyter-nfs-pv &>/dev/null; then
kubectl patch pv jupyter-nfs-pv -p '{"spec":{"claimRef":null}}'
fi
@@ -213,39 +221,43 @@ push-kernel-images:
setup-vault-integration root_token='':
#!/bin/bash
set -euo pipefail
echo "Setting up Vault integration for JupyterHub..."
# Create Kubernetes role for JupyterHub in Vault
echo "Creating Kubernetes authentication role for JupyterHub..."
echo " Service Account: hub"
echo " Namespace: jupyter"
echo " Policies: admin"
echo " TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
echo " Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
export VAULT_TOKEN="{{ root_token }}"
while [ -z "${VAULT_TOKEN}" ]; do
VAULT_TOKEN=$(gum input --prompt="Vault root token: " --password --width=100)
done
vault write auth/kubernetes/role/jupyterhub \
echo "Setting up Vault integration for JupyterHub..."
# Create JupyterHub-specific policy and Kubernetes role in Vault
echo "Creating JupyterHub-specific Vault policy and Kubernetes role..."
echo " Service Account: hub"
echo " Namespace: jupyter"
echo " Policy: jupyterhub-admin (custom policy with extended max TTL)"
echo " TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
# Create JupyterHub-specific policy
echo "Creating jupyterhub-admin policy..."
vault policy write jupyterhub-admin jupyterhub-admin-policy.hcl
# Create Kubernetes role (use system-safe max_ttl to avoid warnings)
echo "Creating Kubernetes role..."
vault write auth/kubernetes/role/jupyterhub-admin \
bound_service_account_names=hub \
bound_service_account_namespaces=jupyter \
policies=admin \
policies=jupyterhub-admin \
ttl=${JUPYTERHUB_VAULT_TOKEN_TTL} \
max_ttl=${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}
max_ttl=720h
# Create Vault Agent configuration with gomplate
echo "Creating Vault Agent configuration..."
gomplate -f vault-agent-config.gomplate.hcl -o vault-agent-config.hcl
# Create ConfigMap with token renewal script
echo "Creating ConfigMap with token renewal script..."
kubectl create configmap vault-agent-config -n ${JUPYTERHUB_NAMESPACE} \
--from-file=agent.hcl=vault-agent-config.hcl \
--from-file=token-monitor.tpl=token-monitor.tpl \
--from-file=vault-token-renewer.sh=vault-token-renewer.sh \
--dry-run=client -o yaml | kubectl apply -f -
echo "✓ Vault integration configured (user-specific tokens + auto-renewal)"
echo ""
echo "Configuration Summary:"
echo " JupyterHub Token TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
echo " JupyterHub Token Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
echo " User Token TTL: ${NOTEBOOK_VAULT_TOKEN_TTL}"
echo " User Token Max TTL: ${NOTEBOOK_VAULT_TOKEN_MAX_TTL}"
echo " Vault Agent Log Level: ${VAULT_AGENT_LOG_LEVEL}"
@@ -257,19 +269,60 @@ setup-vault-integration root_token='':
echo " # Each user gets their own isolated Vault token and policy"
echo " # Admin token is automatically renewed by Vault Agent"
# Create JupyterHub Vault token (uses admin policy for JWT operations)
create-jupyterhub-vault-token:
# Create JupyterHub Vault token (renewable with unlimited Max TTL)
create-jupyterhub-vault-token root_token='':
#!/bin/bash
set -euo pipefail
echo "Creating JupyterHub Vault token with admin policy..."
echo " TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
echo " Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
export VAULT_TOKEN="{{ root_token }}"
while [ -z "${VAULT_TOKEN}" ]; do
VAULT_TOKEN=$(gum input --prompt="Vault root token: " --password --width=100)
done
# JupyterHub needs admin privileges to read Keycloak credentials from Vault
# Create token and store in Vault
just vault::create-token-and-store admin jupyterhub/vault-token ${JUPYTERHUB_VAULT_TOKEN_TTL} ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}
echo "Creating JupyterHub admin Vault token"
echo "✓ JupyterHub Vault token created and stored"
# jupyterhub-admin policy should exist (created by setup-vault-integration)
# Check if token already exists
if vault kv get secret/jupyterhub/vault-token >/dev/null 2>&1; then
echo "Existing admin token found at secret/jupyterhub/vault-token"
if gum confirm "Replace existing token with new one?"; then
echo "Creating new admin token..."
else
echo "Using existing token"
return 0
fi
fi
# Create admin vault token with unlimited max TTL
echo ""
echo "To use in JupyterHub deployment:"
echo " JUPYTERHUB_VAULT_TOKEN=\$(just vault::get jupyterhub/vault-token token)"
echo "Creating admin token (TTL: 24h, Max TTL: unlimited)..."
TOKEN_RESPONSE=$(vault token create \
-policy=jupyterhub-admin \
-ttl=24h \
-explicit-max-ttl=0 \
-display-name="jupyterhub-admin" \
-renewable=true \
-format=json)
# Extract token
ADMIN_TOKEN=$(echo "$TOKEN_RESPONSE" | jq -r .auth.client_token)
if [ -z "$ADMIN_TOKEN" ] || [ "$ADMIN_TOKEN" = "null" ]; then
echo "❌ Failed to create admin token"
exit 1
fi
# Store token in Vault for JupyterHub to retrieve
echo "Storing admin token in Vault..."
vault kv put secret/jupyterhub/vault-token token="$ADMIN_TOKEN"
echo ""
echo "✅ Admin token created and stored successfully!"
echo ""
echo "Token behavior:"
echo " - TTL: 24 hours (will expire in 24h without renewal)"
echo " - Max TTL: Unlimited (can be renewed forever)"
echo " - Vault Agent will renew every 12 hours"
echo " - No more 30-day limitation!"
echo ""
echo "Token stored at: secret/jupyterhub/vault-token"

73
jupyterhub/monitor-vault-token.sh
View File

@@ -1,73 +0,0 @@
#!/bin/bash
# JupyterHub Vault Token Monitor Script
# Usage: ./monitor-vault-token.sh [pod-name]
set -euo pipefail
NAMESPACE="jupyter"
POD_NAME=${1:-$(kubectl get pods -n ${NAMESPACE} -l app.kubernetes.io/component=hub -o jsonpath='{.items[0].metadata.name}')}
echo "🔍 Monitoring Vault Agent for JupyterHub Pod: ${POD_NAME}"
echo "=================================================="
# Check if pod exists and is running
if ! kubectl get pod ${POD_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then
echo "❌ Pod ${POD_NAME} not found in namespace ${NAMESPACE}"
exit 1
fi
echo "📊 Pod Status:"
kubectl get pod ${POD_NAME} -n ${NAMESPACE}
echo ""
echo "📄 Vault Secrets Directory:"
kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- ls -la /vault/secrets/ 2>/dev/null || echo "❌ Cannot access /vault/secrets/"
echo ""
echo "🔐 Current Token Info:"
kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c '
if [ -f /vault/secrets/vault-token ]; then
echo "Token file exists ($(wc -c < /vault/secrets/vault-token) bytes)"
echo "Last modified: $(stat -c %y /vault/secrets/vault-token 2>/dev/null || stat -f %Sm /vault/secrets/vault-token)"
# Test token validity
if command -v curl >/dev/null 2>&1; then
echo ""
echo "Token validation:"
RESPONSE=$(curl -s -w "%{http_code}" -H "X-Vault-Token: $(cat /vault/secrets/vault-token)" $VAULT_ADDR/v1/auth/token/lookup-self)
HTTP_CODE="${RESPONSE: -3}"
if [ "$HTTP_CODE" = "200" ]; then
echo "✅ Token is valid"
echo "$RESPONSE" | head -c -3 | grep -E "(ttl|expire_time|renewable)" | head -3
else
echo "❌ Token validation failed (HTTP $HTTP_CODE)"
fi
fi
else
echo "❌ Token file not found"
fi
' 2>/dev/null || echo "❌ Cannot check token info"
echo ""
echo "📋 Recent Vault Agent Logs:"
kubectl logs -n ${NAMESPACE} ${POD_NAME} -c vault-agent --tail=10 2>/dev/null || echo "❌ Cannot access vault-agent logs"
echo ""
echo "📋 Token Renewal Log (if exists):"
kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c '
if [ -f /vault/secrets/renewal.log ]; then
echo "Recent renewal events:"
tail -10 /vault/secrets/renewal.log
else
echo "No renewal log file found yet"
fi
' 2>/dev/null || echo "❌ Cannot check renewal logs"
echo ""
echo "🔄 To monitor token renewals in real-time, run:"
echo " kubectl logs -n ${NAMESPACE} ${POD_NAME} -c vault-agent -f | grep 'renewed auth token'"
echo ""
echo "🔍 To check token info periodically, run:"
echo " watch -n 30 \"kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c 'curl -s -H \\\"X-Vault-Token: \\\$(cat /vault/secrets/vault-token)\\\" \\\$VAULT_ADDR/v1/auth/token/lookup-self | grep -E \\\"(ttl|expire_time)\\\"'\""

11
jupyterhub/token-monitor.tpl
View File

@@ -1,11 +0,0 @@
{{- with secret "auth/token/lookup-self" -}}
=== Vault Token Status ===
TTL: {{ .Data.ttl }} seconds
Renewable: {{ .Data.renewable }}
Expire Time: {{ .Data.expire_time }}
Policies: {{ range .Data.policies }}{{ . }} {{ end }}
Display Name: {{ .Data.display_name }}
Entity ID: {{ .Data.entity_id }}
Token Type: {{ .Data.type }}
===========================
{{- end -}}

38
jupyterhub/vault-agent-config.gomplate.hcl
View File

@@ -1,38 +0,0 @@
vault {
address = "{{ .Env.VAULT_ADDR }}"
}
# Enable detailed logging
log_level = "{{ .Env.VAULT_AGENT_LOG_LEVEL }}"
log_format = "standard"
auto_auth {
method "kubernetes" {
mount_path = "auth/kubernetes"
config = {
role = "jupyterhub"
}
}
sink "file" {
config = {
path = "/vault/secrets/vault-token"
}
}
}
cache {
use_auto_auth_token = true
}
listener "tcp" {
address = "127.0.0.1:8100"
tls_disable = true
}
# Add template for token monitoring
template {
source = "/vault/config/token-monitor.tpl"
destination = "/vault/secrets/token-info.log"
perms = 0644
}

47
jupyterhub/vault-token-renewer.sh Normal file
View File

@@ -0,0 +1,47 @@
#!/bin/sh
# Script to handle admin token retrieval and renewal
set -e
echo "Starting Vault token management..."
export VAULT_ADDR="${VAULT_ADDR}"
# Wait for ExternalSecret to create the secret
echo "Waiting for admin token from ExternalSecret..."
while [ ! -f /vault/admin-token/token ]; do
echo "Waiting for /vault/admin-token/token..."
sleep 5
done
# Read admin token from mounted secret
ADMIN_TOKEN=$(cat /vault/admin-token/token)
if [ -z "$ADMIN_TOKEN" ]; then
echo "ERROR: No admin token found in mounted secret"
exit 1
fi
echo "Admin token retrieved from ExternalSecret"
echo "$ADMIN_TOKEN" > /vault/secrets/vault-token
# Start token renewal loop
export VAULT_TOKEN="$ADMIN_TOKEN"
while true; do
echo "$(date): Renewing admin token..."
if vault token renew >/dev/null 2>&1; then
echo "$(date): Token renewed successfully"
else
echo "$(date): Token renewal failed - trying to retrieve token again from ExternalSecret"
# Re-read token from mounted secret
ADMIN_TOKEN=$(cat /vault/admin-token/token 2>/dev/null || echo "")
if [ -n "$ADMIN_TOKEN" ]; then
echo "$ADMIN_TOKEN" > /vault/secrets/vault-token
export VAULT_TOKEN="$ADMIN_TOKEN"
echo "$(date): Token re-retrieved successfully from ExternalSecret"
else
echo "$(date): Failed to re-retrieve token from ExternalSecret"
fi
fi
sleep 43200 # 12 hours
done