From 2bf82c7f3805fed69bcbed7a7e31cee3e625e8b7 Mon Sep 17 00:00:00 2001 From: Masaki Yatsu Date: Mon, 8 Sep 2025 14:06:35 +0900 Subject: [PATCH] feat(jupyterhub): admin vault token renewal --- docs/jupyterhub.md | 98 ++++++++++++++++++++-- jupyterhub/.gitignore | 1 + jupyterhub/jupyterhub-values.gomplate.yaml | 84 ++++++++++++++++++- jupyterhub/justfile | 95 ++++++++++++++------- jupyterhub/monitor-vault-token.sh | 73 ++++++++++++++++ jupyterhub/token-monitor.tpl | 11 +++ jupyterhub/vault-agent-config.gomplate.hcl | 38 +++++++++ 7 files changed, 360 insertions(+), 40 deletions(-) create mode 100755 jupyterhub/monitor-vault-token.sh create mode 100644 jupyterhub/token-monitor.tpl create mode 100644 jupyterhub/vault-agent-config.gomplate.hcl diff --git a/docs/jupyterhub.md b/docs/jupyterhub.md index 0f33abd..c3f9416 100644 --- a/docs/jupyterhub.md +++ b/docs/jupyterhub.md @@ -150,7 +150,7 @@ export JUPYTERHUB_VAULT_INTEGRATION_ENABLED=true just jupyterhub::install ``` -**Note**: The `just jupyterhub::setup-vault-jwt-auth` command is called automatically during installation if Vault integration is enabled. This command currently serves as a placeholder for future JWT-based authentication enhancements. +**Note**: The `just jupyterhub::setup-vault-integration` command is called automatically during installation if Vault integration is enabled. This configures Vault Agent for automatic token renewal and user-specific token management. ### Usage in Notebooks @@ -183,7 +183,8 @@ secrets.delete('api-keys', field='github') # Delete only github field ### Security Features - **User isolation**: Each user receives a unique Vault token with access only to their own secrets -- **Automatic token renewal**: Tokens can be renewed to extend session lifetime +- **Automatic token renewal**: Both admin and user tokens are automatically renewed by Vault Agent +- **Vault Agent integration**: JupyterHub admin token is automatically renewed using Kubernetes authentication - **Audit trail**: All secret access is logged in Vault - **Individual policies**: Each user has their own Vault policy restricting access to their namespace @@ -236,13 +237,16 @@ JUPYTER_PYTHON_KERNEL_TAG=python-3.12-28 IMAGE_REGISTRY=localhost:30500 # Vault token TTL settings -JUPYTERHUB_VAULT_TOKEN_TTL=720h # Admin token: 30 days (effective limit) -JUPYTERHUB_VAULT_TOKEN_MAX_TTL=8760h # Admin token: 1 year (currently unused - no auto-renewal) +JUPYTERHUB_VAULT_TOKEN_TTL=24h # Admin token: 1 day (auto-renewed by Vault Agent) +JUPYTERHUB_VAULT_TOKEN_MAX_TTL=720h # Admin token: 30 days (max renewal limit) NOTEBOOK_VAULT_TOKEN_TTL=24h # User token: 1 day (auto-renewed) NOTEBOOK_VAULT_TOKEN_MAX_TTL=168h # User token: 7 days (max renewal limit) -# Logging -JUPYTER_BUUNSTACK_LOG_LEVEL=warning # Options: debug, info, warning, error +# Vault Agent logging +VAULT_AGENT_LOG_LEVEL=info # Options: trace, debug, info, warn, error + +# Application logging +JUPYTER_BUUNSTACK_LOG_LEVEL=warning # Options: debug, info, warning, error ``` ### Advanced Configuration @@ -351,9 +355,10 @@ The `buunstack` SecretStore uses pre-created user-specific Vault tokens that are **Key Components**: -- **JupyterHub Admin Token**: Created with admin policy, stored at `jupyterhub/vault-token`, available as `JUPYTERHUB_VAULT_TOKEN` environment variable -- **User-Specific Tokens**: Created dynamically during notebook spawn, available as `NOTEBOOK_VAULT_TOKEN` environment variable +- **JupyterHub Admin Token**: Automatically renewed by Vault Agent, read from file at `/vault/secrets/vault-token` +- **User-Specific Tokens**: Created dynamically during notebook spawn, available as `NOTEBOOK_VAULT_TOKEN` environment variable - **User Policies**: Restrict access to `secret/data/jupyter/users/{username}/*` +- **Vault Agent**: Sidecar container that handles automatic token renewal using Kubernetes authentication #### Token Lifecycle @@ -511,10 +516,85 @@ For production deployments, consider: - Setting up monitoring and alerts - Monitoring Vault token expiration and renewal patterns +## Vault Agent Integration + +### Overview + +JupyterHub now uses Vault Agent for automatic token renewal, eliminating the need for manual token management. Vault Agent runs as a sidecar container in the JupyterHub hub pod and automatically renews the admin token using Kubernetes authentication. + +### Architecture + +```plain +┌─────────────────────────────────────────────────────────────┐ +│ JupyterHub Hub Pod │ +│ │ +│ ┌─────────────────┐ ┌─────────────────────┐ │ +│ │ Hub Container │ │ Vault Agent Sidecar │ │ +│ │ │ │ │ │ +│ │ Reads token │◄─────────────┤ Writes token │ │ +│ │ from file │ │ to shared volume │ │ +│ │ │ │ │ │ +│ └─────────────────┘ └─────────────────────┘ │ +│ │ │ │ +│ │ │ │ +│ │ │ │ +│ ┌─────────▼─────────┐ ┌─────────▼─────────┐ │ +│ │ /vault/secrets/ │ │ Kubernetes Auth │ │ +│ │ vault-token │ │ with Vault │ │ +│ └───────────────────┘ └───────────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Key Features + +- **Automatic Renewal**: Admin token is automatically renewed every TTL/2 interval +- **Kubernetes Authentication**: Uses Kubernetes ServiceAccount for secure token acquisition +- **File-based Token Sharing**: Vault Agent writes tokens to shared volume, Hub reads from file +- **Zero Downtime**: Token renewal happens in background without service interruption +- **Configurable Logging**: Vault Agent log level can be configured via `VAULT_AGENT_LOG_LEVEL` + +### Monitoring Token Renewal + +Check Vault Agent status and token renewal: + +```bash +# Monitor Vault Agent logs +kubectl logs -n jupyter -l app.kubernetes.io/component=hub -c vault-agent -f + +# Use monitoring script +cd jupyterhub +./monitor-vault-token.sh + +# Check token details +kubectl exec -n jupyter -c hub -- curl -s -H "X-Vault-Token: $(cat /vault/secrets/vault-token)" $VAULT_ADDR/v1/auth/token/lookup-self +``` + +### Testing Token Renewal + +For testing purposes, you can use shorter TTL values to observe rapid token renewal: + +```bash +# Test with 1-minute TTL (renews every 30 seconds) +JUPYTERHUB_VAULT_TOKEN_TTL=1m VAULT_AGENT_LOG_LEVEL=debug just jupyterhub::install + +# Monitor renewal activity +kubectl logs -n jupyter -l app.kubernetes.io/component=hub -c vault-agent -f | grep "renewed auth token" +``` + +### Configuration Files + +The Vault Agent integration uses several configuration files: + +- `vault-agent-config.gomplate.hcl`: Vault Agent configuration template +- `token-monitor.tpl`: Template for logging token information +- `monitor-vault-token.sh`: Monitoring script for token status + ## Known Limitations -1. **Admin Token Refresh**: JupyterHub's admin Vault token (`JUPYTERHUB_VAULT_TOKEN`) does not auto-refresh. You must redeploy JupyterHub before the token expires (default TTL: 720h/30 days). The `JUPYTERHUB_VAULT_TOKEN_MAX_TTL` setting is currently not utilized since automatic renewal is not implemented. Monitor the token expiration and schedule redeployments accordingly. +1. **Token Max TTL**: Even with Vault Agent auto-renewal, tokens cannot be renewed beyond `JUPYTERHUB_VAULT_TOKEN_MAX_TTL` (default: 720h/30 days). After this period, JupyterHub must be redeployed to acquire a new token from Vault. With the default 30-day limit, this requires monthly maintenance. 2. **Cull Settings**: Server idle timeout is set to 2 hours by default. Adjust `cull.timeout` and `cull.every` in the Helm values for different requirements. 3. **NFS Storage**: When using NFS storage, ensure proper permissions are set on the NFS server. The default `JUPYTER_FSGID` is 100. + +4. **Vault Agent Resource Usage**: The Vault Agent sidecar uses minimal resources (50m CPU, 64Mi memory) but adds slight overhead to the hub pod. diff --git a/jupyterhub/.gitignore b/jupyterhub/.gitignore index e2969b3..1650ce5 100644 --- a/jupyterhub/.gitignore +++ b/jupyterhub/.gitignore @@ -1,2 +1,3 @@ jupyterhub-values.yaml +vault-agent-config.hcl /notebooks/ diff --git a/jupyterhub/jupyterhub-values.gomplate.yaml b/jupyterhub/jupyterhub-values.gomplate.yaml index 30098e8..eaa56cb 100644 --- a/jupyterhub/jupyterhub-values.gomplate.yaml +++ b/jupyterhub/jupyterhub-values.gomplate.yaml @@ -1,10 +1,16 @@ hub: extraEnv: JUPYTERHUB_CRYPT_KEY: {{ .Env.JUPYTERHUB_CRYPT_KEY | quote }} - JUPYTERHUB_VAULT_TOKEN: {{ .Env.JUPYTERHUB_VAULT_TOKEN | quote }} VAULT_ADDR: {{ .Env.VAULT_ADDR | quote }} NOTEBOOK_VAULT_TOKEN_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_TTL | quote }} NOTEBOOK_VAULT_TOKEN_MAX_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_MAX_TTL | quote }} + {{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }} + # Vault Agent will provide token via file + VAULT_TOKEN_FILE: "/vault/secrets/vault-token" + {{- else }} + # Traditional token via environment variable + JUPYTERHUB_VAULT_TOKEN: {{ .Env.JUPYTERHUB_VAULT_TOKEN | quote }} + {{- end }} # Install packages at container startup extraFiles: @@ -57,6 +63,25 @@ hub: # Set environment variables for spawned containers import hvac + def get_vault_token(): + """Read Vault token from file written by Vault Agent""" + import os + token_file = os.environ.get('VAULT_TOKEN_FILE', '/vault/secrets/vault-token') + try: + with open(token_file, 'r') as f: + token = f.read().strip() + if token: + return token + else: + raise Exception(f"Empty token file: {token_file}") + except FileNotFoundError: + # Fallback to environment variable for backward compatibility + return os.environ.get("JUPYTERHUB_VAULT_TOKEN") + except Exception as e: + # Log error but attempt fallback + print(f"Error reading token file {token_file}: {e}") + return os.environ.get("JUPYTERHUB_VAULT_TOKEN") + async def pre_spawn_hook(spawner): """Set essential environment variables for spawned containers""" # PostgreSQL configuration @@ -73,15 +98,19 @@ hub: try: username = spawner.user.name - # Step 1: Initialize admin Vault client + # Step 1: Initialize admin Vault client with file-based token import os vault_addr = os.environ.get("VAULT_ADDR", "{{ .Env.VAULT_ADDR }}") - vault_token = os.environ.get("JUPYTERHUB_VAULT_TOKEN", "{{ .Env.JUPYTERHUB_VAULT_TOKEN }}") + vault_token = get_vault_token() spawner.log.info(f"pre_spawn_hook starting for {username}") spawner.log.info(f"Vault address: {vault_addr}") + spawner.log.info(f"Vault token source: {'file' if os.path.exists(os.environ.get('VAULT_TOKEN_FILE', '/vault/secrets/vault-token')) else 'env'}") spawner.log.info(f"Vault token present: {bool(vault_token)}, length: {len(vault_token) if vault_token else 0}") + if not vault_token: + raise Exception("No Vault token available from file or environment") + vault_client = hvac.Client(url=vault_addr, verify=False) vault_client.token = vault_token @@ -135,6 +164,55 @@ hub: c.KubeSpawner.pre_spawn_hook = pre_spawn_hook + {{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }} + # Vault Agent sidecar configuration + extraVolumes: + - name: vault-secrets + emptyDir: {} + - name: vault-config + configMap: + name: vault-agent-config + + extraVolumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + - name: vault-config + mountPath: /vault/config + + extraContainers: + - name: vault-agent + image: hashicorp/vault:1.15.2 + securityContext: + runAsUser: 100 + runAsGroup: 101 + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: + - ALL + command: + - /bin/sh + - -c + - | + # Start Vault Agent + vault agent -config=/vault/config/agent.hcl + env: + - name: VAULT_ADDR + value: {{ .Env.VAULT_ADDR | quote }} + volumeMounts: + - name: vault-secrets + mountPath: /vault/secrets + - name: vault-config + mountPath: /vault/config + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + {{- end }} podSecurityContext: fsGroup: {{ .Env.JUPYTER_FSGID }} diff --git a/jupyterhub/justfile b/jupyterhub/justfile index 7218707..0317d6d 100644 --- a/jupyterhub/justfile +++ b/jupyterhub/justfile @@ -19,15 +19,17 @@ export JUPYTER_PROFILE_PYTORCH_ENABLED := env("JUPYTER_PROFILE_PYTORCH_ENABLED", export JUPYTER_PROFILE_TENSORFLOW_ENABLED := env("JUPYTER_PROFILE_TENSORFLOW_ENABLED", "false") export JUPYTER_PROFILE_BUUN_STACK_ENABLED := env("JUPYTER_PROFILE_BUUN_STACK_ENABLED", "false") export JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED := env("JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED", "false") -export IMAGE_REGISTRY := env("IMAGE_REGISTRY", "localhost:30500") -export JUPYTERHUB_VAULT_TOKEN_TTL := env("JUPYTERHUB_VAULT_TOKEN_TTL", "720h") # 30 days -export JUPYTERHUB_VAULT_TOKEN_MAX_TTL := env("JUPYTERHUB_VAULT_TOKEN_MAX_TTL", "8760h") # 1 year -export NOTEBOOK_VAULT_TOKEN_TTL := env("NOTEBOOK_VAULT_TOKEN_TTL", "24h") # 1 day -export NOTEBOOK_VAULT_TOKEN_MAX_TTL := env("NOTEBOOK_VAULT_TOKEN_MAX_TTL", "168h") # 7 days -export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") -export LONGHORN_NAMESPACE := env("LONGHORN_NAMESPACE", "longhorn") -export VAULT_ADDR := env("VAULT_ADDR", "http://vault.vault.svc:8200") +export JUPYTERHUB_VAULT_TOKEN_TTL := env("JUPYTERHUB_VAULT_TOKEN_TTL", "24h") +export JUPYTERHUB_VAULT_TOKEN_MAX_TTL := env("JUPYTERHUB_VAULT_TOKEN_MAX_TTL", "720h") +export NOTEBOOK_VAULT_TOKEN_TTL := env("NOTEBOOK_VAULT_TOKEN_TTL", "24h") +export NOTEBOOK_VAULT_TOKEN_MAX_TTL := env("NOTEBOOK_VAULT_TOKEN_MAX_TTL", "168h") +export VAULT_AGENT_LOG_LEVEL := env("VAULT_AGENT_LOG_LEVEL", "info") export JUPYTER_BUUNSTACK_LOG_LEVEL := env("JUPYTER_BUUNSTACK_LOG_LEVEL", "warning") +export IMAGE_REGISTRY := env("IMAGE_REGISTRY", "localhost:30500") +export LONGHORN_NAMESPACE := env("LONGHORN_NAMESPACE", "longhorn") +export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") +export VAULT_HOST := env("VAULT_HOST", "") +export VAULT_ADDR := "https://" + VAULT_HOST [private] default: @@ -116,13 +118,26 @@ install: kubectl apply -n ${JUPYTERHUB_NAMESPACE} -f nfs-pvc.yaml fi - # Always create new JupyterHub Vault token on deployment - echo "Creating new JupyterHub Vault token for this deployment..." - just create-jupyterhub-vault-token - export JUPYTERHUB_VAULT_TOKEN=$(just vault::get jupyterhub/vault-token token) + # Setup Vault Agent for automatic token management + if [ -z "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" ]; then + if gum confirm "Are you going to enable Vault integration?"; then + JUPYTERHUB_VAULT_INTEGRATION_ENABLED=true + else + JUPYTERHUB_VAULT_INTEGRATION_ENABLED=false + fi + fi + if [ "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" = "true" ]; then + echo "Setting up Vault Agent for automatic token management..." + echo " Token TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}" + echo " Token Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}" + just setup-vault-integration - # Read user policy template for Vault - export USER_POLICY_HCL=$(cat user_policy.hcl) + # Read user policy template for Vault + export USER_POLICY_HCL=$(cat user_policy.hcl) + else + echo "Vault integration disabled - deploying without Vault support" + export USER_POLICY_HCL="" + fi # https://z2jh.jupyter.org/en/stable/ gomplate -f jupyterhub-values.gomplate.yaml -o jupyterhub-values.yaml @@ -133,17 +148,6 @@ install: # wait deployments manually because `helm upgrade --wait` does not work for JupyterHub just k8s::wait-deployments-ready ${JUPYTERHUB_NAMESPACE} hub proxy - if [ -z "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" ]; then - if gum confirm "Are you going to enable Vault integration?"; then - JUPYTERHUB_VAULT_INTEGRATION_ENABLED=true - else - JUPYTERHUB_VAULT_INTEGRATION_ENABLED=false - fi - fi - if [ "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" = "true" ]; then - just setup-vault-jwt-auth - fi - # Uninstall JupyterHub uninstall: #!/bin/bash @@ -205,18 +209,53 @@ push-kernel-images: docker push ${IMAGE_REGISTRY}/${KERNEL_IMAGE_BUUN_STACK_CUDA_REPOSITORY}:${JUPYTER_PYTHON_KERNEL_TAG} fi -# Setup Vault integration for JupyterHub (user-specific tokens) -setup-vault-jwt-auth: +# Setup Vault integration for JupyterHub (user-specific tokens + auto-renewal) +setup-vault-integration root_token='': #!/bin/bash set -euo pipefail echo "Setting up Vault integration for JupyterHub..." - echo "✓ Vault integration configured (user-specific tokens)" + # Create Kubernetes role for JupyterHub in Vault + echo "Creating Kubernetes authentication role for JupyterHub..." + echo " Service Account: hub" + echo " Namespace: jupyter" + echo " Policies: admin" + echo " TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}" + echo " Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}" + export VAULT_TOKEN="{{ root_token }}" + while [ -z "${VAULT_TOKEN}" ]; do + VAULT_TOKEN=$(gum input --prompt="Vault root token: " --password --width=100) + done + vault write auth/kubernetes/role/jupyterhub \ + bound_service_account_names=hub \ + bound_service_account_namespaces=jupyter \ + policies=admin \ + ttl=${JUPYTERHUB_VAULT_TOKEN_TTL} \ + max_ttl=${JUPYTERHUB_VAULT_TOKEN_MAX_TTL} + + # Create Vault Agent configuration with gomplate + echo "Creating Vault Agent configuration..." + gomplate -f vault-agent-config.gomplate.hcl -o vault-agent-config.hcl + kubectl create configmap vault-agent-config -n ${JUPYTERHUB_NAMESPACE} \ + --from-file=agent.hcl=vault-agent-config.hcl \ + --from-file=token-monitor.tpl=token-monitor.tpl \ + --dry-run=client -o yaml | kubectl apply -f - + + echo "✓ Vault integration configured (user-specific tokens + auto-renewal)" + echo "" + echo "Configuration Summary:" + echo " JupyterHub Token TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}" + echo " JupyterHub Token Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}" + echo " User Token TTL: ${NOTEBOOK_VAULT_TOKEN_TTL}" + echo " User Token Max TTL: ${NOTEBOOK_VAULT_TOKEN_MAX_TTL}" + echo " Vault Agent Log Level: ${VAULT_AGENT_LOG_LEVEL}" + echo " Auto-renewal: Every $(( $(echo ${JUPYTERHUB_VAULT_TOKEN_TTL} | sed 's/m/*60/g; s/h/*3600/g; s/s//g' | bc) / 2 ))s (TTL/2)" echo "" echo "Users can now access Vault from notebooks using:" echo " from buunstack import SecretStore" echo " secrets = SecretStore()" echo " # Each user gets their own isolated Vault token and policy" + echo " # Admin token is automatically renewed by Vault Agent" # Create JupyterHub Vault token (uses admin policy for JWT operations) create-jupyterhub-vault-token: diff --git a/jupyterhub/monitor-vault-token.sh b/jupyterhub/monitor-vault-token.sh new file mode 100755 index 0000000..42fb162 --- /dev/null +++ b/jupyterhub/monitor-vault-token.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# JupyterHub Vault Token Monitor Script +# Usage: ./monitor-vault-token.sh [pod-name] + +set -euo pipefail + +NAMESPACE="jupyter" +POD_NAME=${1:-$(kubectl get pods -n ${NAMESPACE} -l app.kubernetes.io/component=hub -o jsonpath='{.items[0].metadata.name}')} + +echo "🔍 Monitoring Vault Agent for JupyterHub Pod: ${POD_NAME}" +echo "==================================================" + +# Check if pod exists and is running +if ! kubectl get pod ${POD_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then + echo "❌ Pod ${POD_NAME} not found in namespace ${NAMESPACE}" + exit 1 +fi + +echo "📊 Pod Status:" +kubectl get pod ${POD_NAME} -n ${NAMESPACE} +echo "" + +echo "📄 Vault Secrets Directory:" +kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- ls -la /vault/secrets/ 2>/dev/null || echo "❌ Cannot access /vault/secrets/" +echo "" + +echo "🔐 Current Token Info:" +kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c ' + if [ -f /vault/secrets/vault-token ]; then + echo "Token file exists ($(wc -c < /vault/secrets/vault-token) bytes)" + echo "Last modified: $(stat -c %y /vault/secrets/vault-token 2>/dev/null || stat -f %Sm /vault/secrets/vault-token)" + + # Test token validity + if command -v curl >/dev/null 2>&1; then + echo "" + echo "Token validation:" + RESPONSE=$(curl -s -w "%{http_code}" -H "X-Vault-Token: $(cat /vault/secrets/vault-token)" $VAULT_ADDR/v1/auth/token/lookup-self) + HTTP_CODE="${RESPONSE: -3}" + if [ "$HTTP_CODE" = "200" ]; then + echo "✅ Token is valid" + echo "$RESPONSE" | head -c -3 | grep -E "(ttl|expire_time|renewable)" | head -3 + else + echo "❌ Token validation failed (HTTP $HTTP_CODE)" + fi + fi + else + echo "❌ Token file not found" + fi +' 2>/dev/null || echo "❌ Cannot check token info" + +echo "" +echo "📋 Recent Vault Agent Logs:" +kubectl logs -n ${NAMESPACE} ${POD_NAME} -c vault-agent --tail=10 2>/dev/null || echo "❌ Cannot access vault-agent logs" + +echo "" +echo "📋 Token Renewal Log (if exists):" +kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c ' + if [ -f /vault/secrets/renewal.log ]; then + echo "Recent renewal events:" + tail -10 /vault/secrets/renewal.log + else + echo "No renewal log file found yet" + fi +' 2>/dev/null || echo "❌ Cannot check renewal logs" + +echo "" +echo "🔄 To monitor token renewals in real-time, run:" +echo " kubectl logs -n ${NAMESPACE} ${POD_NAME} -c vault-agent -f | grep 'renewed auth token'" +echo "" +echo "🔍 To check token info periodically, run:" +echo " watch -n 30 \"kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c 'curl -s -H \\\"X-Vault-Token: \\\$(cat /vault/secrets/vault-token)\\\" \\\$VAULT_ADDR/v1/auth/token/lookup-self | grep -E \\\"(ttl|expire_time)\\\"'\"" + diff --git a/jupyterhub/token-monitor.tpl b/jupyterhub/token-monitor.tpl new file mode 100644 index 0000000..4a45bed --- /dev/null +++ b/jupyterhub/token-monitor.tpl @@ -0,0 +1,11 @@ +{{- with secret "auth/token/lookup-self" -}} +=== Vault Token Status === +TTL: {{ .Data.ttl }} seconds +Renewable: {{ .Data.renewable }} +Expire Time: {{ .Data.expire_time }} +Policies: {{ range .Data.policies }}{{ . }} {{ end }} +Display Name: {{ .Data.display_name }} +Entity ID: {{ .Data.entity_id }} +Token Type: {{ .Data.type }} +=========================== +{{- end -}} \ No newline at end of file diff --git a/jupyterhub/vault-agent-config.gomplate.hcl b/jupyterhub/vault-agent-config.gomplate.hcl new file mode 100644 index 0000000..6d557a8 --- /dev/null +++ b/jupyterhub/vault-agent-config.gomplate.hcl @@ -0,0 +1,38 @@ +vault { + address = "{{ .Env.VAULT_ADDR }}" +} + +# Enable detailed logging +log_level = "{{ .Env.VAULT_AGENT_LOG_LEVEL }}" +log_format = "standard" + +auto_auth { + method "kubernetes" { + mount_path = "auth/kubernetes" + config = { + role = "jupyterhub" + } + } + + sink "file" { + config = { + path = "/vault/secrets/vault-token" + } + } +} + +cache { + use_auto_auth_token = true +} + +listener "tcp" { + address = "127.0.0.1:8100" + tls_disable = true +} + +# Add template for token monitoring +template { + source = "/vault/config/token-monitor.tpl" + destination = "/vault/secrets/token-info.log" + perms = 0644 +} \ No newline at end of file