feat(jupyterhub): admin vault token renewal

jupyterhub/.gitignore

@@ -1,2 +1,3 @@
 jupyterhub-values.yaml
+vault-agent-config.hcl
 /notebooks/

jupyterhub/jupyterhub-values.gomplate.yaml

@@ -1,10 +1,16 @@
 hub:
   extraEnv:
     JUPYTERHUB_CRYPT_KEY: {{ .Env.JUPYTERHUB_CRYPT_KEY | quote }}
-    JUPYTERHUB_VAULT_TOKEN: {{ .Env.JUPYTERHUB_VAULT_TOKEN | quote }}
     VAULT_ADDR: {{ .Env.VAULT_ADDR | quote }}
     NOTEBOOK_VAULT_TOKEN_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_TTL | quote }}
     NOTEBOOK_VAULT_TOKEN_MAX_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_MAX_TTL | quote }}
+    {{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
+    # Vault Agent will provide token via file
+    VAULT_TOKEN_FILE: "/vault/secrets/vault-token"
+    {{- else }}
+    # Traditional token via environment variable
+    JUPYTERHUB_VAULT_TOKEN: {{ .Env.JUPYTERHUB_VAULT_TOKEN | quote }}
+    {{- end }}
 
   # Install packages at container startup
   extraFiles:
@@ -57,6 +63,25 @@ hub:
       # Set environment variables for spawned containers
       import hvac
 
+      def get_vault_token():
+          """Read Vault token from file written by Vault Agent"""
+          import os
+          token_file = os.environ.get('VAULT_TOKEN_FILE', '/vault/secrets/vault-token')
+          try:
+              with open(token_file, 'r') as f:
+                  token = f.read().strip()
+                  if token:
+                      return token
+                  else:
+                      raise Exception(f"Empty token file: {token_file}")
+          except FileNotFoundError:
+              # Fallback to environment variable for backward compatibility
+              return os.environ.get("JUPYTERHUB_VAULT_TOKEN")
+          except Exception as e:
+              # Log error but attempt fallback
+              print(f"Error reading token file {token_file}: {e}")
+              return os.environ.get("JUPYTERHUB_VAULT_TOKEN")
+
       async def pre_spawn_hook(spawner):
           """Set essential environment variables for spawned containers"""
           # PostgreSQL configuration
@@ -73,15 +98,19 @@ hub:
           try:
               username = spawner.user.name
 
-              # Step 1: Initialize admin Vault client
+              # Step 1: Initialize admin Vault client with file-based token
               import os
               vault_addr = os.environ.get("VAULT_ADDR", "{{ .Env.VAULT_ADDR }}")
-              vault_token = os.environ.get("JUPYTERHUB_VAULT_TOKEN", "{{ .Env.JUPYTERHUB_VAULT_TOKEN }}")
+              vault_token = get_vault_token()
 
               spawner.log.info(f"pre_spawn_hook starting for {username}")
               spawner.log.info(f"Vault address: {vault_addr}")
+              spawner.log.info(f"Vault token source: {'file' if os.path.exists(os.environ.get('VAULT_TOKEN_FILE', '/vault/secrets/vault-token')) else 'env'}")
               spawner.log.info(f"Vault token present: {bool(vault_token)}, length: {len(vault_token) if vault_token else 0}")
 
+              if not vault_token:
+                  raise Exception("No Vault token available from file or environment")
+
               vault_client = hvac.Client(url=vault_addr, verify=False)
               vault_client.token = vault_token
 
@@ -135,6 +164,55 @@ hub:
 
       c.KubeSpawner.pre_spawn_hook = pre_spawn_hook
 
+  {{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
+  # Vault Agent sidecar configuration
+  extraVolumes:
+    - name: vault-secrets
+      emptyDir: {}
+    - name: vault-config
+      configMap:
+        name: vault-agent-config
+
+  extraVolumeMounts:
+    - name: vault-secrets
+      mountPath: /vault/secrets
+    - name: vault-config
+      mountPath: /vault/config
+
+  extraContainers:
+    - name: vault-agent
+      image: hashicorp/vault:1.15.2
+      securityContext:
+        runAsUser: 100
+        runAsGroup: 101
+        runAsNonRoot: true
+        allowPrivilegeEscalation: false
+        readOnlyRootFilesystem: false
+        capabilities:
+          drop:
+          - ALL
+      command:
+        - /bin/sh
+        - -c
+        - |
+          # Start Vault Agent
+          vault agent -config=/vault/config/agent.hcl
+      env:
+        - name: VAULT_ADDR
+          value: {{ .Env.VAULT_ADDR | quote }}
+      volumeMounts:
+        - name: vault-secrets
+          mountPath: /vault/secrets
+        - name: vault-config
+          mountPath: /vault/config
+      resources:
+        requests:
+          cpu: 50m
+          memory: 64Mi
+        limits:
+          cpu: 100m
+          memory: 128Mi
+  {{- end }}
 
   podSecurityContext:
     fsGroup: {{ .Env.JUPYTER_FSGID }}

jupyterhub/justfile

@@ -19,15 +19,17 @@ export JUPYTER_PROFILE_PYTORCH_ENABLED := env("JUPYTER_PROFILE_PYTORCH_ENABLED",
 export JUPYTER_PROFILE_TENSORFLOW_ENABLED := env("JUPYTER_PROFILE_TENSORFLOW_ENABLED", "false")
 export JUPYTER_PROFILE_BUUN_STACK_ENABLED := env("JUPYTER_PROFILE_BUUN_STACK_ENABLED", "false")
 export JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED := env("JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED", "false")
-export IMAGE_REGISTRY := env("IMAGE_REGISTRY", "localhost:30500")
-export JUPYTERHUB_VAULT_TOKEN_TTL := env("JUPYTERHUB_VAULT_TOKEN_TTL", "720h")  # 30 days
-export JUPYTERHUB_VAULT_TOKEN_MAX_TTL := env("JUPYTERHUB_VAULT_TOKEN_MAX_TTL", "8760h")  # 1 year
-export NOTEBOOK_VAULT_TOKEN_TTL := env("NOTEBOOK_VAULT_TOKEN_TTL", "24h")  # 1 day
-export NOTEBOOK_VAULT_TOKEN_MAX_TTL := env("NOTEBOOK_VAULT_TOKEN_MAX_TTL", "168h")  # 7 days
-export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack")
-export LONGHORN_NAMESPACE := env("LONGHORN_NAMESPACE", "longhorn")
-export VAULT_ADDR := env("VAULT_ADDR", "http://vault.vault.svc:8200")
+export JUPYTERHUB_VAULT_TOKEN_TTL := env("JUPYTERHUB_VAULT_TOKEN_TTL", "24h")
+export JUPYTERHUB_VAULT_TOKEN_MAX_TTL := env("JUPYTERHUB_VAULT_TOKEN_MAX_TTL", "720h")
+export NOTEBOOK_VAULT_TOKEN_TTL := env("NOTEBOOK_VAULT_TOKEN_TTL", "24h")
+export NOTEBOOK_VAULT_TOKEN_MAX_TTL := env("NOTEBOOK_VAULT_TOKEN_MAX_TTL", "168h")
+export VAULT_AGENT_LOG_LEVEL := env("VAULT_AGENT_LOG_LEVEL", "info")
 export JUPYTER_BUUNSTACK_LOG_LEVEL := env("JUPYTER_BUUNSTACK_LOG_LEVEL", "warning")
+export IMAGE_REGISTRY := env("IMAGE_REGISTRY", "localhost:30500")
+export LONGHORN_NAMESPACE := env("LONGHORN_NAMESPACE", "longhorn")
+export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack")
+export VAULT_HOST := env("VAULT_HOST", "")
+export VAULT_ADDR := "https://" + VAULT_HOST
 
 [private]
 default:
@@ -116,13 +118,26 @@ install:
         kubectl apply -n ${JUPYTERHUB_NAMESPACE} -f nfs-pvc.yaml
     fi
 
-    # Always create new JupyterHub Vault token on deployment
-    echo "Creating new JupyterHub Vault token for this deployment..."
-    just create-jupyterhub-vault-token
-    export JUPYTERHUB_VAULT_TOKEN=$(just vault::get jupyterhub/vault-token token)
+    # Setup Vault Agent for automatic token management
+    if [ -z "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" ]; then
+        if gum confirm "Are you going to enable Vault integration?"; then
+            JUPYTERHUB_VAULT_INTEGRATION_ENABLED=true
+        else
+            JUPYTERHUB_VAULT_INTEGRATION_ENABLED=false
+        fi
+    fi
+    if [ "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" = "true" ]; then
+        echo "Setting up Vault Agent for automatic token management..."
+        echo "  Token TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
+        echo "  Token Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
+        just setup-vault-integration
 
-    # Read user policy template for Vault
-    export USER_POLICY_HCL=$(cat user_policy.hcl)
+        # Read user policy template for Vault
+        export USER_POLICY_HCL=$(cat user_policy.hcl)
+    else
+        echo "Vault integration disabled - deploying without Vault support"
+        export USER_POLICY_HCL=""
+    fi
 
     # https://z2jh.jupyter.org/en/stable/
     gomplate -f jupyterhub-values.gomplate.yaml -o jupyterhub-values.yaml
@@ -133,17 +148,6 @@ install:
     # wait deployments manually because `helm upgrade --wait` does not work for JupyterHub
     just k8s::wait-deployments-ready ${JUPYTERHUB_NAMESPACE} hub proxy
 
-    if [ -z "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" ]; then
-        if gum confirm "Are you going to enable Vault integration?"; then
-            JUPYTERHUB_VAULT_INTEGRATION_ENABLED=true
-        else
-            JUPYTERHUB_VAULT_INTEGRATION_ENABLED=false
-        fi
-    fi
-    if [ "${JUPYTERHUB_VAULT_INTEGRATION_ENABLED}" = "true" ]; then
-        just setup-vault-jwt-auth
-    fi
-
 # Uninstall JupyterHub
 uninstall:
     #!/bin/bash
@@ -205,18 +209,53 @@ push-kernel-images:
         docker push ${IMAGE_REGISTRY}/${KERNEL_IMAGE_BUUN_STACK_CUDA_REPOSITORY}:${JUPYTER_PYTHON_KERNEL_TAG}
     fi
 
-# Setup Vault integration for JupyterHub (user-specific tokens)
-setup-vault-jwt-auth:
+# Setup Vault integration for JupyterHub (user-specific tokens + auto-renewal)
+setup-vault-integration root_token='':
     #!/bin/bash
     set -euo pipefail
     echo "Setting up Vault integration for JupyterHub..."
 
-    echo "✓ Vault integration configured (user-specific tokens)"
+    # Create Kubernetes role for JupyterHub in Vault
+    echo "Creating Kubernetes authentication role for JupyterHub..."
+    echo "  Service Account: hub"
+    echo "  Namespace: jupyter"
+    echo "  Policies: admin"
+    echo "  TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
+    echo "  Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
+    export VAULT_TOKEN="{{ root_token }}"
+    while [ -z "${VAULT_TOKEN}" ]; do
+        VAULT_TOKEN=$(gum input --prompt="Vault root token: " --password --width=100)
+    done
+    vault write auth/kubernetes/role/jupyterhub \
+        bound_service_account_names=hub \
+        bound_service_account_namespaces=jupyter \
+        policies=admin \
+        ttl=${JUPYTERHUB_VAULT_TOKEN_TTL} \
+        max_ttl=${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}
+
+    # Create Vault Agent configuration with gomplate
+    echo "Creating Vault Agent configuration..."
+    gomplate -f vault-agent-config.gomplate.hcl -o vault-agent-config.hcl
+    kubectl create configmap vault-agent-config -n ${JUPYTERHUB_NAMESPACE} \
+        --from-file=agent.hcl=vault-agent-config.hcl \
+        --from-file=token-monitor.tpl=token-monitor.tpl \
+        --dry-run=client -o yaml | kubectl apply -f -
+
+    echo "✓ Vault integration configured (user-specific tokens + auto-renewal)"
+    echo ""
+    echo "Configuration Summary:"
+    echo "  JupyterHub Token TTL: ${JUPYTERHUB_VAULT_TOKEN_TTL}"
+    echo "  JupyterHub Token Max TTL: ${JUPYTERHUB_VAULT_TOKEN_MAX_TTL}"
+    echo "  User Token TTL: ${NOTEBOOK_VAULT_TOKEN_TTL}"
+    echo "  User Token Max TTL: ${NOTEBOOK_VAULT_TOKEN_MAX_TTL}"
+    echo "  Vault Agent Log Level: ${VAULT_AGENT_LOG_LEVEL}"
+    echo "  Auto-renewal: Every $(( $(echo ${JUPYTERHUB_VAULT_TOKEN_TTL} | sed 's/m/*60/g; s/h/*3600/g; s/s//g' | bc) / 2 ))s (TTL/2)"
     echo ""
     echo "Users can now access Vault from notebooks using:"
     echo "  from buunstack import SecretStore"
     echo "  secrets = SecretStore()"
     echo "  # Each user gets their own isolated Vault token and policy"
+    echo "  # Admin token is automatically renewed by Vault Agent"
 
 # Create JupyterHub Vault token (uses admin policy for JWT operations)
 create-jupyterhub-vault-token:

jupyterhub/monitor-vault-token.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# JupyterHub Vault Token Monitor Script
+# Usage: ./monitor-vault-token.sh [pod-name]
+
+set -euo pipefail
+
+NAMESPACE="jupyter"
+POD_NAME=${1:-$(kubectl get pods -n ${NAMESPACE} -l app.kubernetes.io/component=hub -o jsonpath='{.items[0].metadata.name}')}
+
+echo "🔍 Monitoring Vault Agent for JupyterHub Pod: ${POD_NAME}"
+echo "=================================================="
+
+# Check if pod exists and is running
+if ! kubectl get pod ${POD_NAME} -n ${NAMESPACE} >/dev/null 2>&1; then
+  echo "❌ Pod ${POD_NAME} not found in namespace ${NAMESPACE}"
+  exit 1
+fi
+
+echo "📊 Pod Status:"
+kubectl get pod ${POD_NAME} -n ${NAMESPACE}
+echo ""
+
+echo "📄 Vault Secrets Directory:"
+kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- ls -la /vault/secrets/ 2>/dev/null || echo "❌ Cannot access /vault/secrets/"
+echo ""
+
+echo "🔐 Current Token Info:"
+kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c '
+    if [ -f /vault/secrets/vault-token ]; then
+        echo "Token file exists ($(wc -c < /vault/secrets/vault-token) bytes)"
+        echo "Last modified: $(stat -c %y /vault/secrets/vault-token 2>/dev/null || stat -f %Sm /vault/secrets/vault-token)"
+
+        # Test token validity
+        if command -v curl >/dev/null 2>&1; then
+            echo ""
+            echo "Token validation:"
+            RESPONSE=$(curl -s -w "%{http_code}" -H "X-Vault-Token: $(cat /vault/secrets/vault-token)" $VAULT_ADDR/v1/auth/token/lookup-self)
+            HTTP_CODE="${RESPONSE: -3}"
+            if [ "$HTTP_CODE" = "200" ]; then
+                echo "✅ Token is valid"
+                echo "$RESPONSE" | head -c -3 | grep -E "(ttl|expire_time|renewable)" | head -3
+            else
+                echo "❌ Token validation failed (HTTP $HTTP_CODE)"
+            fi
+        fi
+    else
+        echo "❌ Token file not found"
+    fi
+' 2>/dev/null || echo "❌ Cannot check token info"
+
+echo ""
+echo "📋 Recent Vault Agent Logs:"
+kubectl logs -n ${NAMESPACE} ${POD_NAME} -c vault-agent --tail=10 2>/dev/null || echo "❌ Cannot access vault-agent logs"
+
+echo ""
+echo "📋 Token Renewal Log (if exists):"
+kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c '
+    if [ -f /vault/secrets/renewal.log ]; then
+        echo "Recent renewal events:"
+        tail -10 /vault/secrets/renewal.log
+    else
+        echo "No renewal log file found yet"
+    fi
+' 2>/dev/null || echo "❌ Cannot check renewal logs"
+
+echo ""
+echo "🔄 To monitor token renewals in real-time, run:"
+echo "  kubectl logs -n ${NAMESPACE} ${POD_NAME} -c vault-agent -f | grep 'renewed auth token'"
+echo ""
+echo "🔍 To check token info periodically, run:"
+echo "  watch -n 30 \"kubectl exec -n ${NAMESPACE} ${POD_NAME} -c hub -- sh -c 'curl -s -H \\\"X-Vault-Token: \\\$(cat /vault/secrets/vault-token)\\\" \\\$VAULT_ADDR/v1/auth/token/lookup-self | grep -E \\\"(ttl|expire_time)\\\"'\""
+

jupyterhub/token-monitor.tpl

@@ -0,0 +1,11 @@
+{{- with secret "auth/token/lookup-self" -}}
+=== Vault Token Status ===
+TTL: {{ .Data.ttl }} seconds
+Renewable: {{ .Data.renewable }}
+Expire Time: {{ .Data.expire_time }}
+Policies: {{ range .Data.policies }}{{ . }} {{ end }}
+Display Name: {{ .Data.display_name }}
+Entity ID: {{ .Data.entity_id }}
+Token Type: {{ .Data.type }}
+===========================
+{{- end -}}

jupyterhub/vault-agent-config.gomplate.hcl

@@ -0,0 +1,38 @@
+vault {
+  address = "{{ .Env.VAULT_ADDR }}"
+}
+
+# Enable detailed logging
+log_level = "{{ .Env.VAULT_AGENT_LOG_LEVEL }}"
+log_format = "standard"
+
+auto_auth {
+  method "kubernetes" {
+    mount_path = "auth/kubernetes"
+    config = {
+      role = "jupyterhub"
+    }
+  }
+
+  sink "file" {
+    config = {
+      path = "/vault/secrets/vault-token"
+    }
+  }
+}
+
+cache {
+  use_auto_auth_token = true
+}
+
+listener "tcp" {
+  address     = "127.0.0.1:8100"
+  tls_disable = true
+}
+
+# Add template for token monitoring
+template {
+  source      = "/vault/config/token-monitor.tpl"
+  destination = "/vault/secrets/token-info.log"
+  perms       = 0644
+}