436 lines
13 KiB
YAML
436 lines
13 KiB
YAML
hub:
|
|
extraEnv:
|
|
JUPYTERHUB_CRYPT_KEY:
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: jupyterhub-crypt-key
|
|
key: crypt-key
|
|
VAULT_ADDR: {{ .Env.VAULT_ADDR | quote }}
|
|
NOTEBOOK_VAULT_TOKEN_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_TTL | quote }}
|
|
NOTEBOOK_VAULT_TOKEN_MAX_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_MAX_TTL | quote }}
|
|
{{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
|
|
# Vault Agent provides renewable token via file (unlimited max TTL)
|
|
VAULT_TOKEN_FILE: "/vault/secrets/vault-token"
|
|
{{- end }}
|
|
|
|
# Install packages at container startup
|
|
extraFiles:
|
|
startup.sh:
|
|
mountPath: /usr/local/bin/startup.sh
|
|
mode: 0755
|
|
stringData: |
|
|
#!/bin/bash
|
|
pip install --no-cache-dir hvac==2.3.0
|
|
exec jupyterhub --config /usr/local/etc/jupyterhub/jupyterhub_config.py --upgrade-db
|
|
{{- if .Env.USER_POLICY_HCL }}
|
|
user_policy.hcl:
|
|
mountPath: /srv/jupyterhub/user_policy.hcl
|
|
mode: 0644
|
|
stringData: |
|
|
{{ .Env.USER_POLICY_HCL | strings.Indent 8 }}
|
|
{{- end }}
|
|
pre_spawn_hook.py:
|
|
mountPath: /srv/jupyterhub/pre_spawn_hook.py
|
|
mode: 0644
|
|
stringData: |
|
|
{{ file.Read "pre_spawn_hook.py" | strings.Indent 8 }}
|
|
|
|
# Override the default command to run our startup script first
|
|
command:
|
|
- /usr/local/bin/startup.sh
|
|
|
|
config:
|
|
JupyterHub:
|
|
authenticator_class: generic-oauth
|
|
admin_access: false
|
|
authenticate_prometheus: false
|
|
|
|
Authenticator:
|
|
enable_auth_state: true
|
|
allow_all: true # allow all Keycloak users
|
|
|
|
GenericOAuthenticator:
|
|
client_id: {{ .Env.JUPYTERHUB_OIDC_CLIENT_ID }}
|
|
oauth_callback_url: "https://{{ .Env.JUPYTERHUB_HOST }}/hub/oauth_callback"
|
|
authorize_url: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid-connect/auth"
|
|
token_url: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid-connect/token"
|
|
userdata_url: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid-connect/userinfo"
|
|
login_service: keycloak
|
|
# username_claim: email
|
|
username_claim: preferred_username
|
|
auth_refresh_age: 300 # Refresh auth token every 5 minutes
|
|
refresh_pre_spawn: true # Refresh token before spawning server
|
|
|
|
OAuthenticator:
|
|
scope:
|
|
- openid
|
|
- profile
|
|
- email
|
|
|
|
extraConfig:
|
|
load-pre-spawn-hook: |
|
|
# Load pre_spawn_hook from external file
|
|
with open('/srv/jupyterhub/pre_spawn_hook.py', 'r') as f:
|
|
exec(f.read())
|
|
|
|
configure-security-context: |
|
|
# Configure container security context for restricted Pod Security Standard
|
|
c.KubeSpawner.container_security_context = {
|
|
'capabilities': {
|
|
'drop': ['ALL']
|
|
}
|
|
}
|
|
|
|
{{- if eq .Env.JUPYTERHUB_VAULT_INTEGRATION_ENABLED "true" }}
|
|
# Vault token renewal sidecar configuration
|
|
extraVolumes:
|
|
- name: vault-secrets
|
|
emptyDir: {}
|
|
- name: vault-config
|
|
configMap:
|
|
name: vault-token-renewer-config
|
|
- name: vault-admin-token
|
|
secret:
|
|
secretName: jupyterhub-vault-token
|
|
|
|
extraVolumeMounts:
|
|
- name: vault-secrets
|
|
mountPath: /vault/secrets
|
|
- name: vault-config
|
|
mountPath: /vault/config
|
|
- name: vault-admin-token
|
|
mountPath: /vault/admin-token
|
|
readOnly: true
|
|
|
|
extraContainers:
|
|
- name: vault-token-renewer
|
|
image: hashicorp/vault:1.17.5
|
|
securityContext:
|
|
runAsUser: 100
|
|
runAsGroup: 101
|
|
runAsNonRoot: true
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
command:
|
|
- /bin/sh
|
|
- -c
|
|
- |
|
|
# Start token renewal script (handles both retrieval and renewal)
|
|
exec sh /vault/config/vault-token-renewer.sh
|
|
env:
|
|
- name: VAULT_ADDR
|
|
value: {{ .Env.VAULT_ADDR | quote }}
|
|
- name: JUPYTERHUB_VAULT_TOKEN_TTL
|
|
value: {{ .Env.JUPYTERHUB_VAULT_TOKEN_TTL | quote }}
|
|
volumeMounts:
|
|
- name: vault-secrets
|
|
mountPath: /vault/secrets
|
|
- name: vault-config
|
|
mountPath: /vault/config
|
|
- name: vault-admin-token
|
|
mountPath: /vault/admin-token
|
|
readOnly: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
{{- end }}
|
|
|
|
podSecurityContext:
|
|
fsGroup: {{ .Env.JUPYTER_FSGID }}
|
|
|
|
networkPolicy:
|
|
ingress:
|
|
- from:
|
|
- podSelector:
|
|
matchLabels:
|
|
hub.jupyter.org/network-access-hub: "true"
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: {{ .Env.PROMETHEUS_NAMESPACE }}
|
|
ports:
|
|
- port: http
|
|
protocol: TCP
|
|
|
|
proxy:
|
|
service:
|
|
type: ClusterIP
|
|
|
|
singleuser:
|
|
# Disable block-cloud-metadata sidecar for restricted Pod Security Standard compliance
|
|
# Not needed in self-hosted environments without cloud metadata services
|
|
cloudMetadata:
|
|
blockWithIptables: false
|
|
|
|
# Pod Security Standard (restricted) compliance
|
|
allowPrivilegeEscalation: false
|
|
|
|
# Additional security context via extraPodConfig
|
|
extraPodConfig:
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
|
|
storage:
|
|
{{ if env.Getenv "PVC_NAME" -}}
|
|
type: static
|
|
static:
|
|
pvcName: {{ .Env.PVC_NAME }}
|
|
{{ else -}}
|
|
type: dynamic
|
|
dynamic:
|
|
{{ if env.Getenv "JUPYTERHUB_STORAGE_CLASS" -}}
|
|
storageClass: {{ .Env.JUPYTERHUB_STORAGE_CLASS }}
|
|
{{ end -}}
|
|
storageAccessModes:
|
|
- ReadWriteOnce
|
|
{{ end -}}
|
|
capacity: 10Gi
|
|
|
|
extraEnv:
|
|
VAULT_ADDR: "{{ .Env.VAULT_ADDR }}"
|
|
NOTEBOOK_VAULT_TOKEN_TTL: "{{ .Env.NOTEBOOK_VAULT_TOKEN_TTL }}"
|
|
NOTEBOOK_VAULT_TOKEN_MAX_TTL: "{{ .Env.NOTEBOOK_VAULT_TOKEN_MAX_TTL }}"
|
|
# JUPYTERHUB_SINGLEUSER_EXTENSION: "0"
|
|
|
|
{{- if eq .Env.JUPYTERHUB_GPU_ENABLED "true" }}
|
|
extraPodConfig:
|
|
runtimeClassName: nvidia
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
extraResource:
|
|
limits:
|
|
nvidia.com/gpu: "{{ .Env.JUPYTERHUB_GPU_LIMIT }}"
|
|
{{- end }}
|
|
|
|
storage:
|
|
{{ if env.Getenv "PVC_NAME" -}}
|
|
type: static
|
|
static:
|
|
pvcName: {{ .Env.PVC_NAME }}
|
|
{{ else -}}
|
|
type: dynamic
|
|
dynamic:
|
|
{{ if env.Getenv "JUPYTERHUB_STORAGE_CLASS" -}}
|
|
storageClass: {{ .Env.JUPYTERHUB_STORAGE_CLASS }}
|
|
{{ end -}}
|
|
storageAccessModes:
|
|
- ReadWriteOnce
|
|
{{ end -}}
|
|
capacity: 10Gi
|
|
{{- if eq .Env.JUPYTERHUB_AIRFLOW_DAGS_PERSISTENCE_ENABLED "true" }}
|
|
# Mount Airflow DAGs when both are in the same namespace (jupyter)
|
|
extraVolumes:
|
|
- name: airflow-dags
|
|
persistentVolumeClaim:
|
|
claimName: airflow-dags-pvc
|
|
extraVolumeMounts:
|
|
- name: airflow-dags
|
|
mountPath: /home/jovyan/airflow-dags
|
|
readOnly: false
|
|
{{- end }}
|
|
|
|
networkPolicy:
|
|
egress:
|
|
{{- if eq .Env.JUPYTERHUB_AIRFLOW_DAGS_PERSISTENCE_ENABLED "true" }}
|
|
# Allow communication with Airflow API server in the same namespace
|
|
- to:
|
|
- podSelector:
|
|
matchLabels:
|
|
release: airflow
|
|
component: api-server
|
|
ports:
|
|
- port: 8080
|
|
protocol: TCP
|
|
{{- end }}
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: postgres
|
|
ports:
|
|
- port: 5432
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: qdrant
|
|
ports:
|
|
- port: 6333
|
|
protocol: TCP
|
|
- port: 6334
|
|
protocol: TCP
|
|
- port: 6335
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: litellm
|
|
ports:
|
|
- port: 4000
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: vault
|
|
ports:
|
|
- port: 8200
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: clickhouse
|
|
ports:
|
|
- port: 8123
|
|
protocol: TCP
|
|
- port: 9000
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: airflow
|
|
ports:
|
|
- port: 8080
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: minio
|
|
ports:
|
|
- port: 9000
|
|
protocol: TCP
|
|
- port: 9001
|
|
protocol: TCP
|
|
- to:
|
|
- namespaceSelector:
|
|
matchLabels:
|
|
kubernetes.io/metadata.name: lakekeeper
|
|
ports:
|
|
- port: 8181
|
|
protocol: TCP
|
|
# Allow DNS resolution
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
ports:
|
|
- port: 53
|
|
protocol: UDP
|
|
- port: 53
|
|
protocol: TCP
|
|
# Allow HTTP traffic
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
ports:
|
|
- port: 80
|
|
protocol: TCP
|
|
# Allow HTTPS traffic
|
|
- to:
|
|
- ipBlock:
|
|
cidr: 0.0.0.0/0
|
|
ports:
|
|
- port: 443
|
|
protocol: TCP
|
|
|
|
image:
|
|
pullPolicy: IfNotPresent
|
|
|
|
profileList:
|
|
# https://quay.io/repository/jupyter/pyspark-notebook
|
|
{{- if eq .Env.JUPYTER_PROFILE_MINIMAL_ENABLED "true" }}
|
|
- display_name: "Minimal Jupyter Notebook Stack"
|
|
description: "Minimal Jupyter Notebook Stack"
|
|
kubespawner_override:
|
|
image: quay.io/jupyter/minimal-notebook
|
|
{{- end }}
|
|
{{ if eq .Env.JUPYTER_PROFILE_BASE_ENABLED "true" }}
|
|
- display_name: "Base Jupyter Notebook Stack"
|
|
description: "Base Jupyter Notebook Stack"
|
|
kubespawner_override:
|
|
image: quay.io/jupyter/base-notebook
|
|
{{- end }}
|
|
{{- if eq .Env.JUPYTER_PROFILE_DATASCIENCE_ENABLED "true" }}
|
|
- display_name: "Jupyter Notebook Data Science Stack"
|
|
description: "Jupyter Notebook Data Science Stack"
|
|
kubespawner_override:
|
|
image: quay.io/jupyter/datascience-notebook
|
|
{{- end }}
|
|
{{- if eq .Env.JUPYTER_PROFILE_PYSPARK_ENABLED "true" }}
|
|
- display_name: "Jupyter Notebook Python, Spark Stack"
|
|
description: "Jupyter Notebook Python, Spark Stack"
|
|
kubespawner_override:
|
|
image: quay.io/jupyter/pyspark-notebook
|
|
{{- end }}
|
|
{{- if eq .Env.JUPYTER_PROFILE_PYTORCH_ENABLED "true" }}
|
|
- display_name: "Jupyter Notebook PyTorch Deep Learning Stack"
|
|
description: "Jupyter Notebook PyTorch Deep Learning Stack"
|
|
kubespawner_override:
|
|
image: quay.io/jupyter/pytorch-notebook
|
|
{{- end }}
|
|
{{- if eq .Env.JUPYTER_PROFILE_TENSORFLOW_ENABLED "true" }}
|
|
- display_name: "Jupyter Notebook TensorFlow Deep Learning Stack"
|
|
description: "Jupyter Notebook TensorFlow Deep Learning Stack"
|
|
kubespawner_override:
|
|
image: quay.io/jupyter/tensorflow-notebook
|
|
{{- end }}
|
|
{{- if eq .Env.JUPYTER_PROFILE_BUUN_STACK_ENABLED "true" }}
|
|
- display_name: "Buun-stack"
|
|
description: "Jupyter Notebook with buun-stack"
|
|
kubespawner_override:
|
|
image: "{{ .Env.IMAGE_REGISTRY }}/{{ .Env.KERNEL_IMAGE_BUUN_STACK_REPOSITORY }}:{{ .Env.JUPYTER_PYTHON_KERNEL_TAG }}"
|
|
{{- end }}
|
|
{{- if eq .Env.JUPYTER_PROFILE_BUUN_STACK_CUDA_ENABLED "true" }}
|
|
- display_name: "Buun-stack with CUDA"
|
|
description: "Jupyter Notebook with buun-stack and CUDA support"
|
|
kubespawner_override:
|
|
image: "{{ .Env.IMAGE_REGISTRY }}/{{ .Env.KERNEL_IMAGE_BUUN_STACK_CUDA_REPOSITORY }}:{{ .Env.JUPYTER_PYTHON_KERNEL_TAG }}"
|
|
# resources:
|
|
# requests:
|
|
# nvidia.com/gpu: "1"
|
|
{{- end }}
|
|
|
|
cull:
|
|
enabled: true
|
|
|
|
# for production
|
|
timeout: 7200 # 2 hours idle timeout
|
|
every: 600 # Check every 10 minutes
|
|
|
|
# for testing
|
|
# timeout: 300 # 5 minutes idle timeout (for testing) │ │
|
|
# every: 60 # Check every 1 minute (for testing) │ │
|
|
|
|
# Maximum age of a server pod before forced restart
|
|
# IMPORTANT: This must be less than NOTEBOOK_VAULT_TOKEN_MAX_TTL to prevent token expiry
|
|
# - NOTEBOOK_VAULT_TOKEN_MAX_TTL: {{ .Env.NOTEBOOK_VAULT_TOKEN_MAX_TTL }} (7 days = 604800s)
|
|
# - JUPYTERHUB_CULL_MAX_AGE: {{ .Env.JUPYTERHUB_CULL_MAX_AGE }}s (6 days = 518400s)
|
|
# Pod restart creates new user token, preventing 7-day token expiry
|
|
maxAge: {{ .Env.JUPYTERHUB_CULL_MAX_AGE }}
|
|
|
|
adminUsers: true # Also cull admin users' server pods
|
|
users: false # Don't delete user accounts, only stop server pods
|
|
|
|
# imagePullSecrets:
|
|
# - name: regcred
|
|
|
|
ingress:
|
|
enabled: true
|
|
annotations:
|
|
kubernetes.io/ingress.class: traefik
|
|
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
|
ingressClassName: traefik
|
|
hosts:
|
|
- {{ .Env.JUPYTERHUB_HOST }}
|
|
pathType: Prefix
|
|
tls:
|
|
- hosts:
|
|
- {{ .Env.JUPYTERHUB_HOST }}
|