set fallback := true export LITELLM_NAMESPACE := env("LITELLM_NAMESPACE", "litellm") export LITELLM_CHART_VERSION := env("LITELLM_CHART_VERSION", "0.1.825") export LITELLM_HOST := env("LITELLM_HOST", "") export LITELLM_OIDC_CLIENT_ID := env("LITELLM_OIDC_CLIENT_ID", "litellm") export LITELLM_OIDC_ENABLED := env("LITELLM_OIDC_ENABLED", "") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") export OLLAMA_NAMESPACE := env("OLLAMA_NAMESPACE", "ollama") export PROMETHEUS_NAMESPACE := env("PROMETHEUS_NAMESPACE", "monitoring") export MONITORING_ENABLED := env("MONITORING_ENABLED", "") export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") export KEYCLOAK_HOST := env("KEYCLOAK_HOST", "") export K8S_VAULT_NAMESPACE := env("K8S_VAULT_NAMESPACE", "vault") [private] default: @just --list --unsorted --list-submodules # Create LiteLLM namespace create-namespace: kubectl get namespace ${LITELLM_NAMESPACE} &>/dev/null || \ kubectl create namespace ${LITELLM_NAMESPACE} # Delete LiteLLM namespace delete-namespace: kubectl delete namespace ${LITELLM_NAMESPACE} --ignore-not-found # Check prerequisites check-prerequisites: #!/bin/bash set -euo pipefail if ! helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "Error: External Secrets Operator is required but not installed." echo "Please install External Secrets Operator first:" echo " just external-secrets::install" exit 1 fi if [ ! -f models.yaml ]; then echo "Error: models.yaml not found." echo "Please create models.yaml from the example:" echo " cp models.example.yaml models.yaml" echo "Then edit models.yaml to configure your models." exit 1 fi echo "Prerequisites check passed." # Extract required providers from models.yaml [private] get-required-providers: #!/bin/bash set -euo pipefail # Extract providers that require API keys (exclude ollama) yq -r '.[] | select(.litellm_params.api_key != null) | .litellm_params.model' models.yaml | \ cut -d'/' -f1 | sort -u # Verify all required API keys are set in Vault verify-api-keys: #!/bin/bash set -euo pipefail providers=$(just get-required-providers) missing=() for provider in $providers; do if ! just vault::get "litellm/${provider}" apikey &>/dev/null; then missing+=("$provider") fi done if [ ${#missing[@]} -gt 0 ]; then echo "Error: Missing API keys for the following providers:" for p in "${missing[@]}"; do echo " - $p" done echo "" echo "Please set the API keys:" for p in "${missing[@]}"; do echo " just litellm::set-api-key provider=$p" done exit 1 fi echo "All required API keys are configured." # Set API key for a provider set-api-key provider='': #!/bin/bash set -euo pipefail provider="{{ provider }}" if [ -z "${provider}" ]; then available=$(just get-required-providers 2>/dev/null || echo "anthropic openai mistral groq cohere") provider=$(echo "$available" | tr ' ' '\n' | gum choose --header="Select provider:") fi apikey=$(gum input --prompt="${provider} API key: " --password --width=80) if [ -z "${apikey}" ]; then echo "Error: API key cannot be empty" exit 1 fi just vault::put "litellm/${provider}" apikey="${apikey}" echo "API key for ${provider} has been stored in Vault location 'litellm/${provider}'." # Get API key for a provider get-api-key provider='': #!/bin/bash set -euo pipefail provider="{{ provider }}" if [ -z "${provider}" ]; then echo "Usage: just litellm::get-api-key provider=" exit 1 fi just vault::get "litellm/${provider}" apikey # Add a model interactively add-model: #!/bin/bash set -euo pipefail if [ ! -f models.yaml ]; then echo "Creating models.yaml from example..." cp models.example.yaml models.yaml fi echo "Add a new model to LiteLLM" echo "" provider=$(gum choose --header="Select provider:" \ "anthropic" "openai" "ollama" "mistral" "groq" "cohere" "azure" "bedrock" "vertexai") model_name=$(gum input --prompt="Model alias (e.g., claude-sonnet): " --width=60) if [ -z "${model_name}" ]; then echo "Error: Model name is required" exit 1 fi case $provider in anthropic) model=$(gum choose --header="Select Anthropic model:" \ "claude-sonnet-4-20250514" \ "claude-haiku-4-20251015" \ "claude-opus-4-20250514") api_key_line=" api_key: os.environ/ANTHROPIC_API_KEY" ;; openai) model=$(gum choose --header="Select OpenAI model:" \ "gpt-4o" \ "gpt-4o-mini" \ "o3" \ "o4-mini") api_key_line=" api_key: os.environ/OPENAI_API_KEY" ;; ollama) model=$(gum input --prompt="Ollama model name: " --width=60 --placeholder="qwen3:8b") api_key_line=" api_base: http://ollama.${OLLAMA_NAMESPACE}:11434" ;; mistral) model=$(gum choose --header="Select Mistral model:" \ "mistral-large-latest" \ "ministral-8b-latest" \ "codestral-latest") api_key_line=" api_key: os.environ/MISTRAL_API_KEY" ;; groq) model=$(gum choose --header="Select Groq model:" \ "meta-llama/llama-4-scout-17b-16e-instruct" \ "llama-3.3-70b-versatile" \ "llama-3.1-8b-instant") api_key_line=" api_key: os.environ/GROQ_API_KEY" ;; cohere) model=$(gum choose --header="Select Cohere model:" \ "command-r-plus" \ "command-r" \ "command-light") api_key_line=" api_key: os.environ/COHERE_API_KEY" ;; *) model=$(gum input --prompt="Model identifier: " --width=60) api_key_line=" api_key: os.environ/${provider^^}_API_KEY" ;; esac echo "" >> models.yaml echo "- model_name: ${model_name}" >> models.yaml echo " litellm_params:" >> models.yaml echo " model: ${provider}/${model}" >> models.yaml echo "${api_key_line}" >> models.yaml echo "" echo "Model '${model_name}' added to models.yaml" if [ "$provider" != "ollama" ]; then echo "" echo "Don't forget to set the API key if not already done:" echo " just litellm::set-api-key provider=${provider}" fi echo "" echo "Run 'just litellm::install' or 'just litellm::upgrade' to apply changes." # Remove a model interactively remove-model: #!/bin/bash set -euo pipefail if [ ! -f models.yaml ]; then echo "Error: models.yaml not found" exit 1 fi models=$(yq -r '.[].model_name' models.yaml) if [ -z "$models" ]; then echo "No models configured." exit 0 fi model=$(echo "$models" | gum choose --header="Select model to remove:") if gum confirm "Remove model '${model}'?"; then yq -i "del(.[] | select(.model_name == \"${model}\"))" models.yaml echo "Model '${model}' removed from models.yaml" echo "Run 'just litellm::upgrade' to apply changes." else echo "Cancelled." fi # List configured models list-models: #!/bin/bash set -euo pipefail if [ ! -f models.yaml ]; then echo "No models.yaml found. Create one with:" echo " cp models.example.yaml models.yaml" exit 0 fi echo "Configured models:" yq -r '.[] | " - \(.model_name): \(.litellm_params.model)"' models.yaml # Create API key external secret create-api-key-external-secret: #!/bin/bash set -euo pipefail gomplate -d models=models.yaml -f apikey-external-secret.gomplate.yaml -o apikey-external-secret.yaml kubectl apply -f apikey-external-secret.yaml echo "Waiting for API key secret to be ready..." kubectl wait --for=condition=Ready externalsecret/apikey-external-secret \ -n ${LITELLM_NAMESPACE} --timeout=60s # Delete API key external secret delete-api-key-external-secret: kubectl delete externalsecret apikey-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found kubectl delete secret apikey -n ${LITELLM_NAMESPACE} --ignore-not-found # Create Postgres user and database create-postgres-user-and-db: #!/bin/bash set -euo pipefail if just postgres::user-exists litellm &>/dev/null; then echo "PostgreSQL user 'litellm' already exists" else echo "Creating PostgreSQL user and database..." PG_PASSWORD=$(just utils::random-password) just postgres::create-user-and-db litellm litellm "${PG_PASSWORD}" just vault::put litellm/db username=litellm password="${PG_PASSWORD}" echo "PostgreSQL user and database created." fi # Delete Postgres user and database delete-postgres-user-and-db: #!/bin/bash set -euo pipefail if gum confirm "Delete PostgreSQL user and database 'litellm'?"; then just postgres::delete-user-and-db litellm litellm || true just vault::delete litellm/db || true echo "PostgreSQL user and database deleted." else echo "Cancelled." fi # Create Postgres secret create-postgres-secret: #!/bin/bash set -euo pipefail if kubectl get secret postgres-auth -n ${LITELLM_NAMESPACE} &>/dev/null; then echo "Postgres auth secret already exists" exit 0 fi PG_USERNAME=$(just vault::get litellm/db username) PG_PASSWORD=$(just vault::get litellm/db password) kubectl create secret generic postgres-auth \ --from-literal=username="${PG_USERNAME}" \ --from-literal=password="${PG_PASSWORD}" \ -n ${LITELLM_NAMESPACE} # Delete Postgres secret delete-postgres-secret: kubectl delete secret postgres-auth -n ${LITELLM_NAMESPACE} --ignore-not-found # Install LiteLLM install: #!/bin/bash set -euo pipefail just check-prerequisites just verify-api-keys while [ -z "${LITELLM_HOST}" ]; do LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80 \ --placeholder="e.g., litellm.example.com") done while [ -z "${KEYCLOAK_HOST}" ]; do KEYCLOAK_HOST=$(gum input --prompt="Keycloak host (FQDN): " --width=80 \ --placeholder="e.g., auth.example.com") done if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then if [ -z "${MONITORING_ENABLED}" ]; then if gum confirm "Enable Prometheus monitoring?"; then MONITORING_ENABLED="true" fi fi fi echo "Installing LiteLLM..." just create-namespace # Note: LiteLLM requires baseline due to Prisma needing /.cache write access kubectl label namespace ${LITELLM_NAMESPACE} \ pod-security.kubernetes.io/enforce=baseline --overwrite if [ "${MONITORING_ENABLED}" = "true" ]; then kubectl label namespace ${LITELLM_NAMESPACE} \ buun.channel/enable-monitoring=true --overwrite fi just create-api-key-external-secret echo "Setting up PostgreSQL database..." just create-postgres-user-and-db just create-postgres-secret echo "Setting up Keycloak OIDC authentication..." just create-keycloak-client just create-keycloak-auth-secret LITELLM_OIDC_ENABLED="true" echo "Generating Helm values..." gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml echo "Installing LiteLLM Helm chart..." helm upgrade --install litellm oci://ghcr.io/berriai/litellm-helm \ --version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \ -f litellm-values.yaml echo "" echo "LiteLLM installed successfully!" echo "Access LiteLLM at: https://${LITELLM_HOST}" echo "SSO Callback URL: https://${LITELLM_HOST}/sso/callback" # Upgrade LiteLLM upgrade: #!/bin/bash set -euo pipefail just check-prerequisites just verify-api-keys while [ -z "${LITELLM_HOST}" ]; do LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80) done if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then if [ -z "${MONITORING_ENABLED}" ]; then if gum confirm "Enable Prometheus monitoring?"; then MONITORING_ENABLED="true" fi fi fi echo "Upgrading LiteLLM..." if [ "${MONITORING_ENABLED}" = "true" ]; then kubectl label namespace ${LITELLM_NAMESPACE} \ buun.channel/enable-monitoring=true --overwrite fi just create-api-key-external-secret echo "Generating Helm values..." gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml # Delete the migration job as it's immutable and blocks helm upgrade kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found echo "Upgrading LiteLLM Helm chart..." helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \ --version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \ -f litellm-values.yaml echo "" echo "LiteLLM upgraded successfully!" echo "Access LiteLLM at: https://${LITELLM_HOST}" # Uninstall LiteLLM uninstall: #!/bin/bash set -euo pipefail if ! gum confirm "Uninstall LiteLLM?"; then echo "Cancelled." exit 0 fi echo "Uninstalling LiteLLM..." helm uninstall litellm -n ${LITELLM_NAMESPACE} --ignore-not-found --wait just delete-api-key-external-secret just delete-postgres-secret just delete-namespace echo "LiteLLM uninstalled." # Clean up all resources including database cleanup: #!/bin/bash set -euo pipefail echo "This will delete:" echo " - LiteLLM deployment" echo " - LiteLLM database" echo " - All API keys from Vault" echo "" if ! gum confirm "Are you sure?"; then echo "Cancelled." exit 0 fi just uninstall || true just postgres::delete-db litellm || true # Clean up API keys from Vault providers=$(just get-required-providers 2>/dev/null || true) for provider in $providers; do just vault::delete "litellm/${provider}" || true done echo "Cleanup completed." # Generate virtual key generate-virtual-key user='' model='': #!/bin/bash set -euo pipefail user="{{ user }}" while [ -z "${user}" ]; do user=$(gum input --prompt="Username: " --width=80) done model="{{ model }}" if [ -z "${model}" ]; then models=$(yq -r '.[].model_name' models.yaml 2>/dev/null || echo "") if [ -n "$models" ]; then model=$(echo "$models" | gum choose --header="Select model:") else model=$(gum input --prompt="Model: " --width=80) fi fi master_key=$(kubectl get secret litellm-masterkey -n ${LITELLM_NAMESPACE} \ -o jsonpath="{.data.masterkey}" | base64 --decode) response=$(curl -s "https://${LITELLM_HOST}/key/generate" \ --header "Authorization: Bearer ${master_key}" \ --header "Content-Type: application/json" \ --data-raw "{\"models\": [\"${model}\"], \"metadata\": {\"user\": \"${user}\"}}") echo "${response}" | jq . echo "" echo "API Key: $(echo "${response}" | jq -r '.key')" # Get master key master-key: @kubectl get secret litellm-masterkey -n ${LITELLM_NAMESPACE} \ -o jsonpath="{.data.masterkey}" | base64 --decode # List users list-users: kubectl exec -n postgres postgres-cluster-1 -- psql -U postgres -d litellm -c \ "SELECT user_id, user_email, user_role FROM \"LiteLLM_UserTable\";" # Assign role to user assign-role user='' role='': #!/bin/bash set -euo pipefail user="{{ user }}" role="{{ role }}" if [ -z "${user}" ]; then users=$(kubectl exec -n postgres postgres-cluster-1 -- psql -U postgres -d litellm -t -c \ "SELECT user_id FROM \"LiteLLM_UserTable\";" 2>/dev/null | tr -d ' ' | grep -v '^$') if [ -z "${users}" ]; then echo "No users found" exit 1 fi user=$(echo "${users}" | gum choose --header="Select user:") fi if [ -z "${role}" ]; then role=$(gum choose --header="Select role:" \ "proxy_admin" \ "proxy_admin_viewer" \ "internal_user" \ "internal_user_viewer") fi kubectl exec -n postgres postgres-cluster-1 -- psql -U postgres -d litellm -c \ "UPDATE \"LiteLLM_UserTable\" SET user_role = '${role}' WHERE user_id = '${user}';" echo "Assigned role '${role}' to user '${user}'" # Create Keycloak client for LiteLLM OIDC create-keycloak-client: #!/bin/bash set -euo pipefail while [ -z "${LITELLM_HOST}" ]; do LITELLM_HOST=$( gum input --prompt="LiteLLM host (FQDN): " --width=100 \ --placeholder="e.g., litellm.example.com" ) done echo "Creating Keycloak client for LiteLLM..." just keycloak::delete-client ${KEYCLOAK_REALM} ${LITELLM_OIDC_CLIENT_ID} || true CLIENT_SECRET=$(just utils::random-password) just keycloak::create-client \ realm=${KEYCLOAK_REALM} \ client_id=${LITELLM_OIDC_CLIENT_ID} \ redirect_url="https://${LITELLM_HOST}/sso/callback" \ client_secret="${CLIENT_SECRET}" # Store temporarily in k8s secret for create-keycloak-auth-secret to pick up kubectl delete secret litellm-oauth-temp -n ${LITELLM_NAMESPACE} --ignore-not-found kubectl create secret generic litellm-oauth-temp -n ${LITELLM_NAMESPACE} \ --from-literal=client_id="${LITELLM_OIDC_CLIENT_ID}" \ --from-literal=client_secret="${CLIENT_SECRET}" echo "Keycloak client created successfully" echo "Client ID: ${LITELLM_OIDC_CLIENT_ID}" echo "Redirect URI: https://${LITELLM_HOST}/sso/callback" # Delete Keycloak client for LiteLLM delete-keycloak-client: #!/bin/bash set -euo pipefail echo "Deleting Keycloak client for LiteLLM..." just keycloak::delete-client ${KEYCLOAK_REALM} ${LITELLM_OIDC_CLIENT_ID} || true kubectl delete secret litellm-oauth-temp -n ${LITELLM_NAMESPACE} --ignore-not-found if just vault::exist keycloak/client/litellm &>/dev/null; then just vault::delete keycloak/client/litellm fi # Create Keycloak auth secret create-keycloak-auth-secret: #!/bin/bash set -euo pipefail # Prioritize temporary secret (freshly created) over Vault (potentially stale) if kubectl get secret litellm-oauth-temp -n ${LITELLM_NAMESPACE} &>/dev/null; then oauth_client_id=$(kubectl get secret litellm-oauth-temp -n ${LITELLM_NAMESPACE} \ -o jsonpath='{.data.client_id}' | base64 -d) oauth_client_secret=$(kubectl get secret litellm-oauth-temp -n ${LITELLM_NAMESPACE} \ -o jsonpath='{.data.client_secret}' | base64 -d) elif helm status vault -n ${K8S_VAULT_NAMESPACE} &>/dev/null && \ just vault::get keycloak/client/litellm client_secret &>/dev/null; then oauth_client_id=$(just vault::get keycloak/client/litellm client_id) oauth_client_secret=$(just vault::get keycloak/client/litellm client_secret) else echo "Error: Cannot retrieve OAuth client secret. Please run 'just litellm::create-keycloak-client' first." exit 1 fi if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then echo "External Secrets Operator detected. Storing secrets in Vault..." # Store OAuth credentials in Vault just vault::put keycloak/client/litellm \ client_id="${oauth_client_id}" \ client_secret="${oauth_client_secret}" # Delete existing secrets and ExternalSecrets kubectl delete secret keycloak-auth -n ${LITELLM_NAMESPACE} --ignore-not-found kubectl delete externalsecret keycloak-auth-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found # Create ExternalSecret gomplate -f keycloak-auth-external-secret.gomplate.yaml | kubectl apply -f - echo "Waiting for ExternalSecret to sync..." kubectl wait --for=condition=Ready externalsecret/keycloak-auth-external-secret \ -n ${LITELLM_NAMESPACE} --timeout=60s echo "ExternalSecret created successfully" else echo "External Secrets Operator not found. Creating Kubernetes Secret directly..." # Create Keycloak OAuth Secret kubectl delete secret keycloak-auth -n ${LITELLM_NAMESPACE} --ignore-not-found kubectl create secret generic keycloak-auth -n ${LITELLM_NAMESPACE} \ --from-literal=GENERIC_CLIENT_ID="${oauth_client_id}" \ --from-literal=GENERIC_CLIENT_SECRET="${oauth_client_secret}" # Store credentials in Vault if available (backup for admin credentials) if helm status vault -n ${K8S_VAULT_NAMESPACE} &>/dev/null; then just vault::put keycloak/client/litellm \ client_id="${oauth_client_id}" \ client_secret="${oauth_client_secret}" fi echo "Kubernetes Secret created successfully" fi # Clean up temporary OAuth secret kubectl delete secret litellm-oauth-temp -n ${LITELLM_NAMESPACE} --ignore-not-found # Delete Keycloak auth secret delete-keycloak-auth-secret: kubectl delete externalsecret keycloak-auth-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found kubectl delete secret keycloak-auth -n ${LITELLM_NAMESPACE} --ignore-not-found # Setup OIDC authentication for Admin UI setup-oidc: #!/bin/bash set -euo pipefail echo "Setting up OIDC authentication for LiteLLM Admin UI..." while [ -z "${LITELLM_HOST}" ]; do LITELLM_HOST=$( gum input --prompt="LiteLLM host (FQDN): " --width=100 \ --placeholder="e.g., litellm.example.com" ) done while [ -z "${KEYCLOAK_HOST}" ]; do KEYCLOAK_HOST=$( gum input --prompt="Keycloak host (FQDN): " --width=100 \ --placeholder="e.g., auth.example.com" ) done just create-keycloak-client just create-keycloak-auth-secret LITELLM_OIDC_ENABLED="true" if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then if [ -z "${MONITORING_ENABLED}" ]; then if gum confirm "Enable Prometheus monitoring?"; then MONITORING_ENABLED="true" fi fi fi echo "Generating Helm values with OIDC configuration..." gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml # Delete the migration job as it's immutable and blocks helm upgrade kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found echo "Upgrading LiteLLM with OIDC configuration..." helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \ --version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \ -f litellm-values.yaml echo "" echo "OIDC authentication configured successfully!" echo "Access LiteLLM at: https://${LITELLM_HOST}" echo "SSO Callback URL: https://${LITELLM_HOST}/sso/callback" # Disable OIDC authentication disable-oidc: #!/bin/bash set -euo pipefail echo "Disabling OIDC authentication for LiteLLM Admin UI..." LITELLM_OIDC_ENABLED="" while [ -z "${LITELLM_HOST}" ]; do LITELLM_HOST=$( gum input --prompt="LiteLLM host (FQDN): " --width=100 \ --placeholder="e.g., litellm.example.com" ) done if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then if [ -z "${MONITORING_ENABLED}" ]; then if gum confirm "Enable Prometheus monitoring?"; then MONITORING_ENABLED="true" fi fi fi echo "Generating Helm values without OIDC..." gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml # Delete the migration job as it's immutable and blocks helm upgrade kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found echo "Upgrading LiteLLM without OIDC..." helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \ --version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \ -f litellm-values.yaml echo "" echo "OIDC authentication disabled." echo "Note: Keycloak client was NOT deleted. To clean up, run:" echo " just litellm::delete-keycloak-client"