From f3bc41e9ebee372d80c5a1386c08fa63558faf5b Mon Sep 17 00:00:00 2001 From: Masaki Yatsu Date: Fri, 5 Dec 2025 10:11:46 +0900 Subject: [PATCH] feat(litellm): litellm -> langfuse integration and some fixes --- CLAUDE.md | 3 +- litellm/README.md | 122 ++++- litellm/justfile | 438 ++++++++++++++++-- ...angfuse-auth-external-secret.gomplate.yaml | 22 + litellm/litellm-values.gomplate.yaml | 39 +- 5 files changed, 589 insertions(+), 35 deletions(-) create mode 100644 litellm/langfuse-auth-external-secret.gomplate.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 83cc4df..29798e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -429,6 +429,7 @@ When writing Markdown documentation: ``` 2. **Always validate with markdownlint-cli2**: - - Run `markdownlint-cli2 ` before committing any Markdown files + - Run from the project root directory to use `.markdownlint.yaml` config: + `cd && markdownlint-cli2 ` - Fix all linting errors to ensure consistent formatting - Pay attention to code block language specifications (MD040) and list formatting (MD029) diff --git a/litellm/README.md b/litellm/README.md index 174a2c3..e6cd40f 100644 --- a/litellm/README.md +++ b/litellm/README.md @@ -241,7 +241,7 @@ just litellm::master-key just litellm::generate-virtual-key buun ``` -This will prompt for a model selection and generate an API key for the specified user. +This will prompt for a model selection and generate an API key for the specified user. Select `all` to grant access to all models. ### OpenAI SDK Example @@ -272,6 +272,123 @@ curl https://litellm.example.com/v1/chat/completions \ }' ``` +## Team Management + +Teams allow you to group users and configure team-specific settings such as Langfuse projects for observability. + +### Create a Team + +```bash +just litellm::create-team +``` + +Or with a name directly: + +```bash +just litellm::create-team name="project-alpha" +``` + +### List Teams + +```bash +just litellm::list-teams +``` + +### Get Team Info + +```bash +just litellm::get-team team_id= +``` + +### Delete a Team + +```bash +just litellm::delete-team team_id= +``` + +### Generate Virtual Key for a Team + +```bash +just litellm::generate-team-key +``` + +This will prompt for team selection and username. The generated key inherits the team's settings (including Langfuse project configuration). + +## Langfuse Integration + +[Langfuse](https://langfuse.com/) provides LLM observability with tracing, monitoring, and analytics. LiteLLM can send traces to Langfuse for every API call. + +### Enable Langfuse Integration + +During installation (`just litellm::install`) or upgrade (`just litellm::upgrade`), you will be prompted to enable Langfuse integration. Alternatively: + +```bash +just litellm::setup-langfuse +``` + +You will need Langfuse API keys (Public Key and Secret Key) from the Langfuse UI: **Settings > API Keys**. + +### Set Langfuse API Keys + +```bash +just litellm::set-langfuse-keys +``` + +### Disable Langfuse Integration + +```bash +just litellm::disable-langfuse +``` + +### Per-Team Langfuse Projects + +Each team can have its own Langfuse project for isolated observability. This is useful when different projects or departments need separate trace data. + +#### Setup Flow + +1. Create a team: + + ```bash + just litellm::create-team name="project-alpha" + ``` + +2. Create a Langfuse project for the team and get API keys from Langfuse UI + +3. Configure the team's Langfuse project: + + ```bash + just litellm::set-team-langfuse-project + ``` + + This will prompt for team selection and Langfuse API keys. + +4. Generate a key for the team: + + ```bash + just litellm::generate-team-key + ``` + +5. Use the team key for API calls - traces will be sent to the team's Langfuse project + +#### Architecture + +```plain +LiteLLM Proxy + | + +-- Default Langfuse Project (for keys without team) + | + +-- Team A --> Langfuse Project A + | + +-- Team B --> Langfuse Project B +``` + +### Environment Variables + +| Variable | Default | Description | +| -------- | ------- | ----------- | +| `LITELLM_LANGFUSE_INTEGRATION_ENABLED` | (prompt) | Enable Langfuse integration | +| `LANGFUSE_HOST` | (prompt) | Langfuse instance hostname | + ## Supported Providers | Provider | Model Prefix | API Key Required | @@ -408,6 +525,7 @@ kubectl exec -n litellm deployment/litellm -- \ | `litellm-values.gomplate.yaml` | Helm values template | | `apikey-external-secret.gomplate.yaml` | ExternalSecret for API keys | | `keycloak-auth-external-secret.gomplate.yaml` | ExternalSecret for Keycloak OIDC | +| `langfuse-auth-external-secret.gomplate.yaml` | ExternalSecret for Langfuse API keys | ## Security Considerations @@ -425,3 +543,5 @@ kubectl exec -n litellm deployment/litellm -- \ - [LiteLLM Helm Chart](https://github.com/BerriAI/litellm/tree/main/deploy/charts/litellm-helm) - [Supported Models](https://docs.litellm.ai/docs/providers) - [Virtual Keys](https://docs.litellm.ai/docs/proxy/virtual_keys) +- [Langfuse Integration](https://docs.litellm.ai/docs/proxy/logging#langfuse) +- [Team-based Logging](https://docs.litellm.ai/docs/proxy/team_logging) diff --git a/litellm/justfile b/litellm/justfile index 29ac361..d2d2b62 100644 --- a/litellm/justfile +++ b/litellm/justfile @@ -5,12 +5,15 @@ export LITELLM_CHART_VERSION := env("LITELLM_CHART_VERSION", "0.1.825") export LITELLM_HOST := env("LITELLM_HOST", "") export LITELLM_OIDC_CLIENT_ID := env("LITELLM_OIDC_CLIENT_ID", "litellm") export LITELLM_OIDC_ENABLED := env("LITELLM_OIDC_ENABLED", "") +export LITELLM_LANGFUSE_INTEGRATION_ENABLED := env("LITELLM_LANGFUSE_INTEGRATION_ENABLED", "") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") export OLLAMA_NAMESPACE := env("OLLAMA_NAMESPACE", "ollama") export PROMETHEUS_NAMESPACE := env("PROMETHEUS_NAMESPACE", "monitoring") export MONITORING_ENABLED := env("MONITORING_ENABLED", "") export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") export KEYCLOAK_HOST := env("KEYCLOAK_HOST", "") +export LANGFUSE_HOST := env("LANGFUSE_HOST", "") +export LANGFUSE_NAMESPACE := env("LANGFUSE_NAMESPACE", "langfuse") export K8S_VAULT_NAMESPACE := env("K8S_VAULT_NAMESPACE", "vault") [private] @@ -131,9 +134,9 @@ add-model: case $provider in anthropic) model=$(gum choose --header="Select Anthropic model:" \ - "claude-sonnet-4-20250514" \ - "claude-haiku-4-20251015" \ - "claude-opus-4-20250514") + "claude-sonnet-4-5" \ + "claude-haiku-4-5" \ + "claude-opus-4-5") api_key_line=" api_key: os.environ/ANTHROPIC_API_KEY" ;; openai) @@ -311,6 +314,19 @@ install: fi fi + if [ -z "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" ]; then + if gum confirm "Enable Langfuse integration?"; then + LITELLM_LANGFUSE_INTEGRATION_ENABLED="true" + fi + fi + + if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then + while [ -z "${LANGFUSE_HOST}" ]; do + LANGFUSE_HOST=$(gum input --prompt="Langfuse host (FQDN): " --width=80 \ + --placeholder="e.g., langfuse.example.com") + done + fi + echo "Installing LiteLLM..." just create-namespace @@ -335,6 +351,18 @@ install: just create-keycloak-auth-secret LITELLM_OIDC_ENABLED="true" + if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then + echo "Setting up Langfuse integration..." + if ! just vault::exist litellm/langfuse &>/dev/null; then + echo "" + echo "Langfuse API keys are required." + echo "You can get these from Langfuse UI: Settings > API Keys" + echo "" + just set-langfuse-keys + fi + just create-langfuse-auth-secret + fi + echo "Generating Helm values..." gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml @@ -347,6 +375,9 @@ install: echo "LiteLLM installed successfully!" echo "Access LiteLLM at: https://${LITELLM_HOST}" echo "SSO Callback URL: https://${LITELLM_HOST}/sso/callback" + if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then + echo "Langfuse integration: enabled (https://${LANGFUSE_HOST})" + fi # Upgrade LiteLLM upgrade: @@ -360,6 +391,18 @@ upgrade: LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80) done + # Detect existing OIDC configuration + if [ -z "${LITELLM_OIDC_ENABLED}" ]; then + if kubectl get secret keycloak-auth -n ${LITELLM_NAMESPACE} &>/dev/null; then + echo "Detected existing OIDC configuration" + LITELLM_OIDC_ENABLED="true" + if [ -z "${KEYCLOAK_HOST}" ]; then + KEYCLOAK_HOST=$(gum input --prompt="Keycloak host (FQDN): " --width=80 \ + --placeholder="e.g., auth.example.com") + fi + fi + fi + if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then if [ -z "${MONITORING_ENABLED}" ]; then if gum confirm "Enable Prometheus monitoring?"; then @@ -368,6 +411,23 @@ upgrade: fi fi + # Detect existing Langfuse configuration + if [ -z "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" ]; then + if kubectl get secret langfuse-auth -n ${LITELLM_NAMESPACE} &>/dev/null; then + echo "Detected existing Langfuse configuration" + LITELLM_LANGFUSE_INTEGRATION_ENABLED="true" + elif gum confirm "Enable Langfuse integration?"; then + LITELLM_LANGFUSE_INTEGRATION_ENABLED="true" + fi + fi + + if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then + while [ -z "${LANGFUSE_HOST}" ]; do + LANGFUSE_HOST=$(gum input --prompt="Langfuse host (FQDN): " --width=80 \ + --placeholder="e.g., langfuse.example.com") + done + fi + echo "Upgrading LiteLLM..." if [ "${MONITORING_ENABLED}" = "true" ]; then @@ -377,6 +437,17 @@ upgrade: just create-api-key-external-secret + if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then + if ! just vault::exist litellm/langfuse &>/dev/null; then + echo "" + echo "Langfuse API keys are required." + echo "You can get these from Langfuse UI: Settings > API Keys" + echo "" + just set-langfuse-keys + fi + just create-langfuse-auth-secret + fi + echo "Generating Helm values..." gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml @@ -391,9 +462,15 @@ upgrade: echo "" echo "LiteLLM upgraded successfully!" echo "Access LiteLLM at: https://${LITELLM_HOST}" + if [ "${LITELLM_OIDC_ENABLED}" = "true" ]; then + echo "SSO authentication: enabled" + fi + if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then + echo "Langfuse integration: enabled (https://${LANGFUSE_HOST})" + fi -# Uninstall LiteLLM -uninstall: +# Uninstall LiteLLM (delete-data: true to delete database and Vault secrets) +uninstall delete-data='false': #!/bin/bash set -euo pipefail if ! gum confirm "Uninstall LiteLLM?"; then @@ -404,30 +481,33 @@ uninstall: helm uninstall litellm -n ${LITELLM_NAMESPACE} --ignore-not-found --wait just delete-api-key-external-secret just delete-postgres-secret + just delete-keycloak-client || true + just delete-keycloak-auth-secret || true + just delete-langfuse-auth-secret || true just delete-namespace - echo "LiteLLM uninstalled." -# Clean up all resources including database -cleanup: - #!/bin/bash - set -euo pipefail - echo "This will delete:" - echo " - LiteLLM deployment" - echo " - LiteLLM database" - echo " - All API keys from Vault" - echo "" - if ! gum confirm "Are you sure?"; then - echo "Cancelled." - exit 0 + if [ "{{ delete-data }}" = "true" ]; then + echo "Deleting database and Vault secrets..." + just postgres::delete-user-and-db litellm litellm || true + just vault::delete litellm/db || true + just vault::delete keycloak/client/litellm || true + just vault::delete litellm/langfuse || true + # Clean up API keys from Vault + providers=$(just get-required-providers 2>/dev/null || true) + for provider in $providers; do + just vault::delete "litellm/${provider}" || true + done + echo "LiteLLM uninstalled with all data deleted." + else + echo "LiteLLM uninstalled." + echo "" + echo "Note: The following resources were NOT deleted:" + echo " - PostgreSQL user and database (litellm)" + echo " - Vault secrets (litellm/*, keycloak/client/litellm)" + echo "" + echo "To delete all data, run:" + echo " just litellm::uninstall true" fi - just uninstall || true - just postgres::delete-db litellm || true - # Clean up API keys from Vault - providers=$(just get-required-providers 2>/dev/null || true) - for provider in $providers; do - just vault::delete "litellm/${provider}" || true - done - echo "Cleanup completed." # Generate virtual key generate-virtual-key user='' model='': @@ -441,17 +521,22 @@ generate-virtual-key user='' model='': if [ -z "${model}" ]; then models=$(yq -r '.[].model_name' models.yaml 2>/dev/null || echo "") if [ -n "$models" ]; then - model=$(echo "$models" | gum choose --header="Select model:") + model=$(echo -e "all\n$models" | gum choose --header="Select model (or 'all' for all models):") else - model=$(gum input --prompt="Model: " --width=80) + model=$(gum input --prompt="Model (or 'all' for all models): " --width=80) fi fi master_key=$(kubectl get secret litellm-masterkey -n ${LITELLM_NAMESPACE} \ -o jsonpath="{.data.masterkey}" | base64 --decode) + if [ "${model}" = "all" ]; then + data="{\"metadata\": {\"user\": \"${user}\"}}" + else + data="{\"models\": [\"${model}\"], \"metadata\": {\"user\": \"${user}\"}}" + fi response=$(curl -s "https://${LITELLM_HOST}/key/generate" \ --header "Authorization: Bearer ${master_key}" \ --header "Content-Type: application/json" \ - --data-raw "{\"models\": [\"${model}\"], \"metadata\": {\"user\": \"${user}\"}}") + --data-raw "${data}") echo "${response}" | jq . echo "" echo "API Key: $(echo "${response}" | jq -r '.key')" @@ -692,3 +777,298 @@ disable-oidc: echo "OIDC authentication disabled." echo "Note: Keycloak client was NOT deleted. To clean up, run:" echo " just litellm::delete-keycloak-client" + +# Set Langfuse API keys for LiteLLM integration +set-langfuse-keys public_key='' secret_key='': + #!/bin/bash + set -euo pipefail + public_key="{{ public_key }}" + secret_key="{{ secret_key }}" + if [ -z "${public_key}" ]; then + public_key=$(gum input --prompt="Langfuse Public Key: " --width=80) + fi + if [ -z "${secret_key}" ]; then + secret_key=$(gum input --prompt="Langfuse Secret Key: " --password --width=80) + fi + if [ -z "${public_key}" ] || [ -z "${secret_key}" ]; then + echo "Error: Both public key and secret key are required" + exit 1 + fi + just vault::put litellm/langfuse public_key="${public_key}" secret_key="${secret_key}" + echo "Langfuse API keys stored in Vault at 'litellm/langfuse'" + +# Get Langfuse API keys +get-langfuse-keys: + #!/bin/bash + set -euo pipefail + echo "Public Key: $(just vault::get litellm/langfuse public_key)" + echo "Secret Key: $(just vault::get litellm/langfuse secret_key)" + +# Delete Langfuse API keys +delete-langfuse-keys: + #!/bin/bash + set -euo pipefail + if just vault::exist litellm/langfuse &>/dev/null; then + just vault::delete litellm/langfuse + echo "Langfuse API keys deleted from Vault" + else + echo "Langfuse API keys not found in Vault" + fi + +# Create Langfuse auth external secret +create-langfuse-auth-secret: + #!/bin/bash + set -euo pipefail + if ! just vault::exist litellm/langfuse &>/dev/null; then + echo "Error: Langfuse API keys not found in Vault." + echo "Please set the keys first:" + echo " just litellm::set-langfuse-keys" + exit 1 + fi + kubectl delete externalsecret langfuse-auth-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found + kubectl delete secret langfuse-auth -n ${LITELLM_NAMESPACE} --ignore-not-found + gomplate -f langfuse-auth-external-secret.gomplate.yaml | kubectl apply -f - + echo "Waiting for Langfuse auth secret to be ready..." + kubectl wait --for=condition=Ready externalsecret/langfuse-auth-external-secret \ + -n ${LITELLM_NAMESPACE} --timeout=60s + echo "Langfuse auth secret created" + +# Delete Langfuse auth external secret +delete-langfuse-auth-secret: + kubectl delete externalsecret langfuse-auth-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found + kubectl delete secret langfuse-auth -n ${LITELLM_NAMESPACE} --ignore-not-found + +# Setup Langfuse integration +setup-langfuse: + #!/bin/bash + set -euo pipefail + + echo "Setting up Langfuse integration for LiteLLM..." + + if ! helm status langfuse -n ${LANGFUSE_NAMESPACE} &>/dev/null; then + echo "Warning: Langfuse is not installed." + echo "Please install Langfuse first or provide external Langfuse credentials." + fi + + while [ -z "${LANGFUSE_HOST}" ]; do + LANGFUSE_HOST=$( + gum input --prompt="Langfuse host (FQDN): " --width=100 \ + --placeholder="e.g., langfuse.example.com" + ) + done + + if ! just vault::exist litellm/langfuse &>/dev/null; then + echo "" + echo "Langfuse API keys are required." + echo "You can get these from Langfuse UI: Settings > API Keys" + echo "" + just set-langfuse-keys + fi + + just create-langfuse-auth-secret + + while [ -z "${LITELLM_HOST}" ]; do + LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80) + done + + if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then + if [ -z "${MONITORING_ENABLED}" ]; then + if gum confirm "Enable Prometheus monitoring?"; then + MONITORING_ENABLED="true" + fi + fi + fi + + LITELLM_LANGFUSE_INTEGRATION_ENABLED="true" + + echo "Generating Helm values with Langfuse integration..." + gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml + + kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found + + echo "Upgrading LiteLLM with Langfuse integration..." + helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \ + --version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \ + -f litellm-values.yaml + + echo "" + echo "Langfuse integration configured successfully!" + echo "LiteLLM will now send traces to: https://${LANGFUSE_HOST}" + +# Disable Langfuse integration +disable-langfuse: + #!/bin/bash + set -euo pipefail + + echo "Disabling Langfuse integration for LiteLLM..." + + LITELLM_LANGFUSE_INTEGRATION_ENABLED="" + + while [ -z "${LITELLM_HOST}" ]; do + LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80) + done + + if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then + if [ -z "${MONITORING_ENABLED}" ]; then + if gum confirm "Enable Prometheus monitoring?"; then + MONITORING_ENABLED="true" + fi + fi + fi + + echo "Generating Helm values without Langfuse..." + gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml + + kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found + + echo "Upgrading LiteLLM without Langfuse..." + helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \ + --version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \ + -f litellm-values.yaml + + echo "" + echo "Langfuse integration disabled." + echo "Note: Langfuse API keys were NOT deleted. To clean up, run:" + echo " just litellm::delete-langfuse-keys" + +# Create a team +create-team name='': + #!/bin/bash + set -euo pipefail + name="{{ name }}" + while [ -z "${name}" ]; do + name=$(gum input --prompt="Team name: " --width=80) + done + master_key=$(just master-key) + response=$(curl -s "https://${LITELLM_HOST}/team/new" \ + --header "Authorization: Bearer ${master_key}" \ + --header "Content-Type: application/json" \ + --data-raw "{\"team_alias\": \"${name}\"}") + echo "${response}" | jq . + team_id=$(echo "${response}" | jq -r '.team_id') + echo "" + echo "Team created: ${name}" + echo "Team ID: ${team_id}" + +# List teams +list-teams: + #!/bin/bash + set -euo pipefail + master_key=$(just master-key) + curl -s "https://${LITELLM_HOST}/team/list" \ + --header "Authorization: Bearer ${master_key}" | jq . + +# Get team info +get-team team_id='': + #!/bin/bash + set -euo pipefail + team_id="{{ team_id }}" + if [ -z "${team_id}" ]; then + echo "Usage: just litellm::get-team team_id=" + exit 1 + fi + master_key=$(just master-key) + curl -s "https://${LITELLM_HOST}/team/info?team_id=${team_id}" \ + --header "Authorization: Bearer ${master_key}" | jq . + +# Delete a team +delete-team team_id='': + #!/bin/bash + set -euo pipefail + team_id="{{ team_id }}" + if [ -z "${team_id}" ]; then + echo "Usage: just litellm::delete-team team_id=" + exit 1 + fi + if ! gum confirm "Delete team ${team_id}?"; then + echo "Cancelled." + exit 0 + fi + master_key=$(just master-key) + curl -s "https://${LITELLM_HOST}/team/delete" \ + --header "Authorization: Bearer ${master_key}" \ + --header "Content-Type: application/json" \ + --data-raw "{\"team_ids\": [\"${team_id}\"]}" | jq . + echo "Team deleted." + +# Set Langfuse project for a team +set-team-langfuse-project team_id='' public_key='' secret_key='': + #!/bin/bash + set -euo pipefail + team_id="{{ team_id }}" + public_key="{{ public_key }}" + secret_key="{{ secret_key }}" + if [ -z "${team_id}" ]; then + master_key=$(just master-key) + teams=$(curl -s "https://${LITELLM_HOST}/team/list" \ + --header "Authorization: Bearer ${master_key}") + team_aliases=$(echo "${teams}" | jq -r '.[] | "\(.team_id) (\(.team_alias // "no alias"))"') + if [ -z "${team_aliases}" ]; then + echo "No teams found. Create a team first:" + echo " just litellm::create-team" + exit 1 + fi + selected=$(echo "${team_aliases}" | gum choose --header="Select team:") + team_id=$(echo "${selected}" | cut -d' ' -f1) + fi + if [ -z "${public_key}" ]; then + public_key=$(gum input --prompt="Langfuse Public Key for this team: " --width=80) + fi + if [ -z "${secret_key}" ]; then + secret_key=$(gum input --prompt="Langfuse Secret Key for this team: " --password --width=80) + fi + if [ -z "${public_key}" ] || [ -z "${secret_key}" ]; then + echo "Error: Both public key and secret key are required" + exit 1 + fi + master_key=$(just master-key) + response=$(curl -s "https://${LITELLM_HOST}/team/update" \ + --header "Authorization: Bearer ${master_key}" \ + --header "Content-Type: application/json" \ + --data-raw "{ + \"team_id\": \"${team_id}\", + \"metadata\": { + \"logging\": [{ + \"callback_name\": \"langfuse\", + \"callback_vars\": { + \"langfuse_public_key\": \"${public_key}\", + \"langfuse_secret\": \"${secret_key}\", + \"langfuse_host\": \"https://${LANGFUSE_HOST}\" + } + }] + } + }") + echo "${response}" | jq . + echo "" + echo "Langfuse project configured for team ${team_id}" + +# Generate virtual key for a team +generate-team-key team_id='' user='': + #!/bin/bash + set -euo pipefail + team_id="{{ team_id }}" + user="{{ user }}" + if [ -z "${team_id}" ]; then + master_key=$(just master-key) + teams=$(curl -s "https://${LITELLM_HOST}/team/list" \ + --header "Authorization: Bearer ${master_key}") + team_aliases=$(echo "${teams}" | jq -r '.[] | "\(.team_id) (\(.team_alias // "no alias"))"') + if [ -z "${team_aliases}" ]; then + echo "No teams found. Create a team first:" + echo " just litellm::create-team" + exit 1 + fi + selected=$(echo "${team_aliases}" | gum choose --header="Select team:") + team_id=$(echo "${selected}" | cut -d' ' -f1) + fi + while [ -z "${user}" ]; do + user=$(gum input --prompt="Username: " --width=80) + done + master_key=$(just master-key) + response=$(curl -s "https://${LITELLM_HOST}/key/generate" \ + --header "Authorization: Bearer ${master_key}" \ + --header "Content-Type: application/json" \ + --data-raw "{\"team_id\": \"${team_id}\", \"metadata\": {\"user\": \"${user}\"}}") + echo "${response}" | jq . + echo "" + echo "API Key: $(echo "${response}" | jq -r '.key')" diff --git a/litellm/langfuse-auth-external-secret.gomplate.yaml b/litellm/langfuse-auth-external-secret.gomplate.yaml new file mode 100644 index 0000000..910e6d7 --- /dev/null +++ b/litellm/langfuse-auth-external-secret.gomplate.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: langfuse-auth-external-secret + namespace: {{ .Env.LITELLM_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: langfuse-auth + creationPolicy: Owner + data: + - secretKey: LANGFUSE_PUBLIC_KEY + remoteRef: + key: litellm/langfuse + property: public_key + - secretKey: LANGFUSE_SECRET_KEY + remoteRef: + key: litellm/langfuse + property: secret_key diff --git a/litellm/litellm-values.gomplate.yaml b/litellm/litellm-values.gomplate.yaml index 765d31f..440c2ad 100644 --- a/litellm/litellm-values.gomplate.yaml +++ b/litellm/litellm-values.gomplate.yaml @@ -10,6 +10,16 @@ podSecurityContext: {} securityContext: {} +# Resource recommendations from Goldilocks VPA +# litellm target: cpu=11m, memory=549Mi +resources: + requests: + cpu: 25m + memory: 512Mi + limits: + cpu: 100m + memory: 1Gi + migrationJob: resources: requests: @@ -18,12 +28,17 @@ migrationJob: limits: memory: 1Gi -{{- if .Env.LITELLM_OIDC_ENABLED }} environmentSecrets: - apikey +{{- if .Env.LITELLM_OIDC_ENABLED }} - keycloak-auth +{{- end }} +{{- if .Env.LITELLM_LANGFUSE_INTEGRATION_ENABLED }} + - langfuse-auth +{{- end }} extraEnvVars: +{{- if .Env.LITELLM_OIDC_ENABLED }} - name: PROXY_BASE_URL value: "https://{{ .Env.LITELLM_HOST }}" - name: GENERIC_AUTHORIZATION_ENDPOINT @@ -34,14 +49,20 @@ extraEnvVars: value: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid-connect/userinfo" - name: GENERIC_SCOPE value: "openid email profile" -{{- else }} -environmentSecrets: - - apikey +{{- end }} +{{- if .Env.LITELLM_LANGFUSE_INTEGRATION_ENABLED }} + - name: LANGFUSE_HOST + value: "https://{{ .Env.LANGFUSE_HOST }}" {{- end }} proxy_config: model_list: {{ file.Read "models.yaml" | indent 4 }} +{{- if .Env.LITELLM_LANGFUSE_INTEGRATION_ENABLED }} + litellm_settings: + success_callback: ["langfuse"] + failure_callback: ["langfuse"] +{{- end }} db: useExisting: true @@ -57,6 +78,16 @@ db: redis: enabled: true + # Resource recommendations from Goldilocks VPA + # redis target: cpu=15m, memory=100Mi + master: + resources: + requests: + cpu: 25m + memory: 128Mi + limits: + cpu: 100m + memory: 256Mi ingress: enabled: true