baschno/buun-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(litellm): litellm -> langfuse integration and some fixes

Browse Source
This commit is contained in:
Masaki Yatsu
2025-12-05 10:11:46 +09:00
parent 98b03704d7
commit f3bc41e9eb
5 changed files with 589 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CLAUDE.md
View File

@@ -429,6 +429,7 @@ When writing Markdown documentation:
``` ```
2. **Always validate with markdownlint-cli2**: 2. **Always validate with markdownlint-cli2**:
- Run `markdownlint-cli2 <file>` before committing any Markdown files - Run from the project root directory to use `.markdownlint.yaml` config:
`cd <top-dir> && markdownlint-cli2 <relative-path>`
- Fix all linting errors to ensure consistent formatting - Fix all linting errors to ensure consistent formatting
- Pay attention to code block language specifications (MD040) and list formatting (MD029) - Pay attention to code block language specifications (MD040) and list formatting (MD029)

122
litellm/README.md
View File

@@ -241,7 +241,7 @@ just litellm::master-key
just litellm::generate-virtual-key buun just litellm::generate-virtual-key buun
``` ```
This will prompt for a model selection and generate an API key for the specified user. This will prompt for a model selection and generate an API key for the specified user. Select `all` to grant access to all models.
### OpenAI SDK Example ### OpenAI SDK Example
@@ -272,6 +272,123 @@ curl https://litellm.example.com/v1/chat/completions \
}' }'
``` ```
## Team Management
Teams allow you to group users and configure team-specific settings such as Langfuse projects for observability.
### Create a Team
```bash
just litellm::create-team
```
Or with a name directly:
```bash
just litellm::create-team name="project-alpha"
```
### List Teams
```bash
just litellm::list-teams
```
### Get Team Info
```bash
just litellm::get-team team_id=<team-id>
```
### Delete a Team
```bash
just litellm::delete-team team_id=<team-id>
```
### Generate Virtual Key for a Team
```bash
just litellm::generate-team-key
```
This will prompt for team selection and username. The generated key inherits the team's settings (including Langfuse project configuration).
## Langfuse Integration
[Langfuse](https://langfuse.com/) provides LLM observability with tracing, monitoring, and analytics. LiteLLM can send traces to Langfuse for every API call.
### Enable Langfuse Integration
During installation (`just litellm::install`) or upgrade (`just litellm::upgrade`), you will be prompted to enable Langfuse integration. Alternatively:
```bash
just litellm::setup-langfuse
```
You will need Langfuse API keys (Public Key and Secret Key) from the Langfuse UI: **Settings > API Keys**.
### Set Langfuse API Keys
```bash
just litellm::set-langfuse-keys
```
### Disable Langfuse Integration
```bash
just litellm::disable-langfuse
```
### Per-Team Langfuse Projects
Each team can have its own Langfuse project for isolated observability. This is useful when different projects or departments need separate trace data.
#### Setup Flow
1. Create a team:
```bash
just litellm::create-team name="project-alpha"
```
2. Create a Langfuse project for the team and get API keys from Langfuse UI
3. Configure the team's Langfuse project:
```bash
just litellm::set-team-langfuse-project
```
This will prompt for team selection and Langfuse API keys.
4. Generate a key for the team:
```bash
just litellm::generate-team-key
```
5. Use the team key for API calls - traces will be sent to the team's Langfuse project
#### Architecture
```plain
LiteLLM Proxy
|
+-- Default Langfuse Project (for keys without team)
|
+-- Team A --> Langfuse Project A
|
+-- Team B --> Langfuse Project B
```
### Environment Variables
| Variable | Default | Description |
| -------- | ------- | ----------- |
| `LITELLM_LANGFUSE_INTEGRATION_ENABLED` | (prompt) | Enable Langfuse integration |
| `LANGFUSE_HOST` | (prompt) | Langfuse instance hostname |
## Supported Providers ## Supported Providers
| Provider | Model Prefix | API Key Required | | Provider | Model Prefix | API Key Required |
@@ -408,6 +525,7 @@ kubectl exec -n litellm deployment/litellm -- \
| `litellm-values.gomplate.yaml` | Helm values template | | `litellm-values.gomplate.yaml` | Helm values template |
| `apikey-external-secret.gomplate.yaml` | ExternalSecret for API keys | | `apikey-external-secret.gomplate.yaml` | ExternalSecret for API keys |
| `keycloak-auth-external-secret.gomplate.yaml` | ExternalSecret for Keycloak OIDC | | `keycloak-auth-external-secret.gomplate.yaml` | ExternalSecret for Keycloak OIDC |
| `langfuse-auth-external-secret.gomplate.yaml` | ExternalSecret for Langfuse API keys |
## Security Considerations ## Security Considerations
@@ -425,3 +543,5 @@ kubectl exec -n litellm deployment/litellm -- \
- [LiteLLM Helm Chart](https://github.com/BerriAI/litellm/tree/main/deploy/charts/litellm-helm) - [LiteLLM Helm Chart](https://github.com/BerriAI/litellm/tree/main/deploy/charts/litellm-helm)
- [Supported Models](https://docs.litellm.ai/docs/providers) - [Supported Models](https://docs.litellm.ai/docs/providers)
- [Virtual Keys](https://docs.litellm.ai/docs/proxy/virtual_keys) - [Virtual Keys](https://docs.litellm.ai/docs/proxy/virtual_keys)
- [Langfuse Integration](https://docs.litellm.ai/docs/proxy/logging#langfuse)
- [Team-based Logging](https://docs.litellm.ai/docs/proxy/team_logging)

438
litellm/justfile
View File

@@ -5,12 +5,15 @@ export LITELLM_CHART_VERSION := env("LITELLM_CHART_VERSION", "0.1.825")
export LITELLM_HOST := env("LITELLM_HOST", "") export LITELLM_HOST := env("LITELLM_HOST", "")
export LITELLM_OIDC_CLIENT_ID := env("LITELLM_OIDC_CLIENT_ID", "litellm") export LITELLM_OIDC_CLIENT_ID := env("LITELLM_OIDC_CLIENT_ID", "litellm")
export LITELLM_OIDC_ENABLED := env("LITELLM_OIDC_ENABLED", "") export LITELLM_OIDC_ENABLED := env("LITELLM_OIDC_ENABLED", "")
export LITELLM_LANGFUSE_INTEGRATION_ENABLED := env("LITELLM_LANGFUSE_INTEGRATION_ENABLED", "")
export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets")
export OLLAMA_NAMESPACE := env("OLLAMA_NAMESPACE", "ollama") export OLLAMA_NAMESPACE := env("OLLAMA_NAMESPACE", "ollama")
export PROMETHEUS_NAMESPACE := env("PROMETHEUS_NAMESPACE", "monitoring") export PROMETHEUS_NAMESPACE := env("PROMETHEUS_NAMESPACE", "monitoring")
export MONITORING_ENABLED := env("MONITORING_ENABLED", "") export MONITORING_ENABLED := env("MONITORING_ENABLED", "")
export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack")
export KEYCLOAK_HOST := env("KEYCLOAK_HOST", "") export KEYCLOAK_HOST := env("KEYCLOAK_HOST", "")
export LANGFUSE_HOST := env("LANGFUSE_HOST", "")
export LANGFUSE_NAMESPACE := env("LANGFUSE_NAMESPACE", "langfuse")
export K8S_VAULT_NAMESPACE := env("K8S_VAULT_NAMESPACE", "vault") export K8S_VAULT_NAMESPACE := env("K8S_VAULT_NAMESPACE", "vault")
[private] [private]
@@ -131,9 +134,9 @@ add-model:
case $provider in case $provider in
anthropic) anthropic)
model=$(gum choose --header="Select Anthropic model:" \ model=$(gum choose --header="Select Anthropic model:" \
"claude-sonnet-4-20250514" \ "claude-sonnet-4-5" \
"claude-haiku-4-20251015" \ "claude-haiku-4-5" \
"claude-opus-4-20250514") "claude-opus-4-5")
api_key_line=" api_key: os.environ/ANTHROPIC_API_KEY" api_key_line=" api_key: os.environ/ANTHROPIC_API_KEY"
;; ;;
openai) openai)
@@ -311,6 +314,19 @@ install:
fi fi
fi fi
if [ -z "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" ]; then
if gum confirm "Enable Langfuse integration?"; then
LITELLM_LANGFUSE_INTEGRATION_ENABLED="true"
fi
fi
if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then
while [ -z "${LANGFUSE_HOST}" ]; do
LANGFUSE_HOST=$(gum input --prompt="Langfuse host (FQDN): " --width=80 \
--placeholder="e.g., langfuse.example.com")
done
fi
echo "Installing LiteLLM..." echo "Installing LiteLLM..."
just create-namespace just create-namespace
@@ -335,6 +351,18 @@ install:
just create-keycloak-auth-secret just create-keycloak-auth-secret
LITELLM_OIDC_ENABLED="true" LITELLM_OIDC_ENABLED="true"
if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then
echo "Setting up Langfuse integration..."
if ! just vault::exist litellm/langfuse &>/dev/null; then
echo ""
echo "Langfuse API keys are required."
echo "You can get these from Langfuse UI: Settings > API Keys"
echo ""
just set-langfuse-keys
fi
just create-langfuse-auth-secret
fi
echo "Generating Helm values..." echo "Generating Helm values..."
gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml
@@ -347,6 +375,9 @@ install:
echo "LiteLLM installed successfully!" echo "LiteLLM installed successfully!"
echo "Access LiteLLM at: https://${LITELLM_HOST}" echo "Access LiteLLM at: https://${LITELLM_HOST}"
echo "SSO Callback URL: https://${LITELLM_HOST}/sso/callback" echo "SSO Callback URL: https://${LITELLM_HOST}/sso/callback"
if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then
echo "Langfuse integration: enabled (https://${LANGFUSE_HOST})"
fi
# Upgrade LiteLLM # Upgrade LiteLLM
upgrade: upgrade:
@@ -360,6 +391,18 @@ upgrade:
LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80) LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80)
done done
# Detect existing OIDC configuration
if [ -z "${LITELLM_OIDC_ENABLED}" ]; then
if kubectl get secret keycloak-auth -n ${LITELLM_NAMESPACE} &>/dev/null; then
echo "Detected existing OIDC configuration"
LITELLM_OIDC_ENABLED="true"
if [ -z "${KEYCLOAK_HOST}" ]; then
KEYCLOAK_HOST=$(gum input --prompt="Keycloak host (FQDN): " --width=80 \
--placeholder="e.g., auth.example.com")
fi
fi
fi
if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then
if [ -z "${MONITORING_ENABLED}" ]; then if [ -z "${MONITORING_ENABLED}" ]; then
if gum confirm "Enable Prometheus monitoring?"; then if gum confirm "Enable Prometheus monitoring?"; then
@@ -368,6 +411,23 @@ upgrade:
fi fi
fi fi
# Detect existing Langfuse configuration
if [ -z "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" ]; then
if kubectl get secret langfuse-auth -n ${LITELLM_NAMESPACE} &>/dev/null; then
echo "Detected existing Langfuse configuration"
LITELLM_LANGFUSE_INTEGRATION_ENABLED="true"
elif gum confirm "Enable Langfuse integration?"; then
LITELLM_LANGFUSE_INTEGRATION_ENABLED="true"
fi
fi
if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then
while [ -z "${LANGFUSE_HOST}" ]; do
LANGFUSE_HOST=$(gum input --prompt="Langfuse host (FQDN): " --width=80 \
--placeholder="e.g., langfuse.example.com")
done
fi
echo "Upgrading LiteLLM..." echo "Upgrading LiteLLM..."
if [ "${MONITORING_ENABLED}" = "true" ]; then if [ "${MONITORING_ENABLED}" = "true" ]; then
@@ -377,6 +437,17 @@ upgrade:
just create-api-key-external-secret just create-api-key-external-secret
if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then
if ! just vault::exist litellm/langfuse &>/dev/null; then
echo ""
echo "Langfuse API keys are required."
echo "You can get these from Langfuse UI: Settings > API Keys"
echo ""
just set-langfuse-keys
fi
just create-langfuse-auth-secret
fi
echo "Generating Helm values..." echo "Generating Helm values..."
gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml
@@ -391,9 +462,15 @@ upgrade:
echo "" echo ""
echo "LiteLLM upgraded successfully!" echo "LiteLLM upgraded successfully!"
echo "Access LiteLLM at: https://${LITELLM_HOST}" echo "Access LiteLLM at: https://${LITELLM_HOST}"
if [ "${LITELLM_OIDC_ENABLED}" = "true" ]; then
echo "SSO authentication: enabled"
fi
if [ "${LITELLM_LANGFUSE_INTEGRATION_ENABLED}" = "true" ]; then
echo "Langfuse integration: enabled (https://${LANGFUSE_HOST})"
fi
# Uninstall LiteLLM # Uninstall LiteLLM (delete-data: true to delete database and Vault secrets)
uninstall: uninstall delete-data='false':
#!/bin/bash #!/bin/bash
set -euo pipefail set -euo pipefail
if ! gum confirm "Uninstall LiteLLM?"; then if ! gum confirm "Uninstall LiteLLM?"; then
@@ -404,30 +481,33 @@ uninstall:
helm uninstall litellm -n ${LITELLM_NAMESPACE} --ignore-not-found --wait helm uninstall litellm -n ${LITELLM_NAMESPACE} --ignore-not-found --wait
just delete-api-key-external-secret just delete-api-key-external-secret
just delete-postgres-secret just delete-postgres-secret
just delete-keycloak-client || true
just delete-keycloak-auth-secret || true
just delete-langfuse-auth-secret || true
just delete-namespace just delete-namespace
echo "LiteLLM uninstalled."
# Clean up all resources including database if [ "{{ delete-data }}" = "true" ]; then
cleanup: echo "Deleting database and Vault secrets..."
#!/bin/bash just postgres::delete-user-and-db litellm litellm || true
set -euo pipefail just vault::delete litellm/db || true
echo "This will delete:" just vault::delete keycloak/client/litellm || true
echo " - LiteLLM deployment" just vault::delete litellm/langfuse || true
echo " - LiteLLM database" # Clean up API keys from Vault
echo " - All API keys from Vault" providers=$(just get-required-providers 2>/dev/null || true)
echo "" for provider in $providers; do
if ! gum confirm "Are you sure?"; then just vault::delete "litellm/${provider}" || true
echo "Cancelled." done
exit 0 echo "LiteLLM uninstalled with all data deleted."
else
echo "LiteLLM uninstalled."
echo ""
echo "Note: The following resources were NOT deleted:"
echo " - PostgreSQL user and database (litellm)"
echo " - Vault secrets (litellm/*, keycloak/client/litellm)"
echo ""
echo "To delete all data, run:"
echo " just litellm::uninstall true"
fi fi
just uninstall || true
just postgres::delete-db litellm || true
# Clean up API keys from Vault
providers=$(just get-required-providers 2>/dev/null || true)
for provider in $providers; do
just vault::delete "litellm/${provider}" || true
done
echo "Cleanup completed."
# Generate virtual key # Generate virtual key
generate-virtual-key user='' model='': generate-virtual-key user='' model='':
@@ -441,17 +521,22 @@ generate-virtual-key user='' model='':
if [ -z "${model}" ]; then if [ -z "${model}" ]; then
models=$(yq -r '.[].model_name' models.yaml 2>/dev/null || echo "") models=$(yq -r '.[].model_name' models.yaml 2>/dev/null || echo "")
if [ -n "$models" ]; then if [ -n "$models" ]; then
model=$(echo "$models" | gum choose --header="Select model:") model=$(echo -e "all\n$models" | gum choose --header="Select model (or 'all' for all models):")
else else
model=$(gum input --prompt="Model: " --width=80) model=$(gum input --prompt="Model (or 'all' for all models): " --width=80)
fi fi
fi fi
master_key=$(kubectl get secret litellm-masterkey -n ${LITELLM_NAMESPACE} \ master_key=$(kubectl get secret litellm-masterkey -n ${LITELLM_NAMESPACE} \
-o jsonpath="{.data.masterkey}" | base64 --decode) -o jsonpath="{.data.masterkey}" | base64 --decode)
if [ "${model}" = "all" ]; then
data="{\"metadata\": {\"user\": \"${user}\"}}"
else
data="{\"models\": [\"${model}\"], \"metadata\": {\"user\": \"${user}\"}}"
fi
response=$(curl -s "https://${LITELLM_HOST}/key/generate" \ response=$(curl -s "https://${LITELLM_HOST}/key/generate" \
--header "Authorization: Bearer ${master_key}" \ --header "Authorization: Bearer ${master_key}" \
--header "Content-Type: application/json" \ --header "Content-Type: application/json" \
--data-raw "{\"models\": [\"${model}\"], \"metadata\": {\"user\": \"${user}\"}}") --data-raw "${data}")
echo "${response}" | jq . echo "${response}" | jq .
echo "" echo ""
echo "API Key: $(echo "${response}" | jq -r '.key')" echo "API Key: $(echo "${response}" | jq -r '.key')"
@@ -692,3 +777,298 @@ disable-oidc:
echo "OIDC authentication disabled." echo "OIDC authentication disabled."
echo "Note: Keycloak client was NOT deleted. To clean up, run:" echo "Note: Keycloak client was NOT deleted. To clean up, run:"
echo " just litellm::delete-keycloak-client" echo " just litellm::delete-keycloak-client"
# Set Langfuse API keys for LiteLLM integration
set-langfuse-keys public_key='' secret_key='':
#!/bin/bash
set -euo pipefail
public_key="{{ public_key }}"
secret_key="{{ secret_key }}"
if [ -z "${public_key}" ]; then
public_key=$(gum input --prompt="Langfuse Public Key: " --width=80)
fi
if [ -z "${secret_key}" ]; then
secret_key=$(gum input --prompt="Langfuse Secret Key: " --password --width=80)
fi
if [ -z "${public_key}" ] || [ -z "${secret_key}" ]; then
echo "Error: Both public key and secret key are required"
exit 1
fi
just vault::put litellm/langfuse public_key="${public_key}" secret_key="${secret_key}"
echo "Langfuse API keys stored in Vault at 'litellm/langfuse'"
# Get Langfuse API keys
get-langfuse-keys:
#!/bin/bash
set -euo pipefail
echo "Public Key: $(just vault::get litellm/langfuse public_key)"
echo "Secret Key: $(just vault::get litellm/langfuse secret_key)"
# Delete Langfuse API keys
delete-langfuse-keys:
#!/bin/bash
set -euo pipefail
if just vault::exist litellm/langfuse &>/dev/null; then
just vault::delete litellm/langfuse
echo "Langfuse API keys deleted from Vault"
else
echo "Langfuse API keys not found in Vault"
fi
# Create Langfuse auth external secret
create-langfuse-auth-secret:
#!/bin/bash
set -euo pipefail
if ! just vault::exist litellm/langfuse &>/dev/null; then
echo "Error: Langfuse API keys not found in Vault."
echo "Please set the keys first:"
echo " just litellm::set-langfuse-keys"
exit 1
fi
kubectl delete externalsecret langfuse-auth-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found
kubectl delete secret langfuse-auth -n ${LITELLM_NAMESPACE} --ignore-not-found
gomplate -f langfuse-auth-external-secret.gomplate.yaml | kubectl apply -f -
echo "Waiting for Langfuse auth secret to be ready..."
kubectl wait --for=condition=Ready externalsecret/langfuse-auth-external-secret \
-n ${LITELLM_NAMESPACE} --timeout=60s
echo "Langfuse auth secret created"
# Delete Langfuse auth external secret
delete-langfuse-auth-secret:
kubectl delete externalsecret langfuse-auth-external-secret -n ${LITELLM_NAMESPACE} --ignore-not-found
kubectl delete secret langfuse-auth -n ${LITELLM_NAMESPACE} --ignore-not-found
# Setup Langfuse integration
setup-langfuse:
#!/bin/bash
set -euo pipefail
echo "Setting up Langfuse integration for LiteLLM..."
if ! helm status langfuse -n ${LANGFUSE_NAMESPACE} &>/dev/null; then
echo "Warning: Langfuse is not installed."
echo "Please install Langfuse first or provide external Langfuse credentials."
fi
while [ -z "${LANGFUSE_HOST}" ]; do
LANGFUSE_HOST=$(
gum input --prompt="Langfuse host (FQDN): " --width=100 \
--placeholder="e.g., langfuse.example.com"
)
done
if ! just vault::exist litellm/langfuse &>/dev/null; then
echo ""
echo "Langfuse API keys are required."
echo "You can get these from Langfuse UI: Settings > API Keys"
echo ""
just set-langfuse-keys
fi
just create-langfuse-auth-secret
while [ -z "${LITELLM_HOST}" ]; do
LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80)
done
if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then
if [ -z "${MONITORING_ENABLED}" ]; then
if gum confirm "Enable Prometheus monitoring?"; then
MONITORING_ENABLED="true"
fi
fi
fi
LITELLM_LANGFUSE_INTEGRATION_ENABLED="true"
echo "Generating Helm values with Langfuse integration..."
gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml
kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found
echo "Upgrading LiteLLM with Langfuse integration..."
helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \
--version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \
-f litellm-values.yaml
echo ""
echo "Langfuse integration configured successfully!"
echo "LiteLLM will now send traces to: https://${LANGFUSE_HOST}"
# Disable Langfuse integration
disable-langfuse:
#!/bin/bash
set -euo pipefail
echo "Disabling Langfuse integration for LiteLLM..."
LITELLM_LANGFUSE_INTEGRATION_ENABLED=""
while [ -z "${LITELLM_HOST}" ]; do
LITELLM_HOST=$(gum input --prompt="LiteLLM host (FQDN): " --width=80)
done
if helm status kube-prometheus-stack -n ${PROMETHEUS_NAMESPACE} &>/dev/null; then
if [ -z "${MONITORING_ENABLED}" ]; then
if gum confirm "Enable Prometheus monitoring?"; then
MONITORING_ENABLED="true"
fi
fi
fi
echo "Generating Helm values without Langfuse..."
gomplate -f litellm-values.gomplate.yaml -o litellm-values.yaml
kubectl delete job litellm-migrations -n ${LITELLM_NAMESPACE} --ignore-not-found
echo "Upgrading LiteLLM without Langfuse..."
helm upgrade litellm oci://ghcr.io/berriai/litellm-helm \
--version ${LITELLM_CHART_VERSION} -n ${LITELLM_NAMESPACE} --wait \
-f litellm-values.yaml
echo ""
echo "Langfuse integration disabled."
echo "Note: Langfuse API keys were NOT deleted. To clean up, run:"
echo " just litellm::delete-langfuse-keys"
# Create a team
create-team name='':
#!/bin/bash
set -euo pipefail
name="{{ name }}"
while [ -z "${name}" ]; do
name=$(gum input --prompt="Team name: " --width=80)
done
master_key=$(just master-key)
response=$(curl -s "https://${LITELLM_HOST}/team/new" \
--header "Authorization: Bearer ${master_key}" \
--header "Content-Type: application/json" \
--data-raw "{\"team_alias\": \"${name}\"}")
echo "${response}" | jq .
team_id=$(echo "${response}" | jq -r '.team_id')
echo ""
echo "Team created: ${name}"
echo "Team ID: ${team_id}"
# List teams
list-teams:
#!/bin/bash
set -euo pipefail
master_key=$(just master-key)
curl -s "https://${LITELLM_HOST}/team/list" \
--header "Authorization: Bearer ${master_key}" | jq .
# Get team info
get-team team_id='':
#!/bin/bash
set -euo pipefail
team_id="{{ team_id }}"
if [ -z "${team_id}" ]; then
echo "Usage: just litellm::get-team team_id=<team_id>"
exit 1
fi
master_key=$(just master-key)
curl -s "https://${LITELLM_HOST}/team/info?team_id=${team_id}" \
--header "Authorization: Bearer ${master_key}" | jq .
# Delete a team
delete-team team_id='':
#!/bin/bash
set -euo pipefail
team_id="{{ team_id }}"
if [ -z "${team_id}" ]; then
echo "Usage: just litellm::delete-team team_id=<team_id>"
exit 1
fi
if ! gum confirm "Delete team ${team_id}?"; then
echo "Cancelled."
exit 0
fi
master_key=$(just master-key)
curl -s "https://${LITELLM_HOST}/team/delete" \
--header "Authorization: Bearer ${master_key}" \
--header "Content-Type: application/json" \
--data-raw "{\"team_ids\": [\"${team_id}\"]}" | jq .
echo "Team deleted."
# Set Langfuse project for a team
set-team-langfuse-project team_id='' public_key='' secret_key='':
#!/bin/bash
set -euo pipefail
team_id="{{ team_id }}"
public_key="{{ public_key }}"
secret_key="{{ secret_key }}"
if [ -z "${team_id}" ]; then
master_key=$(just master-key)
teams=$(curl -s "https://${LITELLM_HOST}/team/list" \
--header "Authorization: Bearer ${master_key}")
team_aliases=$(echo "${teams}" | jq -r '.[] | "\(.team_id) (\(.team_alias // "no alias"))"')
if [ -z "${team_aliases}" ]; then
echo "No teams found. Create a team first:"
echo " just litellm::create-team"
exit 1
fi
selected=$(echo "${team_aliases}" | gum choose --header="Select team:")
team_id=$(echo "${selected}" | cut -d' ' -f1)
fi
if [ -z "${public_key}" ]; then
public_key=$(gum input --prompt="Langfuse Public Key for this team: " --width=80)
fi
if [ -z "${secret_key}" ]; then
secret_key=$(gum input --prompt="Langfuse Secret Key for this team: " --password --width=80)
fi
if [ -z "${public_key}" ] || [ -z "${secret_key}" ]; then
echo "Error: Both public key and secret key are required"
exit 1
fi
master_key=$(just master-key)
response=$(curl -s "https://${LITELLM_HOST}/team/update" \
--header "Authorization: Bearer ${master_key}" \
--header "Content-Type: application/json" \
--data-raw "{
\"team_id\": \"${team_id}\",
\"metadata\": {
\"logging\": [{
\"callback_name\": \"langfuse\",
\"callback_vars\": {
\"langfuse_public_key\": \"${public_key}\",
\"langfuse_secret\": \"${secret_key}\",
\"langfuse_host\": \"https://${LANGFUSE_HOST}\"
}
}]
}
}")
echo "${response}" | jq .
echo ""
echo "Langfuse project configured for team ${team_id}"
# Generate virtual key for a team
generate-team-key team_id='' user='':
#!/bin/bash
set -euo pipefail
team_id="{{ team_id }}"
user="{{ user }}"
if [ -z "${team_id}" ]; then
master_key=$(just master-key)
teams=$(curl -s "https://${LITELLM_HOST}/team/list" \
--header "Authorization: Bearer ${master_key}")
team_aliases=$(echo "${teams}" | jq -r '.[] | "\(.team_id) (\(.team_alias // "no alias"))"')
if [ -z "${team_aliases}" ]; then
echo "No teams found. Create a team first:"
echo " just litellm::create-team"
exit 1
fi
selected=$(echo "${team_aliases}" | gum choose --header="Select team:")
team_id=$(echo "${selected}" | cut -d' ' -f1)
fi
while [ -z "${user}" ]; do
user=$(gum input --prompt="Username: " --width=80)
done
master_key=$(just master-key)
response=$(curl -s "https://${LITELLM_HOST}/key/generate" \
--header "Authorization: Bearer ${master_key}" \
--header "Content-Type: application/json" \
--data-raw "{\"team_id\": \"${team_id}\", \"metadata\": {\"user\": \"${user}\"}}")
echo "${response}" | jq .
echo ""
echo "API Key: $(echo "${response}" | jq -r '.key')"

22
litellm/langfuse-auth-external-secret.gomplate.yaml Normal file
View File

@@ -0,0 +1,22 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: langfuse-auth-external-secret
namespace: {{ .Env.LITELLM_NAMESPACE }}
spec:
refreshInterval: 1h
secretStoreRef:
name: vault-secret-store
kind: ClusterSecretStore
target:
name: langfuse-auth
creationPolicy: Owner
data:
- secretKey: LANGFUSE_PUBLIC_KEY
remoteRef:
key: litellm/langfuse
property: public_key
- secretKey: LANGFUSE_SECRET_KEY
remoteRef:
key: litellm/langfuse
property: secret_key

39
litellm/litellm-values.gomplate.yaml
View File

@@ -10,6 +10,16 @@ podSecurityContext: {}
securityContext: {} securityContext: {}
# Resource recommendations from Goldilocks VPA
# litellm target: cpu=11m, memory=549Mi
resources:
requests:
cpu: 25m
memory: 512Mi
limits:
cpu: 100m
memory: 1Gi
migrationJob: migrationJob:
resources: resources:
requests: requests:
@@ -18,12 +28,17 @@ migrationJob:
limits: limits:
memory: 1Gi memory: 1Gi
{{- if .Env.LITELLM_OIDC_ENABLED }}
environmentSecrets: environmentSecrets:
- apikey - apikey
{{- if .Env.LITELLM_OIDC_ENABLED }}
- keycloak-auth - keycloak-auth
{{- end }}
{{- if .Env.LITELLM_LANGFUSE_INTEGRATION_ENABLED }}
- langfuse-auth
{{- end }}
extraEnvVars: extraEnvVars:
{{- if .Env.LITELLM_OIDC_ENABLED }}
- name: PROXY_BASE_URL - name: PROXY_BASE_URL
value: "https://{{ .Env.LITELLM_HOST }}" value: "https://{{ .Env.LITELLM_HOST }}"
- name: GENERIC_AUTHORIZATION_ENDPOINT - name: GENERIC_AUTHORIZATION_ENDPOINT
@@ -34,14 +49,20 @@ extraEnvVars:
value: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid-connect/userinfo" value: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid-connect/userinfo"
- name: GENERIC_SCOPE - name: GENERIC_SCOPE
value: "openid email profile" value: "openid email profile"
{{- else }} {{- end }}
environmentSecrets: {{- if .Env.LITELLM_LANGFUSE_INTEGRATION_ENABLED }}
- apikey - name: LANGFUSE_HOST
value: "https://{{ .Env.LANGFUSE_HOST }}"
{{- end }} {{- end }}
proxy_config: proxy_config:
model_list: model_list:
{{ file.Read "models.yaml" | indent 4 }} {{ file.Read "models.yaml" | indent 4 }}
{{- if .Env.LITELLM_LANGFUSE_INTEGRATION_ENABLED }}
litellm_settings:
success_callback: ["langfuse"]
failure_callback: ["langfuse"]
{{- end }}
db: db:
useExisting: true useExisting: true
@@ -57,6 +78,16 @@ db:
redis: redis:
enabled: true enabled: true
# Resource recommendations from Goldilocks VPA
# redis target: cpu=15m, memory=100Mi
master:
resources:
requests:
cpu: 25m
memory: 128Mi
limits:
cpu: 100m
memory: 256Mi
ingress: ingress:
enabled: true enabled: true