From 6abde4ed59317da11e48a6ef785e095c54d69404 Mon Sep 17 00:00:00 2001 From: Masaki Yatsu Date: Fri, 12 Sep 2025 16:09:34 +0900 Subject: [PATCH] examples: add Cube --- custom-example/cube/.gitignore | 2 + custom-example/cube/README.md | 135 ++++++++++ .../cube-api-external-secret.gomplate.yaml | 22 ++ custom-example/cube/cube-values.gomplate.yaml | 99 ++++++++ custom-example/cube/cubestore-values.yaml | 66 +++++ custom-example/cube/justfile | 237 ++++++++++++++++++ ...ostgres-cube-external-secret.gomplate.yaml | 22 ++ custom-example/justfile | 1 + 8 files changed, 584 insertions(+) create mode 100644 custom-example/cube/.gitignore create mode 100644 custom-example/cube/README.md create mode 100644 custom-example/cube/cube-api-external-secret.gomplate.yaml create mode 100644 custom-example/cube/cube-values.gomplate.yaml create mode 100644 custom-example/cube/cubestore-values.yaml create mode 100644 custom-example/cube/justfile create mode 100644 custom-example/cube/postgres-cube-external-secret.gomplate.yaml diff --git a/custom-example/cube/.gitignore b/custom-example/cube/.gitignore new file mode 100644 index 0000000..f07f71a --- /dev/null +++ b/custom-example/cube/.gitignore @@ -0,0 +1,2 @@ +# Generated Helm values +cube-values.yaml \ No newline at end of file diff --git a/custom-example/cube/README.md b/custom-example/cube/README.md new file mode 100644 index 0000000..10f5b97 --- /dev/null +++ b/custom-example/cube/README.md @@ -0,0 +1,135 @@ +# Cube.dev Setup + +Cube.dev universal semantic layer with Cubestore cache engine. + +## Prerequisites + +- Keycloak installed and configured +- `oauth2c` tool available via mise +- PostgreSQL or other data source for Cube.dev + +## Setup + +1. **Configure environment variables**: + + ```bash + # Required for Ingress + export CUBE_HOST=cube.your-domain.com + + # Optional: Customize storage and callback port + export CUBE_STORAGE_SIZE=2Gi + export CUBE_OIDC_CALLBACK_PORT=9877 + ``` + +2. **Create Keycloak client**: + + ```bash + just cube::create-keycloak-client + ``` + +3. **Install Cube.dev and Cubestore**: + + ```bash + just cube::install + ``` + +4. **Access Cube Playground**: + + ```bash + # Via Ingress (if CUBE_HOST is set) + open https://${CUBE_HOST} + + # Via port-forward (for local development) + just cube::port-forward + ``` + +5. **Get JWT token for authentication**: + + ```bash + just cube::show-token + ``` + +## Authentication Flow + +1. Run `just cube::get-token` to authenticate with Keycloak via browser +2. Copy the JWT token to Cube Playground +3. Use the token in Playground > Add Security Context > Token tab + +## Architecture + +``` +Frontend App → Keycloak (OIDC) → JWT Token → Cube.dev API + ↓ + Cubestore Cache + ↓ + Data Warehouse +``` + +## Configuration + +- **Namespace**: `cube` +- **Keycloak OIDC Client**: `cube-cli` (public client) +- **JWT Verification**: Uses Keycloak JWKS endpoint +- **Cache**: Cubestore cluster with 2 workers +- **OAuth2c Callback Port**: `9876` (customizable via `CUBE_OIDC_CALLBACK_PORT`) +- **Ingress**: Automatically enabled when `CUBE_HOST` is set +- **Persistent Storage**: `1Gi` PVC for schema files and configuration (customizable via `CUBE_STORAGE_SIZE`) + +## Commands + +- `just cube::install` - Install Cube.dev and Cubestore +- `just cube::get-token` - Get JWT token via oauth2c +- `just cube::show-token` - Display token for Playground +- `just cube::port-forward` - Access Playground (localhost:4000) +- `just cube::status` - Check installation status +- `just cube::logs` - View Cube.dev logs +- `just cube::test-api` - Test API connection +- `just cube::uninstall` - Remove everything + +## Data Sources Configuration + +### Option 1: Playground Setup Wizard (Recommended) + +1. Access Cube Playground and follow the Setup Wizard +2. Select your database type (PostgreSQL, MySQL, BigQuery, etc.) +3. Enter connection details +4. Test connection and auto-generate schema + +### Option 2: Environment Variables + +Edit `cube-values.gomplate.yaml` or use kubectl: + +```bash +kubectl create configmap cube-db-config -n cube \ + --from-literal=CUBEJS_DB_TYPE=postgres \ + --from-literal=CUBEJS_DB_HOST=your-host \ + --from-literal=CUBEJS_DB_NAME=your-database + +kubectl create secret generic cube-db-secret -n cube \ + --from-literal=CUBEJS_DB_USER=your-user \ + --from-literal=CUBEJS_DB_PASS=your-password +``` + +### Option 3: Multiple Data Sources + +Use `cube.js` configuration file for advanced setups with multiple databases. + +## Persistent Storage + +The PVC stores: + +- **Schema files**: Generated data models from Setup Wizard +- **Configuration files**: `cube.js`, custom settings +- **Custom schemas**: Hand-written data models +- **Cache metadata**: Query optimization data + +Storage is mounted at `/cube/conf` and persists across pod restarts. + +## Security Context + +JWT tokens are verified using Keycloak's JWKS endpoint. The security context includes: + +- `sub` - User ID +- `realm_access.roles` - User roles +- `email` - User email +- Custom claims as configured in Keycloak diff --git a/custom-example/cube/cube-api-external-secret.gomplate.yaml b/custom-example/cube/cube-api-external-secret.gomplate.yaml new file mode 100644 index 0000000..1f7a06f --- /dev/null +++ b/custom-example/cube/cube-api-external-secret.gomplate.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: cube-api-external-secret + namespace: {{ .Env.CUBE_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: cube-api-secret + creationPolicy: Owner + template: + type: Opaque + data: + api-secret: "{{ `{{ .api_secret }}` }}" + data: + - secretKey: api_secret + remoteRef: + key: cube/api + property: api-secret \ No newline at end of file diff --git a/custom-example/cube/cube-values.gomplate.yaml b/custom-example/cube/cube-values.gomplate.yaml new file mode 100644 index 0000000..780d04e --- /dev/null +++ b/custom-example/cube/cube-values.gomplate.yaml @@ -0,0 +1,99 @@ +# Cube.dev Helm Chart Values +# https://github.com/gadsme/charts/tree/main/charts/cube + +# Image configuration +image: + repository: cubejs/cube + tag: v0.35.78 + +# Replica count +replicaCount: 1 + +# Resources +resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + +# Service configuration +service: + type: ClusterIP + port: 4000 + +# Environment variables for Cube.dev +extraEnvVars: + # JWT Authentication + - name: CUBEJS_JWT_KEY + value: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}/protocol/openid_connect/certs" + - name: CUBEJS_JWT_AUDIENCE + value: "{{ .Env.CUBE_OIDC_CLIENT_ID }}" + - name: CUBEJS_JWT_ISSUER + value: "https://{{ .Env.KEYCLOAK_HOST }}/realms/{{ .Env.KEYCLOAK_REALM }}" + + # Cubestore connection + - name: CUBEJS_CUBESTORE_HOST + value: cubestore-router.{{ .Env.CUBE_NAMESPACE }}.svc.cluster.local + + # API settings (loaded from Secret) + - name: CUBEJS_API_SECRET + valueFrom: + secretKeyRef: + name: cube-api-secret + key: api-secret + + - name: CUBEJS_WEB_SOCKETS + value: "true" + - name: CUBEJS_DEV_MODE + value: "true" + +# Override default log level +config: + logLevel: "info" + + # Database connection - minimal configuration for initial deployment + # Configure via Playground Setup Wizard after deployment: http://localhost:4000 + +# Datasource configuration (required by Helm chart) +datasources: + default: + type: postgres + host: postgres-cluster-rw.postgres.svc.cluster.local + port: 5432 + name: cube + user: cube + passFromSecret: + name: postgres-cube-secret + key: password + +# Ingress configuration +ingress: + enabled: false # disabled for now + hostname: {{ .Env.CUBE_HOST | default "cube.local" }} + path: / + pathType: Prefix + ingressClassName: traefik + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + tls: {{ if .Env.CUBE_HOST }}true{{ else }}false{{ end }} + +# Persistence for schema files and configuration +persistence: + enabled: true + size: {{ .Env.CUBE_STORAGE_SIZE | default "1Gi" }} + storageClass: longhorn + accessMode: ReadWriteOnce + # Mount path for Cube configuration and schema files + mountPath: /cube/conf + +# Custom schema mounting (if you have schema files) +extraVolumes: [] +# - name: schema +# configMap: +# name: cube-schema + +extraVolumeMounts: [] +# - name: schema +# mountPath: /cube/conf/schema diff --git a/custom-example/cube/cubestore-values.yaml b/custom-example/cube/cubestore-values.yaml new file mode 100644 index 0000000..c6c2942 --- /dev/null +++ b/custom-example/cube/cubestore-values.yaml @@ -0,0 +1,66 @@ +# Cubestore Helm Chart Values +# https://github.com/gadsme/charts/tree/main/charts/cubestore + +# Router configuration +router: + replicaCount: 1 + image: + repository: cubejs/cubestore + tag: v0.35.78 + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +# Worker configuration +workers: + replicaCount: 2 + image: + repository: cubejs/cubestore + tag: v0.35.78 + + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + + # Persistent storage for workers + persistence: + enabled: true + size: 10Gi + storageClass: longhorn + +# Storage configuration +storage: + # Use local storage for development + type: local + + # For production with S3: + # type: s3 + # s3: + # bucket: your-cubestore-bucket + # region: us-east-1 + # accessKeyId: "" + # secretAccessKey: "" + +# Service configuration +service: + type: ClusterIP + port: 3030 + +# Metrics configuration +metrics: + enabled: false + statsd: + enabled: false + +# Disable statsd exporter +exporter: + enabled: false \ No newline at end of file diff --git a/custom-example/cube/justfile b/custom-example/cube/justfile new file mode 100644 index 0000000..203c06f --- /dev/null +++ b/custom-example/cube/justfile @@ -0,0 +1,237 @@ +set fallback := true + +export CUBE_NAMESPACE := env("CUBE_NAMESPACE", "cube") +export CUBESTORE_CHART_VERSION := env("CUBESTORE_CHART_VERSION", "1.1.0") +export CUBE_CHART_VERSION := env("CUBE_CHART_VERSION", "3.2.0") +export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack") +export KEYCLOAK_HOST := env("KEYCLOAK_HOST", "") +export CUBE_OIDC_CLIENT_ID := env("CUBE_OIDC_CLIENT_ID", "cube-cli") +export CUBE_OIDC_CALLBACK_PORT := env("CUBE_OIDC_CALLBACK_PORT", "9876") +export CUBE_STORAGE_SIZE := env("CUBE_STORAGE_SIZE", "1Gi") +export K8S_VAULT_NAMESPACE := env("K8S_VAULT_NAMESPACE", "vault") +export EXTERNAL_SECRETS_NAMESPACE := env("EXTERNAL_SECRETS_NAMESPACE", "external-secrets") + +[private] +default: + @just --list --unsorted --list-submodules + +# Create Cube namespace +create-namespace: + @kubectl get namespace ${CUBE_NAMESPACE} &>/dev/null || \ + kubectl create namespace ${CUBE_NAMESPACE} + +# Delete Cube namespace +delete-namespace: + @kubectl delete namespace ${CUBE_NAMESPACE} --ignore-not-found + +# Install Cube.dev and Cubestore +install: + #!/bin/bash + set -euo pipefail + just add-helm-repo + just create-credentials + just create-database + just install-cubestore + just install-cube + echo "Cube.dev and Cubestore installed successfully" + echo "Access Cube Playground: http://localhost:4000 (after port-forward)" + echo "Run: just cube::port-forward" + +# Add Helm repository +add-helm-repo: + @echo "Adding gadsme Helm repository..." + helm repo add gadsme https://gadsme.github.io/charts + helm repo update + +# Install Cubestore cluster +install-cubestore: + @echo "Installing Cubestore cluster..." + just create-namespace + helm upgrade --install cubestore gadsme/cubestore --namespace ${CUBE_NAMESPACE} \ + --version ${CUBESTORE_CHART_VERSION} --values cubestore-values.yaml --wait --timeout=5m + +# Install Cube.dev +install-cube: + #!/bin/bash + set -euo pipefail + echo "Installing Cube.dev..." + export CUBE_HOST=${CUBE_HOST:-} + while [ -z "${CUBE_HOST}" ]; do + CUBE_HOST=$( + gum input --prompt="Cube host (FQDN): " --width=100 \ + --placeholder="e.g., cube.example.com" \ + ) + done + gomplate -f cube-values.gomplate.yaml -o cube-values.yaml + helm upgrade --install cube gadsme/cube --namespace ${CUBE_NAMESPACE} \ + --version ${CUBE_CHART_VERSION} --values cube-values.yaml --wait --timeout=5m + +# Create Cube database and user +create-database: + #!/bin/bash + set -euo pipefail + echo "Creating Cube database and user..." + password=$(just utils::random-password) + + # Create database if not exists + if ! just postgres::db-exists cube &>/dev/null; then + just postgres::create-db cube + else + echo "Database cube already exists" + fi + + # Handle existing user - update password instead of recreating + if just postgres::user-exists cube &>/dev/null; then + echo "User cube already exists, updating password..." + just postgres::change-password cube "${password}" + else + # Create new user + echo "Creating new user cube..." + just postgres::create-user cube "${password}" + just postgres::grant cube cube + fi + + just create-namespace + + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "External Secrets Operator detected. Creating ExternalSecret..." + just vault::put postgres/cube password="${password}" + + kubectl delete secret postgres-cube-secret -n ${CUBE_NAMESPACE} --ignore-not-found + kubectl delete externalsecret postgres-cube-external-secret -n ${CUBE_NAMESPACE} \ + --ignore-not-found + + gomplate -f postgres-cube-external-secret.gomplate.yaml | kubectl apply -f - + + echo "Waiting for ExternalSecret to sync..." + kubectl wait --for=condition=Ready externalsecret/postgres-cube-external-secret \ + -n ${CUBE_NAMESPACE} --timeout=60s + else + echo "External Secrets Operator not found. Creating secret directly..." + kubectl delete secret postgres-cube-secret -n ${CUBE_NAMESPACE} --ignore-not-found + kubectl create secret generic postgres-cube-secret -n ${CUBE_NAMESPACE} \ + --from-literal=password="${password}" + + if helm status vault -n ${K8S_VAULT_NAMESPACE} &>/dev/null; then + just vault::put postgres/cube password="${password}" + fi + fi + + echo "Cube database and user created successfully" + +# Create Cube API secret +create-credentials: + #!/bin/bash + set -euo pipefail + api_secret=$(just utils::random-password) + just create-namespace + + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "External Secrets Operator detected. Creating ExternalSecret..." + just put-api-secret-to-vault "${api_secret}" + + kubectl delete secret cube-api-secret -n ${CUBE_NAMESPACE} --ignore-not-found + kubectl delete externalsecret cube-api-external-secret -n ${CUBE_NAMESPACE} \ + --ignore-not-found + + gomplate -f cube-api-external-secret.gomplate.yaml | kubectl apply -f - + + echo "Waiting for ExternalSecret to sync..." + kubectl wait --for=condition=Ready externalsecret/cube-api-external-secret \ + -n ${CUBE_NAMESPACE} --timeout=60s + else + echo "External Secrets Operator not found. Creating secret directly..." + if kubectl get secret cube-api-secret -n ${CUBE_NAMESPACE} &>/dev/null; then + kubectl delete --ignore-not-found secret cube-api-secret -n ${CUBE_NAMESPACE} + fi + kubectl create secret generic cube-api-secret -n ${CUBE_NAMESPACE} \ + --from-literal=api-secret="${api_secret}" + + if helm status vault -n ${K8S_VAULT_NAMESPACE} &>/dev/null; then + just put-api-secret-to-vault "${api_secret}" + fi + fi + +# Delete Cube API secret +delete-credentials: + @kubectl delete secret cube-api-secret -n ${CUBE_NAMESPACE} --ignore-not-found + @kubectl delete externalsecret cube-api-secret -n ${CUBE_NAMESPACE} --ignore-not-found + +# Create Keycloak client for Cube +create-keycloak-client: + @just keycloak::create-client ${KEYCLOAK_REALM} ${CUBE_OIDC_CLIENT_ID} \ + "http://localhost:${CUBE_OIDC_CALLBACK_PORT}/callback" + +# Get JWT token using oauth2c +get-token: + #!/bin/bash + set -euo pipefail + TOKEN=$(oauth2c \ + --issuer-url "https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM}" \ + --client-id ${CUBE_OIDC_CLIENT_ID} \ + --redirect-url "http://localhost:${CUBE_OIDC_CALLBACK_PORT}/callback" \ + --scopes "openid profile email" \ + --response-types code \ + --grant-type authorization_code \ + --auth-method none \ + --browser-timeout 300s \ + --silent 2>/dev/null | jq -r '.access_token') + echo "${TOKEN}" + +# Show JWT token for Playground +show-token: + @echo "JWT Token for Cube Playground:" + @echo "Copy this token to Playground > Add Security Context > Token tab:" + @echo + @just cube::get-token + +# Uninstall Cube.dev +uninstall-cube: + @echo "Deleting Cube.dev..." + helm uninstall cube -n ${CUBE_NAMESPACE} --ignore-not-found + +# Uninstall Cubestore +uninstall-cubestore: + @echo "Deleting Cubestore..." + helm uninstall cubestore -n ${CUBE_NAMESPACE} --ignore-not-found + +# Uninstall everything +uninstall: uninstall-cube uninstall-cubestore delete-credentials delete-namespace + +# Test connection to Cube API +test-api: + #!/bin/bash + set -euo pipefail + echo "Testing Cube API connection..." + TOKEN=$(just cube::get-token) + if [ -n "${CUBE_HOST}" ]; then + API_URL="https://${CUBE_HOST}/cubejs-api/v1/meta" + echo "Testing via Ingress: ${API_URL}" + else + API_URL="http://localhost:4000/cubejs-api/v1/meta" + echo "Testing via port-forward: ${API_URL}" + fi + curl -s -H "Authorization: Bearer ${TOKEN}" \ + -H "Content-Type: application/json" \ + "${API_URL}" || echo "API test failed - check connection method" + +# Show configuration +config: + @echo "Cube Configuration:" + @echo "Namespace: ${CUBE_NAMESPACE}" + @echo "Cube Host: ${CUBE_HOST}" + @echo "Keycloak Host: ${KEYCLOAK_HOST}" + @echo "Keycloak Realm: ${KEYCLOAK_REALM}" + @echo "Cube OIDC Client ID: ${CUBE_OIDC_CLIENT_ID}" + @echo "OIDC Callback Port: ${CUBE_OIDC_CALLBACK_PORT}" + @echo "Storage Size: ${CUBE_STORAGE_SIZE}" + +# Put API secret to Vault +put-api-secret-to-vault secret: + @just vault::put cube/api api-secret={{ secret }} + @echo "API secret stored in Vault under 'cube/api'." + +# Delete API secret from Vault +delete-api-secret-from-vault: + @just vault::delete cube/api + @echo "API secret deleted from Vault." diff --git a/custom-example/cube/postgres-cube-external-secret.gomplate.yaml b/custom-example/cube/postgres-cube-external-secret.gomplate.yaml new file mode 100644 index 0000000..0dad3bb --- /dev/null +++ b/custom-example/cube/postgres-cube-external-secret.gomplate.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgres-cube-external-secret + namespace: {{ .Env.CUBE_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: postgres-cube-secret + creationPolicy: Owner + template: + type: Opaque + data: + password: "{{ `{{ .password }}` }}" + data: + - secretKey: password + remoteRef: + key: postgres/cube + property: password \ No newline at end of file diff --git a/custom-example/justfile b/custom-example/justfile index 8a31228..64166d4 100644 --- a/custom-example/justfile +++ b/custom-example/justfile @@ -4,6 +4,7 @@ set fallback := true default: @just --list --unsorted --list-submodules +mod cube 'cube/justfile' mod bytebase 'bytebase/justfile' mod miniflux 'miniflux/justfile' mod reddit-rss 'reddit-rss/justfile'