baschno/buun-stack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dagster): setting extra env secrets

Browse Source
This commit is contained in:
Masaki Yatsu
2025-09-16 00:36:30 +09:00
parent 26c90a1c0b
commit 6da1fac457
4 changed files with 108 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
dagster/.gitignore vendored
View File

@@ -4,3 +4,4 @@ dagster-minio-external-secret.yaml
dagster-oauth-external-secret.yaml
dagster-storage-pvc.yaml
dagster-user-code-pvc.yaml
dagster-env-external-secret.yaml

46
dagster/dagster-env-external-secret.gomplate.yaml Normal file
View File

@@ -0,0 +1,46 @@
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: dagster-env-external-secret
namespace: {{ .Env.DAGSTER_NAMESPACE }}
spec:
refreshInterval: 1h
secretStoreRef:
name: vault-secret-store
kind: ClusterSecretStore
target:
name: dagster-env-secret
creationPolicy: Owner
template:
data:
# Fixed values - customize as needed
AWS_ENDPOINT_URL: "http://minio.minio.svc.cluster.local:9000"
DESTINATION__POSTGRES__DATA_WRITER__INSERT_VALUES_MAX_ROWS: "10000"
# Template values from Vault - reference via {{ .postgres_user }}
POSTGRES_USER: "{{ .postgres_user }}"
POSTGRES_PASSWORD: "{{ .postgres_password }}"
# Add more fixed values here:
# SOME_CONFIG_VALUE: "fixed-value"
#
# Add more Vault references here:
# AWS_ACCESS_KEY_ID: "{{ .aws_access_key_id }}"
# AWS_SECRET_ACCESS_KEY: "{{ .aws_secret_access_key }}"
data:
# PostgreSQL configuration - fetch from Vault
- secretKey: postgres_user
remoteRef:
key: postgres/admin
property: username
- secretKey: postgres_password
remoteRef:
key: postgres/admin
property: password
# Add more Vault references here:
# - secretKey: aws_access_key_id
# remoteRef:
# key: minio
# property: access_key_id
# - secretKey: aws_secret_access_key
# remoteRef:
# key: minio
# property: secret_access_key

20
dagster/dagster-values.gomplate.yaml
View File

@@ -45,6 +45,11 @@ dagsterWebserver:
persistentVolumeClaim:
claimName: dagster-user-code-pvc
{{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
envSecrets:
- name: dagster-env-secret
{{- end }}
workspace:
enabled: true
servers: []
@@ -74,6 +79,11 @@ dagsterDaemon:
persistentVolumeClaim:
claimName: dagster-user-code-pvc
{{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
envSecrets:
- name: dagster-env-secret
{{- end }}
env:
- name: DAGSTER_HOME
value: /opt/dagster/dagster_home
@@ -99,14 +109,14 @@ runLauncher:
- name: user-code
persistentVolumeClaim:
claimName: dagster-user-code-pvc
{{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }}
envSecrets:
- name: dagster-database-secret
{{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }}
- name: dagster-minio-secret
{{- else }}
envSecrets:
- name: dagster-database-secret
{{- end }}
{{- end }}
{{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
- name: dagster-env-secret
{{- end }}
postgresql:
enabled: false

46
dagster/justfile
View File

@@ -148,6 +148,43 @@ delete-oauth-secret:
@kubectl delete secret dagster-oauth-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
@kubectl delete externalsecret dagster-oauth-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
# Create environment variables secret example (customize as needed)
create-env-secrets-example:
#!/bin/bash
set -euo pipefail
echo "Creating Dagster environment secrets example..."
echo "This is an example - customize the environment variables as needed"
if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then
echo "External Secrets available. Creating ExternalSecret using template..."
echo "Edit dagster-env-external-secret.gomplate.yaml to customize environment variables"
kubectl delete externalsecret dagster-env-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
gomplate -f dagster-env-external-secret.gomplate.yaml -o dagster-env-external-secret.yaml
kubectl apply -f dagster-env-external-secret.yaml
echo "Waiting for environment secret to be ready..."
kubectl wait --for=condition=Ready externalsecret/dagster-env-external-secret \
-n ${DAGSTER_NAMESPACE} --timeout=60s
else
echo "External Secrets not available. Creating Kubernetes Secret directly..."
POSTGRES_USER="buun"
POSTGRES_PASSWORD="buunpass"
kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
kubectl create secret generic dagster-env-secret -n ${DAGSTER_NAMESPACE} \
--from-literal=POSTGRES_USER="$POSTGRES_USER" \
--from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD"
# Add more environment variables here:
# --from-literal=AWS_ACCESS_KEY_ID="your_value" \
# --from-literal=AWS_SECRET_ACCESS_KEY="your_value"
echo "Environment secret created directly in Kubernetes"
fi
echo "Example environment secrets created successfully"
echo "Customize the environment variables in this recipe as needed for your project"
# Delete environment secrets
delete-env-secrets:
@kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
@kubectl delete externalsecret dagster-env-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
# Setup MinIO storage for Dagster
setup-minio-storage:
#!/bin/bash
@@ -567,6 +604,15 @@ install:
fi
just setup-user-code-pvc
export DAGSTER_ENV_SECRETS_EXIST="false"
if kubectl get secret dagster-env-secret -n ${DAGSTER_NAMESPACE} &>/dev/null; then
echo "Environment secrets found - will include in deployment"
export DAGSTER_ENV_SECRETS_EXIST="true"
else
echo "No environment secrets found - use 'just dagster::create-env-secrets-example' to create them if needed"
export DAGSTER_ENV_SECRETS_EXIST="false"
fi
just add-helm-repo
gomplate -f dagster-values.gomplate.yaml -o dagster-values.yaml
helm upgrade --install dagster dagster/dagster \