From 6da1fac457eee07d41ff6f2df9a9a4377e2771aa Mon Sep 17 00:00:00 2001 From: Masaki Yatsu Date: Tue, 16 Sep 2025 00:36:30 +0900 Subject: [PATCH] feat(dagster): setting extra env secrets --- dagster/.gitignore | 1 + .../dagster-env-external-secret.gomplate.yaml | 46 +++++++++++++++++++ dagster/dagster-values.gomplate.yaml | 20 ++++++-- dagster/justfile | 46 +++++++++++++++++++ 4 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 dagster/dagster-env-external-secret.gomplate.yaml diff --git a/dagster/.gitignore b/dagster/.gitignore index efbc485..8292a87 100644 --- a/dagster/.gitignore +++ b/dagster/.gitignore @@ -4,3 +4,4 @@ dagster-minio-external-secret.yaml dagster-oauth-external-secret.yaml dagster-storage-pvc.yaml dagster-user-code-pvc.yaml +dagster-env-external-secret.yaml diff --git a/dagster/dagster-env-external-secret.gomplate.yaml b/dagster/dagster-env-external-secret.gomplate.yaml new file mode 100644 index 0000000..301c396 --- /dev/null +++ b/dagster/dagster-env-external-secret.gomplate.yaml @@ -0,0 +1,46 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: dagster-env-external-secret + namespace: {{ .Env.DAGSTER_NAMESPACE }} +spec: + refreshInterval: 1h + secretStoreRef: + name: vault-secret-store + kind: ClusterSecretStore + target: + name: dagster-env-secret + creationPolicy: Owner + template: + data: + # Fixed values - customize as needed + AWS_ENDPOINT_URL: "http://minio.minio.svc.cluster.local:9000" + DESTINATION__POSTGRES__DATA_WRITER__INSERT_VALUES_MAX_ROWS: "10000" + # Template values from Vault - reference via {{ .postgres_user }} + POSTGRES_USER: "{{ .postgres_user }}" + POSTGRES_PASSWORD: "{{ .postgres_password }}" + # Add more fixed values here: + # SOME_CONFIG_VALUE: "fixed-value" + # + # Add more Vault references here: + # AWS_ACCESS_KEY_ID: "{{ .aws_access_key_id }}" + # AWS_SECRET_ACCESS_KEY: "{{ .aws_secret_access_key }}" + data: + # PostgreSQL configuration - fetch from Vault + - secretKey: postgres_user + remoteRef: + key: postgres/admin + property: username + - secretKey: postgres_password + remoteRef: + key: postgres/admin + property: password + # Add more Vault references here: + # - secretKey: aws_access_key_id + # remoteRef: + # key: minio + # property: access_key_id + # - secretKey: aws_secret_access_key + # remoteRef: + # key: minio + # property: secret_access_key diff --git a/dagster/dagster-values.gomplate.yaml b/dagster/dagster-values.gomplate.yaml index 4ba335d..dea5767 100644 --- a/dagster/dagster-values.gomplate.yaml +++ b/dagster/dagster-values.gomplate.yaml @@ -45,6 +45,11 @@ dagsterWebserver: persistentVolumeClaim: claimName: dagster-user-code-pvc + {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }} + envSecrets: + - name: dagster-env-secret + {{- end }} + workspace: enabled: true servers: [] @@ -74,6 +79,11 @@ dagsterDaemon: persistentVolumeClaim: claimName: dagster-user-code-pvc + {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }} + envSecrets: + - name: dagster-env-secret + {{- end }} + env: - name: DAGSTER_HOME value: /opt/dagster/dagster_home @@ -99,14 +109,14 @@ runLauncher: - name: user-code persistentVolumeClaim: claimName: dagster-user-code-pvc - {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }} envSecrets: - name: dagster-database-secret + {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }} - name: dagster-minio-secret - {{- else }} - envSecrets: - - name: dagster-database-secret - {{- end }} + {{- end }} + {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }} + - name: dagster-env-secret + {{- end }} postgresql: enabled: false diff --git a/dagster/justfile b/dagster/justfile index 33045d2..df5ad23 100644 --- a/dagster/justfile +++ b/dagster/justfile @@ -148,6 +148,43 @@ delete-oauth-secret: @kubectl delete secret dagster-oauth-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found @kubectl delete externalsecret dagster-oauth-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found +# Create environment variables secret example (customize as needed) +create-env-secrets-example: + #!/bin/bash + set -euo pipefail + echo "Creating Dagster environment secrets example..." + echo "This is an example - customize the environment variables as needed" + if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then + echo "External Secrets available. Creating ExternalSecret using template..." + echo "Edit dagster-env-external-secret.gomplate.yaml to customize environment variables" + kubectl delete externalsecret dagster-env-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found + kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found + gomplate -f dagster-env-external-secret.gomplate.yaml -o dagster-env-external-secret.yaml + kubectl apply -f dagster-env-external-secret.yaml + echo "Waiting for environment secret to be ready..." + kubectl wait --for=condition=Ready externalsecret/dagster-env-external-secret \ + -n ${DAGSTER_NAMESPACE} --timeout=60s + else + echo "External Secrets not available. Creating Kubernetes Secret directly..." + POSTGRES_USER="buun" + POSTGRES_PASSWORD="buunpass" + kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found + kubectl create secret generic dagster-env-secret -n ${DAGSTER_NAMESPACE} \ + --from-literal=POSTGRES_USER="$POSTGRES_USER" \ + --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD" + # Add more environment variables here: + # --from-literal=AWS_ACCESS_KEY_ID="your_value" \ + # --from-literal=AWS_SECRET_ACCESS_KEY="your_value" + echo "Environment secret created directly in Kubernetes" + fi + echo "Example environment secrets created successfully" + echo "Customize the environment variables in this recipe as needed for your project" + +# Delete environment secrets +delete-env-secrets: + @kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found + @kubectl delete externalsecret dagster-env-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found + # Setup MinIO storage for Dagster setup-minio-storage: #!/bin/bash @@ -567,6 +604,15 @@ install: fi just setup-user-code-pvc + export DAGSTER_ENV_SECRETS_EXIST="false" + if kubectl get secret dagster-env-secret -n ${DAGSTER_NAMESPACE} &>/dev/null; then + echo "Environment secrets found - will include in deployment" + export DAGSTER_ENV_SECRETS_EXIST="true" + else + echo "No environment secrets found - use 'just dagster::create-env-secrets-example' to create them if needed" + export DAGSTER_ENV_SECRETS_EXIST="false" + fi + just add-helm-repo gomplate -f dagster-values.gomplate.yaml -o dagster-values.yaml helm upgrade --install dagster dagster/dagster \