feat(dagster): setting extra env secrets

dagster/.gitignore

@@ -4,3 +4,4 @@ dagster-minio-external-secret.yaml
 dagster-oauth-external-secret.yaml
 dagster-storage-pvc.yaml
 dagster-user-code-pvc.yaml
+dagster-env-external-secret.yaml

dagster/dagster-env-external-secret.gomplate.yaml

@@ -0,0 +1,46 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: dagster-env-external-secret
+  namespace: {{ .Env.DAGSTER_NAMESPACE }}
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vault-secret-store
+    kind: ClusterSecretStore
+  target:
+    name: dagster-env-secret
+    creationPolicy: Owner
+    template:
+      data:
+        # Fixed values - customize as needed
+        AWS_ENDPOINT_URL: "http://minio.minio.svc.cluster.local:9000"
+        DESTINATION__POSTGRES__DATA_WRITER__INSERT_VALUES_MAX_ROWS: "10000"
+        # Template values from Vault - reference via {{ .postgres_user }}
+        POSTGRES_USER: "{{ .postgres_user }}"
+        POSTGRES_PASSWORD: "{{ .postgres_password }}"
+        # Add more fixed values here:
+        # SOME_CONFIG_VALUE: "fixed-value"
+        #
+        # Add more Vault references here:
+        # AWS_ACCESS_KEY_ID: "{{ .aws_access_key_id }}"
+        # AWS_SECRET_ACCESS_KEY: "{{ .aws_secret_access_key }}"
+  data:
+  # PostgreSQL configuration - fetch from Vault
+  - secretKey: postgres_user
+    remoteRef:
+      key: postgres/admin
+      property: username
+  - secretKey: postgres_password
+    remoteRef:
+      key: postgres/admin
+      property: password
+  # Add more Vault references here:
+  # - secretKey: aws_access_key_id
+  #   remoteRef:
+  #     key: minio
+  #     property: access_key_id
+  # - secretKey: aws_secret_access_key
+  #   remoteRef:
+  #     key: minio
+  #     property: secret_access_key

dagster/dagster-values.gomplate.yaml

@@ -45,6 +45,11 @@ dagsterWebserver:
       persistentVolumeClaim:
         claimName: dagster-user-code-pvc
 
+  {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
+  envSecrets:
+    - name: dagster-env-secret
+  {{- end }}
+
   workspace:
     enabled: true
     servers: []
@@ -74,6 +79,11 @@ dagsterDaemon:
       persistentVolumeClaim:
         claimName: dagster-user-code-pvc
 
+  {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
+  envSecrets:
+    - name: dagster-env-secret
+  {{- end }}
+
   env:
     - name: DAGSTER_HOME
       value: /opt/dagster/dagster_home
@@ -99,14 +109,14 @@ runLauncher:
         - name: user-code
           persistentVolumeClaim:
             claimName: dagster-user-code-pvc
-      {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }}
       envSecrets:
         - name: dagster-database-secret
+        {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }}
         - name: dagster-minio-secret
-      {{- else }}
+        {{- end }}
-      envSecrets:
+        {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
-        - name: dagster-database-secret
+        - name: dagster-env-secret
-      {{- end }}
+        {{- end }}
 
 postgresql:
   enabled: false

dagster/justfile

@@ -148,6 +148,43 @@ delete-oauth-secret:
     @kubectl delete secret dagster-oauth-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
     @kubectl delete externalsecret dagster-oauth-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
 
+# Create environment variables secret example (customize as needed)
+create-env-secrets-example:
+    #!/bin/bash
+    set -euo pipefail
+    echo "Creating Dagster environment secrets example..."
+    echo "This is an example - customize the environment variables as needed"
+    if helm status external-secrets -n ${EXTERNAL_SECRETS_NAMESPACE} &>/dev/null; then
+        echo "External Secrets available. Creating ExternalSecret using template..."
+        echo "Edit dagster-env-external-secret.gomplate.yaml to customize environment variables"
+        kubectl delete externalsecret dagster-env-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
+        kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
+        gomplate -f dagster-env-external-secret.gomplate.yaml -o dagster-env-external-secret.yaml
+        kubectl apply -f dagster-env-external-secret.yaml
+        echo "Waiting for environment secret to be ready..."
+        kubectl wait --for=condition=Ready externalsecret/dagster-env-external-secret \
+            -n ${DAGSTER_NAMESPACE} --timeout=60s
+    else
+        echo "External Secrets not available. Creating Kubernetes Secret directly..."
+        POSTGRES_USER="buun"
+        POSTGRES_PASSWORD="buunpass"
+        kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
+        kubectl create secret generic dagster-env-secret -n ${DAGSTER_NAMESPACE} \
+            --from-literal=POSTGRES_USER="$POSTGRES_USER" \
+            --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD"
+        # Add more environment variables here:
+        # --from-literal=AWS_ACCESS_KEY_ID="your_value" \
+        # --from-literal=AWS_SECRET_ACCESS_KEY="your_value"
+        echo "Environment secret created directly in Kubernetes"
+    fi
+    echo "Example environment secrets created successfully"
+    echo "Customize the environment variables in this recipe as needed for your project"
+
+# Delete environment secrets
+delete-env-secrets:
+    @kubectl delete secret dagster-env-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
+    @kubectl delete externalsecret dagster-env-external-secret -n ${DAGSTER_NAMESPACE} --ignore-not-found
+
 # Setup MinIO storage for Dagster
 setup-minio-storage:
     #!/bin/bash
@@ -567,6 +604,15 @@ install:
     fi
     just setup-user-code-pvc
 
+    export DAGSTER_ENV_SECRETS_EXIST="false"
+    if kubectl get secret dagster-env-secret -n ${DAGSTER_NAMESPACE} &>/dev/null; then
+        echo "Environment secrets found - will include in deployment"
+        export DAGSTER_ENV_SECRETS_EXIST="true"
+    else
+        echo "No environment secrets found - use 'just dagster::create-env-secrets-example' to create them if needed"
+        export DAGSTER_ENV_SECRETS_EXIST="false"
+    fi
+
     just add-helm-repo
     gomplate -f dagster-values.gomplate.yaml -o dagster-values.yaml
     helm upgrade --install dagster dagster/dagster \