diff --git a/dagster/dagster-values.gomplate.yaml b/dagster/dagster-values.gomplate.yaml index f522ef2..d6f87cf 100644 --- a/dagster/dagster-values.gomplate.yaml +++ b/dagster/dagster-values.gomplate.yaml @@ -28,6 +28,7 @@ dagsterWebserver: memory: "1Gi" cpu: "1000m" + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} extraPrependedInitContainers: - name: install-packages image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}" @@ -39,27 +40,37 @@ dagsterWebserver: volumeMounts: - name: extra-packages mountPath: /opt/dagster/site-packages + {{- end }} env: - name: DAGSTER_HOME value: /opt/dagster/dagster_home + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: PYTHONPATH value: /opt/dagster/site-packages:/opt/dagster/user-code + {{- else }} + - name: PYTHONPATH + value: /opt/dagster/user-code + {{- end }} - name: PIP_USER value: "true" volumeMounts: - name: user-code mountPath: /opt/dagster/user-code + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: extra-packages mountPath: /opt/dagster/site-packages + {{- end }} volumes: - name: user-code persistentVolumeClaim: claimName: dagster-user-code-pvc + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: extra-packages emptyDir: {} + {{- end }} {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }} envSecrets: @@ -86,6 +97,7 @@ dagsterDaemon: memory: "1Gi" cpu: "1000m" + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} extraPrependedInitContainers: - name: install-packages image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}" @@ -97,19 +109,24 @@ dagsterDaemon: volumeMounts: - name: extra-packages mountPath: /opt/dagster/site-packages + {{- end }} volumeMounts: - name: user-code mountPath: /opt/dagster/user-code + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: extra-packages mountPath: /opt/dagster/site-packages + {{- end }} volumes: - name: user-code persistentVolumeClaim: claimName: dagster-user-code-pvc + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: extra-packages emptyDir: {} + {{- end }} {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }} envSecrets: @@ -119,8 +136,13 @@ dagsterDaemon: env: - name: DAGSTER_HOME value: /opt/dagster/dagster_home + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: PYTHONPATH value: /opt/dagster/site-packages:/opt/dagster/user-code + {{- else }} + - name: PYTHONPATH + value: /opt/dagster/user-code + {{- end }} - name: PIP_USER value: "true" @@ -137,16 +159,22 @@ runLauncher: volumeMounts: - name: user-code mountPath: /opt/dagster/user-code + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: extra-packages mountPath: /opt/dagster/site-packages + {{- end }} volumes: - name: user-code persistentVolumeClaim: claimName: dagster-user-code-pvc + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} - name: extra-packages emptyDir: {} + {{- end }} + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} envVars: - "PYTHONPATH=/opt/dagster/site-packages:/opt/dagster/user-code" + {{- end }} envSecrets: - name: dagster-database-secret {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }} @@ -155,6 +183,7 @@ runLauncher: {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }} - name: dagster-env-secret {{- end }} + {{- if .Env.DAGSTER_EXTRA_PACKAGES }} runK8sConfig: podSpecConfig: initContainers: @@ -168,6 +197,7 @@ runLauncher: volumeMounts: - name: extra-packages mountPath: /opt/dagster/site-packages + {{- end }} postgresql: enabled: false diff --git a/dagster/image/Dockerfile b/dagster/image/Dockerfile index 602fe85..360847c 100644 --- a/dagster/image/Dockerfile +++ b/dagster/image/Dockerfile @@ -1,3 +1,3 @@ -FROM docker.io/dagster/dagster-k8s:1.11.10 +FROM docker.io/dagster/dagster-k8s:1.11.13 -RUN pip install dagster-duckdb dagster-postgres pandas 'dlt[filesystem,postgres,s3]' +RUN pip install "dlt[duckdb]" pyarrow pyiceberg s3fs simple-salesforce diff --git a/dagster/justfile b/dagster/justfile index 1e5caec..2fb5030 100644 --- a/dagster/justfile +++ b/dagster/justfile @@ -12,7 +12,9 @@ export DAGSTER_STORAGE_SIZE := env("DAGSTER_STORAGE_SIZE", "20Gi") export DAGSTER_CODE_STORAGE_SIZE := env("DAGSTER_CODE_STORAGE_SIZE", "10Gi") export MINIO_NAMESPACE := env("MINIO_NAMESPACE", "minio") export DAGSTER_STORAGE_TYPE := env("DAGSTER_STORAGE_TYPE", "") -export DAGSTER_EXTRA_PACKAGES := env("DAGSTER_EXTRA_PACKAGES", "dlt[duckdb] pyarrow pyiceberg s3fs simple-salesforce") +export DAGSTER_EXTRA_PACKAGES := env("DAGSTER_EXTRA_PACKAGES", "") + +# export DAGSTER_EXTRA_PACKAGES := env("DAGSTER_EXTRA_PACKAGES", "dlt[duckdb] pyarrow pyiceberg s3fs simple-salesforce") [private] default: @@ -379,9 +381,15 @@ add-workspace-module module_name working_directory: kubectl patch configmap dagster-workspace-yaml -n ${DAGSTER_NAMESPACE} --patch "$PATCH_JSON" echo "✓ Module '${MODULE_NAME}' added to workspace" - echo "Restarting Dagster to reload workspace..." + +# Reload workspace configuration (restart webserver and daemon) +reload-workspace: + #!/bin/bash + set -euo pipefail + echo "Reloading Dagster workspace configuration..." kubectl rollout restart deployment/dagster-dagster-webserver -n ${DAGSTER_NAMESPACE} kubectl rollout restart deployment/dagster-daemon -n ${DAGSTER_NAMESPACE} + echo "✓ Workspace reload initiated" # Note: add-workspace-file command has been removed due to sed parsing issues # Use add-workspace-module command instead for adding Python modules to workspace @@ -542,11 +550,6 @@ remove-project project_name='': echo "✓ Module '${PYTHON_MODULE_NAME}' removed from workspace" fi - # Restart Dagster to reload workspace - echo "Restarting Dagster to reload workspace..." - kubectl rollout restart deployment/dagster-dagster-webserver -n ${DAGSTER_NAMESPACE} - kubectl rollout restart deployment/dagster-daemon -n ${DAGSTER_NAMESPACE} - echo "✓ Project '${PROJECT_NAME}' removed successfully" # Setup OAuth2 Proxy for Dagster authentication