feat(dagster): deploy without restart

dagster/dagster-values.gomplate.yaml

@@ -28,6 +28,7 @@ dagsterWebserver:
       memory: "1Gi"
       cpu: "1000m"
 
+  {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
   extraPrependedInitContainers:
     - name: install-packages
       image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}"
@@ -39,27 +40,37 @@ dagsterWebserver:
       volumeMounts:
         - name: extra-packages
           mountPath: /opt/dagster/site-packages
+  {{- end }}
 
   env:
     - name: DAGSTER_HOME
       value: /opt/dagster/dagster_home
+    {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
     - name: PYTHONPATH
       value: /opt/dagster/site-packages:/opt/dagster/user-code
+    {{- else }}
+    - name: PYTHONPATH
+      value: /opt/dagster/user-code
+    {{- end }}
     - name: PIP_USER
       value: "true"
 
   volumeMounts:
     - name: user-code
       mountPath: /opt/dagster/user-code
+    {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
     - name: extra-packages
       mountPath: /opt/dagster/site-packages
+    {{- end }}
 
   volumes:
     - name: user-code
       persistentVolumeClaim:
         claimName: dagster-user-code-pvc
+    {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
     - name: extra-packages
       emptyDir: {}
+    {{- end }}
 
   {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
   envSecrets:
@@ -86,6 +97,7 @@ dagsterDaemon:
       memory: "1Gi"
       cpu: "1000m"
 
+  {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
   extraPrependedInitContainers:
     - name: install-packages
       image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}"
@@ -97,19 +109,24 @@ dagsterDaemon:
       volumeMounts:
         - name: extra-packages
           mountPath: /opt/dagster/site-packages
+  {{- end }}
 
   volumeMounts:
     - name: user-code
       mountPath: /opt/dagster/user-code
+    {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
     - name: extra-packages
       mountPath: /opt/dagster/site-packages
+    {{- end }}
 
   volumes:
     - name: user-code
       persistentVolumeClaim:
         claimName: dagster-user-code-pvc
+    {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
     - name: extra-packages
       emptyDir: {}
+    {{- end }}
 
   {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
   envSecrets:
@@ -119,8 +136,13 @@ dagsterDaemon:
   env:
     - name: DAGSTER_HOME
       value: /opt/dagster/dagster_home
+    {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
     - name: PYTHONPATH
       value: /opt/dagster/site-packages:/opt/dagster/user-code
+    {{- else }}
+    - name: PYTHONPATH
+      value: /opt/dagster/user-code
+    {{- end }}
     - name: PIP_USER
       value: "true"
 
@@ -137,16 +159,22 @@ runLauncher:
       volumeMounts:
         - name: user-code
           mountPath: /opt/dagster/user-code
+        {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
         - name: extra-packages
           mountPath: /opt/dagster/site-packages
+        {{- end }}
       volumes:
         - name: user-code
           persistentVolumeClaim:
             claimName: dagster-user-code-pvc
+        {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
         - name: extra-packages
           emptyDir: {}
+        {{- end }}
+      {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
       envVars:
         - "PYTHONPATH=/opt/dagster/site-packages:/opt/dagster/user-code"
+      {{- end }}
       envSecrets:
         - name: dagster-database-secret
         {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }}
@@ -155,6 +183,7 @@ runLauncher:
         {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
         - name: dagster-env-secret
         {{- end }}
+      {{- if .Env.DAGSTER_EXTRA_PACKAGES }}
       runK8sConfig:
         podSpecConfig:
           initContainers:
@@ -168,6 +197,7 @@ runLauncher:
               volumeMounts:
                 - name: extra-packages
                   mountPath: /opt/dagster/site-packages
+      {{- end }}
 
 postgresql:
   enabled: false

dagster/image/Dockerfile

@@ -1,3 +1,3 @@
-FROM docker.io/dagster/dagster-k8s:1.11.10
+FROM docker.io/dagster/dagster-k8s:1.11.13
 
-RUN pip install dagster-duckdb dagster-postgres pandas 'dlt[filesystem,postgres,s3]'
+RUN pip install "dlt[duckdb]" pyarrow pyiceberg s3fs simple-salesforce

dagster/justfile

@@ -12,7 +12,9 @@ export DAGSTER_STORAGE_SIZE := env("DAGSTER_STORAGE_SIZE", "20Gi")
 export DAGSTER_CODE_STORAGE_SIZE := env("DAGSTER_CODE_STORAGE_SIZE", "10Gi")
 export MINIO_NAMESPACE := env("MINIO_NAMESPACE", "minio")
 export DAGSTER_STORAGE_TYPE := env("DAGSTER_STORAGE_TYPE", "")
-export DAGSTER_EXTRA_PACKAGES := env("DAGSTER_EXTRA_PACKAGES", "dlt[duckdb] pyarrow pyiceberg s3fs simple-salesforce")
+export DAGSTER_EXTRA_PACKAGES := env("DAGSTER_EXTRA_PACKAGES", "")
+
+# export DAGSTER_EXTRA_PACKAGES := env("DAGSTER_EXTRA_PACKAGES", "dlt[duckdb] pyarrow pyiceberg s3fs simple-salesforce")
 
 [private]
 default:
@@ -379,9 +381,15 @@ add-workspace-module module_name working_directory:
     kubectl patch configmap dagster-workspace-yaml -n ${DAGSTER_NAMESPACE} --patch "$PATCH_JSON"
 
     echo "✓ Module '${MODULE_NAME}' added to workspace"
-    echo "Restarting Dagster to reload workspace..."
+
+# Reload workspace configuration (restart webserver and daemon)
+reload-workspace:
+    #!/bin/bash
+    set -euo pipefail
+    echo "Reloading Dagster workspace configuration..."
     kubectl rollout restart deployment/dagster-dagster-webserver -n ${DAGSTER_NAMESPACE}
     kubectl rollout restart deployment/dagster-daemon -n ${DAGSTER_NAMESPACE}
+    echo "✓ Workspace reload initiated"
 
 # Note: add-workspace-file command has been removed due to sed parsing issues
 # Use add-workspace-module command instead for adding Python modules to workspace
@@ -542,11 +550,6 @@ remove-project project_name='':
         echo "✓ Module '${PYTHON_MODULE_NAME}' removed from workspace"
     fi
 
-    # Restart Dagster to reload workspace
-    echo "Restarting Dagster to reload workspace..."
-    kubectl rollout restart deployment/dagster-dagster-webserver -n ${DAGSTER_NAMESPACE}
-    kubectl rollout restart deployment/dagster-daemon -n ${DAGSTER_NAMESPACE}
-
     echo "✓ Project '${PROJECT_NAME}' removed successfully"
 
 # Setup OAuth2 Proxy for Dagster authentication