feat(dagster): install dependencies with initContainers

dagster/dagster-values.gomplate.yaml

@@ -28,22 +28,38 @@ dagsterWebserver:
       memory: "1Gi"
       cpu: "1000m"
 
+  extraPrependedInitContainers:
+    - name: install-packages
+      image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}"
+      command:
+        - /bin/bash
+        - -c
+        - |
+          pip install --target /opt/dagster/site-packages {{ .Env.DAGSTER_EXTRA_PACKAGES }}
+      volumeMounts:
+        - name: extra-packages
+          mountPath: /opt/dagster/site-packages
+
   env:
     - name: DAGSTER_HOME
       value: /opt/dagster/dagster_home
     - name: PYTHONPATH
-      value: /opt/dagster/user-code
+      value: /opt/dagster/site-packages:/opt/dagster/user-code
     - name: PIP_USER
       value: "true"
 
   volumeMounts:
     - name: user-code
       mountPath: /opt/dagster/user-code
+    - name: extra-packages
+      mountPath: /opt/dagster/site-packages
 
   volumes:
     - name: user-code
       persistentVolumeClaim:
         claimName: dagster-user-code-pvc
+    - name: extra-packages
+      emptyDir: {}
 
   {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
   envSecrets:
@@ -70,14 +86,30 @@ dagsterDaemon:
       memory: "1Gi"
       cpu: "1000m"
 
+  extraPrependedInitContainers:
+    - name: install-packages
+      image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}"
+      command:
+        - /bin/bash
+        - -c
+        - |
+          pip install --target /opt/dagster/site-packages {{ .Env.DAGSTER_EXTRA_PACKAGES }}
+      volumeMounts:
+        - name: extra-packages
+          mountPath: /opt/dagster/site-packages
+
   volumeMounts:
     - name: user-code
       mountPath: /opt/dagster/user-code
+    - name: extra-packages
+      mountPath: /opt/dagster/site-packages
 
   volumes:
     - name: user-code
       persistentVolumeClaim:
         claimName: dagster-user-code-pvc
+    - name: extra-packages
+      emptyDir: {}
 
   {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
   envSecrets:
@@ -88,7 +120,7 @@ dagsterDaemon:
     - name: DAGSTER_HOME
       value: /opt/dagster/dagster_home
     - name: PYTHONPATH
-      value: /opt/dagster/user-code
+      value: /opt/dagster/site-packages:/opt/dagster/user-code
     - name: PIP_USER
       value: "true"
 
@@ -105,10 +137,16 @@ runLauncher:
       volumeMounts:
         - name: user-code
           mountPath: /opt/dagster/user-code
+        - name: extra-packages
+          mountPath: /opt/dagster/site-packages
       volumes:
         - name: user-code
           persistentVolumeClaim:
             claimName: dagster-user-code-pvc
+        - name: extra-packages
+          emptyDir: {}
+      envVars:
+        - "PYTHONPATH=/opt/dagster/site-packages:/opt/dagster/user-code"
       envSecrets:
         - name: dagster-database-secret
         {{- if eq (.Env.DAGSTER_STORAGE_TYPE | default "local") "minio" }}
@@ -117,6 +155,19 @@ runLauncher:
         {{- if eq (.Env.DAGSTER_ENV_SECRETS_EXIST | default "false") "true" }}
         - name: dagster-env-secret
         {{- end }}
+      runK8sConfig:
+        podSpecConfig:
+          initContainers:
+            - name: install-packages
+              image: "{{ .Env.DAGSTER_CONTAINER_IMAGE }}:{{ .Env.DAGSTER_CONTAINER_TAG }}"
+              command:
+                - /bin/bash
+                - -c
+                - |
+                  pip install --target /opt/dagster/site-packages {{ .Env.DAGSTER_EXTRA_PACKAGES }}
+              volumeMounts:
+                - name: extra-packages
+                  mountPath: /opt/dagster/site-packages
 
 postgresql:
   enabled: false