fix(jupyterhub): airflow integration

jupyterhub/images/datastack-cuda-notebook/Dockerfile

@@ -77,7 +77,6 @@ RUN pip uninstall -y \
 # 2. Find the pandas version in the file `dev/infra/Dockerfile`.
 RUN mamba install --yes \
     'aif360' \
-    'airflow' \
     'chromadb' \
     'clickhouse-connect' \
     'csvkit' \
@@ -142,6 +141,7 @@ RUN mamba install --yes \
 
 RUN pip install \
     agno \
+    apache-airflow \
     apache-airflow-client \
     fastembed \
     feature-engine \

jupyterhub/images/datastack-notebook/Dockerfile

@@ -77,7 +77,6 @@ RUN pip uninstall -y \
 # 2. Find the pandas version in the file `dev/infra/Dockerfile`.
 RUN mamba install --yes \
     'aif360' \
-    'airflow' \
     'chromadb' \
     'clickhouse-connect' \
     'csvkit' \
@@ -142,6 +141,7 @@ RUN mamba install --yes \
 
 RUN pip install \
     agno \
+    apache-airflow \
     apache-airflow-client \
     fastembed \
     feature-engine \

jupyterhub/jupyterhub-values.gomplate.yaml

@@ -178,12 +178,23 @@ singleuser:
           optional: true  # Don't fail if PVC doesn't exist yet
     extraVolumeMounts:
       - name: airflow-dags
-        mountPath: /opt/airflow-dags
+        mountPath: /home/jovyan/airflow-dags
         readOnly: false
     {{- end }}
 
   networkPolicy:
     egress:
+      {{- if eq .Env.JUPYTERHUB_AIRFLOW_DAGS_PERSISTENCE_ENABLED "true" }}
+      # Allow communication with Airflow API server in the same namespace
+      - to:
+          - podSelector:
+              matchLabels:
+                release: airflow
+                component: api-server
+        ports:
+          - port: 8080
+            protocol: TCP
+      {{- end }}
       - to:
           - namespaceSelector:
               matchLabels:

jupyterhub/justfile

@@ -9,7 +9,7 @@ export JUPYTERHUB_NFS_PV_ENABLED := env("JUPYTERHUB_NFS_PV_ENABLED", "")
 export JUPYTERHUB_STORAGE_CLASS := env("JUPYTERHUB_STORAGE_CLASS", "")
 export JUPYTERHUB_VAULT_INTEGRATION_ENABLED := env("JUPYTERHUB_VAULT_INTEGRATION_ENABLED", "")
 export JUPYTERHUB_AIRFLOW_DAGS_PERSISTENCE_ENABLED := env("JUPYTERHUB_AIRFLOW_DAGS_PERSISTENCE_ENABLED", "")
-export JUPYTER_PYTHON_KERNEL_TAG := env("JUPYTER_PYTHON_KERNEL_TAG", "python-3.12-36")
+export JUPYTER_PYTHON_KERNEL_TAG := env("JUPYTER_PYTHON_KERNEL_TAG", "python-3.12-37")
 export KERNEL_IMAGE_BUUN_STACK_REPOSITORY := env("KERNEL_IMAGE_BUUN_STACK_REPOSITORY", "buun-stack-notebook")
 export KERNEL_IMAGE_BUUN_STACK_CUDA_REPOSITORY := env("KERNEL_IMAGE_BUUN_STACK_CUDA_REPOSITORY", "buun-stack-cuda-notebook")
 export JUPYTER_PROFILE_MINIMAL_ENABLED := env("JUPYTER_PROFILE_MINIMAL_ENABLED", "false")