454 lines
16 KiB
Makefile
454 lines
16 KiB
Makefile
set fallback := true
|
|
|
|
export LOCAL_K8S_HOST := env("LOCAL_K8S_HOST", "")
|
|
export EXTERNAL_K8S_HOST := env("EXTERNAL_K8S_HOST", "")
|
|
export KEYCLOAK_HOST := env("KEYCLOAK_HOST", "")
|
|
export KEYCLOAK_REALM := env("KEYCLOAK_REALM", "buunstack")
|
|
export K8S_OIDC_CLIENT_ID := env('K8S_OIDC_CLIENT_ID', "k8s")
|
|
export K3S_ENABLE_REGISTRY := env("K3S_ENABLE_REGISTRY", "true")
|
|
|
|
[private]
|
|
default:
|
|
@just --list --unsorted --list-submodules
|
|
|
|
# Install k3s cluster
|
|
install:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
just env::check
|
|
username=$(gum input --prompt="SSH username: " --value="${USER}" --width=100)
|
|
kubeconfig=""
|
|
context=""
|
|
if gum confirm "Update KUBECONFIG?"; then
|
|
kubeconfig=$(
|
|
gum input --prompt="KUBECONFIG file: " --value="${HOME}/.kube/config" --width=100
|
|
)
|
|
context=$(
|
|
gum input --prompt="Context name: " --value="${LOCAL_K8S_HOST}" --width=100
|
|
)
|
|
fi
|
|
args=(
|
|
"install"
|
|
"--host" "${LOCAL_K8S_HOST}"
|
|
"--tls-san" "${EXTERNAL_K8S_HOST}"
|
|
"--user" "${username}"
|
|
)
|
|
if [ -n "${context}" ]; then
|
|
args+=("--context" "${context}")
|
|
fi
|
|
|
|
if [ -n "${kubeconfig}" ]; then
|
|
mkdir -p "$(dirname "${kubeconfig}")"
|
|
args+=("--local-path" "${kubeconfig}" "--merge")
|
|
fi
|
|
echo "Running: k3sup ${args[*]}"
|
|
k3sup "${args[@]}"
|
|
|
|
if [ -n "${context}" ]; then
|
|
kubectl config use-context "${context}"
|
|
fi
|
|
|
|
if [ "${K3S_ENABLE_REGISTRY}" = "true" ]; then
|
|
echo "Setting up local Docker registry..."
|
|
|
|
# Deploy Docker registry to cluster
|
|
kubectl apply -f ./registry/registry.yaml
|
|
|
|
# Set Pod Security Standard for registry namespace
|
|
kubectl label namespace registry pod-security.kubernetes.io/enforce=restricted --overwrite
|
|
|
|
# Wait for registry deployment
|
|
echo "Waiting for registry to be ready..."
|
|
kubectl wait --for=condition=available --timeout=60s deployment/registry -n registry
|
|
|
|
# Configure registries.yaml for k3s
|
|
just configure-registry
|
|
|
|
echo "✓ Local Docker registry deployed and configured"
|
|
echo ""
|
|
echo "Registry accessible at:"
|
|
echo " localhost:30500"
|
|
echo ""
|
|
echo "Usage:"
|
|
echo " export DOCKER_HOST=ssh://${LOCAL_K8S_HOST}"
|
|
echo " docker build -t localhost:30500/myapp:latest ."
|
|
echo " docker push localhost:30500/myapp:latest"
|
|
echo " kubectl run myapp --image=localhost:30500/myapp:latest"
|
|
fi
|
|
|
|
echo "k3s cluster installed on ${LOCAL_K8S_HOST}."
|
|
|
|
# Uninstall k3s cluster
|
|
uninstall:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
if gum confirm "Uninstall k3s from ${LOCAL_K8S_HOST}?"; then
|
|
# Check if Longhorn is installed and uninstall it first
|
|
if helm status longhorn -n longhorn-system &>/dev/null; then
|
|
echo "Detected Longhorn installation. Uninstalling Longhorn first to prevent CSI mount issues..."
|
|
just longhorn::uninstall || echo "Warning: Longhorn uninstallation had errors, continuing..."
|
|
|
|
# Wait a bit for CSI cleanup
|
|
echo "Waiting for CSI cleanup..."
|
|
sleep 5
|
|
fi
|
|
|
|
# Force cleanup any remaining CSI mounts
|
|
echo "Cleaning up CSI mounts..."
|
|
ssh "${LOCAL_K8S_HOST}" "sudo pkill -9 umount || true"
|
|
ssh "${LOCAL_K8S_HOST}" "sudo umount -f /var/lib/kubelet/plugins/kubernetes.io/csi/*/globalmount 2>/dev/null || true"
|
|
ssh "${LOCAL_K8S_HOST}" "sudo umount -l /var/lib/kubelet/plugins/kubernetes.io/csi/*/globalmount 2>/dev/null || true"
|
|
|
|
ssh "${LOCAL_K8S_HOST}" "/usr/local/bin/k3s-uninstall.sh"
|
|
echo "Cleaning up kubeconfig entries..."
|
|
cluster_name=$(kubectl config view -o json | jq -r ".contexts[] | select(.name == \"${LOCAL_K8S_HOST}\") | .context.cluster // empty")
|
|
user_name=$(kubectl config view -o json | jq -r ".contexts[] | select(.name == \"${LOCAL_K8S_HOST}\") | .context.user // empty")
|
|
if kubectl config get-contexts "${LOCAL_K8S_HOST}" &>/dev/null; then
|
|
kubectl config delete-context "${LOCAL_K8S_HOST}"
|
|
echo "Deleted context: ${LOCAL_K8S_HOST}"
|
|
fi
|
|
if [ -n "${cluster_name}" ] && kubectl config get-clusters | grep -q "^${cluster_name}$"; then
|
|
kubectl config delete-cluster "${cluster_name}"
|
|
echo "Deleted cluster: ${cluster_name}"
|
|
fi
|
|
if [ -n "${user_name}" ] && kubectl config get-users | grep -q "^${user_name}$"; then
|
|
kubectl config delete-user "${user_name}"
|
|
echo "Deleted user: ${user_name}"
|
|
fi
|
|
echo "k3s cluster uninstalled from ${LOCAL_K8S_HOST}."
|
|
else
|
|
echo "Uninstallation cancelled." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Stop k3s cluster (with CSI cleanup)
|
|
stop:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
echo "Cleaning up CSI mounts before stopping k3s..."
|
|
ssh "${LOCAL_K8S_HOST}" 'bash -s' << 'EOF'
|
|
set +e
|
|
sudo pkill -9 umount 2>/dev/null
|
|
shopt -s nullglob
|
|
for mount in /var/lib/kubelet/plugins/kubernetes.io/csi/*/globalmount; do
|
|
if [ -d "$mount" ]; then
|
|
echo "Unmounting $mount..."
|
|
sudo umount -f "$mount" 2>/dev/null
|
|
sudo umount -l "$mount" 2>/dev/null
|
|
fi
|
|
done
|
|
exit 0
|
|
EOF
|
|
echo "Stopping k3s service..."
|
|
ssh "${LOCAL_K8S_HOST}" "sudo systemctl stop k3s"
|
|
echo "k3s stopped on ${LOCAL_K8S_HOST}."
|
|
echo "You can now safely shutdown the machine."
|
|
|
|
# Start k3s cluster
|
|
start:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
echo "Starting k3s service..."
|
|
ssh "${LOCAL_K8S_HOST}" "sudo systemctl start k3s"
|
|
echo "Waiting for k3s to be ready..."
|
|
sleep 5
|
|
kubectl wait --for=condition=Ready nodes --all --timeout=60s
|
|
echo "k3s started on ${LOCAL_K8S_HOST}."
|
|
|
|
# Restart k3s cluster (with CSI cleanup)
|
|
restart:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
just k8s::stop
|
|
just k8s::start
|
|
|
|
# Setup k8s OIDC authentication
|
|
setup-oidc-auth:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
just env::check
|
|
|
|
# Clear OIDC token cache if it exists
|
|
if [ -d ~/.kube/cache/oidc-login ]; then
|
|
if gum confirm "Clear OIDC token cache? (Recommended after Keycloak reinstall)"; then
|
|
rm -rf ~/.kube/cache/oidc-login/
|
|
echo "OIDC token cache cleared."
|
|
fi
|
|
fi
|
|
|
|
gomplate -f ./k3s/config.gomplate.yaml | \
|
|
ssh ${LOCAL_K8S_HOST} "sudo tee /etc/rancher/k3s/config.yaml > /dev/null"
|
|
ssh ${LOCAL_K8S_HOST} "sudo systemctl restart k3s"
|
|
kubectl config set-credentials ${LOCAL_K8S_HOST}-oidc \
|
|
--exec-api-version=client.authentication.k8s.io/v1beta1 \
|
|
--exec-command=kubectl \
|
|
--exec-arg=oidc-login \
|
|
--exec-arg=get-token \
|
|
--exec-arg=--oidc-issuer-url=https://${KEYCLOAK_HOST}/realms/${KEYCLOAK_REALM} \
|
|
--exec-arg=--oidc-client-id=${K8S_OIDC_CLIENT_ID}
|
|
kubectl config set-cluster ${LOCAL_K8S_HOST}-oidc \
|
|
--server=https://${EXTERNAL_K8S_HOST}
|
|
kubectl config set-context ${LOCAL_K8S_HOST}-oidc \
|
|
--cluster=${LOCAL_K8S_HOST}-oidc --user=${LOCAL_K8S_HOST}-oidc
|
|
kubectl config use-context ${LOCAL_K8S_HOST}-oidc
|
|
|
|
# Create the container registry credentials
|
|
create-regcred namespace='default':
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
while [ -z "${CONTAINER_REGISTRY_SERVER:-}" ]; do
|
|
if ! CONTAINER_REGISTRY_SERVER=$(
|
|
gum input --prompt="Container registry server: " --width=100 \
|
|
--placeholder="e.g., index.docker.io/v1 ghcr.io"
|
|
); then
|
|
echo "Setup cancelled." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
while [ -z "${CONTAINER_REGISTRY_USERNAME:-}" ]; do
|
|
if ! CONTAINER_REGISTRY_USERNAME=$(
|
|
gum input --prompt="Container registry username: " --width=100
|
|
); then
|
|
echo "Setup cancelled." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
while [ -z "${CONTAINER_REGISTRY_PASSWORD:-}" ]; do
|
|
if ! CONTAINER_REGISTRY_PASSWORD=$(
|
|
gum input --prompt="Container registry password or token: " --password --width=100
|
|
); then
|
|
echo "Setup cancelled." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
while [ -z "${CONTAINER_REGISTRY_EMAIL:-}" ]; do
|
|
if ! CONTAINER_REGISTRY_EMAIL=$(
|
|
gum input --prompt="Container registry email: " --width=100
|
|
); then
|
|
echo "Setup cancelled." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
kubectl create -n {{ namespace }} secret docker-registry regcred \
|
|
--docker-server="${CONTAINER_REGISTRY_SERVER}" \
|
|
--docker-username="${CONTAINER_REGISTRY_USERNAME}" \
|
|
--docker-password="${CONTAINER_REGISTRY_PASSWORD}" \
|
|
--docker-email="${CONTAINER_REGISTRY_EMAIL}"
|
|
|
|
# Delete the container registry credentials
|
|
delete-regcred namespace='default':
|
|
kubectl delete -n {{ namespace }} secret regcred --ignore-not-found
|
|
|
|
# Copy the container registry credentials from the default namespace
|
|
copy-regcred namespace:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
if ! kubectl get -n default secret regcred &>/dev/null; then
|
|
just create-regcred default
|
|
fi
|
|
if kubectl get -n {{ namespace }} secret regcred &>/dev/null; then
|
|
kubectl delete -n {{ namespace }} secret regcred
|
|
fi
|
|
kubectl get -n default secret regcred -o json | \
|
|
sed "s/\"namespace\": \"default\"/\"namespace\": \"{{ namespace }}\"/g" | \
|
|
kubectl apply -n {{ namespace }} -f -
|
|
|
|
# Check local Docker registry status
|
|
check-registry:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
echo "Checking local Docker registry status..."
|
|
echo ""
|
|
|
|
# Check if registry deployment exists
|
|
echo "1. Registry deployment status:"
|
|
if kubectl get deployment registry -n registry &>/dev/null; then
|
|
echo "✓ Registry deployment exists"
|
|
kubectl get deployment registry -n registry
|
|
echo ""
|
|
echo "Registry pods:"
|
|
kubectl get pods -n registry -l app=registry
|
|
else
|
|
echo "✗ Registry deployment not found"
|
|
fi
|
|
echo ""
|
|
|
|
# Check registry services
|
|
echo "2. Registry services:"
|
|
if kubectl get service registry -n registry &>/dev/null; then
|
|
echo "✓ Registry service exists"
|
|
kubectl get service registry registry-nodeport -n registry
|
|
else
|
|
echo "✗ Registry services not found"
|
|
fi
|
|
echo ""
|
|
|
|
# Check k3s registries configuration
|
|
echo "3. K3s registries configuration:"
|
|
if ssh "${LOCAL_K8S_HOST}" "sudo test -f /etc/rancher/k3s/registries.yaml"; then
|
|
echo "✓ Registries configuration exists"
|
|
ssh "${LOCAL_K8S_HOST}" "sudo cat /etc/rancher/k3s/registries.yaml"
|
|
else
|
|
echo "✗ Registries configuration not found"
|
|
fi
|
|
echo ""
|
|
|
|
# Test registry accessibility
|
|
echo "4. Registry accessibility test:"
|
|
echo "Testing from k3s host (localhost:30500):"
|
|
if ssh "${LOCAL_K8S_HOST}" "curl -f -s http://localhost:30500/v2/" &>/dev/null; then
|
|
echo "✓ Registry is accessible from k3s host"
|
|
# Show registry catalog
|
|
echo "Registry catalog:"
|
|
ssh "${LOCAL_K8S_HOST}" "curl -s http://localhost:30500/v2/_catalog" 2>/dev/null || \
|
|
echo "Unable to retrieve catalog"
|
|
else
|
|
echo "✗ Registry is not accessible from k3s host"
|
|
fi
|
|
echo ""
|
|
echo "Note: To push images, use:"
|
|
echo " export DOCKER_HOST=ssh://${LOCAL_K8S_HOST}"
|
|
echo " docker push localhost:30500/myimage:tag"
|
|
|
|
# Deploy Docker registry manually
|
|
deploy-registry:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
echo "Deploying local Docker registry..."
|
|
|
|
kubectl apply -f ./registry/registry.yaml
|
|
|
|
# Set Pod Security Standard for registry namespace
|
|
kubectl label namespace registry pod-security.kubernetes.io/enforce=restricted --overwrite
|
|
|
|
echo "Waiting for registry to be ready..."
|
|
kubectl wait --for=condition=available --timeout=60s deployment/registry -n registry
|
|
echo "✓ Registry deployed and ready"
|
|
|
|
# Remove Docker registry
|
|
remove-registry:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
if gum confirm "Remove local Docker registry?"; then
|
|
kubectl delete namespace registry --ignore-not-found
|
|
echo "✓ Registry removed"
|
|
else
|
|
echo "Registry removal cancelled."
|
|
fi
|
|
|
|
# Configure k3s to use local registry
|
|
configure-registry:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
echo "Configuring k3s registries.yaml..."
|
|
|
|
ssh "${LOCAL_K8S_HOST}" "sudo mkdir -p /etc/rancher/k3s"
|
|
gomplate -f ./registry/registries.gomplate.yaml | ssh "${LOCAL_K8S_HOST}" "sudo tee /etc/rancher/k3s/registries.yaml > /dev/null"
|
|
|
|
echo "Restarting k3s to apply registry configuration..."
|
|
ssh "${LOCAL_K8S_HOST}" "sudo systemctl restart k3s"
|
|
echo "✓ Registry configuration applied"
|
|
|
|
[positional-arguments]
|
|
wait-deployments-ready *args:
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
namespace="$1"
|
|
shift
|
|
deployments=("$@")
|
|
check_ready() {
|
|
for deployment in "${deployments[@]}"; do
|
|
ready=$(kubectl get -n ${namespace} deployment "${deployment}" \
|
|
-o jsonpath="{.status.readyReplicas}" 2>/dev/null || true)
|
|
replicas=$(kubectl get -n ${namespace} deployment "${deployment}" \
|
|
-o jsonpath="{.status.replicas}" 2>/dev/null || true)
|
|
if [[ "${ready}" != "${replicas}" || -z "${ready}" ]]; then
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
echo -n "Waiting for deployments $@ to be ready..."
|
|
while check_ready; do
|
|
echo -n "."
|
|
sleep 2
|
|
done
|
|
echo "ok"
|
|
|
|
# Delete completed pods across all namespaces
|
|
delete-completed-pods namespace='':
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
NAMESPACE="${NAMESPACE:-{{ namespace }}}"
|
|
|
|
if [ -n "${NAMESPACE}" ]; then
|
|
echo "Deleting completed pods in namespace '${NAMESPACE}'..."
|
|
SELECTOR="--namespace=${NAMESPACE}"
|
|
else
|
|
echo "Deleting completed pods in all namespaces..."
|
|
SELECTOR="--all-namespaces"
|
|
fi
|
|
|
|
# Find and delete pods with status.phase=Succeeded or status.phase=Failed
|
|
COMPLETED_PODS=$(kubectl get pods ${SELECTOR} -o json | \
|
|
jq -r '.items[] | select(.status.phase == "Succeeded" or .status.phase == "Failed") |
|
|
.metadata.namespace + "/" + .metadata.name')
|
|
|
|
if [ -z "${COMPLETED_PODS}" ]; then
|
|
echo "No completed pods found."
|
|
exit 0
|
|
fi
|
|
|
|
echo "Found completed pods:"
|
|
echo "${COMPLETED_PODS}"
|
|
echo ""
|
|
|
|
if gum confirm "Delete these pods?"; then
|
|
echo "${COMPLETED_PODS}" | while IFS='/' read -r ns pod; do
|
|
echo "Deleting pod ${pod} in namespace ${ns}..."
|
|
kubectl delete pod "${pod}" -n "${ns}" --ignore-not-found
|
|
done
|
|
echo "Completed pods deleted."
|
|
else
|
|
echo "Deletion cancelled."
|
|
fi
|
|
|
|
# Delete completed jobs across all namespaces
|
|
delete-completed-jobs namespace='':
|
|
#!/bin/bash
|
|
set -euo pipefail
|
|
NAMESPACE="${NAMESPACE:-{{ namespace }}}"
|
|
|
|
if [ -n "${NAMESPACE}" ]; then
|
|
echo "Deleting completed jobs in namespace '${NAMESPACE}'..."
|
|
SELECTOR="--namespace=${NAMESPACE}"
|
|
else
|
|
echo "Deleting completed jobs in all namespaces..."
|
|
SELECTOR="--all-namespaces"
|
|
fi
|
|
|
|
# Find and delete jobs with status.succeeded > 0 or status.failed > 0
|
|
COMPLETED_JOBS=$(kubectl get jobs ${SELECTOR} -o json | \
|
|
jq -r '.items[] | select((.status.succeeded // 0) > 0 or (.status.failed // 0) > 0) |
|
|
.metadata.namespace + "/" + .metadata.name')
|
|
|
|
if [ -z "${COMPLETED_JOBS}" ]; then
|
|
echo "No completed jobs found."
|
|
exit 0
|
|
fi
|
|
|
|
echo "Found completed jobs:"
|
|
echo "${COMPLETED_JOBS}"
|
|
echo ""
|
|
|
|
if gum confirm "Delete these jobs?"; then
|
|
echo "${COMPLETED_JOBS}" | while IFS='/' read -r ns job; do
|
|
echo "Deleting job ${job} in namespace ${ns}..."
|
|
kubectl delete job "${job}" -n "${ns}" --ignore-not-found --cascade=foreground
|
|
done
|
|
echo "Completed jobs deleted."
|
|
else
|
|
echo "Deletion cancelled."
|
|
fi
|