Multi-Tenant GPU Kubernetes on Simulated Hardware with Kamaji and HAMi

kind create cluster --name kamaji

kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.21.2/cert-manager.yaml
kubectl -n cert-manager wait pod --all --for=condition=Ready --timeout=180s

kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.16.1/config/manifests/metallb-frr-k8s.yaml
kubectl -n metallb-system wait pod --all --for=condition=Ready --timeout=180s

docker network inspect kind | grep Subnet

cat <<EOF | kubectl apply -f -
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
  name: kind-ip-pool
  namespace: metallb-system
spec:
  addresses:
  - 172.18.255.200-172.18.255.250
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
  name: empty
  namespace: metallb-system
EOF

helm repo add clastix https://clastix.github.io/charts
helm upgrade --install kamaji clastix/kamaji \
  --namespace kamaji-system --create-namespace \
  --set 'resources=null' --version 0.0.0+latest

kubectl -n kamaji-system get pods

export TENANT_VERSION=v1.36.4
kubectl create namespace tenants

for t in tenant-a tenant-b; do
cat <<EOF | kubectl apply -f -
apiVersion: kamaji.clastix.io/v1alpha1
kind: TenantControlPlane
metadata:
  name: ${t}
  namespace: tenants
  labels:
    tenant.clastix.io: ${t}
spec:
  controlPlane:
    deployment:
      replicas: 1
    service:
      serviceType: LoadBalancer
  kubernetes:
    version: "${TENANT_VERSION}"
    kubelet:
      configurationJSONPatches:
        - op: add
          path: /cgroupDriver
          value: systemd
  networkProfile:
    port: 6443
    podCidr: 10.244.0.0/16
  addons:
    coreDNS: {}
    kubeProxy: {}
    konnectivity:
      server:
        port: 8132
      agent:
        mode: DaemonSet
EOF
done

for t in tenant-a tenant-b; do
  kubectl -n tenants get secret ${t}-admin-kubeconfig -o jsonpath='{.data.admin\.conf}' | base64 -d > ${t}.kubeconfig
  kubectl --kubeconfig ${t}.kubeconfig cluster-info
done

docker run -d --name tenant-a-worker --hostname tenant-a-worker \
  --network kind --privileged \
  --security-opt seccomp=unconfined --security-opt apparmor=unconfined \
  --tmpfs /tmp --tmpfs /run -v /var -v /lib/modules:/lib/modules:ro \
  kindest/node:v1.36.4@sha256:099e049362a1526b2db71494e1947aae99bd16290d7c895f2b7ea312e3cbfaed
sleep 15

docker exec -i tenant-a-worker sh -c 'cat > /root/tenant-a.kubeconfig' < tenant-a.kubeconfig
JOIN_CMD=$(docker exec tenant-a-worker kubeadm token create --print-join-command \
  --kubeconfig /root/tenant-a.kubeconfig)

echo "$JOIN_CMD"  
docker exec tenant-a-worker bash -c "${JOIN_CMD} --ignore-preflight-errors=all"

curl -sLO https://raw.githubusercontent.com/projectcalico/calico/v3.31.2/manifests/calico.yaml
sed -i 's|# - name: CALICO_IPV4POOL_CIDR|- name: CALICO_IPV4POOL_CIDR|; s|#   value: "192.168.0.0/16"|  value: "10.244.0.0/16"|' calico.yaml
kubectl --kubeconfig tenant-a.kubeconfig apply -f calico.yaml

kubectl --kubeconfig tenant-a.kubeconfig get nodes

export KUBECONFIG=$PWD/tenant-a.kubeconfig

helm repo add hami-charts https://project-hami.github.io/HAMi/
helm install hami hami-charts/hami -n kube-system
# wait for hami-scheduler Running, and the hami-scheduler-device ConfigMap to exist

kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/mock-device-plugin/master/k8s-mock-rbac.yaml
kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/mock-device-plugin/master/k8s-mock-plugin.yaml

# Simulate 1 x A100 80 GB
NODE=tenant-a-worker
kubectl patch node $NODE --subresource=status --type=json \
  -p '[{"op":"add","path":"/status/capacity/nvidia.com~1gpu","value":"10"}]'

kubectl annotate node $NODE \
  'hami.io/node-nvidia-register=[{"id":"GPU-MOCK-0","count":10,"devmem":81920,"devcore":100,"type":"NVIDIA-A100-SXM4-80GB","health":true,"numa":0,"mode":"hami-core"}]'

{
  "nvidia.com/gpu": "10",
  "nvidia.com/gpucores": "100",
  "nvidia.com/gpumem": "81920",
  "nvidia.com/gpumem-percentage": "100"
}

export KUBECONFIG=$PWD/tenant-a.kubeconfig

for i in 1 2 3; do
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: gpu-share-${i}
spec:
  containers:
    - name: app
      image: busybox
      command: ["sleep", "infinity"]
      resources:
        limits:
          nvidia.com/gpu: 1
          nvidia.com/gpumem: 20000   # MB
          nvidia.com/gpucores: 30    # % of one card
EOF
  kubectl wait pod/gpu-share-${i} --for=condition=PodScheduled --timeout=120s
  kubectl annotate pod gpu-share-${i} hami.io/bind-phase=success --overwrite
  kubectl annotate node tenant-a-worker hami.io/mutex.lock-
done

$ kubectl get pod gpu-share-1 -o jsonpath='{.spec.schedulerName}'
hami-scheduler

Annotations:  hami.io/vgpu-devices-allocated: GPU-MOCK-0,NVIDIA,20000,30:;
              hami.io/vgpu-node: tenant-a-worker
Events:
  Normal  FilteringSucceed  hami-scheduler  find fit node(tenant-a-worker), 0 nodes not fit, 1 nodes fit
  Normal  BindingSucceed    hami-scheduler  Successfully binding node [tenant-a-worker] to default/gpu-share-1

NAME          READY   STATUS    NODE
gpu-share-1   1/1     Running   tenant-a-worker
gpu-share-2   1/1     Running   tenant-a-worker
gpu-share-3   1/1     Running   tenant-a-worker

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: gpu-share-4
spec:
  containers:
    - name: app
      image: busybox
      command: ["sleep", "infinity"]
      resources:
        limits:
          nvidia.com/gpu: 1
          nvidia.com/gpumem: 20000   # MB
          nvidia.com/gpucores: 30    # % of one card
EOF

$ kubectl describe pod gpu-share-4 | sed -n '/Events/,$p'
Events:
  Type     Reason            Age                From            Message
  ----     ------            ----               ----            -------
  Warning  FailedScheduling  15s                hami-scheduler  0/1 nodes are available: 1 1/1 CardInsufficientCore. no new claims to deallocate, preemption: 0/1 nodes are available: 1 No preemption victims found for incoming pod.
  Warning  FilteringFailed   15s (x2 over 15s)  hami-scheduler  1 nodes CardInsufficientCore(tenant-a-worker)
  Warning  FilteringFailed   15s (x2 over 15s)  hami-scheduler  no available node, 1 nodes do not meet

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: gpu-mem-big
spec:
  containers:
    - name: app
      image: busybox
      command: ["sleep", "infinity"]
      resources:
        limits:
          nvidia.com/gpu: 1
          nvidia.com/gpumem: 90000   # MB, more than the whole card
          nvidia.com/gpucores: 10    # % of one card, fits in the free 10%
EOF

for t in a b; do
  echo "===== tenant-$t ====="
  kubectl --kubeconfig tenant-$t.kubeconfig get nodes
  kubectl --kubeconfig tenant-$t.kubeconfig get pods -A | grep -E 'gpu-|NAME'
  kubectl --kubeconfig tenant-$t.kubeconfig get nodes -o json \
    | jq '.items[].status.allocatable | with_entries(select(.key|test("gpumem|gpucores")))'
done

===== tenant-a =====
NAME              STATUS   ROLES    AGE   VERSION
tenant-a-worker   Ready    <none>   29m   v1.36.4
NAMESPACE     NAME          READY   STATUS    RESTARTS   AGE
default       gpu-share-1   1/1     Running   0          14m
default       gpu-share-2   1/1     Running   0          12m
default       gpu-share-3   1/1     Running   0          12m
{
  "nvidia.com/gpucores": "100",
  "nvidia.com/gpumem": "81920",
  "nvidia.com/gpumem-percentage": "100"
}
===== tenant-b =====
NAME              STATUS   ROLES    AGE     VERSION
tenant-b-worker   Ready    <none>   5m28s   v1.36.4
NAMESPACE     NAME          READY   STATUS    RESTARTS   AGE
{
  "nvidia.com/gpucores": "200",
  "nvidia.com/gpumem": "49152",
  "nvidia.com/gpumem-percentage": "200"
}

Warning  BindingFailed  hami-scheduler  node tenant-a-worker has been locked within 5m0s: node lock contention

kubectl annotate pod gpu-share-1 hami.io/bind-phase=success --overwrite
kubectl annotate node tenant-a-worker hami.io/mutex.lock-