Jupyter spawner can not spawn pods with GPU nvidia

JupyterHub Zero to JupyterHub on Kubernetes
, ,
9 
apiVersion: v1
kind: Pod
metadata:
  annotations:
    hub.jupyter.org/jupyterhub-version: 5.4.3
    hub.jupyter.org/kubespawner-version: 7.0.0
    hub.jupyter.org/username: ikol006
  creationTimestamp: "2026-01-22T03:06:14Z"
  labels:
    app: jupyterhub
    app.kubernetes.io/component: singleuser-server
    app.kubernetes.io/instance: jupyterhub
    app.kubernetes.io/managed-by: kubespawner
    app.kubernetes.io/name: jupyterhub
    chart: jupyterhub-4.3.2
    component: singleuser-server
    helm.sh/chart: jupyterhub-4.3.2
    hub.jupyter.org/network-access-hub: "true"
    hub.jupyter.org/servername: ""
    hub.jupyter.org/username: ikol006
    release: jupyterhub
  name: jupyter-ikol006
  namespace: jupyterhub
  resourceVersion: "930369902"
  uid: ccc01307-09bb-4b2a-b870-88c82764d874
spec:
  affinity:
    nodeAffinity:
      preferredDuringSchedulingIgnoredDuringExecution:
      - preference:
          matchExpressions:
          - key: hub.jupyter.org/node-purpose
            operator: In
            values:
            - user
        weight: 100
  automountServiceAccountToken: false
  containers:
  - args:
    - jupyterhub-singleuser
    env:
    - name: JPY_API_TOKEN
      value: 66sdfe0e9e7af47a91b43205aa6dc8c4
    - name: JUPYTERHUB_ACTIVITY_URL
      value: http://hub:8081/hub/api/users/ikol006/activity
    - name: JUPYTERHUB_ADMIN_ACCESS
      value: "1"
    - name: JUPYTERHUB_API_TOKEN
      value: 66sdfe0e9e7af47a91b43205aa6dc8c4
    - name: JUPYTERHUB_API_URL
      value: http://hub:8081/hub/api
    - name: JUPYTERHUB_BASE_URL
      value: /
    - name: JUPYTERHUB_CLIENT_ID
      value: jupyterhub-user-ikol006
    - name: JUPYTERHUB_COOKIE_HOST_PREFIX_ENABLED
      value: "0"
    - name: JUPYTERHUB_DEBUG
      value: "1"
    - name: JUPYTERHUB_HOST
    - name: JUPYTERHUB_OAUTH_ACCESS_SCOPES
      value: '["access:servers!server=ikol006/", "access:servers!user=ikol006"]'
    - name: JUPYTERHUB_OAUTH_CALLBACK_URL
      value: /user/ikol006/oauth_callback
    - name: JUPYTERHUB_OAUTH_CLIENT_ALLOWED_SCOPES
      value: '[]'
    - name: JUPYTERHUB_OAUTH_SCOPES
      value: '["access:servers!server=ikol006/", "access:servers!user=ikol006"]'
    - name: JUPYTERHUB_PUBLIC_HUB_URL
    - name: JUPYTERHUB_PUBLIC_URL
    - name: JUPYTERHUB_SERVER_NAME
    - name: JUPYTERHUB_SERVICE_PREFIX
      value: /user/ikol006/
    - name: JUPYTERHUB_SERVICE_URL
      value: http://0.0.0.0:8888/user/ikol006/
    - name: JUPYTERHUB_USER
      value: ikol006
    - name: JUPYTER_IMAGE
      value: ml-with-gpu-notebook:x86_64-ubuntu-22.04
    - name: JUPYTER_IMAGE_SPEC
      value: ml-with-gpu-notebook:x86_64-ubuntu-22.04
    - name: MEM_GUARANTEE
      value: "1073741824"
    - name: PIP_INDEX_URL
      value: https://nexus.local/repository/pypi-proxy/simple
    - name: PIP_TIMEOUT
      value: "60"
    - name: PIP_TRUSTED_HOST
      value: nexus.local
    - name: SSL_CERT_FILE
      value: /etc/ssl/certs/ca-certificates.crt
    image: ml-with-gpu-notebook:x86_64-ubuntu-22.04
    imagePullPolicy: Always
    lifecycle:
      postStart:
        exec:
          command:
          - sh
          - -c
          - |
            cp /etc/ssl/certs/ca-certificates.crt /opt/conda/lib/python3.11/site-packages/certifi/cacert.pem
    name: notebook
    ports:
    - containerPort: 8888
      name: notebook-port
      protocol: TCP
    resources:
      limits:
        nvidia.com/gpu: "1"
      requests:
        memory: "1073741824"
        nvidia.com/gpu: "1"
    securityContext:
      allowPrivilegeEscalation: false
      runAsUser: 1000
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /etc/ssl/certs/ca-certificates.crt
      name: files
      subPath: ca-certificates.crt
    - mountPath: /home/jovyan
      name: volume-ikol006
  dnsPolicy: ClusterFirst
  enableServiceLinks: true
  initContainers:
  - command:
    - iptables
    - --append
    - OUTPUT
    - --protocol
    - tcp
    - --destination
    - 111.127.222.127
    - --destination-port
    - "80"
    - --jump
    - DROP
    image: quay.io/jupyterhub/k8s-network-tools:4.3.2
    imagePullPolicy: IfNotPresent
    name: block-cloud-metadata
    resources: {}
    securityContext:
      capabilities:
        add:
        - NET_ADMIN
      privileged: true
      runAsUser: 0
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
  preemptionPolicy: PreemptLowerPriority
  priority: 1000
  priorityClassName: develop
  restartPolicy: OnFailure
  schedulerName: default-scheduler
  securityContext:
    fsGroup: 100
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  tolerations:
  - effect: NoSchedule
    key: hub.jupyter.org/dedicated
    operator: Equal
    value: user
  - effect: NoSchedule
    key: hub.jupyter.org_dedicated
    operator: Equal
    value: user
  - effect: NoSchedule
    key: nvidia.com/gpu
    operator: Exists
  - effect: NoExecute
    key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
  - effect: NoExecute
    key: node.kubernetes.io/unreachable
    operator: Exists
    tolerationSeconds: 300
  - effect: NoSchedule
    key: node.kubernetes.io/memory-pressure
    operator: Exists
  volumes:
  - name: files
    secret:
      defaultMode: 420
      items:
      - key: ca-certificates.crt
        mode: 420
        path: ca-certificates.crt
      secretName: singleuser
  - name: volume-ikol006
    persistentVolumeClaim:
      claimName: claim-ikol006
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: "2026-01-22T03:06:15Z"
    message: '0/8 nodes are available: 1 Insufficient nvidia.com/gpu. preemption:
      0/8 nodes are available: 1 No preemption victims found for incoming pod, 7 Preemption
      is not helpful for scheduling..'
    reason: Unschedulable
    status: "False"
    type: PodScheduled
  phase: Pending
  qosClass: Burstable