Broken pre-spawn-hook after upgrade to JH 2.0.0 (kubernetes_asyncio problem)

I have upgraded z2jh deployment to the latest version (2.0.0) which features jupyterHub 3.0.0. As written in all changelogs and upgrade guides, specifically e.g. here in the administrator guide, KubeSpawner has replaced kubernetes library with kubernetes_asyncio.

This change is a problem for us, as we have extended the z2jh config with pre-spawn-hook that performs:

  • check if Secret exists; if yes → mount, if not → create and mount
  • add init container
  • add some volumes

I am unable to perform these actions with the new library as the notebook spawn always timeouts. This is error message I get from hub pod

[I 2022-11-15 12:19:01.941 JupyterHub provider:651] Creating oauth client jupyterhub-user-viktorias-467814-asmsa-fork-pat6ixvn
in mount volume
in mount volume
in mount shared volume
in init cont
<string>:102: RuntimeWarning: coroutine 'get_secret' was never awaited
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
in manage volumes
<string>:192: RuntimeWarning: coroutine 'get_secret' was never awaited
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
in gitlab repo
in manage resources

I think get_namespaced_secret function does not yield the result in time but I don’t know how to fix the problem. I have never used Python’s async libraries and I am lost in how should I refactor the code to be functional again. I went through the github Pr which discusses adopting async library for kubespawner 3.x but it didn’t help me at all. I think it would be great if there was an example in the official documentation how to modify existing extending custom Python codes.

Anyway, this is our pre-spawn-hook (including prints) which currently does not work. there might be a lot of problems (importing libraries/code) but I can’t locate the problem even tho I get a meaningful error message. Could someone please help?

pre-spawn-hook: |
        from kubernetes_asyncio.client import (
        from kubernetes_asyncio import client, config
        from tornado import web                                                   
        from cryptography.hazmat.primitives import serialization as crypto_serialization
        from cryptography.hazmat.primitives.asymmetric import rsa
        from cryptography.hazmat.backends import default_backend as crypto_default_backend
        import subprocess                                                         
        import base64                                                             
        import os
        import re
        import urllib.request

        async def get_secret(v1, namespace, secret_name):
          print("in secrets \n")
          secrets = await v1.list_namespaced_secret(namespace, watch=False)
          for secret in secrets.items:
            if == secret_name:
              return secret                                      
          return None 

        async def create_ssh_pub_secret(v1, namespace, secret_name, key):
            print("in create pub secrets \n")
            pub_key = key.public_key().public_bytes(

            pub_secret = V1Secret()                                               
            pub_secret.metadata = V1ObjectMeta(name=secret_name)     
            pub_secret.type = "Opaque"                                            
   = {                                                   
                    "ssh-publickey": base64.b64encode(pub_key).decode()
            await v1.create_namespaced_secret(namespace=namespace, body=pub_secret)     

        async def create_ssh_priv_secret(v1, namespace, secret_name, key):
          print("in create priv secrets \n")
          priv_key_str = key.private_bytes(
          priv_secret = V1Secret()
          priv_secret.metadata = V1ObjectMeta(name=secret_name)
          priv_secret.type = ""
          priv_secret.string_data = {
                  "ssh-privatekey": priv_key_str
          await v1.create_namespaced_secret(namespace=namespace, body=priv_secret)

        def mount_volume(spawner, secret_name):
          print("in mount volume \n")
          volume = {"name": secret_name, "secret": {"secretName": secret_name}}
          if len(spawner.volumes) == 0:
            spawner.volumes = [volume]

        def mount_shared(spawner):
          print("in mount shared volume \n")
          volume = {"name": "shared", "emptyDir": {}}
          volume_mount = {"mountPath": "/home/jovyan/.ssh", "name": "shared"}
          if len(spawner.volume_mounts) == 0:
            spawner.volume_mounts = [volume_mount]

        def add_init_container(v1, spawner, secret_name):
          print("in init cont  \n")
          volume_mount_priv = V1VolumeMount(mount_path="/.ssh/id_rsa",name=secret_name+"-priv",sub_path="ssh-privatekey")
          volume_mount_pub = V1VolumeMount(mount_path="/.ssh/",name=secret_name+"-pub",sub_path="ssh-publickey")
          shared = V1VolumeMount(mount_path="/ssh",name="shared")
          init_container = V1Container(name="copy-secret", image="busybox",
                           command=["sh", "-c", "cp /.ssh/* /ssh && chown -R 1000 /ssh && chmod 400 /ssh/id_rsa"],
          if len(spawner.init_containers) == 0:
            spawner.init_containers = [init_container]

        def manage_volumes(spawner, namespace, secret_name, v1):
          # TODO check for existence of both before
          # generate key
          key = rsa.generate_private_key(

          secret = get_secret(v1, namespace, secret_name + "-priv")
          if not secret:
            create_ssh_priv_secret(v1, namespace, secret_name + "-priv", key)

          secret = get_secret(v1, namespace, secret_name + "-pub")
          if not secret:
            create_ssh_pub_secret(v1, namespace, secret_name + "-pub", key)
          mount_volume(spawner, secret_name + "-priv")
          mount_volume(spawner, secret_name + "-pub")
          add_init_container(v1, spawner, secret_name)
          print("in manage volumes \n")

        def github_url(repo_url, branch_req):
          repo = repo_url.split("")[1]
          branch = branch_req.split('/')[-1]
          return "" + repo + "/" + branch + "/.resources"

        def gitlab_url(repo_url, branch_req):
          print("in gitlab repo \n")
          branch = branch_req.split('/')[-1]
          if repo_url[-4:] == ".git":
            return repo_url[:-4] + "/-/raw/" + branch + "/.resources"
          return repo_url + "/-/raw/" + branch + "/.resources"

        def manage_resources(spawner, namespace, v1):
          repo_url = spawner.get_env().get('BINDER_REPO_URL','NA')
          branch_req = spawner.get_env().get('BINDER_REQUEST')

          # Many repositories possible with unknown env var structure
          urlname = ""
          if re.compile("").match(repo_url):
            urlname = github_url(repo_url, branch_req)
          elif re.compile("https://gitlab.").match(repo_url):
            urlname = gitlab_url(repo_url, branch_req)

          resources = {"cpul": 8, "cpur": 1, "meml": "16", "memr": "1", "gpu": '0'}
          print("in manage resources \n")

            u = urllib.request.urlopen(urlname)
          except urllib.request.HTTPError:
            print("Resources file does not exist at " + urlname + "\n")

          for line in urllib.request.urlopen(urlname):
            decoded = line.decode('utf-8').strip()
            if len(decoded) > 0:
              if decoded[0] != '#':
                val = ''
                if re.compile("gpu=").match(decoded):
                  val = decoded[4:].strip()
                if re.compile("cpur=").match(decoded) or re.compile("cpul=").match(decoded) or re.compile("memr=").match(decoded) or re.compile("meml=").match(decoded):
                  val = decoded[5:].strip()
                if val != '':
                  key = decoded.split('=')[0].strip()
                  if val.isnumeric():
                    resources[key] = val
          print(" resources " + resources + "\n")
          if resources['gpu'] != '0':
            spawner.extra_resource_guarantees = {"": resources['gpu'] }
            spawner.extra_resource_limits = {"": resources['gpu']}

            volume = {"name": "dshm", "emptyDir": {"medium": "Memory", "sizeLimit": resources['gpu'] + 'G'}}
            volume_mount = {"mountPath": "/dev/shm", "name": "dshm"}          
            volume_exists = False                                             
            for vol in spawner.volumes:                                       
              if "name" in vol and vol["name"] == "dshm":                     
                volume_exists = True                                          
            if volume_exists:                                                 

            meml = int(resources['meml']) + 1
            memr = int(resources['memr']) + 1
            resources["meml"] = str(meml)
            resources["memr"] = str(memr)

          spawner.cpu_limit = float(resources['cpul'])
          spawner.cpu_guarantee = float(resources['cpur'])
          spawner.mem_limit = resources['meml'] + 'G'
          spawner.mem_guarantee = resources['memr'] + 'G'
        def bootstrap_pre_spawn(spawner):
          username =                                      
          namespace = spawner.namespace
          secret_name = username + "-ssh"        
          v1 = client.CoreV1Api()
          manage_volumes(spawner, namespace, secret_name, v1)
          manage_resources(spawner, namespace, v1)
        c.KubeSpawner.pre_spawn_hook = bootstrap_pre_spawn

Thanks for help!

get_secret is async, which means that where it’s called also needs to be async and await get_secret(...). Ultimately, this means that manage_volumes and bootstrap_pre_spawn must also be async and await the async functions they call