Hi,
I have upgraded z2jh deployment to the latest version (2.0.0) which features jupyterHub 3.0.0. As written in all changelogs and upgrade guides, specifically e.g. here in the administrator guide, KubeSpawner has replaced kubernetes library with kubernetes_asyncio.
This change is a problem for us, as we have extended the z2jh config with pre-spawn-hook that performs:
- check if Secret exists; if yes → mount, if not → create and mount
- add init container
- add some volumes
I am unable to perform these actions with the new library as the notebook spawn always timeouts. This is error message I get from hub pod
[I 2022-11-15 12:19:01.941 JupyterHub provider:651] Creating oauth client jupyterhub-user-viktorias-467814-asmsa-fork-pat6ixvn
in mount volume
in mount volume
in mount shared volume
in init cont
<string>:102: RuntimeWarning: coroutine 'get_secret' was never awaited
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
in manage volumes
<string>:192: RuntimeWarning: coroutine 'get_secret' was never awaited
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
in gitlab repo
in manage resources
I think get_namespaced_secret function does not yield the result in time but I don’t know how to fix the problem. I have never used Python’s async libraries and I am lost in how should I refactor the code to be functional again. I went through the github Pr which discusses adopting async library for kubespawner 3.x but it didn’t help me at all. I think it would be great if there was an example in the official documentation how to modify existing extending custom Python codes.
Anyway, this is our pre-spawn-hook (including prints) which currently does not work. there might be a lot of problems (importing libraries/code) but I can’t locate the problem even tho I get a meaningful error message. Could someone please help?
pre-spawn-hook: |
from kubernetes_asyncio.client import (
V1ObjectMeta,
CoreV1Api,
V1Secret,
V1Container,
V1VolumeMount,
)
from kubernetes_asyncio import client, config
from tornado import web
from cryptography.hazmat.primitives import serialization as crypto_serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.backends import default_backend as crypto_default_backend
import subprocess
import base64
import os
import re
import urllib.request
async def get_secret(v1, namespace, secret_name):
print("in secrets \n")
secrets = await v1.list_namespaced_secret(namespace, watch=False)
for secret in secrets.items:
if secret.metadata.name == secret_name:
return secret
return None
async def create_ssh_pub_secret(v1, namespace, secret_name, key):
print("in create pub secrets \n")
pub_key = key.public_key().public_bytes(
crypto_serialization.Encoding.OpenSSH,
crypto_serialization.PublicFormat.OpenSSH)
pub_secret = V1Secret()
pub_secret.metadata = V1ObjectMeta(name=secret_name)
pub_secret.type = "Opaque"
pub_secret.data = {
"ssh-publickey": base64.b64encode(pub_key).decode()
}
await v1.create_namespaced_secret(namespace=namespace, body=pub_secret)
async def create_ssh_priv_secret(v1, namespace, secret_name, key):
print("in create priv secrets \n")
priv_key_str = key.private_bytes(
crypto_serialization.Encoding.PEM,
crypto_serialization.PrivateFormat.PKCS8,
crypto_serialization.NoEncryption()).decode()
priv_secret = V1Secret()
priv_secret.metadata = V1ObjectMeta(name=secret_name)
priv_secret.type = "kubernetes.io/ssh-auth"
priv_secret.string_data = {
"ssh-privatekey": priv_key_str
}
await v1.create_namespaced_secret(namespace=namespace, body=priv_secret)
def mount_volume(spawner, secret_name):
print("in mount volume \n")
volume = {"name": secret_name, "secret": {"secretName": secret_name}}
if len(spawner.volumes) == 0:
spawner.volumes = [volume]
else:
spawner.volumes.append(volume)
def mount_shared(spawner):
print("in mount shared volume \n")
volume = {"name": "shared", "emptyDir": {}}
volume_mount = {"mountPath": "/home/jovyan/.ssh", "name": "shared"}
spawner.volumes.append(volume)
if len(spawner.volume_mounts) == 0:
spawner.volume_mounts = [volume_mount]
else:
spawner.volume_mounts.append(volume_mount)
def add_init_container(v1, spawner, secret_name):
print("in init cont \n")
volume_mount_priv = V1VolumeMount(mount_path="/.ssh/id_rsa",name=secret_name+"-priv",sub_path="ssh-privatekey")
volume_mount_pub = V1VolumeMount(mount_path="/.ssh/id_rsa.pub",name=secret_name+"-pub",sub_path="ssh-publickey")
shared = V1VolumeMount(mount_path="/ssh",name="shared")
init_container = V1Container(name="copy-secret", image="busybox",
command=["sh", "-c", "cp /.ssh/* /ssh && chown -R 1000 /ssh && chmod 400 /ssh/id_rsa"],
volume_mounts=[volume_mount_priv,volume_mount_pub,shared])
if len(spawner.init_containers) == 0:
spawner.init_containers = [init_container]
else:
spawner.init_containers.append(init_container)
def manage_volumes(spawner, namespace, secret_name, v1):
# TODO check for existence of both before
# generate key
key = rsa.generate_private_key(
backend=crypto_default_backend(),
public_exponent=65537,
key_size=2048
)
#private
secret = get_secret(v1, namespace, secret_name + "-priv")
if not secret:
create_ssh_priv_secret(v1, namespace, secret_name + "-priv", key)
#public
secret = get_secret(v1, namespace, secret_name + "-pub")
if not secret:
create_ssh_pub_secret(v1, namespace, secret_name + "-pub", key)
mount_volume(spawner, secret_name + "-priv")
mount_volume(spawner, secret_name + "-pub")
mount_shared(spawner)
add_init_container(v1, spawner, secret_name)
print("in manage volumes \n")
def github_url(repo_url, branch_req):
repo = repo_url.split("https://github.com/")[1]
branch = branch_req.split('/')[-1]
return "https://raw.githubusercontent.com/" + repo + "/" + branch + "/.resources"
def gitlab_url(repo_url, branch_req):
print("in gitlab repo \n")
branch = branch_req.split('/')[-1]
if repo_url[-4:] == ".git":
return repo_url[:-4] + "/-/raw/" + branch + "/.resources"
return repo_url + "/-/raw/" + branch + "/.resources"
def manage_resources(spawner, namespace, v1):
repo_url = spawner.get_env().get('BINDER_REPO_URL','NA')
branch_req = spawner.get_env().get('BINDER_REQUEST')
# Many repositories possible with unknown env var structure
urlname = ""
if re.compile("https://github.com/").match(repo_url):
urlname = github_url(repo_url, branch_req)
elif re.compile("https://gitlab.").match(repo_url):
urlname = gitlab_url(repo_url, branch_req)
else:
return
resources = {"cpul": 8, "cpur": 1, "meml": "16", "memr": "1", "gpu": '0'}
print("in manage resources \n")
try:
u = urllib.request.urlopen(urlname)
except urllib.request.HTTPError:
print("Resources file does not exist at " + urlname + "\n")
return
for line in urllib.request.urlopen(urlname):
decoded = line.decode('utf-8').strip()
if len(decoded) > 0:
if decoded[0] != '#':
val = ''
if re.compile("gpu=").match(decoded):
val = decoded[4:].strip()
if re.compile("cpur=").match(decoded) or re.compile("cpul=").match(decoded) or re.compile("memr=").match(decoded) or re.compile("meml=").match(decoded):
val = decoded[5:].strip()
if val != '':
key = decoded.split('=')[0].strip()
if val.isnumeric():
resources[key] = val
print(" resources " + resources + "\n")
if resources['gpu'] != '0':
spawner.extra_resource_guarantees = {"nvidia.com/gpu": resources['gpu'] }
spawner.extra_resource_limits = {"nvidia.com/gpu": resources['gpu']}
volume = {"name": "dshm", "emptyDir": {"medium": "Memory", "sizeLimit": resources['gpu'] + 'G'}}
volume_mount = {"mountPath": "/dev/shm", "name": "dshm"}
volume_exists = False
for vol in spawner.volumes:
if "name" in vol and vol["name"] == "dshm":
volume_exists = True
if volume_exists:
return
spawner.volumes.append(volume)
spawner.volume_mounts.append(volume_mount)
meml = int(resources['meml']) + 1
memr = int(resources['memr']) + 1
resources["meml"] = str(meml)
resources["memr"] = str(memr)
spawner.cpu_limit = float(resources['cpul'])
spawner.cpu_guarantee = float(resources['cpur'])
spawner.mem_limit = resources['meml'] + 'G'
spawner.mem_guarantee = resources['memr'] + 'G'
def bootstrap_pre_spawn(spawner):
username = spawner.user.name
namespace = spawner.namespace
secret_name = username + "-ssh"
v1 = client.CoreV1Api()
manage_volumes(spawner, namespace, secret_name, v1)
manage_resources(spawner, namespace, v1)
config.load_incluster_config()
c.KubeSpawner.pre_spawn_hook = bootstrap_pre_spawn
Thanks for help!
Viktoria