startvnc

#!/bin/bash
#
# Copyright 2017 Deutsches Klimarechenzentrum GmbH
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.
#
#
# startvnc
#
# This script is intended to be used on your local workstation running
# Ubuntu or Fedora Linux or macOS (tested). Other Unix flavors may
# work as well. It allows you to connect to one of DKRZ's
# visualization nodes to work remotely on your visualizations.
#
# Technically, it starts a VNC server on one of the GPU nodes by
# submitting a generated job script. A local vncviewer client is used
# to connect to the server over an encypted ssh tunnel. After the
# client shuts down, the server job is terminated.
#
# Your local vncviewer client has to support the -via
# option. TigerVNC, TightVNC, and TurboVNC  are known to work.
#
# In case of problems contact Mathis Rosenhauer <rosenhauer@dkrz.de>.
#

set -eu

# Default settings
#
# You can change the settings here or override them with command line
# options. SVNC_ACCTCODE is the only parameter you have to provide if
# your local username matches your username on the frontend and if
# vncviewer is installed in the search path (tight/tigervnc on Linux
# for example).

# Project account code
SVNC_ACCTCODE=""

# LDAP username
#
# Specify your username on the login node if it is not your local
# username.
SVNC_USERNAME=$(id -un)

# Path to local vncviewer
#
# If your local vnc client is not in the search path or named
# differently (e.g. TurboVNC, macOS), then change this parameter.
SVNC_VNCVIEWER="vncviewer"

# Resolution of remote desktop window
#
# This is just the initial resolution. Resizing the vncviewer window
# later should also work.
SVNC_GEOMETRY="1920x1200"

# Session run time in minutes
SVNC_RUNTIME=240

# Number of GPU nodes reserved. Default is the special value "half"
# which allocates 24 cores of a node. To allocate one or more nodes,
# set to the integer number of nodes.
SVNC_NODES="half"

# Quality of service
SVNC_QOS="normal"

# Uncomment this if you use TurboVNC as vncviewer.
#
# TurboVNC will work without this option but then it will use its own
# implementation of ssh which dosen't support public key auth among other
# things.
#
# readonly SVNC_VNC_OPTIONS="-extssh"

# Frontend host
#
# Must be directly accessible from client. The frontend and the node
# where vncserver is running need a shared home file system. You will
# have to change this for other sites than DKRZ (along with many other
# things).
readonly SVNC_FRONTEND_HOST="mistral.dkrz.de"

# Copy vncpassword temporarily to the local workstation
#
# If your vncviewer client has trouble with TurboVNC's password format
# then disable this option (set to "false" or comment).
readonly SVNC_PASSWORD="true"


clean_up () {
    trap - ERR EXIT
    set +e

    if [[ -n ${vnc_host:-} ]]; then
        echo "Killing vncserver :${vnc_display} on ${vnc_host}."
        ssh_frontend "ssh ${vnc_host} \"/opt/TurboVNC/bin/vncserver \
                     -kill :${vnc_display}\""
    fi

    if [[ -n ${job_id:-} ]]; then
        echo "Removing job ${job_id}."
        ssh_frontend "scancel -Q ${job_id}; sleep 1; \
                     cd .startvnc && rm -f out.${job_id}"
    else
        echo "Job ID not available. Make sure the vncjob is not running!"
        ssh_frontend "squeue -u ${SVNC_USERNAME}"
    fi

    ssh_frontend "" "-O exit"
    rmdir "${ssh_socket_dir}"

    # Remove local vnc PasswordFile
    if [[ ${SVNC_PASSWORD} = "true" ]]; then
        rm -f "${HOME}/vnc_passwd"
    fi
    exit
}

usage () {
    printf "Usage: %s [OPTIONS]\n" $(basename $0) >&2
    printf -- "-u username\n-A acctcode\n-g geometry\n-n nodes\n" >&2
    printf -- "-q qos\n-t time\n-v path_to_vncviewer\n" >&2
    printf -- "-x vncviewer_options\n" >&2
}

ssh_frontend () {
    local command="$1"
    local extra_options="${2:-}"
    local options="${extra_options} -o ForwardX11=no \
          -o ControlPath=${ssh_socket_dir}/control:%h:%p:%r"
    ssh ${options} "${SVNC_USERNAME}@${SVNC_FRONTEND_HOST}" "${command}"
}

parse_options () {
    local option
    while getopts 'A:g:n:q:t:u:v:x:' option; do
        case ${option} in
            A) SVNC_ACCTCODE="$OPTARG"
               ;;
            g) SVNC_GEOMETRY="$OPTARG"
               ;;
            n) SVNC_NODES="$OPTARG"
               ;;
            q) SVNC_QOS="$OPTARG"
               ;;
            t) SVNC_RUNTIME="$OPTARG"
               ;;
            u) SVNC_USERNAME="$OPTARG"
               ;;
            v) SVNC_VNCVIEWER="$OPTARG"
               ;;
            x) SVNC_VNC_OPTIONS="$OPTARG"
               ;;
            ?) usage
            exit 1
            ;;
        esac
    done

    readonly SVNC_ACCTCODE
    readonly SVNC_GEOMETRY
    readonly SVNC_QOS
    readonly SVNC_RUNTIME
    readonly SVNC_USERNAME
    readonly SVNC_VNCVIEWER
    readonly SVNC_VNC_OPTIONS

    if [[ -z ${SVNC_ACCTCODE} ]]; then
        printf "ERROR: Please specify an acctcode.\n\n" >&2
        usage
        exit 1
    fi
}

prepare_vncserver () {
    ssh_frontend "mkdir -p .startvnc"
    if ! ssh_frontend "test -s .vnc/passwd"; then
        echo "No VNC password found. Please set now."
        ssh_frontend "salloc -Q -n1 -pgpu -A${SVNC_ACCTCODE} -- /bin/bash -c \
                     'ssh -tt \$SLURM_JOB_NODELIST -- mkdir -p .vnc \
                     && /opt/TurboVNC/bin/vncpasswd'" "-t"
    fi
}

submit_vnc_job () {
    if [[ ${SVNC_NODES} = "half" ]]; then
        local sbatch_resources="#SBATCH --ntasks=24"
    else
        local sbatch_resources="#SBATCH --nodes=${SVNC_NODES}"
        sbatch_resources+=$'\n#SBATCH --exclusive'
    fi
    ssh_frontend "cd .startvnc && sbatch" <<EOF
#!/bin/bash -l
#SBATCH --partition=gpu
${sbatch_resources}
#SBATCH --qos=${SVNC_QOS}
#SBATCH --time=${SVNC_RUNTIME}
#SBATCH --account=${SVNC_ACCTCODE}
#SBATCH --output=out.%j
#SBATCH --parsable
#SBATCH --dkrzepilog=0
cd \${HOME}
/opt/TurboVNC/bin/vncserver -fg -geometry "${SVNC_GEOMETRY}" -localhost
EOF
}

get_vnc_host_and_display () {
    local job_id="$1"
    ssh_frontend "/bin/bash" <<EOF
logfile=\${HOME}/.startvnc/out.${job_id}
while [[ -z \${host_and_display} ]]; do
    sleep 1
    if [[ -f \${logfile} ]]; then
        host_and_display=\$(grep -Po "started on display \Kmg[0-9]+:[0-9]+" \
                          \${logfile})
    fi
done
echo \${host_and_display}
EOF
}

main () {
    parse_options "$@"
    trap clean_up INT QUIT TERM ERR EXIT

    ssh_socket_dir=$(mktemp -d "${HOME}/.ssh/socket.XXXXX")
    ssh_frontend "" "-MNf"
    prepare_vncserver

    echo "Submitting vncserver job."
    job_id=$(submit_vnc_job)

    echo "Waiting for job ${job_id} to start."
    local host_and_display=$(get_vnc_host_and_display "$job_id")
    vnc_host=${host_and_display%:*}
    vnc_display=${host_and_display#*:}
    echo "Vncserver started on node ${vnc_host}.dkrz.de display \
:${vnc_display}."

    local vnc_options="${SVNC_VNC_OPTIONS:-} \
                       -via ${SVNC_USERNAME}@${vnc_host}.dkrz.de"
    if [[ ${SVNC_PASSWORD} = "true" ]]; then
        echo "Fetching password from frontend."
        ssh_frontend "cat .vnc/passwd" > ${HOME}/vnc_passwd
        vnc_options+=" -passwd ${HOME}/vnc_passwd"
    fi
    echo "Connecting vncviewer to ${vnc_host}.dkrz.de"
    "${SVNC_VNCVIEWER}" ${vnc_options} :${vnc_display}
}

main "$@"