diff --git a/base/etc/supervisord.d/10-htcondor-ce.conf b/base/etc/supervisord.d/10-htcondor-ce.conf index 97395d0..962cf88 100644 --- a/base/etc/supervisord.d/10-htcondor-ce.conf +++ b/base/etc/supervisord.d/10-htcondor-ce.conf @@ -2,4 +2,3 @@ command=/usr/share/condor-ce/condor_ce_startup -f autorestart=true startsecs=20 - diff --git a/hosted-ce-sshd/Dockerfile b/hosted-ce-sshd/Dockerfile new file mode 100644 index 0000000..b8da59b --- /dev/null +++ b/hosted-ce-sshd/Dockerfile @@ -0,0 +1,10 @@ +FROM almalinux:9 + +RUN yum install -y openssh-server python3-pyyaml && \ + yum clean all && \ + rm -rf /var/cache/yum/ && \ + adduser sshd-user + +COPY init.sh configure_authorized_keys.py /usr/local/sbin/ + +CMD [ "/usr/local/sbin/init.sh" ] diff --git a/hosted-ce-sshd/configure_authorized_keys.py b/hosted-ce-sshd/configure_authorized_keys.py new file mode 100755 index 0000000..636ac4b --- /dev/null +++ b/hosted-ce-sshd/configure_authorized_keys.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +''' +Util script to copy the public key given in a yaml ConfigMap into the +authorized_hosts file for the sshd daemon user. Assumes config in the +form of + +public_keys: + key_name_1: pubkey_1 + key_name_1: pubkey_2 +''' +import yaml +from pathlib import Path +from os import environ +from sys import exit, argv + +CONFIG_PATH = Path(argv[1]) # eg. /etc/ssh.orig/key-mappings.yaml +AUTHORIZED_KEYS_PATH = Path(argv[2]) # eg. /home/sshd-user/.ssh/authorized_keys +INSTANCE = environ['CE_INSTANCE'] +AUTHORIZED_KEY = environ['AUTHORIZED_KEY'] + +with open(CONFIG_PATH) as f: + config = yaml.load(f.read(), Loader=yaml.Loader) + +AUTHORIZED_KEYS_PATH.parent.mkdir(parents=True, exist_ok=True) +pubkeys: dict[str, str] = config['public_keys'] +pubkey = pubkeys.get(AUTHORIZED_KEY) + +if not pubkey: + print(f"Fatal: No public key found for key {AUTHORIZED_KEY}") + exit(1) + +with open(AUTHORIZED_KEYS_PATH, 'w') as keyf: + keyf.write(pubkey) diff --git a/hosted-ce-sshd/init.sh b/hosted-ce-sshd/init.sh new file mode 100755 index 0000000..3492e01 --- /dev/null +++ b/hosted-ce-sshd/init.sh @@ -0,0 +1,20 @@ +#!/bin/bash +PORT=${SSHD_PORT:-22} + +# Create a new directory to hold original SSH keys (if it doesn't yet exist) +mkdir -p /etc/ssh.orig/etc/ssh + +# Assume SSH host keys have been configured in /etc/ssh.orig but mounted with improper permissions +# Copy the keys and then fix their permissions +cp -p /etc/ssh.orig/ssh_* /etc/ssh/ +chmod 400 /etc/ssh/*key* + +# Assume an authorized_keys file has been configured in /etc/ssh.orig/authorized_keys +mkdir -p /home/sshd-user/.ssh/ +configure_authorized_keys.py /etc/ssh.orig/key-mappings.yaml /home/sshd-user/.ssh/authorized_keys +chown -R sshd-user /home/sshd-user/.ssh +chmod 600 /home/sshd-user/.ssh/authorized_keys + +echo "Starting sshd on port $PORT" +# Start sshd +/usr/sbin/sshd -p $PORT -e -D diff --git a/hosted-ce/Dockerfile b/hosted-ce/Dockerfile index a31dc50..76f6a41 100644 --- a/hosted-ce/Dockerfile +++ b/hosted-ce/Dockerfile @@ -34,3 +34,4 @@ RUN sed -i 's/bosco_cluster/condor_remote_cluster/g' /tmp/*.patch && \ COPY usr/local/bin /usr/local/bin +COPY etc/supervisord.d/* /etc/supervisord.d/ diff --git a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh index b971ee2..c0cea71 100755 --- a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh +++ b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh @@ -70,28 +70,37 @@ setup_user_ssh () { mkdir -p $ssh_dir chmod 700 $ssh_dir - # copy Bosco key - ssh_key=$ssh_dir/id_rsa - cp $BOSCO_KEY $ssh_key - chmod 600 $ssh_key - # HACK: Symlink the Bosco key to the location expected by - # bosco_cluster so it doesn't go and try to generate a new one - ln -s $ssh_key $ssh_dir/bosco_key.rsa - - # copy Bosco certificate - if [[ -f $BOSCO_CERT ]]; then - ssh_cert=${ssh_key}-cert.pub - cp $BOSCO_CERT $ssh_cert - chmod 600 $ssh_cert - fi - - # Write user/host stanza to the global SSH config - cat <> /etc/ssh/ssh_config + # copy Bosco key if not using a forwarded agent + if [[ ${USE_SSH_AGENT_FORWARD:-false} != 'true' ]]; then + ssh_key=$ssh_dir/id_rsa + cp $BOSCO_KEY $ssh_key + chmod 600 $ssh_key + + # HACK: Symlink the Bosco key to the location expected by + # bosco_cluster so it doesn't go and try to generate a new one + ln -s $ssh_key $ssh_dir/bosco_key.rsa + + # copy Bosco certificate + if [[ -f $BOSCO_CERT ]]; then + ssh_cert=${ssh_key}-cert.pub + cp $BOSCO_CERT $ssh_cert + chmod 600 $ssh_cert + fi + + # Write user/host stanza to the global SSH config + cat << EOF >> /etc/ssh/ssh_config Match user "$remote_user" IdentityFile $ssh_key ${extra_config} - EOF + else + # Set IdentityAgent to the forwarded yubikey agent for osg01 (etc) + cat << EOF >> /etc/ssh/ssh_config +Match user "$remote_user" + IdentityAgent /etc/condor-ce/sshd-sock/auth-sock + ${extra_config} +EOF + fi chown -R "${ruser}": "$ssh_dir" @@ -192,16 +201,28 @@ fi # Set up a control master for each rootly SSH connection # Add a sentinel to simplify awk in ssh-to-login-node -cat <> /etc/ssh/ssh_config - +if [[ ${USE_SSH_AGENT_FORWARD:-false} != 'true' ]]; then + cat <> /etc/ssh/ssh_config Host $remote_fqdn # remote login host Port $remote_port IdentitiesOnly yes +EOF +fi + +# Hack to make the forwarded SSH agent carry over to the bosco tools, +# which run as root and spin off their own SSH agents by default +identity_agent_config="" +if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then + identity_agent_config=" IdentityAgent /etc/condor-ce/sshd-sock/auth-sock" +fi + +cat <> /etc/ssh/ssh_config Match localuser root ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m +$identity_agent_config EOF @@ -225,6 +246,30 @@ done ################### test_remote_connect () { + if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then + # Wait for an SSH agent forwarding socket to be established before attempting SSH + echo "Waiting for SSH agent forwarding to be established..." + MAX_RETRIES=100 + SSH_SOCK_DIR=/etc/condor-ce/sshd-sock + for _ in $(seq 1 $MAX_RETRIES); do + if ls $SSH_SOCK_DIR | grep 'ssh-' ; then + TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) + LINK=$SSH_SOCK_DIR/auth-sock + ln -s "$TARGET" "$LINK" + + # Allow non-root users (eg. osg01) to read the auth sock + chmod 755 "$(dirname "$TARGET")" + chmod 666 "$TARGET" + + echo "Got auth-sock: $LINK -> $TARGET" + break + else + echo "No auth socket found yet, retrying in 10 seconds..." + sleep 10 + fi + done + fi + ssh -vvv "$1@$2" true } diff --git a/hosted-ce/etc/supervisord.d/05-agent-sock.conf b/hosted-ce/etc/supervisord.d/05-agent-sock.conf new file mode 100644 index 0000000..d9a598a --- /dev/null +++ b/hosted-ce/etc/supervisord.d/05-agent-sock.conf @@ -0,0 +1,8 @@ +[program:update-agent-sock] +command=/usr/local/bin/update-agent-sock +autostart=true +autorestart=true +# TODO writing a non-condor log to the condor ce location is not ideal but is useful for debugging +stdout_logfile=/var/log/condor-ce/update-agent-sock.log +stdout_logfile_maxbytes=0 +redirect_stderr=true diff --git a/hosted-ce/usr/local/bin/update-agent-sock b/hosted-ce/usr/local/bin/update-agent-sock new file mode 100755 index 0000000..2e298fc --- /dev/null +++ b/hosted-ce/usr/local/bin/update-agent-sock @@ -0,0 +1,32 @@ +#!/bin/bash + +# Util script to update the symlink to the SSH agent socket that +# condor_ce uses to access the remote cluster login host + +SSH_SOCK_DIR=/etc/condor-ce/sshd-sock + +poll_ssh_auth_sock() { + echo "Polling SSH agent socket..." + while true; do + if ls $SSH_SOCK_DIR | grep 'ssh-' ; then + TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) + LINK=$SSH_SOCK_DIR/auth-sock + ln -sf "$TARGET" "$LINK" + + # Allow non-root users (eg. osg01) to read the auth sock + chmod 755 "$(dirname "$TARGET")" + chmod 666 "$TARGET" + echo "Updating SSH_AUTH_SOCK: $LINK -> $TARGET. Checking again in 10 seconds..." + else + echo "No auth socket found yet, retrying in 10 seconds..." + fi + sleep 10 + done + +} + +if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then + poll_ssh_auth_sock +else + sleep infinity +fi