SOFTWARE-6302: yubikey ssh

base/etc/supervisord.d/10-htcondor-ce.conf

@@ -2,4 +2,3 @@
 command=/usr/share/condor-ce/condor_ce_startup -f
 autorestart=true
 startsecs=20
-

hosted-ce-sshd/Dockerfile

@@ -0,0 +1,10 @@
+FROM almalinux:9
+
+RUN yum install -y openssh-server python3-pyyaml && \
+    yum clean all && \
+    rm -rf /var/cache/yum/ && \
+    adduser sshd-user
+
+COPY init.sh configure_authorized_keys.py /usr/local/sbin/
+
+CMD [ "/usr/local/sbin/init.sh" ]

hosted-ce-sshd/configure_authorized_keys.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+'''
+Util script to copy the public key given in a yaml ConfigMap into the
+authorized_hosts file for the sshd daemon user. Assumes config in the
+form of
+
+public_keys:
+  key_name_1: pubkey_1
+  key_name_1: pubkey_2
+'''
+import yaml
+from pathlib import Path
+from os import environ
+from sys import exit, argv
+
+CONFIG_PATH = Path(argv[1])  # eg. /etc/ssh.orig/key-mappings.yaml
+AUTHORIZED_KEYS_PATH = Path(argv[2]) # eg. /home/sshd-user/.ssh/authorized_keys
+INSTANCE = environ['CE_INSTANCE']
+AUTHORIZED_KEY = environ['AUTHORIZED_KEY']
+
+with open(CONFIG_PATH) as f:
+    config = yaml.load(f.read(), Loader=yaml.Loader)
+
+AUTHORIZED_KEYS_PATH.parent.mkdir(parents=True, exist_ok=True)
+pubkeys: dict[str, str] = config['public_keys']
+pubkey = pubkeys.get(AUTHORIZED_KEY)
+
+if not pubkey:
+    print(f"Fatal: No public key found for key {AUTHORIZED_KEY}")
+    exit(1)
+
+with open(AUTHORIZED_KEYS_PATH, 'w') as keyf:
+    keyf.write(pubkey)

hosted-ce-sshd/init.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+PORT=${SSHD_PORT:-22}
+
+# Create a new directory to hold original SSH keys (if it doesn't yet exist)
+mkdir -p /etc/ssh.orig/etc/ssh
+
+# Assume SSH host keys have been configured in /etc/ssh.orig but mounted with improper permissions
+# Copy the keys and then fix their permissions
+cp -p /etc/ssh.orig/ssh_* /etc/ssh/
+chmod 400 /etc/ssh/*key*
+
+# Assume an authorized_keys file has been configured in /etc/ssh.orig/authorized_keys
+mkdir -p /home/sshd-user/.ssh/
+configure_authorized_keys.py /etc/ssh.orig/key-mappings.yaml /home/sshd-user/.ssh/authorized_keys
+chown -R sshd-user /home/sshd-user/.ssh
+chmod 600 /home/sshd-user/.ssh/authorized_keys
+
+echo "Starting sshd on port $PORT"
+# Start sshd
+/usr/sbin/sshd -p $PORT -e -D

hosted-ce/Dockerfile

@@ -34,3 +34,4 @@ RUN sed -i 's/bosco_cluster/condor_remote_cluster/g' /tmp/*.patch && \
 
 
 COPY usr/local/bin /usr/local/bin
+COPY etc/supervisord.d/* /etc/supervisord.d/

hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh

@@ -70,28 +70,37 @@ setup_user_ssh () {
   mkdir -p $ssh_dir
   chmod 700 $ssh_dir
 
-  # copy Bosco key
-  ssh_key=$ssh_dir/id_rsa
-  cp $BOSCO_KEY $ssh_key
-  chmod 600 $ssh_key
-  # HACK: Symlink the Bosco key to the location expected by
-  # bosco_cluster so it doesn't go and try to generate a new one
-  ln -s $ssh_key $ssh_dir/bosco_key.rsa
-
-  # copy Bosco certificate
-  if [[ -f $BOSCO_CERT ]]; then
-      ssh_cert=${ssh_key}-cert.pub
-      cp $BOSCO_CERT $ssh_cert
-      chmod 600 $ssh_cert
-  fi
-
-  # Write user/host stanza to the global SSH config
-  cat <<EOF >> /etc/ssh/ssh_config
+  # copy Bosco key if not using a forwarded agent
+  if [[ ${USE_SSH_AGENT_FORWARD:-false} != 'true' ]]; then
+    ssh_key=$ssh_dir/id_rsa
+    cp $BOSCO_KEY $ssh_key
+    chmod 600 $ssh_key
+
+    # HACK: Symlink the Bosco key to the location expected by
+    # bosco_cluster so it doesn't go and try to generate a new one
+    ln -s $ssh_key $ssh_dir/bosco_key.rsa
+
+    # copy Bosco certificate
+    if [[ -f $BOSCO_CERT ]]; then
+        ssh_cert=${ssh_key}-cert.pub
+        cp $BOSCO_CERT $ssh_cert
+        chmod 600 $ssh_cert
+    fi
+
+    # Write user/host stanza to the global SSH config
+    cat << EOF >> /etc/ssh/ssh_config
 Match user "$remote_user"
   IdentityFile $ssh_key
   ${extra_config}
-
 EOF
+  else
+    # Set IdentityAgent to the forwarded yubikey agent for osg01 (etc)
+    cat << EOF >> /etc/ssh/ssh_config
+Match user "$remote_user"
+  IdentityAgent /etc/condor-ce/sshd-sock/auth-sock
+  ${extra_config}
+EOF
+  fi
 
   chown -R "${ruser}": "$ssh_dir"
 
@@ -192,16 +201,28 @@ fi
 
 # Set up a control master for each rootly SSH connection
 # Add a sentinel to simplify awk in ssh-to-login-node
-cat <<EOF >> /etc/ssh/ssh_config
-
+if [[ ${USE_SSH_AGENT_FORWARD:-false} != 'true' ]]; then
+    cat <<EOF >> /etc/ssh/ssh_config
 Host $remote_fqdn # remote login host
   Port $remote_port
   IdentitiesOnly yes
+EOF
+fi
+
+# Hack to make the forwarded SSH agent carry over to the bosco tools,
+# which run as root and spin off their own SSH agents by default
+identity_agent_config=""
+if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then
+    identity_agent_config="  IdentityAgent /etc/condor-ce/sshd-sock/auth-sock"
+fi
+
+cat <<EOF >> /etc/ssh/ssh_config
 
 Match localuser root
   ControlMaster auto
   ControlPath /tmp/cm-%i-%r@%h:%p
   ControlPersist  15m
+$identity_agent_config
 
 EOF
 
@@ -225,6 +246,30 @@ done
 ###################
 
 test_remote_connect () {
+    if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then
+        # Wait for an SSH agent forwarding socket to be established before attempting SSH
+        echo "Waiting for SSH agent forwarding to be established..."
+        MAX_RETRIES=100
+        SSH_SOCK_DIR=/etc/condor-ce/sshd-sock
+        for _ in $(seq 1 $MAX_RETRIES); do
+            if ls $SSH_SOCK_DIR | grep 'ssh-' ; then
+                TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1)
+                LINK=$SSH_SOCK_DIR/auth-sock
+                ln -s "$TARGET" "$LINK"
+
+                # Allow non-root users (eg. osg01) to read the auth sock
+                chmod 755 "$(dirname "$TARGET")"
+                chmod 666 "$TARGET"
+
+                echo "Got auth-sock: $LINK -> $TARGET"
+                break
+            else
+                echo "No auth socket found yet, retrying in 10 seconds..."
+                sleep 10
+            fi
+        done
+    fi
+
     ssh -vvv "$1@$2" true
 }
 

hosted-ce/etc/supervisord.d/05-agent-sock.conf

@@ -0,0 +1,8 @@
+[program:update-agent-sock]
+command=/usr/local/bin/update-agent-sock
+autostart=true
+autorestart=true
+# TODO writing a non-condor log to the condor ce location is not ideal but is useful for debugging
+stdout_logfile=/var/log/condor-ce/update-agent-sock.log
+stdout_logfile_maxbytes=0
+redirect_stderr=true

hosted-ce/usr/local/bin/update-agent-sock

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Util script to update the symlink to the SSH agent socket that
+# condor_ce uses to access the remote cluster login host
+
+SSH_SOCK_DIR=/etc/condor-ce/sshd-sock
+
+poll_ssh_auth_sock() {
+    echo "Polling SSH agent socket..."
+    while true; do
+        if ls $SSH_SOCK_DIR | grep 'ssh-' ; then
+            TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1)
+            LINK=$SSH_SOCK_DIR/auth-sock
+            ln -sf "$TARGET" "$LINK"
+
+            # Allow non-root users (eg. osg01) to read the auth sock
+            chmod 755 "$(dirname "$TARGET")"
+            chmod 666 "$TARGET"
+            echo "Updating SSH_AUTH_SOCK: $LINK -> $TARGET. Checking again in 10 seconds..."
+        else
+            echo "No auth socket found yet, retrying in 10 seconds..."
+        fi
+        sleep 10
+    done
+
+}
+
+if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then
+    poll_ssh_auth_sock
+else
+    sleep infinity
+fi