From 37c7c2b5689d6743fe7f57cd209c1f58d33e13f2 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 14 Oct 2025 15:42:27 -0500 Subject: [PATCH 01/12] Add sshd to supervisord --- base/etc/supervisord.d/20-sshd.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 base/etc/supervisord.d/20-sshd.conf diff --git a/base/etc/supervisord.d/20-sshd.conf b/base/etc/supervisord.d/20-sshd.conf new file mode 100644 index 0000000..7b44880 --- /dev/null +++ b/base/etc/supervisord.d/20-sshd.conf @@ -0,0 +1,10 @@ +[program:sshd] +user = root +# Writing directly into /etc/ssh overwrites some necessary config files, +# instead stage keys into /etc/ssh.orig.d and then copy to /etc/sh +command = sh -c ' + mkdir -p /mnt/ssh.orig/etc/ssh && + ssh-keygen -A -f /mnt/ssh.orig && + /bin/cp -p /mnt/ssh.orig/etc/ssh/* /etc/ssh/ && + chmod 400 /etc/ssh/*key && + /usr/sbin/sshd -D' From 5a8d5d13349fc2b041dfba02ca086d1665f64494 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Feb 2026 11:07:10 -0600 Subject: [PATCH 02/12] Copy bind-mounted authorized_keys to host --- hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh index b971ee2..e5c95c6 100755 --- a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh +++ b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh @@ -18,6 +18,7 @@ BOSCO_KEY=/etc/osg/bosco.key BOSCO_CERT=${BOSCO_KEY}-cert.pub ENDPOINT_CONFIG=/etc/endpoints.ini KNOWN_HOSTS=/etc/osg/ssh_known_hosts +AUTHORIZED_KEYS=/etc/osg/ssh_authorized_keys SKIP_WN_INSTALL=no function errexit { @@ -74,6 +75,12 @@ setup_user_ssh () { ssh_key=$ssh_dir/id_rsa cp $BOSCO_KEY $ssh_key chmod 600 $ssh_key + + # copy authorized_keys + authorized_keys=$ssh_dir/authorized_keys + cp $AUTHORIZED_KEYS $authorized_keys + chmod 600 $authorized_keys + # HACK: Symlink the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one ln -s $ssh_key $ssh_dir/bosco_key.rsa From 8dfb9b957b5520e2bf88be991632eba5e3dd62d7 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Feb 2026 15:19:35 -0600 Subject: [PATCH 03/12] Refactor startup sequence to start sshd before condor, allow ssh via agent-forwarded credentials --- .../{20-sshd.conf => 05-sshd.conf} | 0 base/etc/supervisord.d/10-htcondor-ce.conf | 7 ++- .../local/bin/remote-site-setup.sh} | 48 ++++++++++++------- 3 files changed, 37 insertions(+), 18 deletions(-) rename base/etc/supervisord.d/{20-sshd.conf => 05-sshd.conf} (100%) rename hosted-ce/{etc/osg/image-config.d/30-remote-site-setup.sh => usr/local/bin/remote-site-setup.sh} (90%) diff --git a/base/etc/supervisord.d/20-sshd.conf b/base/etc/supervisord.d/05-sshd.conf similarity index 100% rename from base/etc/supervisord.d/20-sshd.conf rename to base/etc/supervisord.d/05-sshd.conf diff --git a/base/etc/supervisord.d/10-htcondor-ce.conf b/base/etc/supervisord.d/10-htcondor-ce.conf index 97395d0..48a7492 100644 --- a/base/etc/supervisord.d/10-htcondor-ce.conf +++ b/base/etc/supervisord.d/10-htcondor-ce.conf @@ -1,5 +1,10 @@ [program:htcondor-ce] -command=/usr/share/condor-ce/condor_ce_startup -f +# CE startup depends on sshd becoming available and must start after +# a remote processes has forwarded an agent to the container +command = sh -c ' + source /usr/local/bin/remote-site-setup.sh && + /usr/share/condor-ce/condor_ce_startup -f' + autorestart=true startsecs=20 diff --git a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh b/hosted-ce/usr/local/bin/remote-site-setup.sh similarity index 90% rename from hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh rename to hosted-ce/usr/local/bin/remote-site-setup.sh index e5c95c6..3942fa4 100755 --- a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh +++ b/hosted-ce/usr/local/bin/remote-site-setup.sh @@ -72,9 +72,9 @@ setup_user_ssh () { chmod 700 $ssh_dir # copy Bosco key - ssh_key=$ssh_dir/id_rsa - cp $BOSCO_KEY $ssh_key - chmod 600 $ssh_key +# ssh_key=$ssh_dir/id_rsa +# cp $BOSCO_KEY $ssh_key +# chmod 600 $ssh_key # copy authorized_keys authorized_keys=$ssh_dir/authorized_keys @@ -83,22 +83,22 @@ setup_user_ssh () { # HACK: Symlink the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one - ln -s $ssh_key $ssh_dir/bosco_key.rsa +# ln -s $ssh_key $ssh_dir/bosco_key.rsa # copy Bosco certificate - if [[ -f $BOSCO_CERT ]]; then - ssh_cert=${ssh_key}-cert.pub - cp $BOSCO_CERT $ssh_cert - chmod 600 $ssh_cert - fi +# if [[ -f $BOSCO_CERT ]]; then +# ssh_cert=${ssh_key}-cert.pub +# cp $BOSCO_CERT $ssh_cert +# chmod 600 $ssh_cert +# fi # Write user/host stanza to the global SSH config - cat <> /etc/ssh/ssh_config -Match user "$remote_user" - IdentityFile $ssh_key - ${extra_config} +# cat <> /etc/ssh/ssh_config +# Match user "$remote_user" +# IdentityFile $ssh_key +# ${extra_config} -EOF +# EOF chown -R "${ruser}": "$ssh_dir" @@ -201,9 +201,9 @@ fi # Add a sentinel to simplify awk in ssh-to-login-node cat <> /etc/ssh/ssh_config -Host $remote_fqdn # remote login host - Port $remote_port - IdentitiesOnly yes +# Host $remote_fqdn # remote login host +# Port $remote_port +# IdentitiesOnly yes Match localuser root ControlMaster auto @@ -232,6 +232,20 @@ done ################### test_remote_connect () { + # Wait for an SSH agent forwarding socket to be established before attempting SSH + echo "Waiting for SSH agent forwarding to be established..." + MAX_RETRIES=100 + for _ in $(seq 1 $MAX_RETRIES); do + if ls /tmp/ | grep 'ssh-' ; then + export SSH_AUTH_SOCK=$(ls /tmp/ssh-*/*agent* | head -n1) + echo "Got SSH_AUTH_SOCK: $SSH_AUTH_SOCK" + break + else + echo "No auth socket found yet, retrying in 10 seconds..." + sleep 10 + fi + done + ssh -vvv "$1@$2" true } From 9d90048152da17c807664df83d44eb7619c493ec Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Feb 2026 15:59:18 -0600 Subject: [PATCH 04/12] log remote-site-setup.sh to a file --- base/etc/supervisord.d/10-htcondor-ce.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/etc/supervisord.d/10-htcondor-ce.conf b/base/etc/supervisord.d/10-htcondor-ce.conf index 48a7492..d681cfe 100644 --- a/base/etc/supervisord.d/10-htcondor-ce.conf +++ b/base/etc/supervisord.d/10-htcondor-ce.conf @@ -2,7 +2,7 @@ # CE startup depends on sshd becoming available and must start after # a remote processes has forwarded an agent to the container command = sh -c ' - source /usr/local/bin/remote-site-setup.sh && + source /usr/local/bin/remote-site-setup.sh >> /var/log/remote-site-setup.log && /usr/share/condor-ce/condor_ce_startup -f' autorestart=true From 4d3537c2372fa7aa09faac58afbd18c7b71e3d91 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 10 Feb 2026 16:39:36 -0600 Subject: [PATCH 05/12] Split sshd daemon into separate container from main hosted-ce --- base/etc/supervisord.d/05-sshd.conf | 10 ---------- base/etc/supervisord.d/10-htcondor-ce.conf | 4 +--- docker-compose.yaml | 13 +++++++++++++ hosted-ce-sshd/Dockerfile | 10 ++++++++++ hosted-ce-sshd/init.sh | 18 ++++++++++++++++++ .../image-config.d/30-remote-site-setup.sh} | 11 +++-------- 6 files changed, 45 insertions(+), 21 deletions(-) delete mode 100644 base/etc/supervisord.d/05-sshd.conf create mode 100644 docker-compose.yaml create mode 100644 hosted-ce-sshd/Dockerfile create mode 100755 hosted-ce-sshd/init.sh rename hosted-ce/{usr/local/bin/remote-site-setup.sh => etc/osg/image-config.d/30-remote-site-setup.sh} (97%) diff --git a/base/etc/supervisord.d/05-sshd.conf b/base/etc/supervisord.d/05-sshd.conf deleted file mode 100644 index 7b44880..0000000 --- a/base/etc/supervisord.d/05-sshd.conf +++ /dev/null @@ -1,10 +0,0 @@ -[program:sshd] -user = root -# Writing directly into /etc/ssh overwrites some necessary config files, -# instead stage keys into /etc/ssh.orig.d and then copy to /etc/sh -command = sh -c ' - mkdir -p /mnt/ssh.orig/etc/ssh && - ssh-keygen -A -f /mnt/ssh.orig && - /bin/cp -p /mnt/ssh.orig/etc/ssh/* /etc/ssh/ && - chmod 400 /etc/ssh/*key && - /usr/sbin/sshd -D' diff --git a/base/etc/supervisord.d/10-htcondor-ce.conf b/base/etc/supervisord.d/10-htcondor-ce.conf index d681cfe..0190de9 100644 --- a/base/etc/supervisord.d/10-htcondor-ce.conf +++ b/base/etc/supervisord.d/10-htcondor-ce.conf @@ -1,9 +1,7 @@ [program:htcondor-ce] # CE startup depends on sshd becoming available and must start after # a remote processes has forwarded an agent to the container -command = sh -c ' - source /usr/local/bin/remote-site-setup.sh >> /var/log/remote-site-setup.log && - /usr/share/condor-ce/condor_ce_startup -f' +command=/usr/share/condor-ce/condor_ce_startup -f autorestart=true startsecs=20 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..cf8289a --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + base: + image: base + build: + context: base + hosted-ce: + image: hub.opensciencegrid.org/mwestphall/hosted-ce + build: + context: hosted-ce + hosted-ce-sshd: + image: hub.opensciencegrid.org/mwestphall/hosted-ce-sshd + build: + context: hosted-ce-sshd diff --git a/hosted-ce-sshd/Dockerfile b/hosted-ce-sshd/Dockerfile new file mode 100644 index 0000000..3451476 --- /dev/null +++ b/hosted-ce-sshd/Dockerfile @@ -0,0 +1,10 @@ +FROM almalinux:9 + +RUN yum install -y openssh-server && \ + yum clean all && \ + rm -rf /var/cache/yum/ && \ + adduser sshd-user + +COPY init.sh /usr/local/sbin + +CMD [ "/usr/local/sbin/init.sh" ] diff --git a/hosted-ce-sshd/init.sh b/hosted-ce-sshd/init.sh new file mode 100755 index 0000000..bc47bc3 --- /dev/null +++ b/hosted-ce-sshd/init.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Create a new directory to hold original SSH keys (if it doesn't yet exist) +mkdir -p /etc/ssh.orig/etc/ssh + +# Generate a new set of SSH host keys for the container and copy them to the original location +ssh-keygen -A -f /etc/ssh.orig +/bin/cp -p /etc/ssh.orig/etc/ssh/* /etc/ssh/ +chmod 400 /etc/ssh/*key + +#Assume an authorized_keys file has been configured in /etc/ssh.orig/authorized_keys +mkdir -p /home/sshd-user/.ssh/ +cp /etc/ssh.orig/authorized_keys /home/sshd-user/.ssh/authorized_keys +chown -R sshd-user /home/sshd-user/.ssh +chmod 600 /home/sshd-user/.ssh/authorized_keys + +# Start sshd +/usr/sbin/sshd -e -D diff --git a/hosted-ce/usr/local/bin/remote-site-setup.sh b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh similarity index 97% rename from hosted-ce/usr/local/bin/remote-site-setup.sh rename to hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh index 3942fa4..39ea96f 100755 --- a/hosted-ce/usr/local/bin/remote-site-setup.sh +++ b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh @@ -18,7 +18,6 @@ BOSCO_KEY=/etc/osg/bosco.key BOSCO_CERT=${BOSCO_KEY}-cert.pub ENDPOINT_CONFIG=/etc/endpoints.ini KNOWN_HOSTS=/etc/osg/ssh_known_hosts -AUTHORIZED_KEYS=/etc/osg/ssh_authorized_keys SKIP_WN_INSTALL=no function errexit { @@ -76,11 +75,6 @@ setup_user_ssh () { # cp $BOSCO_KEY $ssh_key # chmod 600 $ssh_key - # copy authorized_keys - authorized_keys=$ssh_dir/authorized_keys - cp $AUTHORIZED_KEYS $authorized_keys - chmod 600 $authorized_keys - # HACK: Symlink the Bosco key to the location expected by # bosco_cluster so it doesn't go and try to generate a new one # ln -s $ssh_key $ssh_dir/bosco_key.rsa @@ -235,9 +229,10 @@ test_remote_connect () { # Wait for an SSH agent forwarding socket to be established before attempting SSH echo "Waiting for SSH agent forwarding to be established..." MAX_RETRIES=100 + SSH_SOCK_DIR=/etc/condor-ce/sshd-sock for _ in $(seq 1 $MAX_RETRIES); do - if ls /tmp/ | grep 'ssh-' ; then - export SSH_AUTH_SOCK=$(ls /tmp/ssh-*/*agent* | head -n1) + if ls $SSH_SOCK_DIR | grep 'ssh-' ; then + export SSH_AUTH_SOCK=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) echo "Got SSH_AUTH_SOCK: $SSH_AUTH_SOCK" break else From 25d07de4304deac2b9db45f28d7c90539310e7d0 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Mar 2026 15:22:08 -0600 Subject: [PATCH 06/12] Update hosted-ce-sshd to read from a yaml config file --- hosted-ce-sshd/Dockerfile | 4 +- hosted-ce-sshd/configure_authorized_keys.py | 46 +++++++++++++++++++++ hosted-ce-sshd/init.sh | 12 +++--- 3 files changed, 54 insertions(+), 8 deletions(-) create mode 100755 hosted-ce-sshd/configure_authorized_keys.py diff --git a/hosted-ce-sshd/Dockerfile b/hosted-ce-sshd/Dockerfile index 3451476..b8da59b 100644 --- a/hosted-ce-sshd/Dockerfile +++ b/hosted-ce-sshd/Dockerfile @@ -1,10 +1,10 @@ FROM almalinux:9 -RUN yum install -y openssh-server && \ +RUN yum install -y openssh-server python3-pyyaml && \ yum clean all && \ rm -rf /var/cache/yum/ && \ adduser sshd-user -COPY init.sh /usr/local/sbin +COPY init.sh configure_authorized_keys.py /usr/local/sbin/ CMD [ "/usr/local/sbin/init.sh" ] diff --git a/hosted-ce-sshd/configure_authorized_keys.py b/hosted-ce-sshd/configure_authorized_keys.py new file mode 100755 index 0000000..7b2780a --- /dev/null +++ b/hosted-ce-sshd/configure_authorized_keys.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +''' +Util script to copy the public key given in a yaml ConfigMap into the +authorized_hosts file for the sshd daemon user. Assumes config in the +form of + + +instances: + key_name_1: instance_1 + key_name_2: instance_2 +publick_keys: + key_name_1: pubkey_1 + key_name_1: pubkey_2 +''' +import yaml +from pathlib import Path +from os import environ +from sys import exit, argv + +CONFIG_PATH = argv[1] # /etc/ssh.orig/key-mappings.yaml +AUTHORIZED_KEYS_PATH = argv[2] # /home/sshd-user/.ssh/authorized_keys +INSTANCE = environ['CE_INSTANCE'] + +with open(CONFIG_PATH) as f: + config = yaml.load(f.read()) + +# Figure out to which instance our key belongs +# Instance dict is in the format key_name : instance +instances: dict[str, str] = config['instances'] +for key_name, instance in instances.items(): + if instance == INSTANCE: + break +else: + print(f"Fatal: No key name found for instance {INSTANCE}.") + exit(1) + +AUTHORIZED_KEYS_PATH.parent.mkdir(parents=True, exist_ok=True) +pubkeys: dict[str, str] = config['public_keys'] +pubkey = pubkeys.get(key_name) + +if not pubkey: + print(f"Fatal: No public key found for key {key_name}") + exit(1) + +with open(AUTHORIZED_KEYS_PATH, 'w') as keyf: + keyf.write(pubkey) diff --git a/hosted-ce-sshd/init.sh b/hosted-ce-sshd/init.sh index bc47bc3..33321fd 100755 --- a/hosted-ce-sshd/init.sh +++ b/hosted-ce-sshd/init.sh @@ -3,14 +3,14 @@ # Create a new directory to hold original SSH keys (if it doesn't yet exist) mkdir -p /etc/ssh.orig/etc/ssh -# Generate a new set of SSH host keys for the container and copy them to the original location -ssh-keygen -A -f /etc/ssh.orig -/bin/cp -p /etc/ssh.orig/etc/ssh/* /etc/ssh/ -chmod 400 /etc/ssh/*key +# Assume SSH host keys have been configured in /etc/ssh.orig but mounted with improper permissions +# Copy the keys and then fix their permissions +cp -p /etc/ssh.orig/ssh_* /etc/ssh/ +chmod 400 /etc/ssh/*key* -#Assume an authorized_keys file has been configured in /etc/ssh.orig/authorized_keys +# Assume an authorized_keys file has been configured in /etc/ssh.orig/authorized_keys mkdir -p /home/sshd-user/.ssh/ -cp /etc/ssh.orig/authorized_keys /home/sshd-user/.ssh/authorized_keys +configure_authorized_keys.py /etc/ssh.orig/key-mappings.yaml /home/sshd-user/.ssh/authorized_keys chown -R sshd-user /home/sshd-user/.ssh chmod 600 /home/sshd-user/.ssh/authorized_keys From f9b101de530628d4ab9f560a836359e30bc7c2c7 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 3 Mar 2026 16:14:45 -0600 Subject: [PATCH 07/12] fix path <-> str conversion --- hosted-ce-sshd/configure_authorized_keys.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hosted-ce-sshd/configure_authorized_keys.py b/hosted-ce-sshd/configure_authorized_keys.py index 7b2780a..07cd556 100755 --- a/hosted-ce-sshd/configure_authorized_keys.py +++ b/hosted-ce-sshd/configure_authorized_keys.py @@ -17,12 +17,12 @@ from os import environ from sys import exit, argv -CONFIG_PATH = argv[1] # /etc/ssh.orig/key-mappings.yaml -AUTHORIZED_KEYS_PATH = argv[2] # /home/sshd-user/.ssh/authorized_keys +CONFIG_PATH = Path(argv[1]) # eg. /etc/ssh.orig/key-mappings.yaml +AUTHORIZED_KEYS_PATH = Path(argv[2]) # eg. /home/sshd-user/.ssh/authorized_keys INSTANCE = environ['CE_INSTANCE'] with open(CONFIG_PATH) as f: - config = yaml.load(f.read()) + config = yaml.load(f.read(), Loader=yaml.Loader) # Figure out to which instance our key belongs # Instance dict is in the format key_name : instance From 7f8d6ffadf2970f36d2bfb95636ba1554a651d17 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 24 Mar 2026 14:03:31 -0500 Subject: [PATCH 08/12] Add support for updating the ssh auth sock (via a symlink) during runtime --- hosted-ce/Dockerfile | 1 + .../image-config.d/30-remote-site-setup.sh | 7 ++++-- .../etc/supervisord.d/05-agent-sock.conf | 8 +++++++ hosted-ce/usr/local/bin/update-agent-sock | 24 +++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 hosted-ce/etc/supervisord.d/05-agent-sock.conf create mode 100755 hosted-ce/usr/local/bin/update-agent-sock diff --git a/hosted-ce/Dockerfile b/hosted-ce/Dockerfile index 4f20b15..1c8f8aa 100644 --- a/hosted-ce/Dockerfile +++ b/hosted-ce/Dockerfile @@ -42,3 +42,4 @@ RUN sed -i 's/bosco_cluster/condor_remote_cluster/g' /tmp/*.patch && \ COPY usr/local/bin /usr/local/bin +COPY etc/supervisord.d/* /etc/supervisord.d/ diff --git a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh index 39ea96f..e7c550e 100755 --- a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh +++ b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh @@ -232,8 +232,11 @@ test_remote_connect () { SSH_SOCK_DIR=/etc/condor-ce/sshd-sock for _ in $(seq 1 $MAX_RETRIES); do if ls $SSH_SOCK_DIR | grep 'ssh-' ; then - export SSH_AUTH_SOCK=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) - echo "Got SSH_AUTH_SOCK: $SSH_AUTH_SOCK" + TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) + LINK=$SSH_SOCK_DIR/auth-sock + ln -s "$TARGET" "$LINK" + export SSH_AUTH_SOCK="$LINK" + echo "Got SSH_AUTH_SOCK: $LINK -> $TARGET" break else echo "No auth socket found yet, retrying in 10 seconds..." diff --git a/hosted-ce/etc/supervisord.d/05-agent-sock.conf b/hosted-ce/etc/supervisord.d/05-agent-sock.conf new file mode 100644 index 0000000..d9a598a --- /dev/null +++ b/hosted-ce/etc/supervisord.d/05-agent-sock.conf @@ -0,0 +1,8 @@ +[program:update-agent-sock] +command=/usr/local/bin/update-agent-sock +autostart=true +autorestart=true +# TODO writing a non-condor log to the condor ce location is not ideal but is useful for debugging +stdout_logfile=/var/log/condor-ce/update-agent-sock.log +stdout_logfile_maxbytes=0 +redirect_stderr=true diff --git a/hosted-ce/usr/local/bin/update-agent-sock b/hosted-ce/usr/local/bin/update-agent-sock new file mode 100755 index 0000000..4c56165 --- /dev/null +++ b/hosted-ce/usr/local/bin/update-agent-sock @@ -0,0 +1,24 @@ +#!/bin/bash + +# Util script to update the symlink to the SSH agent socket that +# condor_ce uses to access the remote cluster login host + +SSH_SOCK_DIR=/etc/condor-ce/sshd-sock + +poll_ssh_auth_sock() { + echo "Polling SSH agent socket..." + while true; do + if ls $SSH_SOCK_DIR | grep 'ssh-' ; then + TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) + LINK=$SSH_SOCK_DIR/auth-sock + ln -sf "$TARGET" "$LINK" + echo "Updating SSH_AUTH_SOCK: $LINK -> $TARGET. Checking again in 10 seconds..." + else + echo "No auth socket found yet, retrying in 10 seconds..." + fi + sleep 10 + done + +} + +poll_ssh_auth_sock From 43eccb27c581c97f93b96206e6930ae2b8dea7e1 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Tue, 7 Apr 2026 14:12:33 -0500 Subject: [PATCH 09/12] Get key name from an env var rather than a configmap --- hosted-ce-sshd/configure_authorized_keys.py | 21 ++++----------------- hosted-ce-sshd/init.sh | 4 +++- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/hosted-ce-sshd/configure_authorized_keys.py b/hosted-ce-sshd/configure_authorized_keys.py index 07cd556..636ac4b 100755 --- a/hosted-ce-sshd/configure_authorized_keys.py +++ b/hosted-ce-sshd/configure_authorized_keys.py @@ -4,11 +4,7 @@ authorized_hosts file for the sshd daemon user. Assumes config in the form of - -instances: - key_name_1: instance_1 - key_name_2: instance_2 -publick_keys: +public_keys: key_name_1: pubkey_1 key_name_1: pubkey_2 ''' @@ -20,26 +16,17 @@ CONFIG_PATH = Path(argv[1]) # eg. /etc/ssh.orig/key-mappings.yaml AUTHORIZED_KEYS_PATH = Path(argv[2]) # eg. /home/sshd-user/.ssh/authorized_keys INSTANCE = environ['CE_INSTANCE'] +AUTHORIZED_KEY = environ['AUTHORIZED_KEY'] with open(CONFIG_PATH) as f: config = yaml.load(f.read(), Loader=yaml.Loader) -# Figure out to which instance our key belongs -# Instance dict is in the format key_name : instance -instances: dict[str, str] = config['instances'] -for key_name, instance in instances.items(): - if instance == INSTANCE: - break -else: - print(f"Fatal: No key name found for instance {INSTANCE}.") - exit(1) - AUTHORIZED_KEYS_PATH.parent.mkdir(parents=True, exist_ok=True) pubkeys: dict[str, str] = config['public_keys'] -pubkey = pubkeys.get(key_name) +pubkey = pubkeys.get(AUTHORIZED_KEY) if not pubkey: - print(f"Fatal: No public key found for key {key_name}") + print(f"Fatal: No public key found for key {AUTHORIZED_KEY}") exit(1) with open(AUTHORIZED_KEYS_PATH, 'w') as keyf: diff --git a/hosted-ce-sshd/init.sh b/hosted-ce-sshd/init.sh index 33321fd..3492e01 100755 --- a/hosted-ce-sshd/init.sh +++ b/hosted-ce-sshd/init.sh @@ -1,4 +1,5 @@ #!/bin/bash +PORT=${SSHD_PORT:-22} # Create a new directory to hold original SSH keys (if it doesn't yet exist) mkdir -p /etc/ssh.orig/etc/ssh @@ -14,5 +15,6 @@ configure_authorized_keys.py /etc/ssh.orig/key-mappings.yaml /home/sshd-user/.ss chown -R sshd-user /home/sshd-user/.ssh chmod 600 /home/sshd-user/.ssh/authorized_keys +echo "Starting sshd on port $PORT" # Start sshd -/usr/sbin/sshd -e -D +/usr/sbin/sshd -p $PORT -e -D From 2e9a53947553fcceb25f66d8dddfb85e4788d602 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Mon, 21 Sep 2026 11:06:58 -0500 Subject: [PATCH 10/12] Untrack dev docker-compose --- docker-compose.yaml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index cf8289a..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,13 +0,0 @@ -services: - base: - image: base - build: - context: base - hosted-ce: - image: hub.opensciencegrid.org/mwestphall/hosted-ce - build: - context: hosted-ce - hosted-ce-sshd: - image: hub.opensciencegrid.org/mwestphall/hosted-ce-sshd - build: - context: hosted-ce-sshd From 633023e9d18aa1c07de7bf5301af91dafacbb557 Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Thu, 24 Sep 2026 10:32:03 -0500 Subject: [PATCH 11/12] Restore non-agent-forward functionality to init scripts via env var switch --- base/etc/supervisord.d/10-htcondor-ce.conf | 4 - .../image-config.d/30-remote-site-setup.sh | 95 ++++++++++--------- hosted-ce/usr/local/bin/update-agent-sock | 6 +- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/base/etc/supervisord.d/10-htcondor-ce.conf b/base/etc/supervisord.d/10-htcondor-ce.conf index 0190de9..962cf88 100644 --- a/base/etc/supervisord.d/10-htcondor-ce.conf +++ b/base/etc/supervisord.d/10-htcondor-ce.conf @@ -1,8 +1,4 @@ [program:htcondor-ce] -# CE startup depends on sshd becoming available and must start after -# a remote processes has forwarded an agent to the container command=/usr/share/condor-ce/condor_ce_startup -f - autorestart=true startsecs=20 - diff --git a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh index e7c550e..fb84f8f 100755 --- a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh +++ b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh @@ -70,29 +70,30 @@ setup_user_ssh () { mkdir -p $ssh_dir chmod 700 $ssh_dir - # copy Bosco key -# ssh_key=$ssh_dir/id_rsa -# cp $BOSCO_KEY $ssh_key -# chmod 600 $ssh_key - - # HACK: Symlink the Bosco key to the location expected by - # bosco_cluster so it doesn't go and try to generate a new one -# ln -s $ssh_key $ssh_dir/bosco_key.rsa - - # copy Bosco certificate -# if [[ -f $BOSCO_CERT ]]; then -# ssh_cert=${ssh_key}-cert.pub -# cp $BOSCO_CERT $ssh_cert -# chmod 600 $ssh_cert -# fi - - # Write user/host stanza to the global SSH config -# cat <> /etc/ssh/ssh_config -# Match user "$remote_user" -# IdentityFile $ssh_key -# ${extra_config} - -# EOF + # copy Bosco key if not using a forwarded agent + if [[ ${USE_SSH_AGENT_FORWARD:-false} != 'true' ]]; then + ssh_key=$ssh_dir/id_rsa + cp $BOSCO_KEY $ssh_key + chmod 600 $ssh_key + + # HACK: Symlink the Bosco key to the location expected by + # bosco_cluster so it doesn't go and try to generate a new one + ln -s $ssh_key $ssh_dir/bosco_key.rsa + + # copy Bosco certificate + if [[ -f $BOSCO_CERT ]]; then + ssh_cert=${ssh_key}-cert.pub + cp $BOSCO_CERT $ssh_cert + chmod 600 $ssh_cert + fi + + # Write user/host stanza to the global SSH config + cat << EOF >> /etc/ssh/ssh_config +Match user "$remote_user" + IdentityFile $ssh_key + ${extra_config} +EOF + fi chown -R "${ruser}": "$ssh_dir" @@ -193,11 +194,15 @@ fi # Set up a control master for each rootly SSH connection # Add a sentinel to simplify awk in ssh-to-login-node -cat <> /etc/ssh/ssh_config +if [[ ${USE_SSH_AGENT_FORWARD:-false} != 'true' ]]; then + cat <> /etc/ssh/ssh_config +Host $remote_fqdn # remote login host + Port $remote_port + IdentitiesOnly yes +EOF +fi -# Host $remote_fqdn # remote login host -# Port $remote_port -# IdentitiesOnly yes +cat <> /etc/ssh/ssh_config Match localuser root ControlMaster auto @@ -226,23 +231,25 @@ done ################### test_remote_connect () { - # Wait for an SSH agent forwarding socket to be established before attempting SSH - echo "Waiting for SSH agent forwarding to be established..." - MAX_RETRIES=100 - SSH_SOCK_DIR=/etc/condor-ce/sshd-sock - for _ in $(seq 1 $MAX_RETRIES); do - if ls $SSH_SOCK_DIR | grep 'ssh-' ; then - TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) - LINK=$SSH_SOCK_DIR/auth-sock - ln -s "$TARGET" "$LINK" - export SSH_AUTH_SOCK="$LINK" - echo "Got SSH_AUTH_SOCK: $LINK -> $TARGET" - break - else - echo "No auth socket found yet, retrying in 10 seconds..." - sleep 10 - fi - done + if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then + # Wait for an SSH agent forwarding socket to be established before attempting SSH + echo "Waiting for SSH agent forwarding to be established..." + MAX_RETRIES=100 + SSH_SOCK_DIR=/etc/condor-ce/sshd-sock + for _ in $(seq 1 $MAX_RETRIES); do + if ls $SSH_SOCK_DIR | grep 'ssh-' ; then + TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) + LINK=$SSH_SOCK_DIR/auth-sock + ln -s "$TARGET" "$LINK" + export SSH_AUTH_SOCK="$LINK" + echo "Got SSH_AUTH_SOCK: $LINK -> $TARGET" + break + else + echo "No auth socket found yet, retrying in 10 seconds..." + sleep 10 + fi + done + fi ssh -vvv "$1@$2" true } diff --git a/hosted-ce/usr/local/bin/update-agent-sock b/hosted-ce/usr/local/bin/update-agent-sock index 4c56165..32050a1 100755 --- a/hosted-ce/usr/local/bin/update-agent-sock +++ b/hosted-ce/usr/local/bin/update-agent-sock @@ -21,4 +21,8 @@ poll_ssh_auth_sock() { } -poll_ssh_auth_sock +if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then + poll_ssh_auth_sock +else + sleep infinity +fi From a04c47d1e1160fcb95d38cf1ab5e7717f584053b Mon Sep 17 00:00:00 2001 From: Matthew Westphall Date: Thu, 24 Sep 2026 17:14:05 -0500 Subject: [PATCH 12/12] Use IdenityAgents inside of ssh config instead of SSH_AUTH_SOCK env var to get auth sock to carry over to bosco init scripts --- .../image-config.d/30-remote-site-setup.sh | 23 +++++++++++++++++-- hosted-ce/usr/local/bin/update-agent-sock | 4 ++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh index fb84f8f..c0cea71 100755 --- a/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh +++ b/hosted-ce/etc/osg/image-config.d/30-remote-site-setup.sh @@ -92,6 +92,13 @@ setup_user_ssh () { Match user "$remote_user" IdentityFile $ssh_key ${extra_config} +EOF + else + # Set IdentityAgent to the forwarded yubikey agent for osg01 (etc) + cat << EOF >> /etc/ssh/ssh_config +Match user "$remote_user" + IdentityAgent /etc/condor-ce/sshd-sock/auth-sock + ${extra_config} EOF fi @@ -202,12 +209,20 @@ Host $remote_fqdn # remote login host EOF fi +# Hack to make the forwarded SSH agent carry over to the bosco tools, +# which run as root and spin off their own SSH agents by default +identity_agent_config="" +if [[ ${USE_SSH_AGENT_FORWARD:-false} == 'true' ]]; then + identity_agent_config=" IdentityAgent /etc/condor-ce/sshd-sock/auth-sock" +fi + cat <> /etc/ssh/ssh_config Match localuser root ControlMaster auto ControlPath /tmp/cm-%i-%r@%h:%p ControlPersist 15m +$identity_agent_config EOF @@ -241,8 +256,12 @@ test_remote_connect () { TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) LINK=$SSH_SOCK_DIR/auth-sock ln -s "$TARGET" "$LINK" - export SSH_AUTH_SOCK="$LINK" - echo "Got SSH_AUTH_SOCK: $LINK -> $TARGET" + + # Allow non-root users (eg. osg01) to read the auth sock + chmod 755 "$(dirname "$TARGET")" + chmod 666 "$TARGET" + + echo "Got auth-sock: $LINK -> $TARGET" break else echo "No auth socket found yet, retrying in 10 seconds..." diff --git a/hosted-ce/usr/local/bin/update-agent-sock b/hosted-ce/usr/local/bin/update-agent-sock index 32050a1..2e298fc 100755 --- a/hosted-ce/usr/local/bin/update-agent-sock +++ b/hosted-ce/usr/local/bin/update-agent-sock @@ -12,6 +12,10 @@ poll_ssh_auth_sock() { TARGET=$(ls $SSH_SOCK_DIR/ssh-*/*agent* | head -n1) LINK=$SSH_SOCK_DIR/auth-sock ln -sf "$TARGET" "$LINK" + + # Allow non-root users (eg. osg01) to read the auth sock + chmod 755 "$(dirname "$TARGET")" + chmod 666 "$TARGET" echo "Updating SSH_AUTH_SOCK: $LINK -> $TARGET. Checking again in 10 seconds..." else echo "No auth socket found yet, retrying in 10 seconds..."