Merge pull request #17 from keybase/david/circleci

ddworken · web-flow · commit 23f3e00cd43d · 2019-08-15T13:33:42.000-04:00
Add CircleCI config
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,34 @@
+version: 2
+jobs:
+  integration:
+    machine:
+      image: circleci/classic:latest
+    steps:
+      - checkout
+      - run: "./integrationTest.sh"
+  unit:
+    docker:
+      - image: circleci/golang:1.11
+    steps:
+      - checkout
+      - run: go test ./...
+  lint:
+    docker:
+      - image: circleci/golang:1.11
+    steps:
+      - checkout
+      - run: "! go fmt ./... 2>&1 | read"
+      - run: "! go vet ./... 2>&1 | read"
+      - run: go get -u golang.org/x/lint/golint
+      - run: "! golint ./... 2>&1 | read"
+      - run: go get golang.org/x/tools/cmd/goimports
+      - run: "! goimports ./... 2>&1 | read"
+      - run: "! go mod tidy ./... 2>&1 | read"
+workflows:
+  version: 2
+  build:
+    jobs:
+      - integration
+      - unit
+      - lint
+      
diff --git a/integrationTest.sh b/integrationTest.sh
@@ -3,14 +3,24 @@ set -euo pipefail
 IFS=$'\n\t'
 
 # Unit tests first
-go test ./... 2>&1 | grep -v 'no test files'
+set +u
+if [ -z "$CIRCLECI" ]; then
+  go test ./... 2>&1 | grep -v 'no test files'
+fi
 
+# Ensure we have the correct environment variables
 if [[ -f "tests/env.sh" ]]; then
-    echo "env.sh file already exists, skipping configuring new accounts..."
+  echo "env.sh already exists, skipping configuring new accounts..."
+  source tests/env.sh
 else
+  if [ -n "$CIRCLECI" ]; then
+    echo "Running in circle with configured environment variables"
+  else
     python3 tests/configure_accounts.py
+    source tests/env.sh
+  fi
 fi
-
+set -u
 
 # Some colors for pretty output
 RED='\033[0;31m'
@@ -19,13 +29,13 @@ NC='\033[0m'
 
 # A function used to indent the log output from the tests
 indent() { sed 's/^/    /'; }
+# Reset docker and wipe all volumes
 reset_docker() {
     docker-compose down -v
     docker system prune -f
 }
 
 cd tests/
-source env.sh
 reset_docker
 
 echo "Building containers..."
diff --git a/tests/Dockerfile-sshd b/tests/Dockerfile-sshd
@@ -1,4 +1,4 @@
-# This dockerfile builds an openssh server that will accept SSH keys signed by the key provided in /mnt/keybase-ca-key.pub
+# This dockerfile builds an openssh server that will accept SSH keys signed by the key provided in /shared/keybase-ca-key.pub
 # It takes in a build argument and only allows keys with the build argument in the principals field
 FROM ubuntu:18.04
 
@@ -26,4 +26,4 @@ RUN echo -n "uniquestring" > /etc/unique
 
 EXPOSE 22
 
-CMD ln -sf /mnt/keybase-ca-key.pub /etc/ssh/ca.pub && /usr/sbin/sshd -D
+CMD ln -sf /shared/keybase-ca-key.pub /etc/ssh/ca.pub && /usr/sbin/sshd -D
diff --git a/tests/bot-entrypoint.py b/tests/bot-entrypoint.py
@@ -24,7 +24,7 @@ def load_env():
         "bin/keybaseca --wipe-all-configs\n"
         "bin/keybaseca --wipe-logs || true\n"
         "bin/keybaseca generate --overwrite-existing-key\n"
-        "echo yes | bin/keybaseca backup > /mnt/cakey.backup\n"
+        "echo yes | bin/keybaseca backup > /shared/cakey.backup\n"
         "bin/keybaseca service &"
     ) % (shlex.quote(path)))
     # Sleep so keybaseca has time to start
diff --git a/tests/bot-entrypoint.sh b/tests/bot-entrypoint.sh
@@ -2,17 +2,16 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-# For some reason it is necessary to touch a file in /mnt/ in order to get the volume permissions to work correctly
+# For some reason it is necessary to touch a file in /shared/ in order to get the volume permissions to work correctly
 # when keybaseca generate runs
-touch /mnt/.keep
+touch /shared/.keep
 
 # Generate the env files that will be used for tests
-source tests/env.sh
 mkdir -p tests/generated-env
 ls tests/envFiles/ | xargs -I {} -- bash -c 'cat tests/envFiles/{} | envsubst > tests/generated-env/{}'
 
 nohup bash -c "run_keybase -g &"
 sleep 3
 keybase oneshot --username $BOT_USERNAME --paperkey "$BOT_PAPERKEY"
-touch /mnt/ready
+touch /shared/ready
 python3 tests/bot-entrypoint.py
diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml
@@ -9,10 +9,11 @@ services:
     environment:
       - BOT_PAPERKEY
       - BOT_USERNAME
+      - SUBTEAM
     volumes:
-      - app-volume:/mnt/
+      - app-volume:/shared/
     user: root
-    command: "sh -c 'chown -R keybase:keybase /mnt && su keybase -c \"bash tests/bot-entrypoint.sh\"'"
+    command: "sh -c 'chown -R keybase:keybase /shared && su keybase -c \"bash tests/bot-entrypoint.sh\"'"
     ports:
       - 8080  # Used for the flask webserver that manages restarting keybaseca with different config options
     depends_on:
@@ -31,7 +32,7 @@ services:
       - SUBTEAM
       - SUBTEAM_SECONDARY
     volumes:
-      - app-volume:/mnt/
+      - app-volume:/shared/
     user: keybase
     command: "bash tests/tester-entrypoint.sh"
     depends_on:
@@ -49,7 +50,7 @@ services:
         user_principal: ${SUBTEAM}.ssh.staging
         root_principal: ${SUBTEAM}.ssh.root_everywhere
     volumes:
-      - app-volume:/mnt/
+      - app-volume:/shared/
   # An ssh server that will accept signed requests with the principal "prod"
   sshd-prod:
     image: sshd-prod
@@ -61,6 +62,6 @@ services:
         user_principal: ${SUBTEAM}.ssh.prod
         root_principal: ${SUBTEAM}.ssh.root_everywhere
     volumes:
-      - app-volume:/mnt/
+      - app-volume:/shared/
 volumes:
   app-volume:
diff --git a/tests/envFiles/README.md b/tests/envFiles/README.md
@@ -1,2 +1,3 @@
 The files in this directory are env files used to run the CA bot. The files are filled in based on
-the values of environment variables `SUBTEAM`, `BOT_USERNAME`, and `BOT_PAPERKEY` from tests/env.sh.
+the values of environment variables `SUBTEAM`, `BOT_USERNAME`, and `BOT_PAPERKEY`. When running
+locally, these environment variables are stored in tests/env.sh. 
diff --git a/tests/envFiles/test_env_1 b/tests/envFiles/test_env_1
@@ -6,3 +6,4 @@ export TEAMS="$SUBTEAM.ssh.staging,$SUBTEAM.ssh.prod,$SUBTEAM.ssh.root_everywher
 export KEYBASE_PAPERKEY="$BOT_PAPERKEY"
 export KEYBASE_USERNAME="$BOT_USERNAME"
 export CHAT_CHANNEL="$SUBTEAM.ssh#ssh-provision"
+export CA_KEY_LOCATION="/shared/keybase-ca-key"
diff --git a/tests/envFiles/test_env_2_local_audit_log b/tests/envFiles/test_env_2_local_audit_log
@@ -1,7 +1,8 @@
 # Used to test sending the audit log to the normal filesystem. The code that handles local audit log writes is different
 # from the code that handles KBFS audit log writes.
 export KEY_EXPIRATION="+1h"
-export LOG_LOCATION="/mnt/ca.log"
+export LOG_LOCATION="/shared/ca.log"
 export TEAMS="$SUBTEAM.ssh.staging,$SUBTEAM.ssh.prod,$SUBTEAM.ssh.root_everywhere"
 export KEYBASE_PAPERKEY="$BOT_PAPERKEY"
 export KEYBASE_USERNAME="$BOT_USERNAME"
+export CA_KEY_LOCATION="/shared/keybase-ca-key"
diff --git a/tests/envFiles/test_env_3_user_not_in_first_team b/tests/envFiles/test_env_3_user_not_in_first_team
@@ -1,7 +1,8 @@
 # Used to test the behavior of the chatbot when the user is not in the first listed team. This used to not work
 # due to the choice of only placing config files in the first team.
 export KEY_EXPIRATION="+1h"
-export LOG_LOCATION="/mnt/ca.log"
+export LOG_LOCATION="/shared/ca.log"
 export TEAMS="$SUBTEAM.ssh.prod,$SUBTEAM.ssh.staging,$SUBTEAM.ssh.root_everywhere"
 export KEYBASE_PAPERKEY="$BOT_PAPERKEY"
 export KEYBASE_USERNAME="$BOT_USERNAME"
+export CA_KEY_LOCATION="/shared/keybase-ca-key"
diff --git a/tests/envFiles/test_env_4_user_not_in_configured_teams b/tests/envFiles/test_env_4_user_not_in_configured_teams
@@ -1,7 +1,8 @@
 # Used to test the behavior of the chatbot when the user is not in any of the listed teams. Meant to ensure that the
 # bot does not respond to messages from users outside of the teams (a security regression test). 
 export KEY_EXPIRATION="+1h"
-export LOG_LOCATION="/mnt/ca.log"
+export LOG_LOCATION="/shared/ca.log"
 export TEAMS="$SUBTEAM.ssh.prod"
 export KEYBASE_PAPERKEY="$BOT_PAPERKEY"
 export KEYBASE_USERNAME="$BOT_USERNAME"
+export CA_KEY_LOCATION="/shared/keybase-ca-key"
diff --git a/tests/tester-entrypoint.sh b/tests/tester-entrypoint.sh
@@ -5,7 +5,7 @@ IFS=$'\n\t'
 nohup bash -c "run_keybase -g &"
 
 # Sleep until the CA bot has started
-while ! [ -f /mnt/ready ];
+while ! [ -f /shared/ready ];
 do
     sleep 1
 done
diff --git a/tests/tests/lib.py b/tests/tests/lib.py
@@ -54,7 +54,8 @@ def run_command(cmd: str, timeout: int=10) -> bytes:
 def read_file(filename: str) -> List[bytes]:
     """
     Read the contents of the given filename to a list of strings. If it is a normal file,
-    uses the standard open() function. Otherwise, uses `keybase fs read`.
+    uses the standard open() function. Otherwise, uses `keybase fs read`. This is because
+    fuse is not running in the container so a normal open call will not work for KBFS.
     :param filename:    The name of the file to read
     :return:            A list of lines in the file
     """
@@ -99,8 +100,8 @@ def simulate_two_teams(tc: TestConfig):
 
 @contextmanager
 def outputs_audit_log(tc: TestConfig, filename: str, expected_number: int):
-    # A context manager that asserts that the given function triggers expected_number of audit logs to be added to '/keybase/team/team.ssh.prod/ca.log'
-    # Note that fuse is not running in the container so this has to use `keybase fs read`
+    # A context manager that asserts that the given function triggers expected_number of audit logs to be added to the
+    # log at the given filename
 
     # Make a set of the lines in the audit log before we ran
     before_lines = set(read_file(filename))
diff --git a/tests/tests/test_env_1.py b/tests/tests/test_env_1.py
@@ -111,10 +111,10 @@ def test_kssh_clear_default_bot(self, test_config):
                 assert b"Found 2 config files" in e.output
 
     def test_keybaseca_backup(self, test_config):
-        # Test the keybaseca backup command by reading and verifying the private key stored in /mnt/cakey.backup
+        # Test the keybaseca backup command by reading and verifying the private key stored in /shared/cakey.backup
         run_command("mkdir -p /tmp/ssh/")
         run_command("chown -R keybase:keybase /tmp/ssh/")
-        with open('/mnt/cakey.backup') as f:
+        with open('/shared/cakey.backup') as f:
             keyLines = []
             add = False
             for line in f:
diff --git a/tests/tests/test_env_2_local_audit_log.py b/tests/tests/test_env_2_local_audit_log.py
@@ -13,5 +13,5 @@ def test_config(self):
 
     def test_kssh(self, test_config):
         # Test ksshing into staging as user
-        with outputs_audit_log(test_config, filename="/mnt/ca.log", expected_number=1):
+        with outputs_audit_log(test_config, filename="/shared/ca.log", expected_number=1):
             assert_contains_hash(test_config.expected_hash, run_command("""bin/kssh -q -o StrictHostKeyChecking=no user@sshd-staging "sha1sum /etc/unique" """))
diff --git a/tests/tests/test_env_3_user_not_in_first_team.py b/tests/tests/test_env_3_user_not_in_first_team.py
@@ -13,7 +13,7 @@ def test_config(self):
 
     def test_kssh(self, test_config):
         # Test ksshing which tests that it is correctly finding a client config
-        with outputs_audit_log(test_config, filename="/mnt/ca.log", expected_number=3):
+        with outputs_audit_log(test_config, filename="/shared/ca.log", expected_number=3):
             clear_keys()
             assert_contains_hash(test_config.expected_hash, run_command("""bin/kssh -q -o StrictHostKeyChecking=no user@sshd-staging "sha1sum /etc/unique" """))
             clear_keys()
diff --git a/tests/tests/test_env_4_user_not_in_configured_teams.py b/tests/tests/test_env_4_user_not_in_configured_teams.py
@@ -16,7 +16,7 @@ def test_config(self):
 
     def test_kssh_no_config_files(self, test_config):
         # Test that it can't find any config files
-        with outputs_audit_log(test_config, filename="/mnt/ca.log", expected_number=0):
+        with outputs_audit_log(test_config, filename="/shared/ca.log", expected_number=0):
             for s in ['user@sshd-staging', 'root@sshd-staging', 'user@sshd-prod', 'root@sshd-prod']:
                 try:
                     run_command(f"""bin/kssh -q -o StrictHostKeyChecking=no {s} "sha1sum /etc/unique" """)
@@ -27,7 +27,7 @@ def test_kssh_no_config_files(self, test_config):
     def test_kssh_spoofed_config(self, test_config):
         # Test that even when kssh is forced to run by a spoofed config, the CA bot ignores messages that are in the
         # wrong channel
-        with outputs_audit_log(test_config, filename="/mnt/ca.log", expected_number=0):
+        with outputs_audit_log(test_config, filename="/shared/ca.log", expected_number=0):
             client_config = json.dumps({'teamname': f"{test_config.subteam}.ssh", "channelname": "", "botname": test_config.bot_username})
             run_command(f"echo '{client_config}' | keybase fs write /keybase/team/{test_config.subteam}.ssh/kssh-client.config")
             for s in ['user@sshd-staging', 'root@sshd-staging', 'user@sshd-prod', 'root@sshd-prod']:
