From c3323f5b46a337790a8ccfd9b81d8823cb854b84 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 22 May 2023 13:00:43 +0800 Subject: [PATCH 001/343] Trigger test workflow of self-hosted-runner branch --- .github/workflows/build_charm.yaml | 4 ++-- .github/workflows/comment.yaml | 9 +++++---- .github/workflows/end_to_end_test.yaml | 2 +- .github/workflows/test.yaml | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml index 41122d32da..35235b8363 100644 --- a/.github/workflows/build_charm.yaml +++ b/.github/workflows/build_charm.yaml @@ -1,7 +1,7 @@ name: Build charm -on: - pull_request: +on: + # pull_request: workflow_call: workflow_dispatch: diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml index 26ac226df8..2e97b3b2f2 100644 --- a/.github/workflows/comment.yaml +++ b/.github/workflows/comment.yaml @@ -1,10 +1,11 @@ name: Comment on the pull request on: - workflow_run: - workflows: ["Tests"] - types: - - completed + workflow_call: + # workflow_run: + # workflows: ["Tests"] + # types: + # - completed jobs: comment-on-pr: diff --git a/.github/workflows/end_to_end_test.yaml b/.github/workflows/end_to_end_test.yaml index 98510a44bf..212741378b 100644 --- a/.github/workflows/end_to_end_test.yaml +++ b/.github/workflows/end_to_end_test.yaml @@ -1,7 +1,7 @@ name: End to end tests on: - pull_request: + # pull_request: workflow_call: workflow_dispatch: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee5fa3eeef..762d491bd8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,5 +5,5 @@ on: jobs: unit-tests: - uses: canonical/operator-workflows/.github/workflows/test.yaml@main + uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner secrets: inherit From f0513335f49f43996873932376db8b7bb77e0251 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 22 May 2023 13:03:02 +0800 Subject: [PATCH 002/343] Trigger test workflow with self-hosted runner --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 762d491bd8..7cc5ed1226 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,4 +6,6 @@ on: jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner + with: + self-hosted-runner: true secrets: inherit From d52289fa5914713af74ab8f8cc37875e93cca8bc Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 25 May 2023 10:09:41 +0800 Subject: [PATCH 003/343] Trigger workflow run --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7cc5ed1226..e66acdc786 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,6 +2,7 @@ name: Tests on: pull_request: + #workflow_call: jobs: unit-tests: From bace817e54462824bbac64671ee3555720c68532 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 25 May 2023 10:54:44 +0800 Subject: [PATCH 004/343] Trigger workflow --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e66acdc786..7cc5ed1226 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,7 +2,6 @@ name: Tests on: pull_request: - #workflow_call: jobs: unit-tests: From 030de3e86c3ba772660bb8281fb52cb33085ae55 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 25 May 2023 15:26:01 +0800 Subject: [PATCH 005/343] Add hello world test --- .github/workflows/check_test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/check_test.yaml diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml new file mode 100644 index 0000000000..a1aadd2530 --- /dev/null +++ b/.github/workflows/check_test.yaml @@ -0,0 +1,10 @@ +name: hello-world + +on: + pull_request: + +jobs: + hello-world: + runs-on: [self-hosted, test-runner] + steps: + - run: echo "hello world" From f0fae10ca742424728c66f577ddf5d36eb5a0d6c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 29 May 2023 14:16:15 +0800 Subject: [PATCH 006/343] Trigger workflow --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7cc5ed1226..d85717b85d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,6 +2,7 @@ name: Tests on: pull_request: + # Trigger jobs: unit-tests: From d5170d0132ed7b74f8b3c33a9bdf30cbe4c43a6d Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 13 Jun 2023 08:51:04 +0800 Subject: [PATCH 007/343] Trigger workflow --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d85717b85d..7cc5ed1226 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,7 +2,6 @@ name: Tests on: pull_request: - # Trigger jobs: unit-tests: From 4822b9cbdbc0482ab9b5e29529bf65715c87778c Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 16 Jun 2023 10:20:54 +0800 Subject: [PATCH 008/343] Add simple test --- .github/workflows/hello_world.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/hello_world.yaml diff --git a/.github/workflows/hello_world.yaml b/.github/workflows/hello_world.yaml new file mode 100644 index 0000000000..36250da2b3 --- /dev/null +++ b/.github/workflows/hello_world.yaml @@ -0,0 +1,10 @@ +name: hello-world + +on: + pull_request: + +jobs: + hello-world: + runs-on: [self-hosted, small] + steps: + - run: echo "hello world" \ No newline at end of file From 400f84ee41bf832109c312012dea4094531d6c50 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:13:27 +0800 Subject: [PATCH 009/343] Trigger tests --- .github/workflows/hello_world.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 .github/workflows/hello_world.yaml diff --git a/.github/workflows/hello_world.yaml b/.github/workflows/hello_world.yaml deleted file mode 100644 index 36250da2b3..0000000000 --- a/.github/workflows/hello_world.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: hello-world - -on: - pull_request: - -jobs: - hello-world: - runs-on: [self-hosted, small] - steps: - - run: echo "hello world" \ No newline at end of file From 2c78ba9c2126d62c159c104b37cbfd31eb7f81f9 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:15:02 +0800 Subject: [PATCH 010/343] Remove conflicting files --- .github/workflows/build_charm.yaml | 32 -------------- .github/workflows/end_to_end_test.yaml | 61 -------------------------- 2 files changed, 93 deletions(-) delete mode 100644 .github/workflows/build_charm.yaml delete mode 100644 .github/workflows/end_to_end_test.yaml diff --git a/.github/workflows/build_charm.yaml b/.github/workflows/build_charm.yaml deleted file mode 100644 index 35235b8363..0000000000 --- a/.github/workflows/build_charm.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Build charm - -on: - # pull_request: - workflow_call: - workflow_dispatch: - -jobs: - get-runner-image: - name: Get runner image - uses: canonical/operator-workflows/.github/workflows/get_runner_image.yaml@main - build-charm: - name: Build the charm - needs: get-runner-image - runs-on: ${{ needs.get-runner-image.outputs.runs-on }} - steps: - - uses: actions/checkout@v3 - - name: Enable network in LXD - run: sudo iptables -I DOCKER-USER -j ACCEPT - - name: Add user to lxd group - run: sudo adduser runner lxd - - name: Initialize LXD - run: sudo -u runner lxd init --auto - - name: Install charmcraft - run: sudo snap install charmcraft --classic - - name: Pack charm - run: sudo -u runner charmcraft pack - - uses: actions/upload-artifact@v3 - with: - name: github-runner-charm - path: github-runner_*.charm - retention-days: 5 diff --git a/.github/workflows/end_to_end_test.yaml b/.github/workflows/end_to_end_test.yaml deleted file mode 100644 index 212741378b..0000000000 --- a/.github/workflows/end_to_end_test.yaml +++ /dev/null @@ -1,61 +0,0 @@ -name: End to end tests - -on: - # pull_request: - workflow_call: - workflow_dispatch: - -jobs: - e2e-test: - name: end to end test - runs-on: [self-hosted, linux, x64, e2e-runner] - steps: - - name: Echo hello world - run: echo "hello world" - - name: File permission for /usr/local/bin - run: ls -ld /usr/local/bin | grep drwxrwxrwx - - name: Test file permission for /usr/local/bin - run: touch /usr/local/bin/test_file - # "Install microk8s" step will test if the proxies settings are correct. - - name: Proxy set in /etc/environment - run: cat /etc/environment | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker default proxy settings due to - # pulling the python image. - - name: Proxy set in docker daemon - run: sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker client default proxy settings. - - name: Proxy set in docker client - run: cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - run: sudo microk8s status --wait-ready - - name: Deploy nginx for testing - run: sudo microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt update - dep-test: - # Test the dependencies in one job to avoid using too many runners at once. - name: dependency test - needs: e2e-test - runs-on: [self-hosted, linux, x64, e2e-runner] - steps: - - name: Docker version - run: docker version - - name: pip version - run: python3 -m pip --version - - name: npm version - run: npm --version - - name: shellcheck version - run: shellcheck --version - - name: jq version - run: jq --version - - name: yq version - run: yq --version - - name: install check-jsonschema - run: python3 -m pip install check-jsonschema - # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. - - name: test check-jsonschema - run: check-jsonschema --version From bc364f037653f4fc2551fac6f2e9f1b66c600f97 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 27 Jun 2023 10:18:00 +0800 Subject: [PATCH 011/343] Add manual test --- .github/workflows/manual_test.yaml | 61 ++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .github/workflows/manual_test.yaml diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml new file mode 100644 index 0000000000..049de55d21 --- /dev/null +++ b/.github/workflows/manual_test.yaml @@ -0,0 +1,61 @@ +name: Manual test + +on: + pull_request: + workflow_call: + workflow_dispatch: + +jobs: + e2e-test: + name: end to end test + runs-on: [self-hosted, linux, x64, test-runner] + steps: + - name: Echo hello world + run: echo "hello world" + - name: File permission for /usr/local/bin + run: ls -ld /usr/local/bin | grep drwxrwxrwx + - name: Test file permission for /usr/local/bin + run: touch /usr/local/bin/test_file + # "Install microk8s" step will test if the proxies settings are correct. + - name: Proxy set in /etc/environment + run: cat /etc/environment | grep HTTP_PROXY + # "Update apt in python docker container" step will test docker default proxy settings due to + # pulling the python image. + - name: Proxy set in docker daemon + run: sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY + # "Update apt in python docker container" step will test docker client default proxy settings. + - name: Proxy set in docker client + run: cat /home/ubuntu/.docker/config.json | grep httpProxy + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + run: sudo microk8s status --wait-ready + - name: Deploy nginx for testing + run: sudo microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: Update apt in python docker container + run: docker run python:3.10-slim apt update + dep-test: + # Test the dependencies in one job to avoid using too many runners at once. + name: dependency test + needs: e2e-test + runs-on: [self-hosted, linux, x64, test-runner] + steps: + - name: Docker version + run: docker version + - name: pip version + run: python3 -m pip --version + - name: npm version + run: npm --version + - name: shellcheck version + run: shellcheck --version + - name: jq version + run: jq --version + - name: yq version + run: yq --version + - name: install check-jsonschema + run: python3 -m pip install check-jsonschema + # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. + - name: test check-jsonschema + run: check-jsonschema --version From daa8f9bb76730e889295af309df593852fcafafa Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 27 Jun 2023 11:00:44 +0800 Subject: [PATCH 012/343] Trigger manual test on runner --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 049de55d21..0cdbc5bf81 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: e2e-test: name: end to end test - runs-on: [self-hosted, linux, x64, test-runner] + runs-on: [self-hosted, linux, x64, runner] steps: - name: Echo hello world run: echo "hello world" @@ -40,7 +40,7 @@ jobs: # Test the dependencies in one job to avoid using too many runners at once. name: dependency test needs: e2e-test - runs-on: [self-hosted, linux, x64, test-runner] + runs-on: [self-hosted, linux, x64, runner] steps: - name: Docker version run: docker version From 89c92531df3d63792503823aa862f8cfa80e5e3e Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 27 Jun 2023 11:09:42 +0800 Subject: [PATCH 013/343] Change name of runner to test --- .github/workflows/check_test.yaml | 2 +- .github/workflows/manual_test.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index a1aadd2530..221ce31488 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -5,6 +5,6 @@ on: jobs: hello-world: - runs-on: [self-hosted, test-runner] + runs-on: [self-hosted, manual-runner] steps: - run: echo "hello world" diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0cdbc5bf81..c8532bb73b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: e2e-test: name: end to end test - runs-on: [self-hosted, linux, x64, runner] + runs-on: [self-hosted, linux, x64, manual-runner] steps: - name: Echo hello world run: echo "hello world" @@ -40,7 +40,7 @@ jobs: # Test the dependencies in one job to avoid using too many runners at once. name: dependency test needs: e2e-test - runs-on: [self-hosted, linux, x64, runner] + runs-on: [self-hosted, linux, x64, manual-runner] steps: - name: Docker version run: docker version From 0232992005e553c92e803750211ea483491dff79 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:26:36 +0800 Subject: [PATCH 014/343] Add disk io test --- .github/workflows/check_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index 221ce31488..791d740f1f 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -4,7 +4,7 @@ on: pull_request: jobs: - hello-world: + disk-io: runs-on: [self-hosted, manual-runner] steps: - - run: echo "hello world" + - run: touch test && dd if=/dev/zero of=test bs=1M count=8192 From 1643b452e48a1ff847d315e536b6a3079c42d8f0 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:07:15 +0800 Subject: [PATCH 015/343] Trigger self-hosted runner on operator workflow test --- .github/workflows/check_test.yaml | 4 ++-- .github/workflows/manual_test.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index 791d740f1f..fad637765d 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -1,10 +1,10 @@ name: hello-world on: - pull_request: + # pull_request: jobs: disk-io: runs-on: [self-hosted, manual-runner] steps: - - run: touch test && dd if=/dev/zero of=test bs=1M count=8192 + - run: touch test && dd if=/dev/zero of=test bs=1M count=8192 || echo "Failed successfully" diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c8532bb73b..db9e319127 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,7 +1,7 @@ name: Manual test on: - pull_request: + # pull_request: workflow_call: workflow_dispatch: From a9edf53f34ab6d81f0650d1ad3c1654980e7d63d Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:09:24 +0800 Subject: [PATCH 016/343] Fix missing action trigger --- .github/workflows/check_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index fad637765d..e620249b6a 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -2,6 +2,7 @@ name: hello-world on: # pull_request: + workflow_dispatch: jobs: disk-io: From bb204aaacfe18eb401a2b9741e78029a327f24c2 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:22:17 +0800 Subject: [PATCH 017/343] Trigger workflow --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7cc5ed1226..05783c0135 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,6 +3,7 @@ name: Tests on: pull_request: + jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner From df19f0734da6d343ba1f3a88dac65abb6f5c4be8 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:31:33 +0800 Subject: [PATCH 018/343] Trigger the workflow --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 05783c0135..7cc5ed1226 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,6 @@ name: Tests on: pull_request: - jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner From e14f97e5246ef593f90e568dbf12983dcc859434 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:47:32 +0800 Subject: [PATCH 019/343] Trigger disk io test --- .github/workflows/check_test.yaml | 2 +- .github/workflows/test.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index e620249b6a..5eeeeac580 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -1,7 +1,7 @@ name: hello-world on: - # pull_request: + pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7cc5ed1226..7eff4a7307 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,8 @@ name: Tests on: - pull_request: + # pull_request: + workflow_dispatch: jobs: unit-tests: From ba79f3b26306fe5e51ae8061fd45ca00048ffd4a Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:54:15 +0800 Subject: [PATCH 020/343] Rename the disk io test --- .github/workflows/check_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index 5eeeeac580..a31cf11c4d 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -1,8 +1,7 @@ -name: hello-world +name: disk io test on: pull_request: - workflow_dispatch: jobs: disk-io: From a2133b62e5852247fcdb7428a29189d2ddd6699f Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:54:52 +0800 Subject: [PATCH 021/343] Trigger workflow --- .github/workflows/check_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml index a31cf11c4d..fad9dd0abe 100644 --- a/.github/workflows/check_test.yaml +++ b/.github/workflows/check_test.yaml @@ -2,6 +2,7 @@ name: disk io test on: pull_request: + workflow_call: jobs: disk-io: From 836d2f68d68bf2e5aadf57f5009ad99680e55bfd Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 12:10:18 +0800 Subject: [PATCH 022/343] Revert "Trigger the workflow" This reverts commit df19f0734da6d343ba1f3a88dac65abb6f5c4be8. --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7eff4a7307..e3c425f676 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -4,6 +4,7 @@ on: # pull_request: workflow_dispatch: + jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner From 0a2112473e094b90fb70a630c1026b4f9c1168f9 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:05:18 +0800 Subject: [PATCH 023/343] Remove unused workflow. --- .github/workflows/check_test.yaml | 11 ----------- .github/workflows/comment.yaml | 13 ------------- .github/workflows/promote_charm.yaml | 26 -------------------------- .github/workflows/test.yaml | 3 +-- 4 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 .github/workflows/check_test.yaml delete mode 100644 .github/workflows/comment.yaml delete mode 100644 .github/workflows/promote_charm.yaml diff --git a/.github/workflows/check_test.yaml b/.github/workflows/check_test.yaml deleted file mode 100644 index fad9dd0abe..0000000000 --- a/.github/workflows/check_test.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: disk io test - -on: - pull_request: - workflow_call: - -jobs: - disk-io: - runs-on: [self-hosted, manual-runner] - steps: - - run: touch test && dd if=/dev/zero of=test bs=1M count=8192 || echo "Failed successfully" diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml deleted file mode 100644 index 2e97b3b2f2..0000000000 --- a/.github/workflows/comment.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Comment on the pull request - -on: - workflow_call: - # workflow_run: - # workflows: ["Tests"] - # types: - # - completed - -jobs: - comment-on-pr: - uses: canonical/operator-workflows/.github/workflows/comment.yaml@main - secrets: inherit diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml deleted file mode 100644 index 390b82d4e1..0000000000 --- a/.github/workflows/promote_charm.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Promote charm - -on: - workflow_dispatch: - inputs: - origin-channel: - type: choice - description: 'Origin Channel' - options: - - latest/edge - destination-channel: - type: choice - description: 'Destination Channel' - options: - - latest/stable - secrets: - CHARMHUB_TOKEN: - required: true - -jobs: - promote-charm: - uses: canonical/operator-workflows/.github/workflows/promote_charm.yaml@promote-charm-base - with: - origin-channel: ${{ github.event.inputs.origin-channel }} - destination-channel: ${{ github.event.inputs.destination-channel }} - secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e3c425f676..05783c0135 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,8 +1,7 @@ name: Tests on: - # pull_request: - workflow_dispatch: + pull_request: jobs: From c1500aa18601d6392ff031a6c5bf5a4da2d94cd4 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:42:06 +0800 Subject: [PATCH 024/343] Trigger workflow --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 05783c0135..7cc5ed1226 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,6 @@ name: Tests on: pull_request: - jobs: unit-tests: uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner From d8598272d60a539de0da1712969a8884a0340083 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 7 Jul 2023 14:50:48 +0800 Subject: [PATCH 025/343] Run hello world on small runner --- .github/workflows/e2e_test.yaml | 204 ----------------------------- .github/workflows/manual_test.yaml | 55 +------- .github/workflows/test.yaml | 13 -- 3 files changed, 4 insertions(+), 268 deletions(-) delete mode 100644 .github/workflows/e2e_test.yaml delete mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml deleted file mode 100644 index 061b010f63..0000000000 --- a/.github/workflows/e2e_test.yaml +++ /dev/null @@ -1,204 +0,0 @@ -name: End-to-End Test - -on: - pull_request: - -jobs: - build-charm: - name: Build Charm - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Remove Unnecessary Components - run: | - rm -rf .git - rm -rf .github - - - name: Write lxd-profile.yaml - run: | - cat << EOF > ./lxd-profile.yaml - config: - security.nesting: true - security.privileged: true - raw.lxc: | - lxc.apparmor.profile=unconfined - lxc.mount.auto=proc:rw sys:rw cgroup:rw - lxc.cgroup.devices.allow=a - lxc.cap.drop= - devices: - kmsg: - path: /dev/kmsg - source: /dev/kmsg - type: unix-char - EOF - - - name: Cache github-runner Charm - uses: actions/cache@v3 - id: cache-charm - with: - path: github-runner_ubuntu-22.04-amd64.charm - key: github-runner-charm-${{ hashFiles('**/*') }} - - - name: Setup LXD - if: steps.cache-charm.outputs.cache-hit != 'true' - uses: canonical/setup-lxd@main - - - name: Install charmcraft - if: steps.cache-charm.outputs.cache-hit != 'true' - run: sudo snap install charmcraft --classic - - - name: Pack github-runner Charm - if: steps.cache-charm.outputs.cache-hit != 'true' - run: charmcraft pack || ( cat ~/.local/state/charmcraft/log/* && exit 1 ) - - - name: Upload github-runner Charm - uses: actions/upload-artifact@v3 - with: - name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm - path: github-runner_ubuntu-22.04-amd64.charm - - run-id: - name: Generate Run ID - runs-on: ubuntu-latest - outputs: - run-id: ${{ steps.run-id.outputs.run-id }} - steps: - - name: Generate Run ID - id: run-id - run: | - echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c8)" >> $GITHUB_OUTPUT - - deploy-e2e-test-runner: - name: Deploy End-to-End Test Runner - runs-on: ubuntu-latest - needs: [ build-charm, run-id ] - steps: - - name: Setup Lxd Juju Controller - uses: charmed-kubernetes/actions-operator@main - with: - juju-channel: 3.1/stable - provider: lxd - - - name: Install GitHub Cli - run: which gh || sudo apt install gh -y - - - name: Create Testing Juju Model - run: juju add-model testing - - - name: Set Testing Model Proxy Configuration - run: | - juju model-config juju-http-proxy=$http_proxy - juju model-config juju-https-proxy=$https_proxy - juju model-config juju-no-proxy=$no_proxy - - - name: Change Testing Model Logging Level - run: juju model-config logging-config="=INFO;unit=DEBUG" - - - name: Download github-runner Charm - uses: actions/download-artifact@v3 - with: - name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm - - - name: Enable br_netfilter - run: sudo modprobe br_netfilter - - - name: Deploy github-runner Charm - run: | - cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm - juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ - ${{ needs.run-id.outputs.run-id }} \ - --base ubuntu@22.04 \ - --config path=${{ secrets.E2E_TESTING_REPO }} \ - --config token=${{ secrets.E2E_TESTING_TOKEN }} \ - --config virtual-machines=1 \ - --config denylist=10.0.0.0/8 \ - --config test-mode=insecure - - - name: Watch github-runner - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 30 - run: | - juju debug-log --replay --tail & - - while :; do - JOBS=$(gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/${{ secrets.E2E_TESTING_REPO }}/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs) - CONCLUSION=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .conclusion') - STATUS=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .status') - if [[ $STATUS != "queued" && $STATUS != "in_progress" ]]; then - break - fi - sleep 10 - done - if [[ $STATUS != "completed" || $CONCLUSION != "success" ]]; then - echo "test workflow failed with status: $STATUS, conclusion: $CONCLUSION" - kill $(jobs -p) - exit 1 - fi - - - name: Show Firewall Rules - run: | - juju ssh ${{ needs.run-id.outputs.run-id }}/0 sudo nft list ruleset - - e2e-test: - name: End-to-End Test - needs: [ build-charm, run-id ] - runs-on: [ self-hosted, linux, x64, "${{ needs.run-id.outputs.run-id }}" ] - steps: - # below is a series of simple tests to assess the functionality of the newly spawned runner. - - name: Echo hello world - run: echo "hello world" - - name: File permission for /usr/local/bin - run: ls -ld /usr/local/bin | grep drwxrwxrwx - - name: Test file permission for /usr/local/bin - run: touch /usr/local/bin/test_file - # "Install microk8s" step will test if the proxies settings are correct. - - name: Proxy set in /etc/environment - run: cat /etc/environment - # "Update apt in python docker container" step will test docker default proxy settings due to - # pulling the python image. - - name: Proxy set in docker daemon - run: | - [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ - || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker client default proxy settings. - - name: Proxy set in docker client - run: | - [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ - || cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: sudo microk8s status --wait-ready - - name: Deploy nginx for testing - run: sudo microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt update - - name: Docker version - run: docker version - - name: pip version - run: python3 -m pip --version - - name: npm version - run: npm --version - - name: shellcheck version - run: shellcheck --version - - name: jq version - run: jq --version - - name: yq version - run: yq --version - - name: install check-jsonschema - run: python3 -m pip install check-jsonschema - # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. - - name: test check-jsonschema - run: check-jsonschema --version - - name: Test Firewall - run: | - HOST_IP=$(ip route | grep default | cut -f 3 -d" ") - [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ] diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index db9e319127..79be4c15c2 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,61 +1,14 @@ name: Manual test on: - # pull_request: + pull_request: workflow_call: workflow_dispatch: jobs: - e2e-test: - name: end to end test - runs-on: [self-hosted, linux, x64, manual-runner] + test: + name: test + runs-on: [self-hosted, small] steps: - name: Echo hello world run: echo "hello world" - - name: File permission for /usr/local/bin - run: ls -ld /usr/local/bin | grep drwxrwxrwx - - name: Test file permission for /usr/local/bin - run: touch /usr/local/bin/test_file - # "Install microk8s" step will test if the proxies settings are correct. - - name: Proxy set in /etc/environment - run: cat /etc/environment | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker default proxy settings due to - # pulling the python image. - - name: Proxy set in docker daemon - run: sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker client default proxy settings. - - name: Proxy set in docker client - run: cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - run: sudo microk8s status --wait-ready - - name: Deploy nginx for testing - run: sudo microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt update - dep-test: - # Test the dependencies in one job to avoid using too many runners at once. - name: dependency test - needs: e2e-test - runs-on: [self-hosted, linux, x64, manual-runner] - steps: - - name: Docker version - run: docker version - - name: pip version - run: python3 -m pip --version - - name: npm version - run: npm --version - - name: shellcheck version - run: shellcheck --version - - name: jq version - run: jq --version - - name: yq version - run: yq --version - - name: install check-jsonschema - run: python3 -m pip install check-jsonschema - # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. - - name: test check-jsonschema - run: check-jsonschema --version diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index 294e33b339..0000000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Tests - -on: - pull_request: {} - -jobs: - unit-tests: - uses: canonical/operator-workflows/.github/workflows/test.yaml@self-hosted-runner - with: - self-hosted-runner: true - secrets: inherit - with: - self-hosted-runner: false From a32ff04173cd497a8f03ba98a3dd55004615f801 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 10 Jul 2023 11:21:15 +0800 Subject: [PATCH 026/343] Test micork8s usage --- .github/workflows/manual_test.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 79be4c15c2..c4e9c7c01a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,16 @@ on: jobs: test: name: test - runs-on: [self-hosted, small] + runs-on: [self-hosted, manual-runner] steps: - name: Echo hello world run: echo "hello world" + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m From 2405500cf961f0d35d3f2d5addac02a93df3c5f4 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 10 Jul 2023 17:30:20 +0800 Subject: [PATCH 027/343] Trigger jobs on kernel runners --- .github/workflows/manual_test.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c4e9c7c01a..2d37c8d63c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,11 @@ on: jobs: test: name: test - runs-on: [self-hosted, manual-runner] + runs-on: [self-hosted, kernel] + strategy: + matrix: + runs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + fail-fast: false steps: - name: Echo hello world run: echo "hello world" From 6ccb8d397ff17ae47b5efeb72e1bc7b1114f3813 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:21:26 +0800 Subject: [PATCH 028/343] Fix matrix test --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 2d37c8d63c..0a9c6d141c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -10,12 +10,12 @@ jobs: name: test runs-on: [self-hosted, kernel] strategy: + fail-fast: false matrix: runs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - fail-fast: false steps: - - name: Echo hello world - run: echo "hello world" + - name: Echo number + run: echo ${{matrix.runs}} - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From c7d4b87cd6c64c605f245bd7cd90fe494f7e3ac4 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:24:39 +0800 Subject: [PATCH 029/343] Debug workflow --- .github/workflows/manual_test.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0a9c6d141c..32f4a74925 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,8 +2,6 @@ name: Manual test on: pull_request: - workflow_call: - workflow_dispatch: jobs: test: @@ -12,10 +10,10 @@ jobs: strategy: fail-fast: false matrix: - runs: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - name: Echo number - run: echo ${{matrix.runs}} + run: echo "${{ matrix.num }}" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From f6af2e6cba50307254649edc6404d1846b940d18 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:25:15 +0800 Subject: [PATCH 030/343] debug workflow --- .github/workflows/manual_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 32f4a74925..92b4409364 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,6 @@ jobs: name: test runs-on: [self-hosted, kernel] strategy: - fail-fast: false matrix: num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: From f57a1a7a143e0d397329e579626cb4c450dd011b Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:25:42 +0800 Subject: [PATCH 031/343] Debug workflow --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 92b4409364..d6db650b53 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,7 +6,7 @@ on: jobs: test: name: test - runs-on: [self-hosted, kernel] + runs-on: ["self-hosted", "kernel"] strategy: matrix: num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From 0115a4bd6c7420291fffaf9147512ef152eebb29 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:26:12 +0800 Subject: [PATCH 032/343] Debug workflow --- .github/workflows/manual_test.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d6db650b53..29199c6a53 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,12 +7,9 @@ jobs: test: name: test runs-on: ["self-hosted", "kernel"] - strategy: - matrix: - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - name: Echo number - run: echo "${{ matrix.num }}" + run: echo "1" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From e6d39c6c6a8a72aab6a5e56e8de66cf6ebb3658a Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:28:25 +0800 Subject: [PATCH 033/343] Debug workflow --- .github/workflows/{manual_test.yaml => matrix_test.yaml} | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) rename .github/workflows/{manual_test.yaml => matrix_test.yaml} (77%) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/matrix_test.yaml similarity index 77% rename from .github/workflows/manual_test.yaml rename to .github/workflows/matrix_test.yaml index 29199c6a53..59ea624319 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/matrix_test.yaml @@ -1,15 +1,18 @@ -name: Manual test +name: Matrix test on: pull_request: jobs: test: - name: test runs-on: ["self-hosted", "kernel"] + strategy: + fail-fast: false + matrix: + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - name: Echo number - run: echo "1" + run: echo "${{matrix.num}}" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From eba1cc568ac4c2a1f5ecfac74d2996cb9329f2ed Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:49:02 +0800 Subject: [PATCH 034/343] Debug workflow --- .../{matrix_test.yaml => manual_test.yaml} | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) rename .github/workflows/{matrix_test.yaml => manual_test.yaml} (67%) diff --git a/.github/workflows/matrix_test.yaml b/.github/workflows/manual_test.yaml similarity index 67% rename from .github/workflows/matrix_test.yaml rename to .github/workflows/manual_test.yaml index 59ea624319..ad971d5562 100644 --- a/.github/workflows/matrix_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,18 +1,17 @@ -name: Matrix test +name: Manual test on: pull_request: + workflow_call: + workflow_dispatch: jobs: test: - runs-on: ["self-hosted", "kernel"] - strategy: - fail-fast: false - matrix: - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + name: test + runs-on: [self-hosted, kernel] steps: - - name: Echo number - run: echo "${{matrix.num}}" + - name: Echo hello world + run: echo "hello world" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From d20ef6ec98a4275fd32f8f9133d6dbe7aab548a3 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 09:49:53 +0800 Subject: [PATCH 035/343] Debug workflow --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ad971d5562..76b61986a6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: test: name: test - runs-on: [self-hosted, kernel] + runs-on: [self-hosted, runner] steps: - name: Echo hello world run: echo "hello world" From ac172eec5cbf95154125df18f766406c7672c9b0 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:03:23 +0800 Subject: [PATCH 036/343] Debug workflow --- .github/workflows/e2e_test.yaml | 204 ++++++++++++++++++++++++++++++++ src/runner.py | 5 +- 2 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/e2e_test.yaml diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml new file mode 100644 index 0000000000..6425417ce2 --- /dev/null +++ b/.github/workflows/e2e_test.yaml @@ -0,0 +1,204 @@ +name: End-to-End Test + +on: + pull_request: + +jobs: + build-charm: + name: Build Charm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Remove Unnecessary Components + run: | + rm -rf .git + rm -rf .github + + - name: Write lxd-profile.yaml + run: | + cat << EOF > ./lxd-profile.yaml + config: + security.nesting: true + security.privileged: true + raw.lxc: | + lxc.apparmor.profile=unconfined + lxc.mount.auto=proc:rw sys:rw cgroup:rw + lxc.cgroup.devices.allow=a + lxc.cap.drop= + devices: + kmsg: + path: /dev/kmsg + source: /dev/kmsg + type: unix-char + EOF + + - name: Cache github-runner Charm + uses: actions/cache@v3 + id: cache-charm + with: + path: github-runner_ubuntu-22.04-amd64.charm + key: github-runner-charm-${{ hashFiles('**/*') }} + + - name: Setup LXD + if: steps.cache-charm.outputs.cache-hit != 'true' + uses: canonical/setup-lxd@main + + - name: Install charmcraft + if: steps.cache-charm.outputs.cache-hit != 'true' + run: sudo snap install charmcraft --classic + + - name: Pack github-runner Charm + if: steps.cache-charm.outputs.cache-hit != 'true' + run: charmcraft pack || ( cat ~/.local/state/charmcraft/log/* && exit 1 ) + + - name: Upload github-runner Charm + uses: actions/upload-artifact@v3 + with: + name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm + path: github-runner_ubuntu-22.04-amd64.charm + + run-id: + name: Generate Run ID + runs-on: ubuntu-latest + outputs: + run-id: ${{ steps.run-id.outputs.run-id }} + steps: + - name: Generate Run ID + id: run-id + run: | + echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c8)" >> $GITHUB_OUTPUT + + deploy-e2e-test-runner: + name: Deploy End-to-End Test Runner + runs-on: ubuntu-latest + needs: [build-charm, run-id] + steps: + - name: Setup Lxd Juju Controller + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd + + - name: Install GitHub Cli + run: which gh || sudo apt install gh -y + + - name: Create Testing Juju Model + run: juju add-model testing + + - name: Set Testing Model Proxy Configuration + run: | + juju model-config juju-http-proxy=$http_proxy + juju model-config juju-https-proxy=$https_proxy + juju model-config juju-no-proxy=$no_proxy + + - name: Change Testing Model Logging Level + run: juju model-config logging-config="=INFO;unit=DEBUG" + + - name: Download github-runner Charm + uses: actions/download-artifact@v3 + with: + name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm + + - name: Enable br_netfilter + run: sudo modprobe br_netfilter + + - name: Deploy github-runner Charm + run: | + cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm + juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ + ${{ needs.run-id.outputs.run-id }} \ + --base ubuntu@22.04 \ + --config path=${{ secrets.E2E_TESTING_REPO }} \ + --config token=${{ secrets.E2E_TESTING_TOKEN }} \ + --config virtual-machines=1 \ + --config denylist=10.0.0.0/8 \ + --config test-mode=insecure + + - name: Watch github-runner + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 30 + run: | + juju debug-log --replay --tail & + + while :; do + JOBS=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ secrets.E2E_TESTING_REPO }}/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs) + CONCLUSION=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .conclusion') + STATUS=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .status') + if [[ $STATUS != "queued" && $STATUS != "in_progress" ]]; then + break + fi + sleep 10 + done + if [[ $STATUS != "completed" || $CONCLUSION != "success" ]]; then + echo "test workflow failed with status: $STATUS, conclusion: $CONCLUSION" + kill $(jobs -p) + exit 1 + fi + + - name: Show Firewall Rules + run: | + juju ssh ${{ needs.run-id.outputs.run-id }}/0 sudo nft list ruleset + + e2e-test: + name: End-to-End Test + needs: [build-charm, run-id] + runs-on: [self-hosted, linux, x64, "${{ needs.run-id.outputs.run-id }}"] + steps: + # below is a series of simple tests to assess the functionality of the newly spawned runner. + - name: Echo hello world + run: echo "hello world" + - name: File permission for /usr/local/bin + run: ls -ld /usr/local/bin | grep drwxrwxrwx + - name: Test file permission for /usr/local/bin + run: touch /usr/local/bin/test_file + # "Install microk8s" step will test if the proxies settings are correct. + - name: Proxy set in /etc/environment + run: cat /etc/environment + # "Update apt in python docker container" step will test docker default proxy settings due to + # pulling the python image. + - name: Proxy set in docker daemon + run: | + [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ + || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY + # "Update apt in python docker container" step will test docker client default proxy settings. + - name: Proxy set in docker client + run: | + [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ + || cat /home/ubuntu/.docker/config.json | grep httpProxy + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: Update apt in python docker container + run: docker run python:3.10-slim apt update + - name: Docker version + run: docker version + - name: pip version + run: python3 -m pip --version + - name: npm version + run: npm --version + - name: shellcheck version + run: shellcheck --version + - name: jq version + run: jq --version + - name: yq version + run: yq --version + - name: install check-jsonschema + run: python3 -m pip install check-jsonschema + # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. + - name: test check-jsonschema + run: check-jsonschema --version + - name: Test Firewall + run: | + HOST_IP=$(ip route | grep default | cut -f 3 -d" ") + [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ] diff --git a/src/runner.py b/src/runner.py index 2815b1188b..0f5932add2 100644 --- a/src/runner.py +++ b/src/runner.py @@ -387,6 +387,10 @@ def _install_binary(self, binary: Path) -> None: # TEMP: Install common tools used in GitHub Actions. This will be removed once virtual # machines are created from custom images/GitHub runner image. + # Pre-create the microk8s group and add the user to it. + self.instance.execute(["/usr/sbin/groupadd", "microk8s"]) + self.instance.execute(["/usr/sbin/usermod", "-aG", "microk8s", "ubuntu"]) + self._apt_install(["docker.io", "npm", "python3-pip", "shellcheck", "jq", "wget"]) self._wget_install( [ @@ -399,7 +403,6 @@ def _install_binary(self, binary: Path) -> None: # Add the user to docker group. self.instance.execute(["/usr/sbin/usermod", "-aG", "docker", "ubuntu"]) - self.instance.execute(["/usr/bin/newgrp", "docker"]) # The LXD instance is meant to run untrusted workload. Hardcoding the tmp directory should # be fine. From 71ca55ab5b922d7b129c6e7e6491e09d827e7960 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:05:23 +0800 Subject: [PATCH 037/343] Debug workflow --- .github/workflows/e2e_test.yaml | 204 ----------------------------- .github/workflows/manual_test.yaml | 10 +- 2 files changed, 7 insertions(+), 207 deletions(-) delete mode 100644 .github/workflows/e2e_test.yaml diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml deleted file mode 100644 index 6425417ce2..0000000000 --- a/.github/workflows/e2e_test.yaml +++ /dev/null @@ -1,204 +0,0 @@ -name: End-to-End Test - -on: - pull_request: - -jobs: - build-charm: - name: Build Charm - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Remove Unnecessary Components - run: | - rm -rf .git - rm -rf .github - - - name: Write lxd-profile.yaml - run: | - cat << EOF > ./lxd-profile.yaml - config: - security.nesting: true - security.privileged: true - raw.lxc: | - lxc.apparmor.profile=unconfined - lxc.mount.auto=proc:rw sys:rw cgroup:rw - lxc.cgroup.devices.allow=a - lxc.cap.drop= - devices: - kmsg: - path: /dev/kmsg - source: /dev/kmsg - type: unix-char - EOF - - - name: Cache github-runner Charm - uses: actions/cache@v3 - id: cache-charm - with: - path: github-runner_ubuntu-22.04-amd64.charm - key: github-runner-charm-${{ hashFiles('**/*') }} - - - name: Setup LXD - if: steps.cache-charm.outputs.cache-hit != 'true' - uses: canonical/setup-lxd@main - - - name: Install charmcraft - if: steps.cache-charm.outputs.cache-hit != 'true' - run: sudo snap install charmcraft --classic - - - name: Pack github-runner Charm - if: steps.cache-charm.outputs.cache-hit != 'true' - run: charmcraft pack || ( cat ~/.local/state/charmcraft/log/* && exit 1 ) - - - name: Upload github-runner Charm - uses: actions/upload-artifact@v3 - with: - name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm - path: github-runner_ubuntu-22.04-amd64.charm - - run-id: - name: Generate Run ID - runs-on: ubuntu-latest - outputs: - run-id: ${{ steps.run-id.outputs.run-id }} - steps: - - name: Generate Run ID - id: run-id - run: | - echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c8)" >> $GITHUB_OUTPUT - - deploy-e2e-test-runner: - name: Deploy End-to-End Test Runner - runs-on: ubuntu-latest - needs: [build-charm, run-id] - steps: - - name: Setup Lxd Juju Controller - uses: charmed-kubernetes/actions-operator@main - with: - juju-channel: 3.1/stable - provider: lxd - - - name: Install GitHub Cli - run: which gh || sudo apt install gh -y - - - name: Create Testing Juju Model - run: juju add-model testing - - - name: Set Testing Model Proxy Configuration - run: | - juju model-config juju-http-proxy=$http_proxy - juju model-config juju-https-proxy=$https_proxy - juju model-config juju-no-proxy=$no_proxy - - - name: Change Testing Model Logging Level - run: juju model-config logging-config="=INFO;unit=DEBUG" - - - name: Download github-runner Charm - uses: actions/download-artifact@v3 - with: - name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm - - - name: Enable br_netfilter - run: sudo modprobe br_netfilter - - - name: Deploy github-runner Charm - run: | - cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm - juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ - ${{ needs.run-id.outputs.run-id }} \ - --base ubuntu@22.04 \ - --config path=${{ secrets.E2E_TESTING_REPO }} \ - --config token=${{ secrets.E2E_TESTING_TOKEN }} \ - --config virtual-machines=1 \ - --config denylist=10.0.0.0/8 \ - --config test-mode=insecure - - - name: Watch github-runner - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 30 - run: | - juju debug-log --replay --tail & - - while :; do - JOBS=$(gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/${{ secrets.E2E_TESTING_REPO }}/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs) - CONCLUSION=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .conclusion') - STATUS=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .status') - if [[ $STATUS != "queued" && $STATUS != "in_progress" ]]; then - break - fi - sleep 10 - done - if [[ $STATUS != "completed" || $CONCLUSION != "success" ]]; then - echo "test workflow failed with status: $STATUS, conclusion: $CONCLUSION" - kill $(jobs -p) - exit 1 - fi - - - name: Show Firewall Rules - run: | - juju ssh ${{ needs.run-id.outputs.run-id }}/0 sudo nft list ruleset - - e2e-test: - name: End-to-End Test - needs: [build-charm, run-id] - runs-on: [self-hosted, linux, x64, "${{ needs.run-id.outputs.run-id }}"] - steps: - # below is a series of simple tests to assess the functionality of the newly spawned runner. - - name: Echo hello world - run: echo "hello world" - - name: File permission for /usr/local/bin - run: ls -ld /usr/local/bin | grep drwxrwxrwx - - name: Test file permission for /usr/local/bin - run: touch /usr/local/bin/test_file - # "Install microk8s" step will test if the proxies settings are correct. - - name: Proxy set in /etc/environment - run: cat /etc/environment - # "Update apt in python docker container" step will test docker default proxy settings due to - # pulling the python image. - - name: Proxy set in docker daemon - run: | - [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ - || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker client default proxy settings. - - name: Proxy set in docker client - run: | - [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ - || cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt update - - name: Docker version - run: docker version - - name: pip version - run: python3 -m pip --version - - name: npm version - run: npm --version - - name: shellcheck version - run: shellcheck --version - - name: jq version - run: jq --version - - name: yq version - run: yq --version - - name: install check-jsonschema - run: python3 -m pip install check-jsonschema - # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. - - name: test check-jsonschema - run: check-jsonschema --version - - name: Test Firewall - run: | - HOST_IP=$(ip route | grep default | cut -f 3 -d" ") - [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ] diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 76b61986a6..952d46e6e3 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,10 +8,14 @@ on: jobs: test: name: test - runs-on: [self-hosted, runner] + strategy: + fail-fast: false + matrix: + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, kernel] steps: - - name: Echo hello world - run: echo "hello world" + - name: Echo number + run: echo "${{matrix.num}}"" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From 12badc8239f39905e67503d615ef5d617e144ce3 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:06:55 +0800 Subject: [PATCH 038/343] Debug workflow --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 952d46e6e3..19fcfc6203 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -20,8 +20,8 @@ jobs: run: sudo snap install microk8s --classic - name: Wait for microk8s timeout-minutes: 10 - run: microk8s status --wait-ready + run: sudo microk8s status --wait-ready - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx + run: sudo microk8s kubectl create deployment nginx --image=nginx - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m + run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m From 99659997adffe5e7a4db07c6ec67252bae044995 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:07:13 +0800 Subject: [PATCH 039/343] Debug workflow --- .github/workflows/e2e_test.yaml | 204 ++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 .github/workflows/e2e_test.yaml diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml new file mode 100644 index 0000000000..6425417ce2 --- /dev/null +++ b/.github/workflows/e2e_test.yaml @@ -0,0 +1,204 @@ +name: End-to-End Test + +on: + pull_request: + +jobs: + build-charm: + name: Build Charm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Remove Unnecessary Components + run: | + rm -rf .git + rm -rf .github + + - name: Write lxd-profile.yaml + run: | + cat << EOF > ./lxd-profile.yaml + config: + security.nesting: true + security.privileged: true + raw.lxc: | + lxc.apparmor.profile=unconfined + lxc.mount.auto=proc:rw sys:rw cgroup:rw + lxc.cgroup.devices.allow=a + lxc.cap.drop= + devices: + kmsg: + path: /dev/kmsg + source: /dev/kmsg + type: unix-char + EOF + + - name: Cache github-runner Charm + uses: actions/cache@v3 + id: cache-charm + with: + path: github-runner_ubuntu-22.04-amd64.charm + key: github-runner-charm-${{ hashFiles('**/*') }} + + - name: Setup LXD + if: steps.cache-charm.outputs.cache-hit != 'true' + uses: canonical/setup-lxd@main + + - name: Install charmcraft + if: steps.cache-charm.outputs.cache-hit != 'true' + run: sudo snap install charmcraft --classic + + - name: Pack github-runner Charm + if: steps.cache-charm.outputs.cache-hit != 'true' + run: charmcraft pack || ( cat ~/.local/state/charmcraft/log/* && exit 1 ) + + - name: Upload github-runner Charm + uses: actions/upload-artifact@v3 + with: + name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm + path: github-runner_ubuntu-22.04-amd64.charm + + run-id: + name: Generate Run ID + runs-on: ubuntu-latest + outputs: + run-id: ${{ steps.run-id.outputs.run-id }} + steps: + - name: Generate Run ID + id: run-id + run: | + echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c8)" >> $GITHUB_OUTPUT + + deploy-e2e-test-runner: + name: Deploy End-to-End Test Runner + runs-on: ubuntu-latest + needs: [build-charm, run-id] + steps: + - name: Setup Lxd Juju Controller + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd + + - name: Install GitHub Cli + run: which gh || sudo apt install gh -y + + - name: Create Testing Juju Model + run: juju add-model testing + + - name: Set Testing Model Proxy Configuration + run: | + juju model-config juju-http-proxy=$http_proxy + juju model-config juju-https-proxy=$https_proxy + juju model-config juju-no-proxy=$no_proxy + + - name: Change Testing Model Logging Level + run: juju model-config logging-config="=INFO;unit=DEBUG" + + - name: Download github-runner Charm + uses: actions/download-artifact@v3 + with: + name: dangerous-test-only-github-runner_ubuntu-22.04-amd64.charm + + - name: Enable br_netfilter + run: sudo modprobe br_netfilter + + - name: Deploy github-runner Charm + run: | + cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm + juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ + ${{ needs.run-id.outputs.run-id }} \ + --base ubuntu@22.04 \ + --config path=${{ secrets.E2E_TESTING_REPO }} \ + --config token=${{ secrets.E2E_TESTING_TOKEN }} \ + --config virtual-machines=1 \ + --config denylist=10.0.0.0/8 \ + --config test-mode=insecure + + - name: Watch github-runner + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 30 + run: | + juju debug-log --replay --tail & + + while :; do + JOBS=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ secrets.E2E_TESTING_REPO }}/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs) + CONCLUSION=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .conclusion') + STATUS=$(echo $JOBS | jq -r '.jobs[] | select(.name == "End-to-End Test") | .status') + if [[ $STATUS != "queued" && $STATUS != "in_progress" ]]; then + break + fi + sleep 10 + done + if [[ $STATUS != "completed" || $CONCLUSION != "success" ]]; then + echo "test workflow failed with status: $STATUS, conclusion: $CONCLUSION" + kill $(jobs -p) + exit 1 + fi + + - name: Show Firewall Rules + run: | + juju ssh ${{ needs.run-id.outputs.run-id }}/0 sudo nft list ruleset + + e2e-test: + name: End-to-End Test + needs: [build-charm, run-id] + runs-on: [self-hosted, linux, x64, "${{ needs.run-id.outputs.run-id }}"] + steps: + # below is a series of simple tests to assess the functionality of the newly spawned runner. + - name: Echo hello world + run: echo "hello world" + - name: File permission for /usr/local/bin + run: ls -ld /usr/local/bin | grep drwxrwxrwx + - name: Test file permission for /usr/local/bin + run: touch /usr/local/bin/test_file + # "Install microk8s" step will test if the proxies settings are correct. + - name: Proxy set in /etc/environment + run: cat /etc/environment + # "Update apt in python docker container" step will test docker default proxy settings due to + # pulling the python image. + - name: Proxy set in docker daemon + run: | + [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ + || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY + # "Update apt in python docker container" step will test docker client default proxy settings. + - name: Proxy set in docker client + run: | + [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ + || cat /home/ubuntu/.docker/config.json | grep httpProxy + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: Update apt in python docker container + run: docker run python:3.10-slim apt update + - name: Docker version + run: docker version + - name: pip version + run: python3 -m pip --version + - name: npm version + run: npm --version + - name: shellcheck version + run: shellcheck --version + - name: jq version + run: jq --version + - name: yq version + run: yq --version + - name: install check-jsonschema + run: python3 -m pip install check-jsonschema + # Test program installed by pip. The directory `~/.local/bin` need to be added to PATH. + - name: test check-jsonschema + run: check-jsonschema --version + - name: Test Firewall + run: | + HOST_IP=$(ip route | grep default | cut -f 3 -d" ") + [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ] From 37f47998052ba03e5798df3a5b71cbdf09cd1684 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 10:22:30 +0800 Subject: [PATCH 040/343] Debug workflow --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 19fcfc6203..908397e807 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,7 +15,7 @@ jobs: runs-on: [self-hosted, kernel] steps: - name: Echo number - run: echo "${{matrix.num}}"" + run: echo "${{matrix.num}}" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s From 74e3493a0d840c315a280ed4b4fb3381513bc549 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 15:40:44 +0800 Subject: [PATCH 041/343] Test prod runners --- .github/workflows/manual_test.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 908397e807..a91485fefe 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,19 @@ on: workflow_dispatch: jobs: + prod-test: + name: prod test + runs-on: [self-hosted, large] + steps: + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: sudo microk8s status --wait-ready + - name: Deploy nginx for testing + run: sudo microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m test: name: test strategy: From 5b583944c7317b45ab98b9cdaf236940750ee936 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Jul 2023 15:43:21 +0800 Subject: [PATCH 042/343] Remove sudo for microk8s commands --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a91485fefe..79aec8c8f6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,11 +14,11 @@ jobs: run: sudo snap install microk8s --classic - name: Wait for microk8s timeout-minutes: 10 - run: sudo microk8s status --wait-ready + run: microk8s status --wait-ready - name: Deploy nginx for testing - run: sudo microk8s kubectl create deployment nginx --image=nginx + run: microk8s kubectl create deployment nginx --image=nginx - name: Wait for nginx to be ready - run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m + run: microk8s kubectl rollout status deployment/nginx --timeout=30m test: name: test strategy: From 58ffdce3a0949b857b2f453adbefa7a10e1a1140 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 12 Jul 2023 21:17:53 +0800 Subject: [PATCH 043/343] Trigger test --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 79aec8c8f6..e23f226b95 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,9 +6,9 @@ on: workflow_dispatch: jobs: - prod-test: - name: prod test - runs-on: [self-hosted, large] + manual-test: + name: manual test + runs-on: [self-hosted, manual-runner] steps: - name: Install microk8s run: sudo snap install microk8s --classic From 9b5d62bd591c3ecfeac739e11465b903aac44cb6 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 14 Jul 2023 17:46:08 +0800 Subject: [PATCH 044/343] Trigger workflow --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e23f226b95..3d3716dd92 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: manual-test: - name: manual test + name: manual tests runs-on: [self-hosted, manual-runner] steps: - name: Install microk8s From e84e27ca9238329a3bd08fae6791d7efd124740c Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 18 Jul 2023 12:10:36 +0800 Subject: [PATCH 045/343] Update test for github-runner --- .github/workflows/manual_test.yaml | 56 ++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 3d3716dd92..b3cd96d396 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,9 +6,9 @@ on: workflow_dispatch: jobs: - manual-test: - name: manual tests - runs-on: [self-hosted, manual-runner] + basic-test-0: + name: basic tests 0 + runs-on: [self-hosted, runner] steps: - name: Install microk8s run: sudo snap install microk8s --classic @@ -19,22 +19,48 @@ jobs: run: microk8s kubectl create deployment nginx --image=nginx - name: Wait for nginx to be ready run: microk8s kubectl rollout status deployment/nginx --timeout=30m - test: - name: test - strategy: - fail-fast: false - matrix: - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, kernel] + basic-test-1: + name: basic tests 1 + runs-on: [self-hosted, runner] steps: - - name: Echo number - run: echo "${{matrix.num}}" - name: Install microk8s run: sudo snap install microk8s --classic - name: Wait for microk8s timeout-minutes: 10 - run: sudo microk8s status --wait-ready + run: microk8s status --wait-ready - name: Deploy nginx for testing - run: sudo microk8s kubectl create deployment nginx --image=nginx + run: microk8s kubectl create deployment nginx --image=nginx - name: Wait for nginx to be ready - run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + basic-test-2: + name: basic tests 2 + runs-on: [self-hosted, runner] + steps: + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + # test: + # name: test + # strategy: + # fail-fast: false + # matrix: + # num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # runs-on: [self-hosted, kernel] + # steps: + # - name: Echo number + # run: echo "${{matrix.num}}" + # - name: Install microk8s + # run: sudo snap install microk8s --classic + # - name: Wait for microk8s + # timeout-minutes: 10 + # run: sudo microk8s status --wait-ready + # - name: Deploy nginx for testing + # run: sudo microk8s kubectl create deployment nginx --image=nginx + # - name: Wait for nginx to be ready + # run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m From db97ee509a1f2f2a538f713d53e93695d4d3cc25 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 18 Jul 2023 14:07:04 +0800 Subject: [PATCH 046/343] Test workflow dispatch --- .github/workflows/promote_charm.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/promote_charm.yaml diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml new file mode 100644 index 0000000000..e56053fcf9 --- /dev/null +++ b/.github/workflows/promote_charm.yaml @@ -0,0 +1,24 @@ +name: Promote charm + +on: + workflow_dispatch: + inputs: + origin-channel: + type: choice + description: "Origin Channel" + options: + - latest/edge + destination-channel: + type: choice + description: "Destination Channel" + options: + - latest/stable + secrets: + CHARMHUB_TOKEN: + required: false + +jobs: + test-workflow-dispatch: + runs-on: [self-hosted, runner] + steps: + - run: echo "TEST PASS" From 43da7445de55e8b8757e445322f4f7012de79245 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 24 Jul 2023 13:19:35 +0800 Subject: [PATCH 047/343] Remove matrix test --- .github/workflows/manual_test.yaml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b3cd96d396..4d409bd5b6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -45,22 +45,3 @@ jobs: run: microk8s kubectl create deployment nginx --image=nginx - name: Wait for nginx to be ready run: microk8s kubectl rollout status deployment/nginx --timeout=30m - # test: - # name: test - # strategy: - # fail-fast: false - # matrix: - # num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # runs-on: [self-hosted, kernel] - # steps: - # - name: Echo number - # run: echo "${{matrix.num}}" - # - name: Install microk8s - # run: sudo snap install microk8s --classic - # - name: Wait for microk8s - # timeout-minutes: 10 - # run: sudo microk8s status --wait-ready - # - name: Deploy nginx for testing - # run: sudo microk8s kubectl create deployment nginx --image=nginx - # - name: Wait for nginx to be ready - # run: sudo microk8s kubectl rollout status deployment/nginx --timeout=30m From d0efc9516cb66677fc5a6cf0256db543ad398430 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:45:21 +0800 Subject: [PATCH 048/343] Test HTTP proxy --- .github/workflows/manual_test.yaml | 47 ++++++------------------------ 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4d409bd5b6..c769018654 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,42 +6,13 @@ on: workflow_dispatch: jobs: - basic-test-0: - name: basic tests 0 - runs-on: [self-hosted, runner] + test: + name: test + runs-on: [self-hosted, xlarge] steps: - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m - basic-test-1: - name: basic tests 1 - runs-on: [self-hosted, runner] - steps: - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m - basic-test-2: - name: basic tests 2 - runs-on: [self-hosted, runner] - steps: - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: curl + run: curl -vvv https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url + - name: Install requests + run: python3 -m pip install requests + - name: Make HTTP request with Python + run: python3 -c "import requests;requests.get('https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url')" From 295f64dda785cbbcde173986aae8f0196cd13af3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:48:08 +0800 Subject: [PATCH 049/343] Test HTTP proxy --- .github/workflows/manual_test.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c769018654..c40464a507 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,12 +2,10 @@ name: Manual test on: pull_request: - workflow_call: - workflow_dispatch: jobs: - test: - name: test + test-proxy: + name: test proxy runs-on: [self-hosted, xlarge] steps: - name: curl From 4e5cffc744d06e61fd6b55cb6cf816f5072937fd Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:50:46 +0800 Subject: [PATCH 050/343] Resolve conflict --- .github/workflows/comment.yaml | 13 ++ .github/workflows/e2e_test.yaml | 114 +++++++++++++- .github/workflows/integration_test.yaml | 26 ++++ .github/workflows/promote_charm.yaml | 20 +-- .github/workflows/test.yaml | 12 ++ .github/workflows/workflow_dispatch_test.yaml | 17 +++ pyproject.toml | 10 ++ requirements.txt | 2 +- src/charm.py | 52 ++++--- src/lxd.py | 7 +- src/runner.py | 10 +- src/runner_manager.py | 105 +++++++++---- src/runner_type.py | 8 + src/utilities.py | 16 +- templates/pre-job.j2 | 54 +++++-- templates/repo-policy-compliance.service.j2 | 2 +- templates/start.j2 | 2 +- tests/conftest.py | 17 +++ tests/integration/conftest.py | 144 ++++++++++++++++-- tests/integration/test_charm.py | 121 +++++++++++++-- tests/integration/test_helpers.py | 47 ++++++ tests/status_name.py | 8 + tests/unit/mock.py | 18 ++- tests/unit/test_runner_manager.py | 20 ++- 24 files changed, 731 insertions(+), 114 deletions(-) create mode 100644 .github/workflows/comment.yaml create mode 100644 .github/workflows/integration_test.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 .github/workflows/workflow_dispatch_test.yaml create mode 100644 tests/conftest.py create mode 100644 tests/integration/test_helpers.py create mode 100644 tests/status_name.py diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml new file mode 100644 index 0000000000..0c35e3bd06 --- /dev/null +++ b/.github/workflows/comment.yaml @@ -0,0 +1,13 @@ +name: Comment on the pull request + +on: + workflow_dispatch: + # workflow_run: + # workflows: ["Tests"] + # types: + # - completed + +jobs: + comment-on-pr: + uses: canonical/operator-workflows/.github/workflows/comment.yaml@main + secrets: inherit diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 6425417ce2..ff6af090e8 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -1,7 +1,8 @@ name: End-to-End Test on: - pull_request: + # pull_request: + workflow_dispatch: jobs: build-charm: @@ -67,12 +68,19 @@ jobs: - name: Generate Run ID id: run-id run: | - echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c8)" >> $GITHUB_OUTPUT + echo "run-id=e2e-$(LC_ALL=C tr -dc 'a-z' < /dev/urandom | head -c4)" >> $GITHUB_OUTPUT deploy-e2e-test-runner: - name: Deploy End-to-End Test Runner + name: Deploy End-to-End Test Runner (${{ matrix.event.name }}) runs-on: ubuntu-latest needs: [build-charm, run-id] + strategy: + matrix: + event: + - name: pull_request + abbreviation: pr + - name: workflow_dispatch + abbreviation: wd steps: - name: Setup Lxd Juju Controller uses: charmed-kubernetes/actions-operator@main @@ -103,11 +111,15 @@ jobs: - name: Enable br_netfilter run: sudo modprobe br_netfilter + - name: Generate Runner Name + id: runner-name + run: echo name=${{ matrix.event.abbreviation }}-${{ needs.run-id.outputs.run-id }} >> $GITHUB_OUTPUT + - name: Deploy github-runner Charm run: | cp github-runner_ubuntu-22.04-amd64.charm /home/$USER/github-runner_ubuntu-22.04-amd64.charm juju deploy /home/$USER/github-runner_ubuntu-22.04-amd64.charm \ - ${{ needs.run-id.outputs.run-id }} \ + ${{ steps.runner-name.outputs.name }} \ --base ubuntu@22.04 \ --config path=${{ secrets.E2E_TESTING_REPO }} \ --config token=${{ secrets.E2E_TESTING_TOKEN }} \ @@ -115,7 +127,8 @@ jobs: --config denylist=10.0.0.0/8 \ --config test-mode=insecure - - name: Watch github-runner + - name: Watch github-runner (Pull Request) + if: matrix.event.name == 'pull_request' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} timeout-minutes: 30 @@ -140,14 +153,101 @@ jobs: exit 1 fi + - name: Watch github-runner (Workflow Dispatch) + if: matrix.event.name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ secrets.E2E_TESTING_TOKEN }} + run: | + juju debug-log --replay --tail & + + # Base any future branches on the current branch + REF_SHA=$(gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ secrets.E2E_TESTING_REPO }}/git/ref/heads/$GITHUB_HEAD_REF \ + --jq .object.sha) + + # Create a temporary reference/branch + gh api \ + --method POST \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/${{ secrets.E2E_TESTING_REPO }}/git/refs \ + -f ref='refs/heads/${{ steps.runner-name.outputs.name }}' \ + -f sha=$REF_SHA + + # Dispatch other workflow + gh workflow run workflow_dispatch_test.yaml \ + -R ${{ secrets.E2E_TESTING_REPO }} \ + --ref ${{ steps.runner-name.outputs.name }} \ + -f runner=${{ steps.runner-name.outputs.name }} + + get-workflow-status() { + # Search recent workflow runs for the one designated by the run-id ref + output=$(gh run list \ + -R ${{ secrets.E2E_TESTING_REPO }} \ + -L 100 \ + --json headBranch,status \ + --jq '[.[] | select(.headBranch=="${{ steps.runner-name.outputs.name }}")]') + + # Workflows that have not started have no status + if [ $(echo "$output" | jq 'length') -eq 0 ] + then + echo "not_started" + else + # Parse output with jq to get the status field of the first object + status=$(echo "$output" | jq -r '.[0].status') + echo "$status" + fi + } + + # Wait for the workflow to start while checking its status + for i in {1..360} + do + status=$(get-workflow-status) + echo "workflow status: $status" + if [[ $status != "not_started" && $status != "queued" && $status != "in_progress" ]]; then + break + fi + sleep 10 + done + + # Make sure the workflow was completed or else consider it failed + conclusion=$(gh run list \ + -R ${{ secrets.E2E_TESTING_REPO }} \ + -L 100 \ + --json headBranch,conclusion \ + --jq '.[] | select(.headBranch=="${{ steps.runner-name.outputs.name }}") | .conclusion') + + if [[ $status != "completed" || $conclusion != "success" ]]; then + echo "test workflow failed with status: $status, conclusion: $conclusion" + kill $(jobs -p) + exit 1 + else + echo "Workflow completed with status: $status, conclusion: $conclusion, run-id: ${{ steps.runner-name.outputs.name }}" + kill $(jobs -p) + fi + - name: Show Firewall Rules run: | - juju ssh ${{ needs.run-id.outputs.run-id }}/0 sudo nft list ruleset + juju ssh ${{ steps.runner-name.outputs.name }}/0 sudo nft list ruleset + + - name: Clean Up + if: always() && matrix.event.name == 'workflow_dispatch' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "/repos/${{ secrets.E2E_TESTING_REPO }}/git/refs/heads/${{ steps.runner-name.outputs.name }}" + echo "Deleted ref ${{ steps.runner-name.outputs.name }}" e2e-test: name: End-to-End Test needs: [build-charm, run-id] - runs-on: [self-hosted, linux, x64, "${{ needs.run-id.outputs.run-id }}"] + runs-on: [self-hosted, linux, x64, "pr-${{ needs.run-id.outputs.run-id }}"] steps: # below is a series of simple tests to assess the functionality of the newly spawned runner. - name: Echo hello world diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml new file mode 100644 index 0000000000..685cf2dcd3 --- /dev/null +++ b/.github/workflows/integration_test.yaml @@ -0,0 +1,26 @@ +name: Integration test + +on: + # pull_request: + workflow_dispatch: + +jobs: + integration-test: + name: Integration test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd + - name: Enable br_netfilter + run: sudo modprobe br_netfilter + - name: Run Integration tests + run: | + tox -e integration -- \ + --keep-models \ + --path ${{secrets.E2E_TESTING_REPO}} \ + --token ${{secrets.E2E_TESTING_TOKEN}} \ + --token-alt ${{secrets.E2E_TESTING_TOKEN_ALT}} diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml index e56053fcf9..390b82d4e1 100644 --- a/.github/workflows/promote_charm.yaml +++ b/.github/workflows/promote_charm.yaml @@ -5,20 +5,22 @@ on: inputs: origin-channel: type: choice - description: "Origin Channel" + description: 'Origin Channel' options: - - latest/edge + - latest/edge destination-channel: type: choice - description: "Destination Channel" + description: 'Destination Channel' options: - - latest/stable + - latest/stable secrets: CHARMHUB_TOKEN: - required: false + required: true jobs: - test-workflow-dispatch: - runs-on: [self-hosted, runner] - steps: - - run: echo "TEST PASS" + promote-charm: + uses: canonical/operator-workflows/.github/workflows/promote_charm.yaml@promote-charm-base + with: + origin-channel: ${{ github.event.inputs.origin-channel }} + destination-channel: ${{ github.event.inputs.destination-channel }} + secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..4aa06569e7 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,12 @@ +name: Tests + +on: + # pull_request: + workflow_dispatch: + +jobs: + unit-tests: + uses: canonical/operator-workflows/.github/workflows/test.yaml@main + secrets: inherit + with: + self-hosted-runner: false diff --git a/.github/workflows/workflow_dispatch_test.yaml b/.github/workflows/workflow_dispatch_test.yaml new file mode 100644 index 0000000000..41db73bef6 --- /dev/null +++ b/.github/workflows/workflow_dispatch_test.yaml @@ -0,0 +1,17 @@ +name: Workflow Dispatch Tests + +on: + # Manually dispatched workflow action + workflow_dispatch: + inputs: + runner: + description: 'Self hosted gh runner' + required: true + +jobs: + workflow-dispatch-tests: + runs-on: [self-hosted, linux, x64, "${{ inputs.runner }}"] + steps: + - name: Echo input variable and message + run: | + echo "Hello, runner: ${{ inputs.runner }}" diff --git a/pyproject.toml b/pyproject.toml index c72f8d0ef2..41a5322ca0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,17 @@ max-line-length = 99 max-doc-length = 99 max-complexity = 10 exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] +select = ["E", "W", "F", "C", "N", "R", "D", "H"] +# Ignore W503, E501 because using black creates errors with this +# Ignore D107 Missing docstring in __init__ +ignore = ["W503", "E501", "D107"] +# D100, D101, D102, D103: Ignore missing docstrings in tests +per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] docstring-convention = "google" +# Check for properly formatted copyright header in each file +copyright-check = "True" +copyright-author = "Canonical Ltd." +copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s" [tool.mypy] ignore_missing_imports = true diff --git a/requirements.txt b/requirements.txt index 76bbb1da03..ede5e44b17 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ ghapi jinja2 ops -pylxd @ git+https://github.com/lxc/pylxd +pylxd @ git+https://github.com/canonical/pylxd requests typing-extensions # Newer version does not work with default OpenSSL version on jammy. diff --git a/src/charm.py b/src/charm.py index 58fa6f52f4..a17886b172 100755 --- a/src/charm.py +++ b/src/charm.py @@ -52,20 +52,17 @@ class UpdateRunnerBinEvent(EventBase): def catch_charm_errors(func: Callable[[CharmT, EventT], None]) -> Callable[[CharmT, EventT], None]: - """Catch unexpected errors in charm. - - This decorator is for unrecoverable errors and sets the charm to - `BlockedStatus`. + """Catch common errors in charm. Args: func: Charm function to be decorated. Returns: - Decorated charm function with catching unexpected errors. + Decorated charm function with catching common errors. """ @functools.wraps(func) - def func_with_catch_unexpected_errors(self, event: EventT) -> None: + def func_with_catch_errors(self, event: EventT) -> None: # Safe guard against unexpected error. try: func(self, event) @@ -75,19 +72,19 @@ def func_with_catch_unexpected_errors(self, event: EventT) -> None: f"Missing required charm configuration: {err.configs}" ) - return func_with_catch_unexpected_errors + return func_with_catch_errors def catch_action_errors( func: Callable[[CharmT, ActionEvent], None] ) -> Callable[[CharmT, ActionEvent], None]: - """Catch unexpected errors in actions. + """Catch common errors in actions. Args: func: Action function to be decorated. Returns: - Decorated charm function with catching unexpected errors. + Decorated charm function with catching common errors. """ @functools.wraps(func) @@ -130,6 +127,7 @@ def __init__(self, *args, **kargs) -> None: self._stored.set_default( path=self.config["path"], # for detecting changes + token=self.config["token"], # for detecting changes runner_bin_url=None, ) @@ -171,6 +169,9 @@ def _create_memory_storage(self, path: Path, size: int) -> None: Raises: RunnerError: Unable to setup storage for runner. """ + if size <= 0: + return + try: # Create tmpfs if not exists, else resize it. if not path.exists(): @@ -202,7 +203,7 @@ def _ensure_service_health(self) -> None: except SubprocessError: logger.exception("Found inactive repo-policy-compliance service") execute_command(["/usr/bin/systemctl", "restart", "repo-policy-compliance"]) - logger.exception("Restart repo-policy-compliance service") + logger.info("Restart repo-policy-compliance service") raise def _get_runner_manager( @@ -317,7 +318,7 @@ def _on_install(self, _event: InstallEvent) -> None: def _upgrade_kernel(self) -> None: """Upgrade the Linux kernel.""" execute_command(["/usr/bin/apt-get", "update"]) - execute_command(["/usr/bin/apt-get", "install", "-qy", "linux-generic-hwe-22.04"]) + execute_command(["/usr/bin/apt-get", "install", "-qy", "linux-generic"]) _, exit_code = execute_command(["ls", "/var/run/reboot-required"], check_exit=False) if exit_code == 0: @@ -351,6 +352,10 @@ def _on_config_changed(self, _event: ConfigChangedEvent) -> None: Args: event: Event of configuration change. """ + if self.config["token"] != self._stored.token: + self._start_services() + self._stored.token = None + self._refresh_firewall() try: self._event_timer.ensure_event_timer( @@ -371,15 +376,20 @@ def _on_config_changed(self, _event: ConfigChangedEvent) -> None: ) # Casting for mypy checks. if prev_runner_manager: self.unit.status = MaintenanceStatus("Removing runners from old org/repo") - prev_runner_manager.flush() + prev_runner_manager.flush(flush_busy=False) self._stored.path = self.config["path"] runner_manager = self._get_runner_manager() if runner_manager: + self._reconcile_runners(runner_manager) self.unit.status = ActiveStatus() else: self.unit.status = BlockedStatus("Missing token or org/repo path config") + if self.config["token"] != self._stored.token: + runner_manager.flush(flush_busy=False) + self._stored.token = self.config["token"] + @catch_charm_errors def _on_update_runner_bin(self, _event: UpdateRunnerBinEvent) -> None: """Handle checking update of runner binary event. @@ -576,7 +586,15 @@ def _install_deps(self) -> None: """Install dependencies.""" logger.info("Installing charm dependencies.") + # Snap and Apt will use any proxies configured in the Juju model. # Binding for snap, apt, and lxd init commands are not available so subprocess.run used. + execute_command(["/usr/bin/apt-get", "update"]) + # install dependencies used by repo-policy-compliance and the firewall + execute_command( + ["/usr/bin/apt-get", "install", "-qy", "gunicorn", "python3-pip", "nftables"] + ) + + # Prepare environment for pip subprocess env = {} if "http" in self.proxies: env["HTTP_PROXY"] = self.proxies["http"] @@ -588,15 +606,12 @@ def _install_deps(self) -> None: env["NO_PROXY"] = self.proxies["no_proxy"] env["no_proxy"] = self.proxies["no_proxy"] - execute_command(["/usr/bin/apt-get", "update"]) - # install dependencies used by repo-policy-compliance and the firewall - execute_command( - ["/usr/bin/apt-get", "install", "-qy", "gunicorn", "python3-pip", "nftables"] - ) + # Install repo-policy-compliance package execute_command( [ "/usr/bin/pip", "install", + "--upgrade", "git+https://github.com/canonical/repo-policy-compliance@main", ], env=env, @@ -669,7 +684,8 @@ def _start_services(self) -> None: ) self.repo_check_systemd_service.write_text(service_content, encoding="utf-8") - execute_command(["/usr/bin/systemctl", "start", "repo-policy-compliance"]) + execute_command(["/usr/bin/systemctl", "daemon-reload"]) + execute_command(["/usr/bin/systemctl", "restart", "repo-policy-compliance"]) execute_command(["/usr/bin/systemctl", "enable", "repo-policy-compliance"]) logger.info("Finished starting charm services") diff --git a/src/lxd.py b/src/lxd.py index a9c1da2a38..2da02e3cbe 100644 --- a/src/lxd.py +++ b/src/lxd.py @@ -222,7 +222,9 @@ def delete(self, wait: bool = False) -> None: logger.exception("Failed to delete LXD instance") raise LxdError(f"Unable to delete LXD instance {self.name}") from err - def execute(self, cmd: list[str], cwd: Optional[str] = None) -> Tuple[int, IO, IO]: + def execute( + self, cmd: list[str], cwd: Optional[str] = None, hide_cmd: bool = False + ) -> Tuple[int, IO, IO]: """Execute a command within the LXD instance. Exceptions are not raise if command execution failed. Caller should check the exit code and @@ -231,6 +233,7 @@ def execute(self, cmd: list[str], cwd: Optional[str] = None) -> Tuple[int, IO, I Args: cmd: Commands to be executed. cwd: Working directory to execute the commands. + hide_cmd: Hide logging of cmd. Returns: Tuple containing the exit code, stdout, stderr. @@ -241,7 +244,7 @@ def execute(self, cmd: list[str], cwd: Optional[str] = None) -> Tuple[int, IO, I lxc_cmd += ["--"] + cmd - result = secure_run_subprocess(lxc_cmd) + result = secure_run_subprocess(lxc_cmd, hide_cmd) return (result.returncode, io.BytesIO(result.stdout), io.BytesIO(result.stderr)) diff --git a/src/runner.py b/src/runner.py index 0f5932add2..b686b809ce 100644 --- a/src/runner.py +++ b/src/runner.py @@ -135,10 +135,10 @@ def remove(self, remove_token: str) -> None: Raises: RunnerRemoveError: Failure in removing runner. """ - logger.info("Removing LXD instance of runner: %s", self.config.name) + logger.info("Removing runner: %s", self.config.name) if self.instance: - # Run script to remove the the runner and cleanup. + logger.info("Executing command to removal of runner and clean up...") self.instance.execute( [ "/usr/bin/sudo", @@ -149,9 +149,11 @@ def remove(self, remove_token: str) -> None: "--token", remove_token, ], + hide_cmd=True, ) if self.instance.status == "Running": + logger.info("Removing LXD instance of runner: %s", self.config.name) try: self.instance.stop(wait=True, timeout=60) except LxdError: @@ -179,6 +181,8 @@ def remove(self, remove_token: str) -> None: if self.status.runner_id is None: return + logger.info("Removing runner on GitHub: %s", self.config.name) + # The runner should cleanup itself. Cleanup on GitHub in case of runner cleanup error. if isinstance(self.config.path, GitHubRepo): logger.debug( @@ -549,9 +553,11 @@ def _register_runner(self, registration_token: str, labels: Sequence[str]) -> No if isinstance(self.config.path, GitHubOrg): register_cmd += ["--runnergroup", self.config.path.group] + logger.info("Executing registration command...") self.instance.execute( register_cmd, cwd=str(self.runner_application), + hide_cmd=True, ) @retry(tries=5, delay=30, local_logger=logger) diff --git a/src/runner_manager.py b/src/runner_manager.py index 460f2369c2..120672e7af 100644 --- a/src/runner_manager.py +++ b/src/runner_manager.py @@ -20,6 +20,7 @@ import requests.adapters import urllib3 from ghapi.all import GhApi +from ghapi.page import pages from typing_extensions import assert_never from errors import RunnerBinaryError, RunnerCreateError @@ -34,7 +35,14 @@ from lxd import LxdClient, LxdInstance from repo_policy_compliance_client import RepoPolicyComplianceClient from runner import Runner, RunnerClients, RunnerConfig, RunnerStatus -from runner_type import GitHubOrg, GitHubPath, GitHubRepo, ProxySetting, VirtualMachineResources +from runner_type import ( + GitHubOrg, + GitHubPath, + GitHubRepo, + ProxySetting, + RunnerByHealth, + VirtualMachineResources, +) from utilities import retry, set_env_var logger = logging.getLogger(__name__) @@ -252,6 +260,26 @@ def get_github_info(self) -> Iterator[RunnerInfo]: remote_runners = self._get_runner_github_info() return iter(RunnerInfo(runner.name, runner.status) for runner in remote_runners.values()) + def _get_runner_health_states(self) -> RunnerByHealth: + local_runners = [ + instance + # Pylint cannot find the `all` method. + for instance in self._clients.lxd.instances.all() # pylint: disable=no-member + if instance.name.startswith(f"{self.instance_name}-") + ] + + healthy = [] + unhealthy = [] + + for runner in local_runners: + _, stdout, _ = runner.execute(["ps", "aux"]) + if f"/bin/bash {Runner.runner_script}" in stdout.read().decode("utf-8"): + healthy.append(runner.name) + else: + unhealthy.append(runner.name) + + return RunnerByHealth(healthy, unhealthy) + def reconcile(self, quantity: int, resources: VirtualMachineResources) -> int: """Bring runners in line with target. @@ -269,40 +297,38 @@ def reconcile(self, quantity: int, resources: VirtualMachineResources) -> int: runner for runner in runners if runner.status.exist and runner.status.online ] - offline_runners = [runner for runner in runners if not runner.status.online] - - local_runners = { - instance.name: instance - # Pylint cannot find the `all` method. - for instance in self._clients.lxd.instances.all() # pylint: disable=no-member - if instance.name.startswith(f"{self.instance_name}-") - } + runner_states = self._get_runner_health_states() logger.info( ( - "Expected runner count: %i, Online runner count: %i, Offline runner count: %i, " - "LXD instance count: %i" + "Expected runner count: %i, Online count: %i, Offline count: %i, " + "healthy count: %i, unhealthy count: %i" ), quantity, len(online_runners), - len(offline_runners), - len(local_runners), + len(runners) - len(online_runners), + len(runner_states.healthy), + len(runner_states.unhealthy), ) # Clean up offline runners - if offline_runners: - logger.info("Cleaning up offline runners.") + if runner_states.unhealthy: + logger.info("Cleaning up unhealthy runners.") remove_token = self._get_github_remove_token() - for runner in offline_runners: + unhealthy_runners = [ + runner for runner in runners if runner.config.name in set(runner_states.unhealthy) + ] + + for runner in unhealthy_runners: runner.remove(remove_token) logger.info("Removed runner: %s", runner.config.name) - delta = quantity - len(online_runners) + delta = quantity - len(runner_states.healthy) # Spawn new runners if delta > 0: - if RunnerManager.runner_bin_path is None: + if not RunnerManager.runner_bin_path.exists(): raise RunnerCreateError("Unable to create runner due to missing runner binary.") logger.info("Getting registration token for GitHub runners.") @@ -310,7 +336,7 @@ def reconcile(self, quantity: int, resources: VirtualMachineResources) -> int: registration_token = self._get_github_registration_token() remove_token = self._get_github_remove_token() - logger.info("Adding %i additional runner(s).", delta) + logger.info("Attempting to add %i runner(s).", delta) for _ in range(delta): config = RunnerConfig( self.app_name, @@ -335,6 +361,7 @@ def reconcile(self, quantity: int, resources: VirtualMachineResources) -> int: raise elif delta < 0: + logger.info("Attempting to remove %i runner(s).", -delta) # Idle runners are online runners that has not taken a job. idle_runners = [runner for runner in online_runners if not runner.status.busy] offset = min(-delta, len(idle_runners)) @@ -349,7 +376,6 @@ def reconcile(self, quantity: int, resources: VirtualMachineResources) -> int: for runner in remove_runners: runner.remove(remove_token) logger.info("Removed runner: %s", runner.config.name) - else: logger.info("There are no idle runner to remove.") else: @@ -397,13 +423,40 @@ def _generate_runner_name(self) -> str: def _get_runner_github_info(self) -> Dict[str, SelfHostedRunner]: remote_runners_list: list[SelfHostedRunner] = [] if isinstance(self.config.path, GitHubRepo): - remote_runners_list = self._clients.github.actions.list_self_hosted_runners_for_repo( - owner=self.config.path.owner, repo=self.config.path.repo - )["runners"] + # The documentation of ghapi for pagination is incorrect and examples will give errors. + # This workaround is a temp solution. Will be moving to PyGitHub in the future. + self._clients.github.actions.list_self_hosted_runners_for_repo( + owner=self.config.path.owner, repo=self.config.path.repo, per_page=100 + ) + num_of_pages = self._clients.github.last_page() + remote_runners_list = [ + item + for page in pages( + self._clients.github.actions.list_self_hosted_runners_for_repo, + num_of_pages + 1, + owner=self.config.path.owner, + repo=self.config.path.repo, + per_page=100, + ) + for item in page["runners"] + ] if isinstance(self.config.path, GitHubOrg): - remote_runners_list = self._clients.github.actions.list_self_hosted_runners_for_org( - org=self.config.path.org - )["runners"] + # The documentation of ghapi for pagination is incorrect and examples will give errors. + # This workaround is a temp solution. Will be moving to PyGitHub in the future. + self._clients.github.actions.list_self_hosted_runners_for_org( + org=self.config.path.org, per_page=100 + ) + num_of_pages = self._clients.github.last_page() + remote_runners_list = [ + item + for page in pages( + self._clients.github.actions.list_self_hosted_runners_for_org, + num_of_pages + 1, + org=self.config.path.org, + per_page=100, + ) + for item in page["runners"] + ] logger.debug("List of runners found on GitHub:%s", remote_runners_list) diff --git a/src/runner_type.py b/src/runner_type.py index 2d11bb32cc..62ce0b18b1 100644 --- a/src/runner_type.py +++ b/src/runner_type.py @@ -15,6 +15,14 @@ from repo_policy_compliance_client import RepoPolicyComplianceClient +@dataclass +class RunnerByHealth: + """Set of runners LXD instance by health state.""" + + healthy: tuple[str] + unhealthy: tuple[str] + + class ProxySetting(TypedDict, total=False): """Represent HTTP-related proxy settings.""" diff --git a/src/utilities.py b/src/utilities.py index 9a1d3ef1ef..a51c11029a 100644 --- a/src/utilities.py +++ b/src/utilities.py @@ -95,7 +95,9 @@ def fn_with_retry(*args, **kwargs) -> ReturnT: return retry_decorator -def secure_run_subprocess(cmd: Sequence[str], **kwargs) -> subprocess.CompletedProcess[bytes]: +def secure_run_subprocess( + cmd: Sequence[str], hide_cmd: bool = False, **kwargs +) -> subprocess.CompletedProcess[bytes]: """Run command in subprocess according to security recommendations. The argument `shell` is set to `False` for security reasons. @@ -105,12 +107,17 @@ def secure_run_subprocess(cmd: Sequence[str], **kwargs) -> subprocess.CompletedP Args: cmd: Command in a list. + hide_cmd: Hide logging of cmd. kwargs: Additional keyword arguments for the `subprocess.run` call. Returns: Object representing the completed process. The outputs subprocess can accessed. """ - logger.info("Executing command %s", cmd) + if not hide_cmd: + logger.info("Executing command %s", cmd) + else: + logger.info("Executing sensitive command") + result = subprocess.run( # nosec B603 cmd, capture_output=True, @@ -119,7 +126,10 @@ def secure_run_subprocess(cmd: Sequence[str], **kwargs) -> subprocess.CompletedP # Disable type check due to the support for unpacking arguments in mypy is experimental. **kwargs, # type: ignore ) - logger.debug("Command %s returns: %s", cmd, result.stdout) + if not hide_cmd: + logger.debug("Command %s returns: %s", cmd, result.stdout) + else: + logger.debug("Command returns: %s", result.stdout) return result diff --git a/templates/pre-job.j2 b/templates/pre-job.j2 index 968a77dbb2..69cf5df811 100644 --- a/templates/pre-job.j2 +++ b/templates/pre-job.j2 @@ -1,11 +1,47 @@ #!/usr/bin/env bash -GITHUB_SOURCE_REPOSITORY=$(cat "${GITHUB_EVENT_PATH}" | jq -r '.pull_request.head.repo.full_name') - -# Request repo-policy-compliance service check. -curl --noproxy '*' \ - --fail-with-body \ - -H 'Authorization: Bearer {{one_time_token}}' \ - -H 'Content-Type: application/json' \ - -d "{\"repository_name\": \"${GITHUB_REPOSITORY}\", \"source_repository_name\": \"${GITHUB_SOURCE_REPOSITORY}\", \"target_branch_name\": \"${GITHUB_BASE_REF}\", \"source_branch_name\": \"${GITHUB_HEAD_REF}\", \"commit_sha\": \"${GITHUB_SHA}\"}" \ - http://{{host_ip}}:8080/check-run +# Log common env variables. +logger -s "GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}, \ + GITHUB_REPOSITORY: ${GITHUB_REPOSITORY}, \ + GITHUB_SHA: ${GITHUB_SHA}" + +# Prepare curl arguments +CURL_ARGS=( + --max-time 60 + --noproxy '*' + --fail-with-body + -H 'Authorization: Bearer {{one_time_token}}' + -H 'Content-Type: application/json' +) + +# Workflow dispatch - Request repo-policy-compliance service check: +if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then + + logger -s "GITHUB_REF_NAME: ${GITHUB_REF_NAME}" + + curl "${CURL_ARGS[@]}" \ + -d "{\"repository_name\": \"${GITHUB_REPOSITORY}\", \"branch_name\": \"${GITHUB_REF_NAME}\", \"commit_sha\": \"${GITHUB_SHA}\"}" \ + http://{{host_ip}}:8080/workflow_dispatch/check-run + +# Pull request - Request repo-policy-compliance service check: +elif [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + + GITHUB_SOURCE_REPOSITORY=$(cat "${GITHUB_EVENT_PATH}" | jq -r '.pull_request.head.repo.full_name') + + logger -s " \ + GITHUB_SOURCE_REPOSITORY: ${GITHUB_SOURCE_REPOSITORY}, \ + GITHUB_SOURCE_REPOSITORY: ${GITHUB_SOURCE_REPOSITORY} \ + GITHUB_BASE_REF: ${GITHUB_BASE_REF}, \ + GITHUB_HEAD_REF: ${GITHUB_HEAD_REF}" + + curl "${CURL_ARGS[@]}" \ + -d "{\"repository_name\": \"${GITHUB_REPOSITORY}\", \"source_repository_name\": \"${GITHUB_SOURCE_REPOSITORY}\", \"target_branch_name\": \"${GITHUB_BASE_REF}\", \"source_branch_name\": \"${GITHUB_HEAD_REF}\", \"commit_sha\": \"${GITHUB_SHA}\"}" \ + http://{{host_ip}}:8080/pull_request/check-run + +else + + logger -p user.error -s "${GITHUB_EVENT_NAME} is not supported yet. Please request it to be added on https://github.com/canonical/github-runner-operator/issues/new/choose" + + return 1 + +fi diff --git a/templates/repo-policy-compliance.service.j2 b/templates/repo-policy-compliance.service.j2 index afde32b2e2..20049e621b 100644 --- a/templates/repo-policy-compliance.service.j2 +++ b/templates/repo-policy-compliance.service.j2 @@ -14,4 +14,4 @@ Environment="NO_PROXY={{proxies['no_proxy']}}" Environment="http_proxy={{proxies['http']}}" Environment="https_proxy={{proxies['https']}}" Environment="no_proxy={{proxies['no_proxy']}}" -ExecStart=/usr/bin/gunicorn --bind 0.0.0.0:8080 app:app +ExecStart=/usr/bin/gunicorn --bind 0.0.0.0:8080 --timeout 60 app:app diff --git a/templates/start.j2 b/templates/start.j2 index 7e6979ce5f..e0d2bbef6e 100644 --- a/templates/start.j2 +++ b/templates/start.j2 @@ -1,3 +1,3 @@ #!/bin/bash -(/home/ubuntu/github-runner/run.sh; sudo systemctl halt -i) &>/dev/null & +(/home/ubuntu/github-runner/run.sh; sudo systemctl poweroff -i) &>/dev/null & diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..2c64509aad --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for github runner charm.""" + + +from pytest import Parser + + +def pytest_addoption(parser: Parser): + """Add options to pytest parser.""" + parser.addoption("--path", action="store") + parser.addoption("--token", action="store") + parser.addoption("--token-alt", action="store") + parser.addoption("--http-proxy", action="store") + parser.addoption("--https-proxy", action="store") + parser.addoption("--no-proxy", action="store") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a74fcf8e51..2bf25bff59 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,32 +1,150 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. +"""Fixtures for github runner charm integration tests.""" + +import secrets +import subprocess from pathlib import Path +from typing import Any, AsyncIterator import pytest +import pytest_asyncio import yaml +from juju.application import Application +from juju.model import Model +from pytest_operator.plugin import OpsTest -@pytest.fixture -def metadata(): +@pytest.fixture(scope="module") +def metadata() -> dict[str, Any]: + """Metadata information of the charm""" metadata = Path("./metadata.yaml") data = yaml.safe_load(metadata.read_text()) return data -@pytest.fixture -def model(ops_test): +@pytest.fixture(scope="module") +def app_name() -> str: + # Randomized app name to avoid collision when connecting to GitHub. + return f"integration-{secrets.token_hex(4)}" + + +@pytest.fixture(scope="module") +def path(pytestconfig: pytest.Config) -> str: + path = pytestconfig.getoption("--path") + assert path, "Please specify the --path command line option" + return path + + +@pytest.fixture(scope="module") +def token(pytestconfig: pytest.Config) -> str: + token = pytestconfig.getoption("--token") + assert token, "Please specify the --token command line option" + return token + + +@pytest.fixture(scope="module") +def token_alt(pytestconfig: pytest.Config, token: str) -> str: + token_alt = pytestconfig.getoption("--token-alt") + assert token_alt, "Please specify the --token-alt command line option" + assert token_alt != token, "Please specify a different token for --token-alt" + return token_alt + + +@pytest.fixture(scope="module") +def http_proxy(pytestconfig: pytest.Config) -> str: + http_proxy = pytestconfig.getoption("--http-proxy") + return "" if http_proxy is None else http_proxy + + +@pytest.fixture(scope="module") +def https_proxy(pytestconfig: pytest.Config) -> str: + https_proxy = pytestconfig.getoption("--https-proxy") + return "" if https_proxy is None else https_proxy + + +@pytest.fixture(scope="module") +def no_proxy(pytestconfig: pytest.Config) -> str: + no_proxy = pytestconfig.getoption("--no-proxy") + return "" if no_proxy is None else no_proxy + + +@pytest.fixture(scope="module") +def model(ops_test: OpsTest) -> Model: + assert ops_test.model is not None return ops_test.model -@pytest.fixture -def application(model, metadata): - charm_name = metadata["name"] - app = model.applications[charm_name] - return app +@pytest_asyncio.fixture(scope="module") +async def lxd_profile() -> AsyncIterator[Path]: + lxd_profile_path = Path("lxd-profile.yaml") + + lxd_profile_path.write_text( + """config: + security.nesting: true + security.privileged: true + raw.lxc: | + lxc.apparmor.profile=unconfined + lxc.mount.auto=proc:rw sys:rw cgroup:rw + lxc.cgroup.devices.allow=a + lxc.cap.drop= +devices: + kmsg: + path: /dev/kmsg + source: /dev/kmsg + type: unix-char +""" + ) + + yield lxd_profile_path + + lxd_profile_path.unlink(missing_ok=True) + + +@pytest_asyncio.fixture(scope="module") +async def app_no_token( + ops_test: OpsTest, + model: Model, + app_name: str, + path: str, + http_proxy: str, + https_proxy: str, + no_proxy: str, + lxd_profile: Path, +) -> AsyncIterator[Application]: + charm = await ops_test.build_charm(".") + + subprocess.run(["sudo", "modprobe", "br_netfilter"]) + await model.set_config( + { + "juju-http-proxy": http_proxy, + "juju-https-proxy": https_proxy, + "juju-no-proxy": no_proxy, + "logging-config": "=INFO;unit=DEBUG", + } + ) + application = await model.deploy( + charm, + application_name=app_name, + series="jammy", + config={ + "path": path, + "virtual-machines": 0, + "test-mode": "insecure", + }, + ) + await model.wait_for_idle() + + yield application + + +@pytest_asyncio.fixture(scope="module") +async def app(model: Model, app_no_token: Application, token: str) -> AsyncIterator[Application]: + await app_no_token.set_config({"token": token}) + await model.wait_for_idle() + yield app_no_token -@pytest.fixture -def units(application): - units = application.units - return units + await app_no_token.set_config({"token": ""}) + await model.wait_for_idle() diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index ad6f7b6a0e..c28c297c3e 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -1,26 +1,121 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -import logging +"""Integration tests for github-runner charm.""" + import pytest +from juju.application import Application +from juju.model import Model + +from runner_manager import RunnerManager +from tests.integration.test_helpers import check_runner_instance, remove_runner_bin +from tests.status_name import ACTIVE_STATUS_NAME, BLOCK_STATUS_NAME -log = logging.getLogger(__name__) +@pytest.mark.asyncio +@pytest.mark.abort_on_fail +async def test_missing_config(app_no_token: Application) -> None: + """ + arrange: An application without token configuration. + act: Check the status the application. + assert: The application is in blocked status. + """ + assert app_no_token.status == BLOCK_STATUS_NAME -async def file_contents(unit, path): - cmd = "cat {}".format(path) - action = await unit.run(cmd) - return action.results["Stdout"] + unit = app_no_token.units[0] + assert unit.workload_status == BLOCK_STATUS_NAME + assert unit.workload_status_message == "Missing required charm configuration: ['token']" +@pytest.mark.asyncio @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test): - my_charm = await ops_test.build_charm(".") - await ops_test.model.deploy(my_charm) - await ops_test.model.wait_for_idle() +async def test_config(model: Model, app: Application, app_name: str, token_alt: str) -> None: + """ + arrange: An working application with no runners. + act: + 1. Nothing + 2. Run update-runner-bin action. + 3. Run check-runners action. + 4. Configure to one runner and run reconcile-runners action. + 5. Run check-runners action. + 6. Change the token in application configuration. + assert: + 1. The application is in active status. + 2. Runner binary exists in the charm. + 3. Action returns result with no runner. + 4. One runner should be spawned. + 5. Action returns result with one runner. + 6. Old runner should be flushed, repo-policy-compliance using the new token. + """ + unit = app.units[0] + + # 1. + assert app.status == ACTIVE_STATUS_NAME + + # 2. + # Ensure there is no runner binary. + await remove_runner_bin(unit) + + action = await unit.run_action("update-runner-bin") + await action.wait() + + await model.wait_for_idle() + + assert app.status == ACTIVE_STATUS_NAME + action = await unit.run(f"test -f {RunnerManager.runner_bin_path}") + await action.wait() + assert action.results["return-code"] == 0 + + # 3. + # Verify there is no runners. + await check_runner_instance(unit, 0) + + action = await unit.run_action("check-runners") + await action.wait() + + assert action.results["online"] == "0" + assert action.results["offline"] == "0" + assert action.results["unknown"] == "0" + assert not action.results["runners"] + + # 4. + await app.set_config({"virtual-machines": "1"}) + + action = await unit.run_action("reconcile-runners") + await action.wait() + await model.wait_for_idle() + + await check_runner_instance(unit, 1) + + # 5. + action = await app.units[0].run_action("check-runners") + await action.wait() + + assert action.status == "completed" + assert action.results["online"] == "1" + assert action.results["offline"] == "0" + assert action.results["unknown"] == "0" + + runner_names = action.results["runners"].split(", ") + assert len(runner_names) == 1 + assert runner_names[0].startswith(f"{app_name}-0") + + # 6. + await app.set_config({"token": token_alt}) + await model.wait_for_idle() + + # The old runner should be flushed out. + action = await app.units[0].run_action("check-runners") + await action.wait() + + assert action.status == "completed" + assert action.results["online"] == "0" + assert action.results["offline"] == "0" + assert action.results["unknown"] == "0" + action = await app.units[0].run("cat /etc/systemd/system/repo-policy-compliance.service") + await action.wait() -async def test_status(units): - assert units[0].workload_status == "blocked" - assert units[0].workload_status_message == "Missing token or org/repo path config" + assert action.status == "completed" + assert f"GITHUB_TOKEN={token_alt}" in action.results["stdout"] diff --git a/tests/integration/test_helpers.py b/tests/integration/test_helpers.py new file mode 100644 index 0000000000..a4588d470c --- /dev/null +++ b/tests/integration/test_helpers.py @@ -0,0 +1,47 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Utilities for integration test.""" + +import json + +from juju.unit import Unit + +from runner import Runner +from runner_manager import RunnerManager +from utilities import retry + + +async def remove_runner_bin(unit: Unit): + action = await unit.run(f"rm {RunnerManager.runner_bin_path}") + await action.wait() + + # No file should exists under with the filename. + action = await unit.run(f"test -f {RunnerManager.runner_bin_path}") + await action.wait() + assert action.results["return-code"] != 0 + + +@retry(tries=30, delay=30) +async def check_runner_instance(unit: Unit, num: int) -> None: + """Helper function to wait for runner instances to be ready. + + Args: + app: Application instance to check the runners. + num: Number of runner instances to check for. + """ + action = await unit.run("lxc list --format json") + await action.wait() + assert action.status == "completed" + + assert action.results["return-code"] == 0 + + lxc_instance = json.loads(action.results["stdout"]) + assert len(lxc_instance) == num + + for instance in lxc_instance: + action = await unit.run(f"lxc exec {instance['name']} -- ps aux") + await action.wait() + assert action.status == "completed" + + assert f"/bin/bash {Runner.runner_script}" in action.results["stdout"] diff --git a/tests/status_name.py b/tests/status_name.py new file mode 100644 index 0000000000..326202cd25 --- /dev/null +++ b/tests/status_name.py @@ -0,0 +1,8 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +from ops.model import ActiveStatus, BlockedStatus + +# mypy can not find type of `name` attribute. +ACTIVE_STATUS_NAME = ActiveStatus.name # type: ignore +BLOCK_STATUS_NAME = BlockedStatus.name # type: ignore diff --git a/tests/unit/mock.py b/tests/unit/mock.py index 7c3bfdac00..cd94cdc6f0 100644 --- a/tests/unit/mock.py +++ b/tests/unit/mock.py @@ -6,9 +6,10 @@ from __future__ import annotations import hashlib +import io import logging import secrets -from typing import Optional, Sequence, Union +from typing import IO, Optional, Sequence, Union from errors import LxdError, RunnerError from github_type import RegistrationToken, RemoveToken, RunnerApplication @@ -102,8 +103,10 @@ def stop(self, wait: bool = True, timeout: int = 60): def delete(self, wait: bool = True): self.deleted = True - def execute(self, cmd: Sequence[str], cwd: Optional[str] = None) -> tuple[int, str, str]: - return 0, "", "" + def execute( + self, cmd: Sequence[str], cwd: Optional[str] = None, hide_cmd: bool = False + ) -> tuple[int, IO, IO]: + return 0, io.BytesIO(b""), io.BytesIO(b"") class MockLxdInstanceFileManager: @@ -186,6 +189,9 @@ def __init__(self, token: str): self.token = token self.actions = MockGhapiActions() + def last_page(self) -> int: + return 0 + class MockGhapiActions: """Mock for actions in Ghapi client.""" @@ -238,10 +244,12 @@ def create_remove_token_for_org(self, org: str): {"token": self.remove_token_org, "expires_at": "2020-01-22T12:13:35.123-08:00"} ) - def list_self_hosted_runners_for_repo(self, owner: str, repo: str): + def list_self_hosted_runners_for_repo( + self, owner: str, repo: str, per_page: int, page: int = 0 + ): return {"runners": []} - def list_self_hosted_runners_for_org(self, org: str): + def list_self_hosted_runners_for_org(self, org: str, per_page: int, page: int = 0): return {"runners": []} def delete_self_hosted_runner_from_repo(self, owner: str, repo: str, runner_id: str): diff --git a/tests/unit/test_runner_manager.py b/tests/unit/test_runner_manager.py index 02358867e2..31c7863e32 100644 --- a/tests/unit/test_runner_manager.py +++ b/tests/unit/test_runner_manager.py @@ -12,7 +12,7 @@ from errors import RunnerBinaryError from runner import Runner, RunnerStatus from runner_manager import RunnerManager, RunnerManagerConfig -from runner_type import GitHubOrg, GitHubRepo, VirtualMachineResources +from runner_type import GitHubOrg, GitHubRepo, RunnerByHealth, VirtualMachineResources from tests.unit.mock import TEST_BINARY @@ -38,12 +38,14 @@ def runner_manager_fixture(request, tmp_path, monkeypatch, token): ) pool_path = tmp_path / "test_storage" pool_path.mkdir(exist_ok=True) + runner_manager = RunnerManager( "test app", "0", RunnerManagerConfig(request.param[0], token, "jammy", secrets.token_hex(16), pool_path), proxies=request.param[1], ) + runner_manager.runner_bin_path.write_bytes(TEST_BINARY) return runner_manager @@ -72,7 +74,7 @@ def test_get_latest_runner_bin_url_missing_binary(runner_manager: RunnerManager) def test_update_runner_bin(runner_manager: RunnerManager): """ - arrange: Nothing. + arrange: Remove the existing runner binary. act: Update runner binary. assert: Runner binary in runner manager is set. """ @@ -84,13 +86,15 @@ def __init__(self, *arg, **kargs): def iter_content(self, *arg, **kargs): return iter([TEST_BINARY]) + runner_manager.runner_bin_path.unlink(missing_ok=True) + runner_manager.session.get = MockRequestLibResponse - # Remove the fake binary in fixture. - runner_manager.runner_bin_path = None runner_bin = runner_manager.get_latest_runner_bin_url(os_name="linux", arch_name="x64") runner_manager.update_runner_bin(runner_bin) + assert runner_manager.runner_bin_path.read_bytes() == TEST_BINARY + def test_reconcile_zero_count(runner_manager: RunnerManager): """ @@ -134,6 +138,14 @@ def mock_get_runners(): # Create online runners. runner_manager._get_runners = mock_get_runners + runner_manager._get_runner_health_states = lambda: RunnerByHealth( + ( + f"{runner_manager.instance_name}-0", + f"{runner_manager.instance_name}-1", + f"{runner_manager.instance_name}-2", + ), + (), + ) delta = runner_manager.reconcile(2, VirtualMachineResources(2, "7GiB", "10Gib")) From e63a43d152d0a87bbd570392d89c7f79813a8b37 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:55:22 +0800 Subject: [PATCH 051/343] Test proxy --- .github/workflows/manual_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c40464a507..dcb51d3ef8 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,6 +11,8 @@ jobs: - name: curl run: curl -vvv https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url - name: Install requests + if: ${{ success() && failure() }} run: python3 -m pip install requests - name: Make HTTP request with Python + if: ${{ success() && failure() }} run: python3 -c "import requests;requests.get('https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url')" From a863075714a8ff417d5b0c3ba2949f72ed7aaa5d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:56:31 +0800 Subject: [PATCH 052/343] Test proxy --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index dcb51d3ef8..873ad7acfa 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,8 +11,8 @@ jobs: - name: curl run: curl -vvv https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url - name: Install requests - if: ${{ success() && failure() }} + if: ${{ success() || failure() }} run: python3 -m pip install requests - name: Make HTTP request with Python - if: ${{ success() && failure() }} + if: ${{ success() || failure() }} run: python3 -c "import requests;requests.get('https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url')" From eb5790b1688da43e9cc592dbf9424291acf412a4 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 7 Aug 2023 10:06:48 +0800 Subject: [PATCH 053/343] Debug HTTP proxy --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 873ad7acfa..29ecba6d29 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,4 +15,4 @@ jobs: run: python3 -m pip install requests - name: Make HTTP request with Python if: ${{ success() || failure() }} - run: python3 -c "import requests;requests.get('https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url')" + run: python3 -c "import logging;logging.basicConfig(level=logging.DEBUG);import requests;resp=requests.get('https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url', proxies={'https':'http://squid.internal:3128'});print(resp.content)" From 8b06570b1495b98ba246421ef21d4b4473e93587 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:09:58 +0800 Subject: [PATCH 054/343] Test rockcraft-pack --- .github/workflows/test.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4aa06569e7..b7fa2b8cf9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,12 +1,15 @@ name: Tests on: - # pull_request: + pull_request: workflow_dispatch: jobs: - unit-tests: - uses: canonical/operator-workflows/.github/workflows/test.yaml@main - secrets: inherit - with: - self-hosted-runner: false + test: + runs-on: [self-hosted, large] + steps: + - uses: canonical/craft-actions/rockcraft-pack@main + with: + path: mlserver-huggingface + verbosity: trace + - run: sleep 10000 From 828bf8c30d68784d96602791deac1c244bf18c3e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:11:48 +0800 Subject: [PATCH 055/343] Resolve conflict --- .github/workflows/integration_tests.yaml | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .github/workflows/integration_tests.yaml diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml new file mode 100644 index 0000000000..b41a32d53a --- /dev/null +++ b/.github/workflows/integration_tests.yaml @@ -0,0 +1,80 @@ +name: Integration tests + +on: + pull_request: + +jobs: + integration-test-charm-no-runner: + name: Integration test charm with no runner + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd + - name: Enable br_netfilter + run: sudo modprobe br_netfilter + - name: Run Integration tests + run: | + tox -e integration -- \ + --keep-models \ + --path ${{secrets.E2E_TESTING_REPO}} \ + --token ${{secrets.E2E_TESTING_TOKEN}} \ + --token-alt ${{secrets.E2E_TESTING_TOKEN_ALT}} \ + tests/integration/test_charm_no_runner.py + integration-test-charm: + name: Integration test charm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd + - name: Enable br_netfilter + run: sudo modprobe br_netfilter + - name: Run Integration tests + run: | + tox -e integration -- \ + --keep-models \ + --path ${{secrets.E2E_TESTING_REPO}} \ + --token ${{secrets.E2E_TESTING_TOKEN}} \ + --token-alt ${{secrets.E2E_TESTING_TOKEN_ALT}} \ + tests/integration/test_charm.py + integration-test-charm-fork-repo: + name: Integration test charm with fork repo + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd + - name: Enable br_netfilter + run: sudo modprobe br_netfilter + - name: Run Integration tests + run: | + tox -e integration -- \ + --keep-models \ + --path ${{secrets.E2E_TESTING_REPO}} \ + --token ${{secrets.E2E_TESTING_TOKEN}} \ + --token-alt ${{secrets.E2E_TESTING_TOKEN_ALT}} \ + tests/integration/test_charm_fork_repo.py + required_status_checks: + name: Required Integration Test Status Checks + runs-on: ubuntu-latest + needs: + - integration-test-charm-no-runner + - integration-test-charm + - integration-test-charm-fork-repo + if: always() && !cancelled() + timeout-minutes: 30 + steps: + - run: | + [ '${{ needs.integration-test-charm-no-runner.result }}' = 'success' ] || (echo integration-test-charm-no-runner failed && false) + [ '${{ needs.integration-test-charm.result }}' = 'success' ] || (echo integration-test-charm failed && false) + [ '${{ needs.integration-test-charm-fork-repo.result }}' = 'success' ] || (echo integration-test-charm-fork-repo failed && false) From 63fff7e978aaea35263a1549687dd4d7e86849a2 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:12:23 +0800 Subject: [PATCH 056/343] Test rockcraft-pack --- .github/workflows/integration_test.yaml | 26 ------------------------ .github/workflows/integration_tests.yaml | 3 ++- 2 files changed, 2 insertions(+), 27 deletions(-) delete mode 100644 .github/workflows/integration_test.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml deleted file mode 100644 index 685cf2dcd3..0000000000 --- a/.github/workflows/integration_test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Integration test - -on: - # pull_request: - workflow_dispatch: - -jobs: - integration-test: - name: Integration test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main - with: - juju-channel: 3.1/stable - provider: lxd - - name: Enable br_netfilter - run: sudo modprobe br_netfilter - - name: Run Integration tests - run: | - tox -e integration -- \ - --keep-models \ - --path ${{secrets.E2E_TESTING_REPO}} \ - --token ${{secrets.E2E_TESTING_TOKEN}} \ - --token-alt ${{secrets.E2E_TESTING_TOKEN_ALT}} diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index b41a32d53a..9a1f4a9e55 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,7 +1,8 @@ name: Integration tests on: - pull_request: + # pull_request: + workflow_dispatch: jobs: integration-test-charm-no-runner: From f40f26a74366f2ae13e7996d8785b3966e8aefe0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:16:11 +0800 Subject: [PATCH 057/343] Test rockcraft-pack --- .github/workflows/test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b7fa2b8cf9..1afe1cffd9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,4 +12,5 @@ jobs: with: path: mlserver-huggingface verbosity: trace - - run: sleep 10000 + - if: failure() + run: sleep 10000 From cc36c9f230da95733cf05cfbc31a2c56612ea44b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:06:34 +0800 Subject: [PATCH 058/343] Shell access to GitHub-hosted runner --- .github/workflows/test.yaml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1afe1cffd9..89e69edff5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,12 +5,17 @@ on: workflow_dispatch: jobs: - test: - runs-on: [self-hosted, large] - steps: - - uses: canonical/craft-actions/rockcraft-pack@main - with: - path: mlserver-huggingface - verbosity: trace - - if: failure() - run: sleep 10000 + ssh: + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + # test: + # runs-on: [self-hosted, large] + # steps: + # - uses: canonical/craft-actions/rockcraft-pack@main + # with: + # path: mlserver-huggingface + # verbosity: trace + # - if: failure() + # run: sleep 10000 From 516b0096e0cbb29a7a94ba0bd2ece56222f2ac36 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:09:25 +0800 Subject: [PATCH 059/343] Shell access to GitHub-hosted runner --- .github/workflows/test.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 89e69edff5..2501a54f33 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,10 +6,12 @@ on: jobs: ssh: - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true + runs-on: ubuntu-latest + steps: + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true # test: # runs-on: [self-hosted, large] # steps: From 24d3757e2adfee976f0ea6d80be5843283fd895e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:44:25 +0800 Subject: [PATCH 060/343] Trigger workflow --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2501a54f33..bd939f2d50 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ on: jobs: ssh: - runs-on: ubuntu-latest + runs-on: [self-hosted, github-runner] steps: - name: Setup tmate session uses: mxschmitt/action-tmate@v3 From 1871cb63c7fab01223fd431eff6339b121e21701 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 29 Aug 2023 15:02:57 +0800 Subject: [PATCH 061/343] Trigger workflow --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd939f2d50..01e542e8c5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,6 +12,7 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true + # test: # runs-on: [self-hosted, large] # steps: From a0e875bcbdf09896812cea0e9908ef4a41bf1d88 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 29 Aug 2023 16:59:13 +0800 Subject: [PATCH 062/343] Trigger workflow --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 01e542e8c5..bd939f2d50 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,7 +12,6 @@ jobs: uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true - # test: # runs-on: [self-hosted, large] # steps: From e8a7e7a96ad7c18d34f79174fe3ebc8280208eb1 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:26:09 +0800 Subject: [PATCH 063/343] Trigger test e2e test --- .github/workflows/e2e_test.yaml | 6 +++--- .github/workflows/test.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 5a995f47d3..ba4f0502d3 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -1,8 +1,7 @@ name: End-to-End Test on: - # pull_request: - workflow_dispatch: + pull_request: jobs: build-charm: @@ -23,7 +22,6 @@ jobs: security.nesting: true security.privileged: true raw.lxc: | - lxc.apparmor.profile=unconfined lxc.mount.auto=proc:rw sys:rw cgroup:rw lxc.cgroup.devices.allow=a lxc.cap.drop= @@ -270,6 +268,8 @@ jobs: run: | [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ || cat /home/ubuntu/.docker/config.json | grep httpProxy + - name: Get status of snapd + run: sudo systemctl status snapd - name: Install microk8s # TMP: Pin version of microk8s to older version due to issues with running microk8s in LXD # containers. Currently, in discussion with microk8s developer on the issue. diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd939f2d50..e06819935d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - pull_request: + # pull_request: workflow_dispatch: jobs: From 82b060d5caf70f902cfc8278aa3c69963b005f6b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:31:22 +0800 Subject: [PATCH 064/343] Trigger workflow --- .github/workflows/e2e_test.yaml | 3 ++- .github/workflows/test.yaml | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index ba4f0502d3..f0ceccd94f 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -1,7 +1,8 @@ name: End-to-End Test on: - pull_request: + # pull_request: + workflow_dispatch: jobs: build-charm: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e06819935d..bf31f221b2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,10 +8,9 @@ jobs: ssh: runs-on: [self-hosted, github-runner] steps: - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true + - run: echo Hello + - if: always() + run: echo world # test: # runs-on: [self-hosted, large] # steps: @@ -21,3 +20,7 @@ jobs: # verbosity: trace # - if: failure() # run: sleep 10000 + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true From e21328c986d100d58f61f36657a7b6b0bbe3bdd9 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:33:41 +0800 Subject: [PATCH 065/343] Trigger workflow --- .github/workflows/manual_test.yaml | 3 ++- .github/workflows/test.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 29ecba6d29..db17a6bb41 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,7 +1,8 @@ name: Manual test on: - pull_request: + # pull_request: + workflow_dispatch: jobs: test-proxy: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bf31f221b2..99fdd27320 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - # pull_request: + pull_request: workflow_dispatch: jobs: From 0bf767d342e8b9fbd745bda9b3503567909a8a02 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:31:34 +0800 Subject: [PATCH 066/343] Trigger some xlarge runners --- .github/workflows/manual_test.yaml | 15 ++++++--------- .github/workflows/test.yaml | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index db17a6bb41..86bcbccd8b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,19 +1,16 @@ name: Manual test on: - # pull_request: + pull_request: workflow_dispatch: jobs: test-proxy: name: test proxy + strategy: + matrix: + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] runs-on: [self-hosted, xlarge] steps: - - name: curl - run: curl -vvv https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url - - name: Install requests - if: ${{ success() || failure() }} - run: python3 -m pip install requests - - name: Make HTTP request with Python - if: ${{ success() || failure() }} - run: python3 -c "import logging;logging.basicConfig(level=logging.DEBUG);import requests;resp=requests.get('https://s3.amazonaws.com/data-charms-testing?list-type=2&prefix=mysql%2F&delimiter=%2F&encoding-type=url', proxies={'https':'http://squid.internal:3128'});print(resp.content)" + - name: Echo + runs: echo "Hello number ${{ matrix.num }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 99fdd27320..bf31f221b2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - pull_request: + # pull_request: workflow_dispatch: jobs: From 558d1d48265bb4560de77be4090508cf23fa3d84 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:34:08 +0800 Subject: [PATCH 067/343] Trigger xlarge runners --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 86bcbccd8b..40f33b3dd4 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,4 +13,4 @@ jobs: runs-on: [self-hosted, xlarge] steps: - name: Echo - runs: echo "Hello number ${{ matrix.num }}" + run: echo "Hello number ${{ matrix.num }}" From 6ae837cc9a4ec3517ea585f345e3d1db47b705c2 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:28:08 +0800 Subject: [PATCH 068/343] Add push test --- .github/workflows/manual_test.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 40f33b3dd4..215069ff13 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,16 +1,14 @@ name: Manual test on: - pull_request: + push: + # pull_request: workflow_dispatch: jobs: - test-proxy: - name: test proxy - strategy: - matrix: - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - runs-on: [self-hosted, xlarge] + test: + name: test + runs-on: [self-hosted, github-runner] steps: - name: Echo - run: echo "Hello number ${{ matrix.num }}" + run: echo "Hello push" From f643309ccce32089d5fc926e6d58f2b09c4da3f8 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:28:53 +0800 Subject: [PATCH 069/343] Remove push test --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 215069ff13..5eb4c9c874 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,7 +1,7 @@ name: Manual test on: - push: + # push: # pull_request: workflow_dispatch: From cc073fd89d0b20b484cfb27a3c150c1e78c50ecf Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 4 Sep 2023 15:29:09 +0800 Subject: [PATCH 070/343] Test push support --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5eb4c9c874..2964f2d368 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,14 +1,14 @@ name: Manual test on: - # push: + push: # pull_request: workflow_dispatch: jobs: test: name: test - runs-on: [self-hosted, github-runner] + runs-on: [self-hosted, large] steps: - name: Echo run: echo "Hello push" From 65c2c65c21b830c993efe3666ef672479c87a278 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 4 Sep 2023 15:30:05 +0800 Subject: [PATCH 071/343] Remove push test --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 2964f2d368..8c2b19514a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -1,7 +1,7 @@ name: Manual test on: - push: + # push: # pull_request: workflow_dispatch: From 124531ea5553755958e26a5265f2aafaccfd8106 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 7 Sep 2023 12:48:31 +0800 Subject: [PATCH 072/343] Test github runner disk usage --- .github/workflows/manual_test.yaml | 9 ++++++--- .github/workflows/test.yaml | 9 ++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8c2b19514a..0a98088314 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,13 +2,16 @@ name: Manual test on: # push: - # pull_request: + pull_request: workflow_dispatch: jobs: test: name: test - runs-on: [self-hosted, large] + runs-on: [self-hosted, kernel] + strategy: + matrix: + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] steps: - name: Echo - run: echo "Hello push" + run: echo "Hello runner number ${{ matrix.num }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bf31f221b2..8f0d24d987 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,16 +1,15 @@ name: Tests on: - # pull_request: + pull_request: workflow_dispatch: jobs: ssh: - runs-on: [self-hosted, github-runner] + runs-on: ubuntu-latest steps: - - run: echo Hello - - if: always() - run: echo world + - run: Disk usage + run: df -h # test: # runs-on: [self-hosted, large] # steps: From 263aa87563ef24404980625f12bec94e87eb21cf Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 7 Sep 2023 12:50:45 +0800 Subject: [PATCH 073/343] Test github runner disk usage --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f0d24d987..52c5a3187f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -8,7 +8,7 @@ jobs: ssh: runs-on: ubuntu-latest steps: - - run: Disk usage + - name: Disk usage run: df -h # test: # runs-on: [self-hosted, large] From ac4a22b40f2317f3260ad51c097fd24fe69669db Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Sep 2023 11:43:09 +0800 Subject: [PATCH 074/343] Test runner --- .github/workflows/test.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 52c5a3187f..a2272d1aaf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,9 +5,14 @@ on: workflow_dispatch: jobs: - ssh: - runs-on: ubuntu-latest + test: + runs-on: [self-hosted, github-runner] + strategy: + matrix: + num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: + - name: Echo test + run: echo hello - name: Disk usage run: df -h # test: From f80885b3060b202751432130161d171ad166dac0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 13 Sep 2023 12:26:58 +0800 Subject: [PATCH 075/343] Test tmate --- .github/workflows/manual_test.yaml | 11 +++++------ .github/workflows/test.yaml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0a98088314..c8838e4bd0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,10 +8,9 @@ on: jobs: test: name: test - runs-on: [self-hosted, kernel] - strategy: - matrix: - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + runs-on: [self-hosted, large] steps: - - name: Echo - run: echo "Hello runner number ${{ matrix.num }}" + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a2272d1aaf..afbe76d780 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - pull_request: + # pull_request: workflow_dispatch: jobs: From 2826a746fed49d896d4dc6085b5f5a923e03d562 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 14 Sep 2023 14:29:30 +0800 Subject: [PATCH 076/343] Test runner --- .github/workflows/manual_test.yaml | 2 +- .github/workflows/test.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c8838e4bd0..844c24cde4 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - pull_request: + #pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index afbe76d780..2684622003 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,12 +1,12 @@ name: Tests on: - # pull_request: + pull_request: workflow_dispatch: jobs: test: - runs-on: [self-hosted, github-runner] + runs-on: [self-hosted, test] strategy: matrix: num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From fdfc4e467d96947e630c5e6a76d97b72de992365 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 18 Sep 2023 15:20:07 +0800 Subject: [PATCH 077/343] Test github-runner --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2684622003..a2272d1aaf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ on: jobs: test: - runs-on: [self-hosted, test] + runs-on: [self-hosted, github-runner] strategy: matrix: num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From 2ff6a43591555e5cd8e4465630dd6c1217f502a5 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 18 Sep 2023 15:59:53 +0800 Subject: [PATCH 078/343] Test e2e test --- .github/workflows/e2e_test.yaml | 6 +++++- .github/workflows/manual_test.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index bf08d84d7c..26f77bad4c 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -1,7 +1,7 @@ name: End-to-End Test on: - # pull_request: + pull_request: workflow_dispatch: jobs: @@ -276,6 +276,10 @@ jobs: || cat /home/ubuntu/.docker/config.json | grep httpProxy - name: Get status of snapd run: sudo systemctl status snapd + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true - name: Install microk8s # TMP: Pin version of microk8s to older version due to issues with running microk8s in LXD # containers. Currently, in discussion with microk8s developer on the issue. diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 844c24cde4..0260ee208b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - #pull_request: + # pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a2272d1aaf..afbe76d780 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - pull_request: + # pull_request: workflow_dispatch: jobs: From 5507544a1fd12ddf5316119ad648a513e7ef9eb8 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:36:37 +0800 Subject: [PATCH 079/343] Test --- .github/workflows/e2e_test.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 26f77bad4c..e0b3441d74 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -78,10 +78,6 @@ jobs: event: - name: pull_request abbreviation: pr - - name: workflow_dispatch - abbreviation: wd - - name: push - abbreviation: push steps: - name: Setup Lxd Juju Controller uses: charmed-kubernetes/actions-operator@main From 90fd983a3c9b12d1b7e4d674fac8beb66526c70f Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 19 Sep 2023 10:55:25 +0800 Subject: [PATCH 080/343] Test with tmate --- .github/workflows/e2e_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 5798356ee0..8fac423aaf 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -465,12 +465,12 @@ jobs: run: | [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ || cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Get status of snapd - run: sudo systemctl status snapd - name: Setup tmate session uses: mxschmitt/action-tmate@v3 with: limit-access-to-actor: true + - name: Get status of snapd + run: sudo systemctl status snapd - name: Install microk8s # TMP: Pin version of microk8s to older version due to issues with running microk8s in LXD # containers. Currently, in discussion with microk8s developer on the issue. From a78dc0429728a76b756ccfbeab12e263555fc116 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:51:44 +0800 Subject: [PATCH 081/343] Test snap fix --- .github/workflows/e2e_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 8fac423aaf..6e757685ad 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -444,6 +444,8 @@ jobs: needs: [build-charm, run-id] runs-on: [self-hosted, linux, x64, "pr-${{ needs.run-id.outputs.run-id }}"] steps: + - name: Fix snap issue in privileged LXD containers + run: ln -s /bin/true /usr/local/bin/udevadm # below is a series of simple tests to assess the functionality of the newly spawned runner. - name: Echo hello world run: echo "hello world" From 819cd02b4012c6d12f326ad61d22108ef89682a0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:10:42 +0800 Subject: [PATCH 082/343] Test unconfined --- .github/workflows/e2e_test.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 6e757685ad..7c6f36f49c 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -23,6 +23,7 @@ jobs: security.nesting: true security.privileged: true raw.lxc: | + lxc.apparmor.profile=unconfined lxc.mount.auto=proc:rw sys:rw cgroup:rw lxc.cgroup.devices.allow=a lxc.cap.drop= @@ -467,16 +468,12 @@ jobs: run: | [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ || cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - name: Get status of snapd run: sudo systemctl status snapd - name: Install microk8s # TMP: Pin version of microk8s to older version due to issues with running microk8s in LXD # containers. Currently, in discussion with microk8s developer on the issue. - run: sudo snap install microk8s --classic --channel=1.24/stable + run: sudo snap install microk8s --classic - name: Wait for microk8s timeout-minutes: 10 run: microk8s status --wait-ready @@ -509,3 +506,8 @@ jobs: run: | HOST_IP=$(ip route | grep default | cut -f 3 -d" ") [ $((ping $HOST_IP -c 5 || :) | grep "Destination Port Unreachable" | wc -l) -eq 5 ] + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: failure() + with: + limit-access-to-actor: true From 7d01568fb1f6a8ffd265df1e52db781d1713f336 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:42:25 +0800 Subject: [PATCH 083/343] Test --- .github/workflows/manual_test.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0260ee208b..a43267cbde 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - # pull_request: + pull_request: workflow_dispatch: jobs: @@ -10,7 +10,9 @@ jobs: name: test runs-on: [self-hosted, large] steps: - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true + - name: test + run: echo "hello world" + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true From ed427cb56dcda32abb439767e44ff4dfd1a6ac5d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 13 Oct 2023 12:53:19 +0800 Subject: [PATCH 084/343] Trigger manual test --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a43267cbde..d4a5725301 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: test: name: test - runs-on: [self-hosted, large] + runs-on: [self-hosted, github-runner] steps: - name: test run: echo "hello world" From 3b0ec2f5d008c3a39d4ced6bf9ed81fbb8291c6e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 20 Oct 2023 08:42:29 +0800 Subject: [PATCH 085/343] Test DOCKERHUB_MIRROR env --- .github/workflows/manual_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d4a5725301..0e41f36991 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,8 +8,10 @@ on: jobs: test: name: test - runs-on: [self-hosted, github-runner] + runs-on: [self-hosted, test] steps: + - name: env + run: echo $DOCKERHUB_MIRROR - name: test run: echo "hello world" # - name: Setup tmate session From f51282e9a9f112ef9f25d4607ed96602a3d71308 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:13:07 +0800 Subject: [PATCH 086/343] Test rust --- .github/workflows/manual_test.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0e41f36991..e9e76c6b0b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,12 +8,10 @@ on: jobs: test: name: test - runs-on: [self-hosted, test] + runs-on: [self-hosted, large] steps: - - name: env - run: echo $DOCKERHUB_MIRROR - - name: test - run: echo "hello world" + - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + - run: cargo install hello-world # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 # with: From 2ea898a466afc60de8dd139e2664bea815fb1161 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:14:44 +0800 Subject: [PATCH 087/343] Test rust install ripgrep --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e9e76c6b0b..4258652d13 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted, large] steps: - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - - run: cargo install hello-world + - run: cargo install ripgrep # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 # with: From ffe94115d14a833c3f0816d0cefa15b31c85e3dd Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:28:35 +0800 Subject: [PATCH 088/343] Fix rustup --- .github/workflows/manual_test.yaml | 3 +- rustup.sh | 731 +++++++++++++++++++++++++++++ 2 files changed, 733 insertions(+), 1 deletion(-) create mode 100644 rustup.sh diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4258652d13..e78e0a4395 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -10,7 +10,8 @@ jobs: name: test runs-on: [self-hosted, large] steps: - - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh + - run: sh rustup.sh -y - run: cargo install ripgrep # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 diff --git a/rustup.sh b/rustup.sh new file mode 100644 index 0000000000..ddc99f2472 --- /dev/null +++ b/rustup.sh @@ -0,0 +1,731 @@ +#!/bin/sh +# shellcheck shell=dash + +# This is just a little script that can be downloaded from the internet to +# install rustup. It just does platform detection, downloads the installer +# and runs it. + +# It runs on Unix shells like {a,ba,da,k,z}sh. It uses the common `local` +# extension. Note: Most shells limit `local` to 1 var per line, contra bash. + +if [ "$KSH_VERSION" = 'Version JM 93t+ 2010-03-05' ]; then + # The version of ksh93 that ships with many illumos systems does not + # support the "local" extension. Print a message rather than fail in + # subtle ways later on: + echo 'rustup does not work with this ksh93 version; please try bash!' >&2 + exit 1 +fi + + +set -u + +# If RUSTUP_UPDATE_ROOT is unset or empty, default it. +RUSTUP_UPDATE_ROOT="${RUSTUP_UPDATE_ROOT:-https://static.rust-lang.org/rustup}" + +# NOTICE: If you change anything here, please make the same changes in setup_mode.rs +usage() { + cat < + Choose a default host triple + + --default-toolchain + Choose a default toolchain to install. Use 'none' to not install any toolchains at all + + --profile + [default: default] [possible values: minimal, default, complete] + + -c, --component ... + Component name to also install + + -t, --target ... + Target name to also install + + --no-update-default-toolchain + Don't update any existing default toolchain after install + + --no-modify-path + Don't configure the PATH environment variable + + -h, --help + Print help information + + -V, --version + Print version information +EOF +} + +main() { + downloader --check + need_cmd uname + need_cmd mktemp + need_cmd chmod + need_cmd mkdir + need_cmd rm + need_cmd rmdir + + get_architecture || return 1 + local _arch="$RETVAL" + assert_nz "$_arch" "arch" + + local _ext="" + case "$_arch" in + *windows*) + _ext=".exe" + ;; + esac + + local _url="${RUSTUP_UPDATE_ROOT}/dist/${_arch}/rustup-init${_ext}" + + local _dir + if ! _dir="$(ensure mktemp -d)"; then + # Because the previous command ran in a subshell, we must manually + # propagate exit status. + exit 1 + fi + local _file="${_dir}/rustup-init${_ext}" + + local _ansi_escapes_are_valid=false + if [ -t 2 ]; then + if [ "${TERM+set}" = 'set' ]; then + case "$TERM" in + xterm*|rxvt*|urxvt*|linux*|vt*) + _ansi_escapes_are_valid=true + ;; + esac + fi + fi + + # check if we have to use /dev/tty to prompt the user + local need_tty=yes + for arg in "$@"; do + case "$arg" in + --help) + usage + exit 0 + ;; + *) + OPTIND=1 + if [ "${arg%%--*}" = "" ]; then + # Long option (other than --help); + # don't attempt to interpret it. + continue + fi + while getopts :hy sub_arg "$arg"; do + case "$sub_arg" in + h) + usage + exit 0 + ;; + y) + # user wants to skip the prompt -- + # we don't need /dev/tty + need_tty=no + ;; + *) + ;; + esac + done + ;; + esac + done + + if $_ansi_escapes_are_valid; then + printf "\33[1minfo:\33[0m downloading installer\n" 1>&2 + else + printf '%s\n' 'info: downloading installer' 1>&2 + fi + + ensure mkdir -p "$_dir" + ensure downloader "$_url" "$_file" "$_arch" + ensure chmod u+x "$_file" + if [ ! -x "$_file" ]; then + printf '%s\n' "Cannot execute $_file (likely because of mounting /tmp as noexec)." 1>&2 + printf '%s\n' "Please copy the file to a location where you can execute binaries and run ./rustup-init${_ext}." 1>&2 + exit 1 + fi + + if [ "$need_tty" = "yes" ] && [ ! -t 0 ]; then + # The installer is going to want to ask for confirmation by + # reading stdin. This script was piped into `sh` though and + # doesn't have stdin to pass to its children. Instead we're going + # to explicitly connect /dev/tty to the installer's stdin. + if [ ! -t 1 ]; then + err "Unable to run interactively. Run with -y to accept defaults, --help for additional options" + fi + + ignore "$_file" "$@" < /dev/tty + else + ignore "$_file" "$@" + fi + + local _retval=$? + + ignore rm "$_file" + ignore rmdir "$_dir" + + return "$_retval" +} + +check_proc() { + # Check for /proc by looking for the /proc/self/exe link + # This is only run on Linux + if ! test -L /proc/self/exe ; then + err "fatal: Unable to find /proc/self/exe. Is /proc mounted? Installation cannot proceed without /proc." + fi +} + +get_bitness() { + need_cmd head + # Architecture detection without dependencies beyond coreutils. + # ELF files start out "\x7fELF", and the following byte is + # 0x01 for 32-bit and + # 0x02 for 64-bit. + # The printf builtin on some shells like dash only supports octal + # escape sequences, so we use those. + local _current_exe_head + _current_exe_head=$(head -c 5 /proc/self/exe ) + if [ "$_current_exe_head" = "$(printf '\177ELF\001')" ]; then + echo 32 + elif [ "$_current_exe_head" = "$(printf '\177ELF\002')" ]; then + echo 64 + else + err "unknown platform bitness" + fi +} + +is_host_amd64_elf() { + need_cmd head + need_cmd tail + # ELF e_machine detection without dependencies beyond coreutils. + # Two-byte field at offset 0x12 indicates the CPU, + # but we're interested in it being 0x3E to indicate amd64, or not that. + local _current_exe_machine + _current_exe_machine=$(head -c 19 /proc/self/exe | tail -c 1) + [ "$_current_exe_machine" = "$(printf '\076')" ] +} + +get_endianness() { + local cputype=$1 + local suffix_eb=$2 + local suffix_el=$3 + + # detect endianness without od/hexdump, like get_bitness() does. + need_cmd head + need_cmd tail + + local _current_exe_endianness + _current_exe_endianness="$(head -c 6 /proc/self/exe | tail -c 1)" + if [ "$_current_exe_endianness" = "$(printf '\001')" ]; then + echo "${cputype}${suffix_el}" + elif [ "$_current_exe_endianness" = "$(printf '\002')" ]; then + echo "${cputype}${suffix_eb}" + else + err "unknown platform endianness" + fi +} + +get_architecture() { + local _ostype _cputype _bitness _arch _clibtype + _ostype="$(uname -s)" + _cputype="$(uname -m)" + _clibtype="gnu" + + if [ "$_ostype" = Linux ]; then + if [ "$(uname -o)" = Android ]; then + _ostype=Android + fi + if ldd --version 2>&1 | grep -q 'musl'; then + _clibtype="musl" + fi + fi + + if [ "$_ostype" = Darwin ] && [ "$_cputype" = i386 ]; then + # Darwin `uname -m` lies + if sysctl hw.optional.x86_64 | grep -q ': 1'; then + _cputype=x86_64 + fi + fi + + if [ "$_ostype" = SunOS ]; then + # Both Solaris and illumos presently announce as "SunOS" in "uname -s" + # so use "uname -o" to disambiguate. We use the full path to the + # system uname in case the user has coreutils uname first in PATH, + # which has historically sometimes printed the wrong value here. + if [ "$(/usr/bin/uname -o)" = illumos ]; then + _ostype=illumos + fi + + # illumos systems have multi-arch userlands, and "uname -m" reports the + # machine hardware name; e.g., "i86pc" on both 32- and 64-bit x86 + # systems. Check for the native (widest) instruction set on the + # running kernel: + if [ "$_cputype" = i86pc ]; then + _cputype="$(isainfo -n)" + fi + fi + + case "$_ostype" in + + Android) + _ostype=linux-android + ;; + + Linux) + check_proc + _ostype=unknown-linux-$_clibtype + _bitness=$(get_bitness) + ;; + + FreeBSD) + _ostype=unknown-freebsd + ;; + + NetBSD) + _ostype=unknown-netbsd + ;; + + DragonFly) + _ostype=unknown-dragonfly + ;; + + Darwin) + _ostype=apple-darwin + ;; + + illumos) + _ostype=unknown-illumos + ;; + + MINGW* | MSYS* | CYGWIN* | Windows_NT) + _ostype=pc-windows-gnu + ;; + + *) + err "unrecognized OS type: $_ostype" + ;; + + esac + + case "$_cputype" in + + i386 | i486 | i686 | i786 | x86) + _cputype=i686 + ;; + + xscale | arm) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + fi + ;; + + armv6l) + _cputype=arm + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + armv7l | armv8l) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + + aarch64 | arm64) + _cputype=aarch64 + ;; + + x86_64 | x86-64 | x64 | amd64) + _cputype=x86_64 + ;; + + mips) + _cputype=$(get_endianness mips '' el) + ;; + + mips64) + if [ "$_bitness" -eq 64 ]; then + # only n64 ABI is supported for now + _ostype="${_ostype}abi64" + _cputype=$(get_endianness mips64 '' el) + fi + ;; + + ppc) + _cputype=powerpc + ;; + + ppc64) + _cputype=powerpc64 + ;; + + ppc64le) + _cputype=powerpc64le + ;; + + s390x) + _cputype=s390x + ;; + riscv64) + _cputype=riscv64gc + ;; + loongarch64) + _cputype=loongarch64 + ;; + *) + err "unknown CPU type: $_cputype" + + esac + + # Detect 64-bit linux with 32-bit userland + if [ "${_ostype}" = unknown-linux-gnu ] && [ "${_bitness}" -eq 32 ]; then + case $_cputype in + x86_64) + if [ -n "${RUSTUP_CPUTYPE:-}" ]; then + _cputype="$RUSTUP_CPUTYPE" + else { + # 32-bit executable for amd64 = x32 + if is_host_amd64_elf; then { + echo "This host is running an x32 userland; as it stands, x32 support is poor," 1>&2 + echo "and there isn't a native toolchain -- you will have to install" 1>&2 + echo "multiarch compatibility with i686 and/or amd64, then select one" 1>&2 + echo "by re-running this script with the RUSTUP_CPUTYPE environment variable" 1>&2 + echo "set to i686 or x86_64, respectively." 1>&2 + echo 1>&2 + echo "You will be able to add an x32 target after installation by running" 1>&2 + echo " rustup target add x86_64-unknown-linux-gnux32" 1>&2 + exit 1 + }; else + _cputype=i686 + fi + }; fi + ;; + mips64) + _cputype=$(get_endianness mips '' el) + ;; + powerpc64) + _cputype=powerpc + ;; + aarch64) + _cputype=armv7 + if [ "$_ostype" = "linux-android" ]; then + _ostype=linux-androideabi + else + _ostype="${_ostype}eabihf" + fi + ;; + riscv64gc) + err "riscv64 with 32-bit userland unsupported" + ;; + esac + fi + + # Detect armv7 but without the CPU features Rust needs in that build, + # and fall back to arm. + # See https://github.com/rust-lang/rustup.rs/issues/587. + if [ "$_ostype" = "unknown-linux-gnueabihf" ] && [ "$_cputype" = armv7 ]; then + if ensure grep '^Features' /proc/cpuinfo | grep -q -v neon; then + # At least one processor does not have NEON. + _cputype=arm + fi + fi + + _arch="${_cputype}-${_ostype}" + + RETVAL="$_arch" +} + +say() { + printf 'rustup: %s\n' "$1" +} + +err() { + say "$1" >&2 + exit 1 +} + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (command not found)" + fi +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 +} + +assert_nz() { + if [ -z "$1" ]; then err "assert_nz $2"; fi +} + +# Run a command that should never fail. If the command fails execution +# will immediately terminate with an error showing the failing +# command. +ensure() { + if ! "$@"; then err "command failed: $*"; fi +} + +# This is just for indicating that commands' results are being +# intentionally ignored. Usually, because it's being executed +# as part of error handling. +ignore() { + "$@" +} + +# This wraps curl or wget. Try curl first, if not installed, +# use wget instead. +downloader() { + local _dld + local _ciphersuites + local _err + local _status + local _retry + if check_cmd curl; then + _dld=curl + elif check_cmd wget; then + _dld=wget + else + _dld='curl or wget' # to be used in error message of need_cmd + fi + + if [ "$1" = --check ]; then + need_cmd "$_dld" + elif [ "$_dld" = curl ]; then + check_curl_for_retry_support + _retry="$RETVAL" + get_ciphersuites_for_curl + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" curl --proto --tlsv1.2; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + fi + fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q 404$; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status + elif [ "$_dld" = wget ]; then + if [ "$(wget -V 2>&1|head -2|tail -1|cut -f1 -d" ")" = "BusyBox" ]; then + echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + get_ciphersuites_for_wget + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" wget --https-only --secure-protocol; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) + _status=$? + fi + fi + fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q ' 404 Not Found$'; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status + else + err "Unknown downloader" # should not reach here + fi +} + +check_help_for() { + local _arch + local _cmd + local _arg + _arch="$1" + shift + _cmd="$1" + shift + + local _category + if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then + _category="all" + else + _category="" + fi + + case "$_arch" in + + *darwin*) + if check_cmd sw_vers; then + case $(sw_vers -productVersion) in + 10.*) + # If we're running on macOS, older than 10.13, then we always + # fail to find these options to force fallback + if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then + # Older than 10.13 + echo "Warning: Detected macOS platform older than 10.13" + return 1 + fi + ;; + 11.*) + # We assume Big Sur will be OK for now + ;; + *) + # Unknown product version, warn and continue + echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" + echo "Warning TLS capabilities detection may fail" + ;; + esac + fi + ;; + + esac + + for _arg in "$@"; do + if ! "$_cmd" --help "$_category" | grep -q -- "$_arg"; then + return 1 + fi + done + + true # not strictly needed +} + +# Check if curl supports the --retry flag, then pass it to the curl invocation. +check_curl_for_retry_support() { + local _retry_supported="" + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--retry"; then + _retry_supported="--retry 3" + if check_help_for "notspecified" "curl" "--continue-at"; then + # "-C -" tells curl to automatically find where to resume the download when retrying. + _retry_supported="--retry 3 -C -" + fi + fi + + RETVAL="$_retry_supported" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these curl backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_curl() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _openssl_syntax="no" + local _gnutls_syntax="no" + local _backend_supported="yes" + if curl -V | grep -q ' OpenSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' LibreSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' BoringSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' GnuTLS/'; then + _gnutls_syntax="yes" + else + _backend_supported="no" + fi + + local _args_supported="no" + if [ "$_backend_supported" = "yes" ]; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then + _args_supported="yes" + fi + fi + + local _cs="" + if [ "$_args_supported" = "yes" ]; then + if [ "$_openssl_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "openssl") + elif [ "$_gnutls_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these wget backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_wget() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _cs="" + if wget -V | grep -q '\-DHAVE_LIBSSL'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "openssl") + fi + elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 +# excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad +# DH params often found on servers (see RFC 7919). Sequence matches or is +# similar to Firefox 68 ESR with weak cipher suites disabled via about:config. +# $1 must be openssl or gnutls. +get_strong_ciphersuites_for() { + if [ "$1" = "openssl" ]; then + # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. + echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" + elif [ "$1" = "gnutls" ]; then + # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. + # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. + echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" + fi +} + +main "$@" || exit 1 From 7583777134287c61bb8bbfd9e3300d4fc12e316f Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:32:01 +0800 Subject: [PATCH 089/343] Fix cargo path --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e78e0a4395..9bd011a335 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -12,7 +12,7 @@ jobs: steps: - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh - run: sh rustup.sh -y - - run: cargo install ripgrep + - run: $HOME/.cargo/bin/cargo install ripgrep # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 # with: From 7e5824699f151d53e1a71d9c0334ffff7cb9aded Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 20 Oct 2023 09:34:06 +0800 Subject: [PATCH 090/343] Test dockerhub mirrort push --- .github/workflows/e2e_test.yaml | 2 +- .github/workflows/manual_test.yaml | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index acc24cba7a..67297c4cd6 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -1,7 +1,7 @@ name: End-to-End Test on: - pull_request: + # pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 9bd011a335..a15ef92a56 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,11 +8,13 @@ on: jobs: test: name: test - runs-on: [self-hosted, large] + runs-on: [self-hosted, test] steps: - - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh - - run: sh rustup.sh -y - - run: $HOME/.cargo/bin/cargo install ripgrep + - run: echo $DOCKERHUB_MIRROR + - run: docker run hello-world + # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh + # - run: sh rustup.sh -y + # - run: $HOME/.cargo/bin/cargo install ripgrep # - name: Setup tmate session # uses: mxschmitt/action-tmate@v3 # with: From 7f79715bcc4750442a5547e38940675b1c838b12 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:31:36 +0800 Subject: [PATCH 091/343] Test github runners --- .github/workflows/manual_test.yaml | 2 +- .github/workflows/test.yaml | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a15ef92a56..d44828b132 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - pull_request: + # pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index afbe76d780..a76c12a5bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -5,17 +5,24 @@ on: workflow_dispatch: jobs: - test: - runs-on: [self-hosted, github-runner] - strategy: - matrix: - num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + github-test: + runs-on: ubuntu steps: - name: Echo test run: echo hello - name: Disk usage run: df -h # test: + # runs-on: [self-hosted, github-runner] + # strategy: + # matrix: + # num: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # steps: + # - name: Echo test + # run: echo hello + # - name: Disk usage + # run: df -h + # test: # runs-on: [self-hosted, large] # steps: # - uses: canonical/craft-actions/rockcraft-pack@main From c78b9a799282e3881132fe58c630d551d191d8d1 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 Nov 2023 16:46:58 +0800 Subject: [PATCH 092/343] Test GitHub runner --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a76c12a5bb..2d16ff367d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - # pull_request: + pull_request: workflow_dispatch: jobs: From a56dc930384f58f557296544bce83f3d9438ee65 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 Nov 2023 22:18:14 +0800 Subject: [PATCH 093/343] Fix test of github runner --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d16ff367d..0d7c32ebaf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ on: jobs: github-test: - runs-on: ubuntu + runs-on: ubuntu-latest steps: - name: Echo test run: echo hello From 0acdecac04929754fe05db86244864fb2e315e27 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 28 Nov 2023 13:04:27 +0800 Subject: [PATCH 094/343] Run integration test --- .github/workflows/integration_tests.yaml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 91fcf6c8fc..3199408578 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,7 +1,7 @@ name: Integration tests on: - # pull_request: + pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0d7c32ebaf..67abd61d98 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,7 +1,7 @@ name: Tests on: - pull_request: + # pull_request: workflow_dispatch: jobs: From e0e3923444926edd7c228a4295fdcad8ef047717 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:01:59 +0800 Subject: [PATCH 095/343] Test integration test with self-hosted runner --- .github/workflows/integration_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 3199408578..927d8c595e 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -7,7 +7,7 @@ on: jobs: integration-tests-juju2: name: Integration test with juju 2.9 - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@self-hosted-runner-integration-test secrets: inherit with: juju-channel: 2.9/stable @@ -17,7 +17,7 @@ jobs: modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics", "test_self_hosted_runner", "test_charm_with_proxy"]' integration-tests-juju3: name: Integration test with juju 3.1 - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@self-hosted-runner-integration-test secrets: inherit with: juju-channel: 3.1/stable From 085c6ff9806ce8149272f5ca6df8af134fdfe5c3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 28 Nov 2023 19:25:34 +0800 Subject: [PATCH 096/343] Test integration test with self-hosted runner --- .github/workflows/integration_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 927d8c595e..26c8d934ea 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -2,7 +2,6 @@ name: Integration tests on: pull_request: - workflow_dispatch: jobs: integration-tests-juju2: From 898c2a19a3dfba6c0c6934cf4eb6f579cb082137 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 29 Nov 2023 15:56:06 +0800 Subject: [PATCH 097/343] Test free disk space --- .github/workflows/integration_tests.yaml | 3 ++- .github/workflows/manual_test.yaml | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 26c8d934ea..10fdc2eed5 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,7 +1,8 @@ name: Integration tests on: - pull_request: + #pull_request: + workflow_dispatch: jobs: integration-tests-juju2: diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d44828b132..5ee5083686 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,16 +2,25 @@ name: Manual test on: # push: - # pull_request: + pull_request: workflow_dispatch: jobs: - test: - name: test + github-test: + runs-on: ubuntu-latest + steps: + - name: Echo test + run: echo hello + - name: Disk usage + run: df -h + self-host-test: + name: self-hosted test runs-on: [self-hosted, test] steps: - - run: echo $DOCKERHUB_MIRROR - - run: docker run hello-world + - name: Echo test + run: echo hello + - name: Disk usage + run: df -h # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y # - run: $HOME/.cargo/bin/cargo install ripgrep From 8732c1457515b7afbc02f2fa5f53eb6f9f4f6f07 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 29 Nov 2023 17:06:32 +0800 Subject: [PATCH 098/343] Test disk space of runners --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5ee5083686..18c4950d73 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,7 +15,7 @@ jobs: run: df -h self-host-test: name: self-hosted test - runs-on: [self-hosted, test] + runs-on: [self-hosted, large] steps: - name: Echo test run: echo hello From a4409ce74e3799952a3925a907b5adcaf51ddefb Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 5 Dec 2023 16:18:16 +0800 Subject: [PATCH 099/343] Test on github-runner charm --- .github/workflows/integration_tests.yaml | 6 +++++- .github/workflows/manual_test.yaml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 10fdc2eed5..454584f05b 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,7 +1,7 @@ name: Integration tests on: - #pull_request: + pull_request: workflow_dispatch: jobs: @@ -15,6 +15,8 @@ jobs: provider: lxd test-tox-env: integration-juju2.9 modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics", "test_self_hosted_runner", "test_charm_with_proxy"]' + self-hosted-runner: true + self-hosted-runner-label: "github-runner" integration-tests-juju3: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@self-hosted-runner-integration-test @@ -25,3 +27,5 @@ jobs: provider: lxd test-tox-env: integration-juju3.1 modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics", "test_self_hosted_runner", "test_charm_with_proxy"]' + self-hosted-runner: true + self-hosted-runner-label: "github-runner" diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 18c4950d73..23ac927a70 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - pull_request: + # pull_request: workflow_dispatch: jobs: From afe8d92be031be439868847e7e8c47d4e6821dcc Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:38:31 +0800 Subject: [PATCH 100/343] Test integration test on edge runner --- .github/workflows/integration_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index 4bd5780499..aded48dbc8 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -16,7 +16,7 @@ jobs: test-tox-env: integration-juju2.9 modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics", "test_self_hosted_runner", "test_charm_with_proxy"]' self-hosted-runner: true - self-hosted-runner-label: "github-runner" + self-hosted-runner-label: "edge" integration-tests-juju3: name: Integration test with juju 3.1 uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@self-hosted-runner-integration-test @@ -28,4 +28,4 @@ jobs: test-tox-env: integration-juju3.1 modules: '["test_charm_fork_repo", "test_charm_no_runner", "test_charm_scheduled_events", "test_charm_one_runner", "test_charm_metrics", "test_self_hosted_runner", "test_charm_with_proxy"]' self-hosted-runner: true - self-hosted-runner-label: "github-runner" + self-hosted-runner-label: "edge" From 0cbaddfd1eced4e7e76b117ab4b9ec1b4a5618f6 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 8 Dec 2023 17:23:33 +0800 Subject: [PATCH 101/343] Test self-hosted runner --- .github/workflows/integration_tests.yaml | 2 +- .github/workflows/manual_test.yaml | 18 ++++++++++-------- src-docs/charm_state.py.md | 12 ++++++------ src-docs/runner.py.md | 14 +++++--------- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml index aded48dbc8..75cf7e04d7 100644 --- a/.github/workflows/integration_tests.yaml +++ b/.github/workflows/integration_tests.yaml @@ -1,7 +1,7 @@ name: integration-tests on: - pull_request: + #pull_request: workflow_dispatch: jobs: diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 23ac927a70..163885b239 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,17 +2,17 @@ name: Manual test on: # push: - # pull_request: + pull_request: workflow_dispatch: jobs: - github-test: - runs-on: ubuntu-latest - steps: - - name: Echo test - run: echo hello - - name: Disk usage - run: df -h + # github-test: + # runs-on: ubuntu-latest + # steps: + # - name: Echo test + # run: echo hello + # - name: Disk usage + # run: df -h self-host-test: name: self-hosted test runs-on: [self-hosted, large] @@ -21,6 +21,8 @@ jobs: run: echo hello - name: Disk usage run: df -h + - name: sleep + run: sleep 60m # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y # - run: $HOME/.cargo/bin/cargo install ripgrep diff --git a/src-docs/charm_state.py.md b/src-docs/charm_state.py.md index e41bba00d3..634235683f 100644 --- a/src-docs/charm_state.py.md +++ b/src-docs/charm_state.py.md @@ -32,7 +32,7 @@ Raised when charm config is invalid. - `msg`: Explanation of the error. - + ### function `__init__` @@ -77,7 +77,7 @@ Return the aproxy address. --- - + ### classmethod `check_fields` @@ -89,7 +89,7 @@ Validate the proxy configuration. --- - + ### classmethod `from_charm` @@ -122,14 +122,14 @@ The charm state. - `is_metrics_logging_available`: Whether the charm is able to issue metrics. - `proxy_config`: Whether aproxy should be used. - - `arch`: The underlying compute architecture, i.e. x64, amd64 + - `arch`: The underlying compute architecture, i.e. x86_64, amd64, arm64/aarch64. --- - + ### classmethod `from_charm` @@ -162,7 +162,7 @@ Raised when given machine charm architecture is unsupported. - `arch`: The current machine architecture. - + ### function `__init__` diff --git a/src-docs/runner.py.md b/src-docs/runner.py.md index fe815da2f8..92f38fda25 100644 --- a/src-docs/runner.py.md +++ b/src-docs/runner.py.md @@ -39,7 +39,7 @@ The configuration values for creating a single runner instance. ## class `Runner` Single instance of GitHub self-hosted runner. - + ### function `__init__` @@ -67,7 +67,7 @@ Construct the runner instance. --- - + ### function `create` @@ -81,11 +81,7 @@ Create the runner instance on LXD and register it on GitHub. **Args:** - - `image`: Name of the image to launch the LXD instance with. - - `resources`: Resource setting for the LXD instance. - - `binary_path`: Path to the runner binary. - - `registration_token`: Token for registering the runner on GitHub. - - `arch`: Current machine architecture. + - `config`: The instance config to create the LXD VMs and configure GitHub runner with. @@ -95,7 +91,7 @@ Create the runner instance on LXD and register it on GitHub. --- - + ### function `remove` @@ -121,7 +117,7 @@ Remove this runner instance from LXD and GitHub. --- ## class `Snap` -Snap(name, channel) +This class represents a snap installation. From 220539599391f104110633171407f92a256d60cf Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:56:07 +0800 Subject: [PATCH 102/343] Test on self-hosted runner --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 163885b239..91ade7cae6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,7 +15,7 @@ jobs: # run: df -h self-host-test: name: self-hosted test - runs-on: [self-hosted, large] + runs-on: [self-hosted, github-runner] steps: - name: Echo test run: echo hello From 1c8732df80c8d428267c2a4d9042796edc60cd9c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:01:44 +0800 Subject: [PATCH 103/343] Test microk8s docker registry --- .github/workflows/manual_test.yaml | 6 ++++-- src-docs/runner.py.md | 5 ----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 91ade7cae6..8aac638daa 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,13 +15,15 @@ jobs: # run: df -h self-host-test: name: self-hosted test - runs-on: [self-hosted, github-runner] + runs-on: [self-hosted, large] steps: - name: Echo test run: echo hello - name: Disk usage run: df -h - - name: sleep + - name: Setup operator environment + uses: yhaliaw/actions-operator@microk8s-docker-registry + - name: Wait run: sleep 60m # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y diff --git a/src-docs/runner.py.md b/src-docs/runner.py.md index 151434d0ab..5d00bd21fe 100644 --- a/src-docs/runner.py.md +++ b/src-docs/runner.py.md @@ -39,7 +39,6 @@ The configuration values for creating a single runner instance. ## class `Runner` Single instance of GitHub self-hosted runner. - ### function `__init__` @@ -68,10 +67,8 @@ Construct the runner instance. --- - - ### function `create` ```python @@ -94,10 +91,8 @@ Create the runner instance on LXD and register it on GitHub. --- - - ### function `remove` ```python From ea9fab2b0ed35ae34776d33981321948ea68f150 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:25:38 +0800 Subject: [PATCH 104/343] Test microk8s docker registry --- .github/workflows/manual_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8aac638daa..faa1f921ab 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -23,6 +23,9 @@ jobs: run: df -h - name: Setup operator environment uses: yhaliaw/actions-operator@microk8s-docker-registry + with: + provider: microk8s + microk8s-docker-registry: https://github-runner-dockerhub-cache.canonical.com:5000 - name: Wait run: sleep 60m # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh From 995e1e1dc2cf108aa33a8ff7278480afcf59c6bd Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 13 Dec 2023 09:55:52 +0800 Subject: [PATCH 105/343] Test DOCKHUB_MIRROR --- .github/workflows/manual_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index faa1f921ab..5c5a7dcefd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -21,6 +21,8 @@ jobs: run: echo hello - name: Disk usage run: df -h + - name: Echo env var + run: echo $DOCKERHUB_MIRROR - name: Setup operator environment uses: yhaliaw/actions-operator@microk8s-docker-registry with: From ff572dbb09fad984d008b274dbafb2f411e79845 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:41:25 +0800 Subject: [PATCH 106/343] Test the above on github-runner --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5c5a7dcefd..be169a9455 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,7 +15,7 @@ jobs: # run: df -h self-host-test: name: self-hosted test - runs-on: [self-hosted, large] + runs-on: [self-hosted, github-runner] steps: - name: Echo test run: echo hello From 6e84babefd78e983d6ea9a57c919c4509918d39d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:10:36 +0800 Subject: [PATCH 107/343] Test with env var --- .github/workflows/manual_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index be169a9455..d6f8c98ad8 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -27,7 +27,6 @@ jobs: uses: yhaliaw/actions-operator@microk8s-docker-registry with: provider: microk8s - microk8s-docker-registry: https://github-runner-dockerhub-cache.canonical.com:5000 - name: Wait run: sleep 60m # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh From 43bf2eda075bbfebd122d65c56894cfff0969c7b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 15 Dec 2023 11:16:13 +0800 Subject: [PATCH 108/343] Test on edge --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d6f8c98ad8..5d3a5eebce 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,7 +15,7 @@ jobs: # run: df -h self-host-test: name: self-hosted test - runs-on: [self-hosted, github-runner] + runs-on: [self-hosted, edge] steps: - name: Echo test run: echo hello From 0d0627a14c130f9e3152b3e83262331a6b8118f2 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 3 Jan 2024 13:27:03 +0800 Subject: [PATCH 109/343] Test charmcraft pack --- .github/workflows/manual_test.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5d3a5eebce..6af31d0d8b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -17,18 +17,12 @@ jobs: name: self-hosted test runs-on: [self-hosted, edge] steps: - - name: Echo test - run: echo hello - - name: Disk usage - run: df -h - - name: Echo env var - run: echo $DOCKERHUB_MIRROR - - name: Setup operator environment - uses: yhaliaw/actions-operator@microk8s-docker-registry - with: - provider: microk8s - - name: Wait - run: sleep 60m + - uses: actions/checkout@v4.1.1 + - uses: canonical/setup-lxd@v0.1.1 + - name: Pack charm + run: | + sudo snap install charmcraft --classic --channel latest/stable + charmcraft pack -v # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y # - run: $HOME/.cargo/bin/cargo install ripgrep From 1374897f27d8efdbf1c91d835396f90899b1bc47 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 3 Jan 2024 16:28:45 +0800 Subject: [PATCH 110/343] Test jammy edge --- .github/workflows/manual_test.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6af31d0d8b..a1b7c3651c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,14 +15,16 @@ jobs: # run: df -h self-host-test: name: self-hosted test - runs-on: [self-hosted, edge] + runs-on: [self-hosted, edge, jammy] steps: - - uses: actions/checkout@v4.1.1 - - uses: canonical/setup-lxd@v0.1.1 - - name: Pack charm - run: | - sudo snap install charmcraft --classic --channel latest/stable - charmcraft pack -v + - run: echo hello + - run: df -h + # - uses: actions/checkout@v4.1.1 + # - uses: canonical/setup-lxd@v0.1.1 + # - name: Pack charm + # run: | + # sudo snap install charmcraft --classic --channel latest/stable + # charmcraft pack -v # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y # - run: $HOME/.cargo/bin/cargo install ripgrep From 09285fa43b56bee3a874978accd4b361914ea343 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 8 Jan 2024 15:22:59 +0800 Subject: [PATCH 111/343] Publish docs on main --- .github/workflows/manual_test.yaml | 35 +++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a1b7c3651c..75412f1659 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,29 @@ on: workflow_dispatch: jobs: + publish-docs: + name: Publish docs + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.1 + with: + ref: main + - name: Search for docs folder + id: docs-exist + run: echo "docs_exist=$([[ -d docs ]] && echo 'True' || echo 'False')" >> $GITHUB_OUTPUT + - name: Publish documentation + if: steps.docs-exist.outputs.docs_exist == 'True' + uses: canonical/discourse-gatekeeper@stable + id: publishDocumentation + with: + discourse_host: discourse.charmhub.io + discourse_api_username: ${{ secrets.DISCOURSE_API_USERNAME }} + discourse_api_key: ${{ secrets.DISCOURSE_API_KEY }} + dry_run: false + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: Show index page + if: steps.docs-exist.outputs.docs_exist == 'True' + run: echo '${{ steps.publishDocumentation.outputs.index_url }}' # github-test: # runs-on: ubuntu-latest # steps: @@ -13,12 +36,12 @@ jobs: # run: echo hello # - name: Disk usage # run: df -h - self-host-test: - name: self-hosted test - runs-on: [self-hosted, edge, jammy] - steps: - - run: echo hello - - run: df -h + # self-host-test: + # name: self-hosted test + # runs-on: [self-hosted, edge, jammy] + # steps: + # - run: echo hello + # - run: df -h # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 46adc732e95a92fedb94f96fc15176f51e718abc Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 29 Jan 2024 12:35:45 +0800 Subject: [PATCH 112/343] Test github runner disk size --- .github/workflows/manual_test.yaml | 35 +++++------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 75412f1659..eabd69a272 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,36 +6,13 @@ on: workflow_dispatch: jobs: - publish-docs: - name: Publish docs - runs-on: ubuntu-22.04 + github-test: + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.1 - with: - ref: main - - name: Search for docs folder - id: docs-exist - run: echo "docs_exist=$([[ -d docs ]] && echo 'True' || echo 'False')" >> $GITHUB_OUTPUT - - name: Publish documentation - if: steps.docs-exist.outputs.docs_exist == 'True' - uses: canonical/discourse-gatekeeper@stable - id: publishDocumentation - with: - discourse_host: discourse.charmhub.io - discourse_api_username: ${{ secrets.DISCOURSE_API_USERNAME }} - discourse_api_key: ${{ secrets.DISCOURSE_API_KEY }} - dry_run: false - github_token: ${{ secrets.GITHUB_TOKEN }} - - name: Show index page - if: steps.docs-exist.outputs.docs_exist == 'True' - run: echo '${{ steps.publishDocumentation.outputs.index_url }}' - # github-test: - # runs-on: ubuntu-latest - # steps: - # - name: Echo test - # run: echo hello - # - name: Disk usage - # run: df -h + - name: Echo test + run: echo hello + - name: Disk usage + run: df -h # self-host-test: # name: self-hosted test # runs-on: [self-hosted, edge, jammy] From 10394a02f27a7d1b112fded61cda452ad7b442c1 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:08:02 +0800 Subject: [PATCH 113/343] Test github provided runner disk size --- .github/workflows/manual_test.yaml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index eabd69a272..74ae9ab81e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,22 +13,23 @@ jobs: run: echo hello - name: Disk usage run: df -h + # self-host-test: # name: self-hosted test # runs-on: [self-hosted, edge, jammy] # steps: # - run: echo hello # - run: df -h - # - uses: actions/checkout@v4.1.1 - # - uses: canonical/setup-lxd@v0.1.1 - # - name: Pack charm - # run: | - # sudo snap install charmcraft --classic --channel latest/stable - # charmcraft pack -v - # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh - # - run: sh rustup.sh -y - # - run: $HOME/.cargo/bin/cargo install ripgrep - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # limit-access-to-actor: true + # - uses: actions/checkout@v4.1.1 + # - uses: canonical/setup-lxd@v0.1.1 + # - name: Pack charm + # run: | + # sudo snap install charmcraft --classic --channel latest/stable + # charmcraft pack -v + # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh + # - run: sh rustup.sh -y + # - run: $HOME/.cargo/bin/cargo install ripgrep + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true From 223dd85da458ebd036648ca5aeb0c8d8867949b6 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 19 Feb 2024 15:13:01 +0800 Subject: [PATCH 114/343] Test number of cpu --- .github/workflows/manual_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 74ae9ab81e..a02d11f9ec 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,6 +13,8 @@ jobs: run: echo hello - name: Disk usage run: df -h + - name: CPU usage + run: lscpu # self-host-test: # name: self-hosted test From 452f2b2e7aecfdf1d2f1095bed49f56089daaa87 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:17:23 +0800 Subject: [PATCH 115/343] Self-hoste ping localhsot --- .github/workflows/manual_test.yaml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a02d11f9ec..fd01af95c7 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,22 +6,22 @@ on: workflow_dispatch: jobs: - github-test: - runs-on: ubuntu-latest - steps: - - name: Echo test - run: echo hello - - name: Disk usage - run: df -h - - name: CPU usage - run: lscpu - - # self-host-test: - # name: self-hosted test - # runs-on: [self-hosted, edge, jammy] + # github-test: + # runs-on: ubuntu-latest # steps: - # - run: echo hello - # - run: df -h + # - name: Echo test + # run: echo hello + # - name: Disk usage + # run: df -h + # - name: CPU usage + # run: lscpu + self-host-test: + name: self-hosted test + runs-on: [self-hosted, edge, jammy] + steps: + - run: echo hello + - run: df -h + - run: ping 127.0.0.1 # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 2ee213e0c66dc2a3f3f3b5ba72ae55daa5a75848 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:16:54 +0800 Subject: [PATCH 116/343] Test URL --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index fd01af95c7..148eca16e7 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -21,6 +21,7 @@ jobs: steps: - run: echo hello - run: df -h + - run: echo $DOCKERHUB_MIRROR - run: ping 127.0.0.1 # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 From f13d8ce08bf790b3ccc3067be8a5643d78006787 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 15 Mar 2024 15:32:55 +0800 Subject: [PATCH 117/343] Test microk8s --- .github/workflows/manual_test.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 148eca16e7..10a851907b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -22,7 +22,17 @@ jobs: - run: echo hello - run: df -h - run: echo $DOCKERHUB_MIRROR - - run: ping 127.0.0.1 + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: Update apt in python docker container + run: docker run python:3.10-slim apt-get update # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 2c430516ee0344b1521a300ec658d34d26fd226f Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 21 Mar 2024 13:27:32 +0800 Subject: [PATCH 118/343] Test disk io --- .github/workflows/manual_test.yaml | 60 +++++++++++++++++------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 10a851907b..ef02108eba 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,33 +6,41 @@ on: workflow_dispatch: jobs: - # github-test: - # runs-on: ubuntu-latest - # steps: - # - name: Echo test - # run: echo hello - # - name: Disk usage - # run: df -h - # - name: CPU usage - # run: lscpu - self-host-test: - name: self-hosted test - runs-on: [self-hosted, edge, jammy] + github-test: + runs-on: ubuntu-latest steps: - - run: echo hello - - run: df -h - - run: echo $DOCKERHUB_MIRROR - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt-get update + - run: sudo apt-get update + - run: sudo apt install -y fio + - name: seq write + run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + - run: sudo rm -f /write* /read* + - name: rand write + run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + - run: sudo rm -f /write* /read* + - name: seq read + run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + - run: sudo rm -f /write* /read* + - name: rand read + run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + - run: sudo rm -f /write* /read* + # self-host-test: + # name: self-hosted test + # runs-on: [self-hosted, edge, jammy] + # steps: + # - run: echo hello + # - run: df -h + # - run: echo $DOCKERHUB_MIRROR + # - name: Install microk8s + # run: sudo snap install microk8s --classic + # - name: Wait for microk8s + # timeout-minutes: 10 + # run: microk8s status --wait-ready + # - name: Deploy nginx for testing + # run: microk8s kubectl create deployment nginx --image=nginx + # - name: Wait for nginx to be ready + # run: microk8s kubectl rollout status deployment/nginx --timeout=30m + # - name: Update apt in python docker container + # run: docker run python:3.10-slim apt-get update # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 2d6fd0b117c3d98680cbcf288c709d52523f62fd Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:20:20 +0800 Subject: [PATCH 119/343] Disk test with less disk usage --- .github/workflows/manual_test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ef02108eba..d54b9d1fe4 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -12,16 +12,16 @@ jobs: - run: sudo apt-get update - run: sudo apt install -y fio - name: seq write - run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - run: sudo rm -f /write* /read* - name: rand write - run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - run: sudo rm -f /write* /read* - name: seq read - run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - run: sudo rm -f /write* /read* - name: rand read - run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=10G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - run: sudo rm -f /write* /read* # self-host-test: # name: self-hosted test From 49c368c1527a1e4fb1dd881fdc3011bc0ae8eae0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:10:58 +0800 Subject: [PATCH 120/343] Test openstack runner --- .github/workflows/manual_test.yaml | 66 +++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d54b9d1fe4..4d33a594e9 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,40 +7,40 @@ on: jobs: github-test: - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + # steps: + # - run: sudo apt-get update + # - run: sudo apt install -y fio + # - name: seq write + # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + # - run: sudo rm -f /write* /read* + # - name: rand write + # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + # - run: sudo rm -f /write* /read* + # - name: seq read + # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + # - run: sudo rm -f /write* /read* + # - name: rand read + # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + # - run: sudo rm -f /write* /read* + self-host-test: + name: self-hosted test + runs-on: [self-hosted, openstack] steps: - - run: sudo apt-get update - - run: sudo apt install -y fio - - name: seq write - run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - - run: sudo rm -f /write* /read* - - name: rand write - run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - - run: sudo rm -f /write* /read* - - name: seq read - run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - - run: sudo rm -f /write* /read* - - name: rand read - run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - - run: sudo rm -f /write* /read* - # self-host-test: - # name: self-hosted test - # runs-on: [self-hosted, edge, jammy] - # steps: - # - run: echo hello - # - run: df -h - # - run: echo $DOCKERHUB_MIRROR - # - name: Install microk8s - # run: sudo snap install microk8s --classic - # - name: Wait for microk8s - # timeout-minutes: 10 - # run: microk8s status --wait-ready - # - name: Deploy nginx for testing - # run: microk8s kubectl create deployment nginx --image=nginx - # - name: Wait for nginx to be ready - # run: microk8s kubectl rollout status deployment/nginx --timeout=30m - # - name: Update apt in python docker container - # run: docker run python:3.10-slim apt-get update + - run: echo hello + - run: df -h + - run: echo $DOCKERHUB_MIRROR + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: Update apt in python docker container + run: docker run python:3.10-slim apt-get update # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 32c9634a942269f671ae41a8ca5247b633e626b3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:17:24 +0800 Subject: [PATCH 121/343] Test openstack runner --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4d33a594e9..fb42c64f2a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - github-test: + # github-test: # runs-on: ubuntu-latest # steps: # - run: sudo apt-get update From b689be679f96582639f4162297c0651b4bb76dce Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 10 Apr 2024 13:00:59 +0800 Subject: [PATCH 122/343] Test --- .github/workflows/manual_test.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index fb42c64f2a..2679c47d55 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -25,22 +25,14 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test: name: self-hosted test - runs-on: [self-hosted, openstack] + runs-on: [self-hosted, openstack-arm] steps: + - run: lscpu - run: echo hello - run: df -h - run: echo $DOCKERHUB_MIRROR - name: Install microk8s run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt-get update # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From f9e54f9338abc69bc18f10aebdb3fbe0e3540f81 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:55:56 +0800 Subject: [PATCH 123/343] Test arm runner --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 2679c47d55..5fbfc7ba25 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -25,7 +25,7 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test: name: self-hosted test - runs-on: [self-hosted, openstack-arm] + runs-on: [self-hosted, medium] steps: - run: lscpu - run: echo hello From 332d850fcc23ab0da732d7f2afd0fdb0380ff7f5 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:00:08 +0800 Subject: [PATCH 124/343] Test arm64 runners --- .github/workflows/manual_test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5fbfc7ba25..05ee99de7f 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -24,6 +24,16 @@ jobs: # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* self-host-test: + name: self-hosted test + runs-on: [self-hosted, openstack-arm] + steps: + - run: lscpu + - run: echo hello + - run: df -h + - run: echo $DOCKERHUB_MIRROR + - name: Install microk8s + run: sudo snap install microk8s --classic + self-host-test-org: name: self-hosted test runs-on: [self-hosted, medium] steps: From 7ad0a3ef4799f964ea5767f7abfd0c8029c984e0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 15 Apr 2024 15:01:09 +0800 Subject: [PATCH 125/343] Test runner label --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 05ee99de7f..fc9252a3b1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -25,7 +25,7 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test: name: self-hosted test - runs-on: [self-hosted, openstack-arm] + runs-on: [self-hosted, openstack-arm, ARM64] steps: - run: lscpu - run: echo hello @@ -35,7 +35,7 @@ jobs: run: sudo snap install microk8s --classic self-host-test-org: name: self-hosted test - runs-on: [self-hosted, medium] + runs-on: [self-hosted, ARM64, medium] steps: - run: lscpu - run: echo hello From 387fa606b86c986c527c33f80337fe6b5359b3dd Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:26:00 +0800 Subject: [PATCH 126/343] test ssh tmate --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index fc9252a3b1..e8f30ffb43 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -41,8 +41,8 @@ jobs: - run: echo hello - run: df -h - run: echo $DOCKERHUB_MIRROR - - name: Install microk8s - run: sudo snap install microk8s --classic + - name: tmate ssh + uses: canonical/action-tmate@main # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 2f91b508a4cb77befa07abb10c26cbed8e414ad0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 17 Apr 2024 19:56:15 +0800 Subject: [PATCH 127/343] Testing SSH connection --- .github/workflows/manual_test.yaml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e8f30ffb43..fa3fa12345 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -23,16 +23,6 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test: - name: self-hosted test - runs-on: [self-hosted, openstack-arm, ARM64] - steps: - - run: lscpu - - run: echo hello - - run: df -h - - run: echo $DOCKERHUB_MIRROR - - name: Install microk8s - run: sudo snap install microk8s --classic self-host-test-org: name: self-hosted test runs-on: [self-hosted, ARM64, medium] @@ -41,6 +31,9 @@ jobs: - run: echo hello - run: df -h - run: echo $DOCKERHUB_MIRROR + - run: echo $TMATE_SERVER_HOST + - run: echo $TMATE_SERVER_PORT + - run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv - name: tmate ssh uses: canonical/action-tmate@main # - uses: actions/checkout@v4.1.1 From a25b2e58f573b911ce20a08382fe46573763fda7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:23:45 +0800 Subject: [PATCH 128/343] Test SSH --- .github/workflows/manual_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index fa3fa12345..d5b7978960 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -33,9 +33,10 @@ jobs: - run: echo $DOCKERHUB_MIRROR - run: echo $TMATE_SERVER_HOST - run: echo $TMATE_SERVER_PORT - - run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv - name: tmate ssh uses: canonical/action-tmate@main + - if: always() + run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 18c8ad132974eadf3a178ab5810300dee374c0b0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 09:57:38 +0800 Subject: [PATCH 129/343] Test proxy --- .github/workflows/manual_test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d5b7978960..dd2141bfdb 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -28,8 +28,11 @@ jobs: runs-on: [self-hosted, ARM64, medium] steps: - run: lscpu - - run: echo hello - run: df -h + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: curl https://github.com -vvv - run: echo $DOCKERHUB_MIRROR - run: echo $TMATE_SERVER_HOST - run: echo $TMATE_SERVER_PORT From b8c401a040fa4be2de20abcf30f5731ef5371f3e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:04:35 +0800 Subject: [PATCH 130/343] Test network for tmate --- .github/workflows/manual_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index dd2141bfdb..96a77bf420 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -32,10 +32,11 @@ jobs: - run: echo $HTTP_PROXY - run: echo $HTTPS_PROXY - run: echo $NO_PROXY - - run: curl https://github.com -vvv - run: echo $DOCKERHUB_MIRROR - run: echo $TMATE_SERVER_HOST - run: echo $TMATE_SERVER_PORT + - name: Test download tmate binary + run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz - name: tmate ssh uses: canonical/action-tmate@main - if: always() From afa3d3e58eba5e2cf983c4fe5b5c53f6976f6627 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:07:12 +0800 Subject: [PATCH 131/343] Test tmate ssh server connection with netcat --- .github/workflows/manual_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 96a77bf420..4e7e0c8cea 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -35,6 +35,9 @@ jobs: - run: echo $DOCKERHUB_MIRROR - run: echo $TMATE_SERVER_HOST - run: echo $TMATE_SERVER_PORT + - run: sudo apt install netcat + - run: Test network connection to tmate ssh server + - run: nc -vz $TMATE_SERVER_PORT $TMATE_SERVER_HOST - name: Test download tmate binary run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz - name: tmate ssh From aa9832982eb220d75728fb14453159e13ef4b5fa Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:08:50 +0800 Subject: [PATCH 132/343] Fix wrong cmd --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4e7e0c8cea..e31d0c10eb 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -36,8 +36,8 @@ jobs: - run: echo $TMATE_SERVER_HOST - run: echo $TMATE_SERVER_PORT - run: sudo apt install netcat - - run: Test network connection to tmate ssh server - - run: nc -vz $TMATE_SERVER_PORT $TMATE_SERVER_HOST + - name: Test network connection to tmate ssh server + run: nc -vz $TMATE_SERVER_PORT $TMATE_SERVER_HOST - name: Test download tmate binary run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz - name: tmate ssh From eae5ccbf6d895773893d534e0421c70e3a33d6ed Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:10:11 +0800 Subject: [PATCH 133/343] Fix port host mixup --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e31d0c10eb..71307152e0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -37,7 +37,7 @@ jobs: - run: echo $TMATE_SERVER_PORT - run: sudo apt install netcat - name: Test network connection to tmate ssh server - run: nc -vz $TMATE_SERVER_PORT $TMATE_SERVER_HOST + run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - name: Test download tmate binary run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz - name: tmate ssh From 0bb2a27dc333d8f3e93685cffce61d42e1d74226 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 11:35:47 +0900 Subject: [PATCH 134/343] fix: arm --- .github/workflows/manual_test.yaml | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 71307152e0..8030b53499 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,22 +7,22 @@ on: jobs: # github-test: - # runs-on: ubuntu-latest - # steps: - # - run: sudo apt-get update - # - run: sudo apt install -y fio - # - name: seq write - # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - # - run: sudo rm -f /write* /read* - # - name: rand write - # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - # - run: sudo rm -f /write* /read* - # - name: seq read - # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - # - run: sudo rm -f /write* /read* - # - name: rand read - # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - # - run: sudo rm -f /write* /read* + # runs-on: ubuntu-latest + # steps: + # - run: sudo apt-get update + # - run: sudo apt install -y fio + # - name: seq write + # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + # - run: sudo rm -f /write* /read* + # - name: rand write + # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + # - run: sudo rm -f /write* /read* + # - name: seq read + # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + # - run: sudo rm -f /write* /read* + # - name: rand read + # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + # - run: sudo rm -f /write* /read* self-host-test-org: name: self-hosted test runs-on: [self-hosted, ARM64, medium] @@ -39,7 +39,7 @@ jobs: - name: Test network connection to tmate ssh server run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - name: Test download tmate binary - run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz + run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64.tar.xz - name: tmate ssh uses: canonical/action-tmate@main - if: always() From 8688c4d536fd8445fb6b45c6909a2b700076944a Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 11:46:57 +0900 Subject: [PATCH 135/343] arm debug --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8030b53499..e2230cd1cf 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -41,7 +41,7 @@ jobs: - name: Test download tmate binary run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64.tar.xz - name: tmate ssh - uses: canonical/action-tmate@main + uses: canonical/action-tmate@debug - if: always() run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv # - uses: actions/checkout@v4.1.1 From cfe936506e141c962e2673ca4471d2fc68df7a96 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 11:47:34 +0900 Subject: [PATCH 136/343] arm debug --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e2230cd1cf..c46f8f8358 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -39,7 +39,7 @@ jobs: - name: Test network connection to tmate ssh server run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - name: Test download tmate binary - run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64.tar.xz + run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From b395c9d184e4827e716588f2d7942f7d0a72738b Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 11:50:33 +0900 Subject: [PATCH 137/343] run tmate only --- .github/workflows/manual_test.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c46f8f8358..f0edb2a72a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -27,17 +27,17 @@ jobs: name: self-hosted test runs-on: [self-hosted, ARM64, medium] steps: - - run: lscpu - - run: df -h - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - run: echo $TMATE_SERVER_HOST - - run: echo $TMATE_SERVER_PORT - - run: sudo apt install netcat - - name: Test network connection to tmate ssh server - run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT + # - run: lscpu + # - run: df -h + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - run: echo $TMATE_SERVER_HOST + # - run: echo $TMATE_SERVER_PORT + # - run: sudo apt install netcat + # - name: Test network connection to tmate ssh server + # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - name: Test download tmate binary run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz - name: tmate ssh From a12b1c391da680a744dd85872b2674a6aea2adb1 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 11:54:25 +0900 Subject: [PATCH 138/343] trigger --- .github/workflows/manual_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f0edb2a72a..d777727bdd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -39,7 +39,9 @@ jobs: # - name: Test network connection to tmate ssh server # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - name: Test download tmate binary - run: wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz + run: | + wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz + echo "success" - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From 29f8812e83b76e32824b570d7be9ca8ac6fb53ed Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:02:09 +0900 Subject: [PATCH 139/343] test api --- .github/workflows/manual_test.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d777727bdd..97fa8a4606 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -42,6 +42,12 @@ jobs: run: | wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz echo "success" + - name: View context attributes + uses: actions/github-script@v7 + with: + script: | + const keys = github.rest.users.listPublicKeysForUser({username: context.actor}); + console.log(keys) - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From bc924431bf3b64f7758103259dc31c66853319d9 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:03:52 +0900 Subject: [PATCH 140/343] test api fix --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 97fa8a4606..1eca9e48ea 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -46,7 +46,7 @@ jobs: uses: actions/github-script@v7 with: script: | - const keys = github.rest.users.listPublicKeysForUser({username: context.actor}); + const keys = await github.rest.users.listPublicKeysForUser({username: context.actor}); console.log(keys) - name: tmate ssh uses: canonical/action-tmate@debug From 1b2526b703a7ec2de3767b46e04de6ac64b6d1c0 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:09:37 +0900 Subject: [PATCH 141/343] trigger --- .github/workflows/manual_test.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 1eca9e48ea..d2b9b90484 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -41,13 +41,6 @@ jobs: - name: Test download tmate binary run: | wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz - echo "success" - - name: View context attributes - uses: actions/github-script@v7 - with: - script: | - const keys = await github.rest.users.listPublicKeysForUser({username: context.actor}); - console.log(keys) - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From f502ff45dafe8307aeb1c82d5acf23c52d4fe24e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:15:50 +0800 Subject: [PATCH 142/343] Test api.github.com --- .github/workflows/manual_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d2b9b90484..590641c09e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -41,6 +41,8 @@ jobs: - name: Test download tmate binary run: | wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz + - name: Test api.github + run: curl https://api.github.com - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From 62a5d269bd3d69ca1464a595f85401b9a9709324 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:16:01 +0900 Subject: [PATCH 143/343] error stat code --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d2b9b90484..d777727bdd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -41,6 +41,7 @@ jobs: - name: Test download tmate binary run: | wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz + echo "success" - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From 30b186d4a6fcb7e0b0d18e154be3bf4216c90200 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 18 Apr 2024 11:25:42 +0800 Subject: [PATCH 144/343] Test x64 runner --- .github/workflows/manual_test.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d777727bdd..3a03b66817 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,12 @@ on: workflow_dispatch: jobs: + self-host-x64: + name: self-hosted x64 test + runs-on: [self-hosted, X64, large] + steps: + - name: tmate ssh + uses: canonical/action-tmate@debug # github-test: # runs-on: ubuntu-latest # steps: From 3e6111d3f18a92623463f08ca290c76ecf6365bd Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:43:22 +0900 Subject: [PATCH 145/343] test main --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 3a03b66817..90bc052d58 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: runs-on: [self-hosted, X64, large] steps: - name: tmate ssh - uses: canonical/action-tmate@debug + uses: canonical/action-tmate@main # github-test: # runs-on: ubuntu-latest # steps: From 3e98047cf3bc1383f58e1fa60999871ddf6928a6 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:50:44 +0900 Subject: [PATCH 146/343] debug --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 90bc052d58..24e64d93b1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,10 +8,10 @@ on: jobs: self-host-x64: name: self-hosted x64 test - runs-on: [self-hosted, X64, large] + runs-on: [self-hosted, ARM64, large] steps: - name: tmate ssh - uses: canonical/action-tmate@main + uses: canonical/action-tmate@debug # github-test: # runs-on: ubuntu-latest # steps: From 9c608cb1053de66fed73ffeca4a4b944cf091ad1 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 12:52:14 +0900 Subject: [PATCH 147/343] revert x64 --- .github/workflows/manual_test.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 24e64d93b1..0770b40acd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: self-host-x64: name: self-hosted x64 test - runs-on: [self-hosted, ARM64, large] + runs-on: [self-hosted, X64, large] steps: - name: tmate ssh uses: canonical/action-tmate@debug @@ -44,10 +44,6 @@ jobs: # - run: sudo apt install netcat # - name: Test network connection to tmate ssh server # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - - name: Test download tmate binary - run: | - wget https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-arm64v8.tar.xz - echo "success" - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From 59ff94977dd87c8f758794e3d6e54f8dbb1f84b1 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 18 Apr 2024 14:57:56 +0900 Subject: [PATCH 148/343] trigger --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0770b40acd..89fb763159 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -32,7 +32,7 @@ jobs: self-host-test-org: name: self-hosted test runs-on: [self-hosted, ARM64, medium] - steps: + steps: # - run: lscpu # - run: df -h # - run: echo $HTTP_PROXY From d6f598b7c39e5db5648f9a4379316236c6cdd124 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 19 Apr 2024 10:39:09 +0800 Subject: [PATCH 149/343] Test dockerhub cache --- .github/workflows/manual_test.yaml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 89fb763159..49398c0780 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - self-host-x64: - name: self-hosted x64 test - runs-on: [self-hosted, X64, large] - steps: - - name: tmate ssh - uses: canonical/action-tmate@debug + # self-host-x64: + # name: self-hosted x64 test + # runs-on: [self-hosted, X64, large] + # steps: + # - name: tmate ssh + # uses: canonical/action-tmate@debug # github-test: # runs-on: ubuntu-latest # steps: @@ -29,16 +29,21 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-org: - name: self-hosted test + self-host-test-arm64: + name: self-hosted arm64 test runs-on: [self-hosted, ARM64, medium] - steps: + steps: # - run: lscpu # - run: df -h # - run: echo $HTTP_PROXY # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR + - run: echo $DOCKERHUB_MIRROR + - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 + - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 + - run: curl $DOCKERHUB_MIRROR -vvv + - run: sudo apt install docker.io -yq + - run: sudo docker image pull github-runner-dockerhub-cache.canonical.com:5000/python # - run: echo $TMATE_SERVER_HOST # - run: echo $TMATE_SERVER_PORT # - run: sudo apt install netcat From a62e49540327d2e1ccaf917b2738d4ed02b585ba Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 19 Apr 2024 11:41:38 +0800 Subject: [PATCH 150/343] Test dockerhub cache --- .github/workflows/manual_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 49398c0780..acd36409cf 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -40,10 +40,9 @@ jobs: # - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - run: curl $DOCKERHUB_MIRROR -vvv - run: sudo apt install docker.io -yq - - run: sudo docker image pull github-runner-dockerhub-cache.canonical.com:5000/python + - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: echo $TMATE_SERVER_HOST # - run: echo $TMATE_SERVER_PORT # - run: sudo apt install netcat From 5f5401c8d783990d7610a3a82a013539fff880b1 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 19 Apr 2024 12:27:13 +0800 Subject: [PATCH 151/343] Demo --- .github/workflows/manual_test.yaml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index acd36409cf..d72c4e4de4 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -31,13 +31,14 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test-arm64: name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] + runs-on: [self-hosted, ARM64, large] steps: - # - run: lscpu - # - run: df -h - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY + - run: lscpu + - run: df -h + - run: free -h + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - run: curl $DOCKERHUB_MIRROR -vvv @@ -46,8 +47,8 @@ jobs: # - run: echo $TMATE_SERVER_HOST # - run: echo $TMATE_SERVER_PORT # - run: sudo apt install netcat - # - name: Test network connection to tmate ssh server - # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT + - name: Test network connection to tmate ssh server + run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - name: tmate ssh uses: canonical/action-tmate@debug - if: always() From 8ce874b4a15a6c9a85fafdd9d2d1a7916b608bc1 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 19 Apr 2024 14:10:06 +0800 Subject: [PATCH 152/343] Test aproxy --- .github/workflows/manual_test.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d72c4e4de4..ed121db2ea 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -40,6 +40,26 @@ jobs: - run: echo $HTTPS_PROXY - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR + - run: sudo snap install aproxy --edge + - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 + - run: |- + sudo nft -f - << EOF + define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') + define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } + table ip aproxy + flush table ip aproxy + table ip aproxy { + chain prerouting { + type nat hook prerouting priority dstnat; policy accept; + ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + } + + chain output { + type nat hook output priority -100; policy accept; + ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + } + } + EOF - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - run: curl $DOCKERHUB_MIRROR -vvv - run: sudo apt install docker.io -yq From f579740aaba92d43293fb90cf6c632ddb1cd233f Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 19 Apr 2024 17:36:52 +0800 Subject: [PATCH 153/343] Test aproxy --- .github/workflows/manual_test.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ed121db2ea..f3f26dadac 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,22 @@ on: workflow_dispatch: jobs: - # self-host-x64: - # name: self-hosted x64 test - # runs-on: [self-hosted, X64, large] - # steps: - # - name: tmate ssh - # uses: canonical/action-tmate@debug + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, test-arm] + steps: + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: curl https://github.com + - name: tmate ssh + uses: canonical/action-tmate@debug + self-host-x64: + name: self-hosted x64 test + runs-on: [self-hosted, X64, large] + steps: + - name: tmate ssh + uses: canonical/action-tmate@debug # github-test: # runs-on: ubuntu-latest # steps: From 9edb332f0d4d750eddb8838f71d4929b8f86f79e Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Sat, 20 Apr 2024 11:18:03 +0800 Subject: [PATCH 154/343] Test staging runner --- .github/workflows/manual_test.yaml | 101 +++++++++++++++-------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f3f26dadac..9061c04de1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,14 +14,17 @@ jobs: - run: echo $HTTPS_PROXY - run: echo $NO_PROXY - run: curl https://github.com + - run: sudo apt -yq install neovim + - run: sudo snap install microk8s --classic --channel=1.30 + - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - name: tmate ssh uses: canonical/action-tmate@debug - self-host-x64: - name: self-hosted x64 test - runs-on: [self-hosted, X64, large] - steps: - - name: tmate ssh - uses: canonical/action-tmate@debug + # self-host-x64: + # name: self-hosted x64 test + # runs-on: [self-hosted, X64, large] + # steps: + # - name: tmate ssh + # uses: canonical/action-tmate@debug # github-test: # runs-on: ubuntu-latest # steps: @@ -39,50 +42,50 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, large] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - run: sudo snap install aproxy --edge - - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 - - run: |- - sudo nft -f - << EOF - define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') - define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } - table ip aproxy - flush table ip aproxy - table ip aproxy { - chain prerouting { - type nat hook prerouting priority dstnat; policy accept; - ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - } + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, large] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - run: sudo snap install aproxy --edge + # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 + # - run: |- + # sudo nft -f - << EOF + # define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') + # define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } + # table ip aproxy + # flush table ip aproxy + # table ip aproxy { + # chain prerouting { + # type nat hook prerouting priority dstnat; policy accept; + # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + # } - chain output { - type nat hook output priority -100; policy accept; - ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - } - } - EOF - - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - - run: curl $DOCKERHUB_MIRROR -vvv - - run: sudo apt install docker.io -yq - - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - run: echo $TMATE_SERVER_HOST - # - run: echo $TMATE_SERVER_PORT - # - run: sudo apt install netcat - - name: Test network connection to tmate ssh server - run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - - name: tmate ssh - uses: canonical/action-tmate@debug - - if: always() - run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv + # chain output { + # type nat hook output priority -100; policy accept; + # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + # } + # } + # EOF + # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 + # - run: curl $DOCKERHUB_MIRROR -vvv + # - run: sudo apt install docker.io -yq + # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # # - run: echo $TMATE_SERVER_HOST + # # - run: echo $TMATE_SERVER_PORT + # # - run: sudo apt install netcat + # - name: Test network connection to tmate ssh server + # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT + # - name: tmate ssh + # uses: canonical/action-tmate@debug + # - if: always() + # run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 9c4370b23e6af7b3aee5ae17079ff8d432a0f10a Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:10:35 +0800 Subject: [PATCH 155/343] Test python installation 3.8.10 --- .github/workflows/manual_test.yaml | 45 +++++++++++++++++------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 9061c04de1..83b67548d1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,28 +6,35 @@ on: workflow_dispatch: jobs: - self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] - steps: - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: curl https://github.com - - run: sudo apt -yq install neovim - - run: sudo snap install microk8s --classic --channel=1.30 - - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - name: tmate ssh - uses: canonical/action-tmate@debug - # self-host-x64: - # name: self-hosted x64 test - # runs-on: [self-hosted, X64, large] + # self-host-stg: + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, test-arm] # steps: + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: curl https://github.com + # - run: sudo apt -yq install neovim + # - run: sudo snap install microk8s --classic --channel=1.30 + # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - name: tmate ssh # uses: canonical/action-tmate@debug - # github-test: - # runs-on: ubuntu-latest - # steps: + self-host-x64: + name: self-hosted x64 test + runs-on: [self-hosted, X64, large, jammy] + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8.10 + github-test: + name: GitHub x64 test + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.8.10 # - run: sudo apt-get update # - run: sudo apt install -y fio # - name: seq write From 18df44a64f51039a93693a59f4c382c55349358a Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:34:02 +0800 Subject: [PATCH 156/343] Test docker pull on self-hosted arm64 --- .github/workflows/manual_test.yaml | 117 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 83b67548d1..8ec15b15cd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -19,22 +19,22 @@ jobs: # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - name: tmate ssh # uses: canonical/action-tmate@debug - self-host-x64: - name: self-hosted x64 test - runs-on: [self-hosted, X64, large, jammy] - steps: - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.8.10 - github-test: - name: GitHub x64 test - runs-on: ubuntu-latest - steps: - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.8.10 + # self-host-x64: + # name: self-hosted x64 test + # runs-on: [self-hosted, X64, large, jammy] + # steps: + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: 3.8.10 + # github-test: + # name: GitHub x64 test + # runs-on: ubuntu-latest + # steps: + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: 3.8.10 # - run: sudo apt-get update # - run: sudo apt install -y fio # - name: seq write @@ -49,50 +49,49 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, large] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - run: sudo snap install aproxy --edge - # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 - # - run: |- - # sudo nft -f - << EOF - # define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') - # define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } - # table ip aproxy - # flush table ip aproxy - # table ip aproxy { - # chain prerouting { - # type nat hook prerouting priority dstnat; policy accept; - # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - # } + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, large] + steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - run: sudo snap install aproxy --edge + # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 + # - run: |- + # sudo nft -f - << EOF + # define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') + # define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } + # table ip aproxy + # flush table ip aproxy + # table ip aproxy { + # chain prerouting { + # type nat hook prerouting priority dstnat; policy accept; + # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + # } - # chain output { - # type nat hook output priority -100; policy accept; - # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - # } - # } - # EOF - # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - # - run: curl $DOCKERHUB_MIRROR -vvv - # - run: sudo apt install docker.io -yq - # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # # - run: echo $TMATE_SERVER_HOST - # # - run: echo $TMATE_SERVER_PORT - # # - run: sudo apt install netcat - # - name: Test network connection to tmate ssh server - # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - # - name: tmate ssh - # uses: canonical/action-tmate@debug - # - if: always() - # run: ssh -p $TMATE_SERVER_PORT $TMATE_SERVER_HOST -vvv + # chain output { + # type nat hook output priority -100; policy accept; + # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + # } + # } + # EOF + # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 + # - run: curl $DOCKERHUB_MIRROR -vvv + - run: sudo apt install docker.io -yq + - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + - run : docker pull python + # - run: echo $TMATE_SERVER_HOST + # - run: echo $TMATE_SERVER_PORT + # - run: sudo apt install netcat + # - name: Test network connection to tmate ssh server + # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT + # - name: tmate ssh + # uses: canonical/action-tmate@debug # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From aaa0c4c240d760535d886e9981d606b14f9ec520 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 22 Apr 2024 16:39:40 +0800 Subject: [PATCH 157/343] Use medium runners --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8ec15b15cd..7ff501248d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -51,7 +51,7 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test-arm64: name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, large] + runs-on: [self-hosted, ARM64, medium] steps: # - run: lscpu # - run: df -h From f716ee8a5ab5ffc3c9d59d2af341597c15229d23 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:55:13 +0800 Subject: [PATCH 158/343] Test stg arm runner --- .github/workflows/manual_test.yaml | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 7ff501248d..c13b3d9450 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,19 +6,19 @@ on: workflow_dispatch: jobs: - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, test-arm] - # steps: - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: curl https://github.com - # - run: sudo apt -yq install neovim - # - run: sudo snap install microk8s --classic --channel=1.30 - # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - name: tmate ssh - # uses: canonical/action-tmate@debug + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, test-arm] + steps: + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: curl https://github.com + - run: sudo apt -yq install neovim + - run: sudo snap install microk8s --classic --channel=1.30 + - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + - name: tmate ssh + uses: canonical/action-tmate@debug # self-host-x64: # name: self-hosted x64 test # runs-on: [self-hosted, X64, large, jammy] @@ -49,10 +49,10 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - steps: + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # steps: # - run: lscpu # - run: df -h # - run: free -h @@ -82,9 +82,9 @@ jobs: # EOF # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 # - run: curl $DOCKERHUB_MIRROR -vvv - - run: sudo apt install docker.io -yq - - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - run : docker pull python + # - run: sudo apt install docker.io -yq + # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - run : docker pull python # - run: echo $TMATE_SERVER_HOST # - run: echo $TMATE_SERVER_PORT # - run: sudo apt install netcat From 2ae107d9d0c732c20ac79fe9f82fb5f2913687d3 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:57:06 +0800 Subject: [PATCH 159/343] Test arm runners --- .github/workflows/manual_test.yaml | 56 +++++++++++++++--------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c13b3d9450..3daa16179d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,19 +6,19 @@ on: workflow_dispatch: jobs: - self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] - steps: - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: curl https://github.com - - run: sudo apt -yq install neovim - - run: sudo snap install microk8s --classic --channel=1.30 - - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - name: tmate ssh - uses: canonical/action-tmate@debug + # self-host-stg: + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, test-arm] + # steps: + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: curl https://github.com + # - run: sudo apt -yq install neovim + # - run: sudo snap install microk8s --classic --channel=1.30 + # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - name: tmate ssh + # uses: canonical/action-tmate@debug # self-host-x64: # name: self-hosted x64 test # runs-on: [self-hosted, X64, large, jammy] @@ -49,17 +49,17 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, medium] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- @@ -82,16 +82,16 @@ jobs: # EOF # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 # - run: curl $DOCKERHUB_MIRROR -vvv - # - run: sudo apt install docker.io -yq - # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + - run: sudo apt install docker.io -yq + - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run : docker pull python # - run: echo $TMATE_SERVER_HOST # - run: echo $TMATE_SERVER_PORT # - run: sudo apt install netcat # - name: Test network connection to tmate ssh server # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - # - name: tmate ssh - # uses: canonical/action-tmate@debug + - name: tmate ssh + uses: canonical/action-tmate@debug # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 22edcee2c833d0fe518d4d776b937732f17d91dc Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:21:30 +0800 Subject: [PATCH 160/343] Test LXD and registry mirror docker configuration --- .github/workflows/manual_test.yaml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 3daa16179d..100ffb6e61 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,19 +6,14 @@ on: workflow_dispatch: jobs: - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, test-arm] - # steps: - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: curl https://github.com - # - run: sudo apt -yq install neovim - # - run: sudo snap install microk8s --classic --channel=1.30 - # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - name: tmate ssh - # uses: canonical/action-tmate@debug + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, test-arm] + steps: + - run: lxd init --auto + - run: lxc list + - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + - run: docker pull jujusolutions/charm-base:ubuntu-20.04 -l debug -d # self-host-x64: # name: self-hosted x64 test # runs-on: [self-hosted, X64, large, jammy] From abf83ea078c76954e5a204dc13d879920b66bfce Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:02:42 +0800 Subject: [PATCH 161/343] Test docker config --- .github/workflows/manual_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 100ffb6e61..975de705b2 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -12,8 +12,9 @@ jobs: steps: - run: lxd init --auto - run: lxc list + - run: cat /home/ubuntu/.docker/config.json - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - run: docker pull jujusolutions/charm-base:ubuntu-20.04 -l debug -d + - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # self-host-x64: # name: self-hosted x64 test # runs-on: [self-hosted, X64, large, jammy] From ef5ad2a16c9a2d8ee4e5b6a66f7d537a1131d590 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:08:02 +0800 Subject: [PATCH 162/343] Test stg --- .github/workflows/manual_test.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 975de705b2..6b4182f1cf 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -45,17 +45,17 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- @@ -86,8 +86,8 @@ jobs: # - run: sudo apt install netcat # - name: Test network connection to tmate ssh server # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - - name: tmate ssh - uses: canonical/action-tmate@debug + # - name: tmate ssh + # uses: canonical/action-tmate@debug # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From 4cf0e044d40059ac7a431f0d6541617b4db3a889 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 24 Apr 2024 13:28:11 +0800 Subject: [PATCH 163/343] Test groups --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6b4182f1cf..98625582dc 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -12,7 +12,7 @@ jobs: steps: - run: lxd init --auto - run: lxc list - - run: cat /home/ubuntu/.docker/config.json + - run: groups - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # self-host-x64: From 9751054f6805f277fbaabd61d9bc4cbde0b73568 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:35:10 +0800 Subject: [PATCH 164/343] Test staging docker config file --- .github/workflows/manual_test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 98625582dc..316f2cd486 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,6 +13,7 @@ jobs: - run: lxd init --auto - run: lxc list - run: groups + - run: cat /etc/docker/daemon.json - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # self-host-x64: @@ -78,8 +79,8 @@ jobs: # EOF # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 # - run: curl $DOCKERHUB_MIRROR -vvv - - run: sudo apt install docker.io -yq - - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - run: sudo apt install docker.io -yq + # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run : docker pull python # - run: echo $TMATE_SERVER_HOST # - run: echo $TMATE_SERVER_PORT From 0cfbb6ba693f9570f003bf5eaf378774612b8cf7 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 29 Apr 2024 12:58:29 +0800 Subject: [PATCH 165/343] Test parallel spawned runners --- .github/workflows/manual_test.yaml | 65 ++++++++++++++++-------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 316f2cd486..54c0350a46 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -9,6 +9,9 @@ jobs: self-host-stg: name: self-hosted stg test runs-on: [self-hosted, ARM64, test-arm] + strategy: + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - run: lxd init --auto - run: lxc list @@ -57,38 +60,38 @@ jobs: # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - # - run: sudo snap install aproxy --edge - # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 - # - run: |- - # sudo nft -f - << EOF - # define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') - # define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } - # table ip aproxy - # flush table ip aproxy - # table ip aproxy { - # chain prerouting { - # type nat hook prerouting priority dstnat; policy accept; - # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - # } + # - run: sudo snap install aproxy --edge + # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 + # - run: |- + # sudo nft -f - << EOF + # define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') + # define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } + # table ip aproxy + # flush table ip aproxy + # table ip aproxy { + # chain prerouting { + # type nat hook prerouting priority dstnat; policy accept; + # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + # } - # chain output { - # type nat hook output priority -100; policy accept; - # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - # } - # } - # EOF - # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - # - run: curl $DOCKERHUB_MIRROR -vvv - # - run: sudo apt install docker.io -yq - # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - run : docker pull python - # - run: echo $TMATE_SERVER_HOST - # - run: echo $TMATE_SERVER_PORT - # - run: sudo apt install netcat - # - name: Test network connection to tmate ssh server - # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - # - name: tmate ssh - # uses: canonical/action-tmate@debug + # chain output { + # type nat hook output priority -100; policy accept; + # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + # } + # } + # EOF + # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 + # - run: curl $DOCKERHUB_MIRROR -vvv + # - run: sudo apt install docker.io -yq + # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - run : docker pull python + # - run: echo $TMATE_SERVER_HOST + # - run: echo $TMATE_SERVER_PORT + # - run: sudo apt install netcat + # - name: Test network connection to tmate ssh server + # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT + # - name: tmate ssh + # uses: canonical/action-tmate@debug # - uses: actions/checkout@v4.1.1 # - uses: canonical/setup-lxd@v0.1.1 # - name: Pack charm From e71e9c8d4347b9f80715fcdf1717d493e075964b Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 29 Apr 2024 16:39:43 +0800 Subject: [PATCH 166/343] Using medium arm64 runners --- .github/workflows/manual_test.yaml | 51 ++++++++++++++++-------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 54c0350a46..ee687db11a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,19 +6,19 @@ on: workflow_dispatch: jobs: - self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] - strategy: - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - steps: - - run: lxd init --auto - - run: lxc list - - run: groups - - run: cat /etc/docker/daemon.json - - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - run: docker pull jujusolutions/charm-base:ubuntu-20.04 + # self-host-stg: + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, test-arm] + # strategy: + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # steps: + # - run: lxd init --auto + # - run: lxc list + # - run: groups + # - run: cat /etc/docker/daemon.json + # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # self-host-x64: # name: self-hosted x64 test # runs-on: [self-hosted, X64, large, jammy] @@ -49,17 +49,20 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, medium] + strategy: + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 172e2f01e53aeeb2e8abab5512527caed55ddfef Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 10:31:20 +0800 Subject: [PATCH 167/343] Test amd64 LXD sudo --- .github/workflows/manual_test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ee687db11a..25cd4b4e0b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -54,7 +54,8 @@ jobs: runs-on: [self-hosted, ARM64, medium] strategy: matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] steps: - run: lscpu - run: df -h @@ -63,6 +64,8 @@ jobs: - run: echo $HTTPS_PROXY - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR + - run: lxd init --auto + - run: lxc list # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From d4ef3f7ea61609c6a4240584e304a2b060af12bd Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 10:37:52 +0800 Subject: [PATCH 168/343] Test install juju for arm64 runners --- .github/workflows/manual_test.yaml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 25cd4b4e0b..60ec0e04c2 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -57,15 +57,16 @@ jobs: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] index: [0] steps: - - run: lscpu - - run: df -h - - run: free -h - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - run: lxd init --auto - - run: lxc list + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + - run: sudo snap install juju + # - run: lxd init --auto + # - run: lxc list # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 3cfc447a127fbc212309d36ce3a8cdd2574fd64f Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 13:05:03 +0800 Subject: [PATCH 169/343] Test more LXD --- .github/workflows/manual_test.yaml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 60ec0e04c2..1fa352611d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,15 +6,18 @@ on: workflow_dispatch: jobs: - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, test-arm] - # strategy: - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # steps: - # - run: lxd init --auto - # - run: lxc list + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, test-arm] + strategy: + matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] + steps: + - run: lxd init --auto + - run: lxc list + - run: sudo snap install juju + - run: time sudo snap install juju # - run: groups # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 @@ -64,7 +67,7 @@ jobs: # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - - run: sudo snap install juju + - run: time sudo snap install juju # - run: lxd init --auto # - run: lxc list # - run: sudo snap install aproxy --edge From 190b656307136689fdd242dee192c5de9e934239 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 13:50:41 +0800 Subject: [PATCH 170/343] Test large runners --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 1fa352611d..f4bc4d1a83 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -54,7 +54,7 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test-arm64: name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] + runs-on: [self-hosted, ARM64, large] strategy: matrix: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] @@ -68,8 +68,8 @@ jobs: # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - run: time sudo snap install juju - # - run: lxd init --auto - # - run: lxc list + - run: lxd init --auto + - run: lxc list # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 9ab4487bec45b83e6216d5f8104f6964fdbab685 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 13:52:40 +0800 Subject: [PATCH 171/343] Test with setup-lxd --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f4bc4d1a83..a12b26ee4f 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -67,6 +67,7 @@ jobs: # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR + - uses: canonical/setup-lxd@v0.1.1 - run: time sudo snap install juju - run: lxd init --auto - run: lxc list From 15b21af6ba870123c98f8ed27ee53dd864502fe0 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 13:53:51 +0800 Subject: [PATCH 172/343] Test lxd init --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a12b26ee4f..fa9f3d05b0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -68,8 +68,8 @@ jobs: # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - uses: canonical/setup-lxd@v0.1.1 - - run: time sudo snap install juju - run: lxd init --auto + - run: time sudo snap install juju - run: lxc list # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 From cabb0dd7439b45d1b88ba8e8310592a44320dfd5 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 13:57:55 +0800 Subject: [PATCH 173/343] Test lxd init then install juju --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index fa9f3d05b0..8b9e67b794 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -70,6 +70,7 @@ jobs: - uses: canonical/setup-lxd@v0.1.1 - run: lxd init --auto - run: time sudo snap install juju + if: ${{ always() }} - run: lxc list # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 From 1b21e5cd32d1b6754daab6a09c19604e7dad7928 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 16:41:36 +0800 Subject: [PATCH 174/343] Test on stg --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8b9e67b794..de55ade54e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,6 +14,7 @@ jobs: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] index: [0] steps: + - uses: canonical/setup-lxd@v0.1.1 - run: lxd init --auto - run: lxc list - run: sudo snap install juju From 4323e479fe85d6e6470252cc0038643fdfdf6ec9 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 17:10:47 +0800 Subject: [PATCH 175/343] Test juju bootstrap --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index de55ade54e..f54e50fab4 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -17,8 +17,7 @@ jobs: - uses: canonical/setup-lxd@v0.1.1 - run: lxd init --auto - run: lxc list - - run: sudo snap install juju - - run: time sudo snap install juju + - run: time juju bootstrap localhost localhost # - run: groups # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 @@ -69,10 +68,11 @@ jobs: # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - uses: canonical/setup-lxd@v0.1.1 - - run: lxd init --auto - run: time sudo snap install juju if: ${{ always() }} + - run: lxd init --auto - run: lxc list + - run: time juju bootstrap localhost localhost # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 0495a7f209664e9c58c735a39e4a5ce8efb77eb6 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 2 May 2024 17:16:57 +0800 Subject: [PATCH 176/343] Test always --- .github/workflows/manual_test.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f54e50fab4..a2e195ac67 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -16,8 +16,11 @@ jobs: steps: - uses: canonical/setup-lxd@v0.1.1 - run: lxd init --auto + if: ${{ always() }} - run: lxc list + if: ${{ always() }} - run: time juju bootstrap localhost localhost + if: ${{ always() }} # - run: groups # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 @@ -68,11 +71,12 @@ jobs: # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - uses: canonical/setup-lxd@v0.1.1 - - run: time sudo snap install juju - if: ${{ always() }} - run: lxd init --auto + if: ${{ always() }} - run: lxc list + if: ${{ always() }} - run: time juju bootstrap localhost localhost + if: ${{ always() }} # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From d38c826fb377a948bd357cfec421c17dde63c42c Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 11:08:50 +0800 Subject: [PATCH 177/343] Test adm group --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a2e195ac67..b06985b4f8 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: self-host-stg: name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] + runs-on: [self-hosted, ARM64, adm-test] strategy: matrix: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From 6e4ed5ba6f704314ae6ca74f5ed7d35623dfa01d Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 11:10:12 +0800 Subject: [PATCH 178/343] Add juju install --- .github/workflows/manual_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b06985b4f8..4f84700625 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -19,6 +19,8 @@ jobs: if: ${{ always() }} - run: lxc list if: ${{ always() }} + - run: time sudo snap install juju + if: ${{ always() }} - run: time juju bootstrap localhost localhost if: ${{ always() }} # - run: groups From a942d753a5da2293a47cc5229f6c7c43800ef3bc Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 11:26:40 +0800 Subject: [PATCH 179/343] Test adm group for lxd --- .github/workflows/manual_test.yaml | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4f84700625..6cb5b05b3b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -57,28 +57,28 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, large] - strategy: - matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - index: [0] - steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - - uses: canonical/setup-lxd@v0.1.1 - - run: lxd init --auto - if: ${{ always() }} - - run: lxc list - if: ${{ always() }} - - run: time juju bootstrap localhost localhost - if: ${{ always() }} + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, large] + # strategy: + # matrix: + # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0] + # steps: + # # - run: lscpu + # # - run: df -h + # # - run: free -h + # # - run: echo $HTTP_PROXY + # # - run: echo $HTTPS_PROXY + # # - run: echo $NO_PROXY + # # - run: echo $DOCKERHUB_MIRROR + # - uses: canonical/setup-lxd@v0.1.1 + # - run: lxd init --auto + # if: ${{ always() }} + # - run: lxc list + # if: ${{ always() }} + # - run: time juju bootstrap localhost localhost + # if: ${{ always() }} # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 78789645e99dae3d2f4dfc6ad4769ddfaf1b2e95 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 12:07:25 +0800 Subject: [PATCH 180/343] Test benchmark on adm --- .github/workflows/manual_test.yaml | 132 ++++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6cb5b05b3b..459a2f9294 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,14 +14,138 @@ jobs: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] index: [0] steps: + ### Network tests + - name: Download indico resource test + if: ${{ always() }} + run : | + python3 -c 'from urllib.request import Request, urlopen; from pathlib import Path; Path("image_location.json").write_bytes(urlopen("https://api.charmhub.io/api/v1/resources/download/charm_18zuSzvljyQo6mJMWjcjN2nMm1TiWKCF.indico-image_179").read())' + export INDICO_IMAGE_NAME=($(jq -r '.ImageName' image_location.json)) + export CHARMHUB_REGISTRY_USERNAME=($(jq -r '.Username' image_location.json)) + export CHARMHUB_REGISTRY_PASSWORD=($(jq -r '.Password' image_location.json)) + sudo apt-get -y install skopeo + sudo skopeo login registry.jujucharms.com -u $CHARMHUB_REGISTRY_USERNAME -p $CHARMHUB_REGISTRY_PASSWORD + for ((i=0; i<10; i++)); do + time sudo skopeo copy docker://$INDICO_IMAGE_NAME docker-archive:./indico_image.tar + rm -f ./indico_image.tar + done + sleep 1 + - name: Network speed test + if: ${{ always() }} + run: | + wget https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_arm64.tar.gz + tar -xvzf librespeed-cli_1.0.10_linux_arm64.tar.gz + echo "---network speed test start---" + ./librespeed-cli + sleep 1 + - name: Docker image pull speed (pull through cache) + if: ${{ always() }} + run: | + echo "---docker image pull speed test start (pull through cache)---" + for ((i=0; i<10; i++)); do + time docker pull busybox --quiet + docker system prune --all -f + done + sleep 1 + - name: Docker image pull speed (no pull through cache) + if: ${{ always() }} + run: | + echo "---docker image pull speed test start (no cache)---" + for ((i=0; i<10; i++)); do + time docker pull registry.hub.docker.com/library/busybox --quiet + docker system prune --all -f + done + sleep 1 + - name: Docker image pull speed (mysql) + if: ${{ always() }} + run: | + echo "---docker image pull speed test start (mysql)---" + for ((i=0; i<10; i++)); do + time docker pull ghcr.io/canonical/charmed-mysql@sha256:5082c99baa7a77c82d73247674e270838dc0a8165c02f7619cf5642d1427cba7 --quiet + docker system prune --all -f + done + sleep 1 + ### CPU TESTS + - name: CPU benchmarks + if: ${{ always() }} + run: | + sudo apt-get install sysbench -y + echo "---CPU benchmark test start---" + sysbench cpu run + sleep 1 - uses: canonical/setup-lxd@v0.1.1 - - run: lxd init --auto if: ${{ always() }} - - run: lxc list + - name: Charmcraft pack speed + if: ${{ always() }} + run: | + lxd init --auto + sudo snap install charmcraft --classic + git clone https://github.com/canonical/github-runner-operator.git + cd github-runner-operator + # First launch + echo "First launch" + echo "---Charmcraft pack first launch test start---" + time sudo charmcraft pack + sleep 1 + # Subsequent launches with cache + echo "---Charmcraft pack with cache start---" + for ((i=0; i<10; i++)); do + time sudo charmcraft pack + done + sleep 1 + # Subsequent launches without cache + echo "---Charmcraft pack without cache start---" + for ((i=0; i<10; i++)); do + sudo charmcraft clean + time sudo charmcraft pack + done + sleep 1 + - name: Postgresql charm deploy speed + if: ${{ always() }} + run: | + time sudo snap install juju + mkdir -p ~/.local/share/juju + time juju bootstrap localhost localhost + sleep 1 + echo "---Postgresql charm deploy speed test start---" + for ((i=0; i<10; i++)); do + juju add-model test + juju deploy postgresql --channel 14/stable + time juju wait-for unit postgresql/0 --timeout=15m0s + juju destroy-model test --no-prompt --destroy-storage + done + sleep 1 + - name: Postgresql charm download speed + if: ${{ always() }} + run: | + echo "---Postgresql charm download speed test start---" + for ((i=0; i<10; i++)); do + time (juju download postgresql --channel "14/stable"; snap download charmed-postgresql --channel "14/edge") + done + sleep 1 + ### DISK TESTS + - run: sudo apt-get update + if: ${{ always() }} + - run: sudo apt install -y fio + if: ${{ always() }} + - name: seq write + if: ${{ always() }} + run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + - run: sudo rm -f /write* /read* + if: ${{ always() }} + - name: rand write + if: ${{ always() }} + run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + - run: sudo rm -f /write* /read* + if: ${{ always() }} + - name: seq read + if: ${{ always() }} + run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 + - run: sudo rm -f /write* /read* if: ${{ always() }} - - run: time sudo snap install juju + - name: rand read if: ${{ always() }} - - run: time juju bootstrap localhost localhost + run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 + - run: sudo rm -f /write* /read* if: ${{ always() }} # - run: groups # - run: cat /etc/docker/daemon.json From 0dded16265b08c08b0c9e45306df8640181d55b9 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 12:49:16 +0800 Subject: [PATCH 181/343] Test arm --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 459a2f9294..8e2610a1fe 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: self-host-stg: name: self-hosted stg test - runs-on: [self-hosted, ARM64, adm-test] + runs-on: [self-hosted, ARM64, arm-test] strategy: matrix: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From dc7deda04dd4a9596c3e4ff774e1d64fab95c963 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 12:50:06 +0800 Subject: [PATCH 182/343] Test arm --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8e2610a1fe..39361e7cf7 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: self-host-stg: name: self-hosted stg test - runs-on: [self-hosted, ARM64, arm-test] + runs-on: [self-hosted, ARM64, test-arm] strategy: matrix: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] From e83cb66c7d5f3cfb8319aa7f62c9c5ee26704843 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 13:28:19 +0800 Subject: [PATCH 183/343] SSH into large runners --- .github/workflows/manual_test.yaml | 169 ++++------------------------- 1 file changed, 19 insertions(+), 150 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 39361e7cf7..f665bed7df 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,147 +6,14 @@ on: workflow_dispatch: jobs: - self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] - strategy: - matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - index: [0] - steps: - ### Network tests - - name: Download indico resource test - if: ${{ always() }} - run : | - python3 -c 'from urllib.request import Request, urlopen; from pathlib import Path; Path("image_location.json").write_bytes(urlopen("https://api.charmhub.io/api/v1/resources/download/charm_18zuSzvljyQo6mJMWjcjN2nMm1TiWKCF.indico-image_179").read())' - export INDICO_IMAGE_NAME=($(jq -r '.ImageName' image_location.json)) - export CHARMHUB_REGISTRY_USERNAME=($(jq -r '.Username' image_location.json)) - export CHARMHUB_REGISTRY_PASSWORD=($(jq -r '.Password' image_location.json)) - sudo apt-get -y install skopeo - sudo skopeo login registry.jujucharms.com -u $CHARMHUB_REGISTRY_USERNAME -p $CHARMHUB_REGISTRY_PASSWORD - for ((i=0; i<10; i++)); do - time sudo skopeo copy docker://$INDICO_IMAGE_NAME docker-archive:./indico_image.tar - rm -f ./indico_image.tar - done - sleep 1 - - name: Network speed test - if: ${{ always() }} - run: | - wget https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_arm64.tar.gz - tar -xvzf librespeed-cli_1.0.10_linux_arm64.tar.gz - echo "---network speed test start---" - ./librespeed-cli - sleep 1 - - name: Docker image pull speed (pull through cache) - if: ${{ always() }} - run: | - echo "---docker image pull speed test start (pull through cache)---" - for ((i=0; i<10; i++)); do - time docker pull busybox --quiet - docker system prune --all -f - done - sleep 1 - - name: Docker image pull speed (no pull through cache) - if: ${{ always() }} - run: | - echo "---docker image pull speed test start (no cache)---" - for ((i=0; i<10; i++)); do - time docker pull registry.hub.docker.com/library/busybox --quiet - docker system prune --all -f - done - sleep 1 - - name: Docker image pull speed (mysql) - if: ${{ always() }} - run: | - echo "---docker image pull speed test start (mysql)---" - for ((i=0; i<10; i++)); do - time docker pull ghcr.io/canonical/charmed-mysql@sha256:5082c99baa7a77c82d73247674e270838dc0a8165c02f7619cf5642d1427cba7 --quiet - docker system prune --all -f - done - sleep 1 - ### CPU TESTS - - name: CPU benchmarks - if: ${{ always() }} - run: | - sudo apt-get install sysbench -y - echo "---CPU benchmark test start---" - sysbench cpu run - sleep 1 - - uses: canonical/setup-lxd@v0.1.1 - if: ${{ always() }} - - name: Charmcraft pack speed - if: ${{ always() }} - run: | - lxd init --auto - sudo snap install charmcraft --classic - git clone https://github.com/canonical/github-runner-operator.git - cd github-runner-operator - # First launch - echo "First launch" - echo "---Charmcraft pack first launch test start---" - time sudo charmcraft pack - sleep 1 - # Subsequent launches with cache - echo "---Charmcraft pack with cache start---" - for ((i=0; i<10; i++)); do - time sudo charmcraft pack - done - sleep 1 - # Subsequent launches without cache - echo "---Charmcraft pack without cache start---" - for ((i=0; i<10; i++)); do - sudo charmcraft clean - time sudo charmcraft pack - done - sleep 1 - - name: Postgresql charm deploy speed - if: ${{ always() }} - run: | - time sudo snap install juju - mkdir -p ~/.local/share/juju - time juju bootstrap localhost localhost - sleep 1 - echo "---Postgresql charm deploy speed test start---" - for ((i=0; i<10; i++)); do - juju add-model test - juju deploy postgresql --channel 14/stable - time juju wait-for unit postgresql/0 --timeout=15m0s - juju destroy-model test --no-prompt --destroy-storage - done - sleep 1 - - name: Postgresql charm download speed - if: ${{ always() }} - run: | - echo "---Postgresql charm download speed test start---" - for ((i=0; i<10; i++)); do - time (juju download postgresql --channel "14/stable"; snap download charmed-postgresql --channel "14/edge") - done - sleep 1 - ### DISK TESTS - - run: sudo apt-get update - if: ${{ always() }} - - run: sudo apt install -y fio - if: ${{ always() }} - - name: seq write - if: ${{ always() }} - run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - - run: sudo rm -f /write* /read* - if: ${{ always() }} - - name: rand write - if: ${{ always() }} - run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - - run: sudo rm -f /write* /read* - if: ${{ always() }} - - name: seq read - if: ${{ always() }} - run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - - run: sudo rm -f /write* /read* - if: ${{ always() }} - - name: rand read - if: ${{ always() }} - run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - - run: sudo rm -f /write* /read* - if: ${{ always() }} + # self-host-stg: + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, test-arm] + # strategy: + # matrix: + # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0] + # steps: # - run: groups # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 @@ -181,15 +48,17 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, large] - # strategy: - # matrix: - # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # index: [0] - # steps: - # # - run: lscpu + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, large] + strategy: + matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] + steps: + - name: tmate ssh + uses: canonical/action-tmate@debug + # - run: lscpu # # - run: df -h # # - run: free -h # # - run: echo $HTTP_PROXY From 2f773f70487cf905ad2e7ce7ba409728c8ce9913 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 17:29:46 +0800 Subject: [PATCH 184/343] Test adm group --- .github/workflows/manual_test.yaml | 32 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f665bed7df..db8ffec9ff 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -50,28 +50,26 @@ jobs: # - run: sudo rm -f /write* /read* self-host-test-arm64: name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, large] + runs-on: [self-hosted, ARM64, medium] strategy: matrix: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] index: [0] steps: - - name: tmate ssh - uses: canonical/action-tmate@debug - # - run: lscpu - # # - run: df -h - # # - run: free -h - # # - run: echo $HTTP_PROXY - # # - run: echo $HTTPS_PROXY - # # - run: echo $NO_PROXY - # # - run: echo $DOCKERHUB_MIRROR - # - uses: canonical/setup-lxd@v0.1.1 - # - run: lxd init --auto - # if: ${{ always() }} - # - run: lxc list - # if: ${{ always() }} - # - run: time juju bootstrap localhost localhost - # if: ${{ always() }} + - run: lscpu + - run: df -h + - run: free -h + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR + - uses: canonical/setup-lxd@v0.1.1 + - run: lxd init --auto + if: ${{ always() }} + - run: lxc list + if: ${{ always() }} + - run: time juju bootstrap localhost localhost + if: ${{ always() }} # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 7d206545a5f5ccb48ff2414250b54614610da9d6 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 17:31:52 +0800 Subject: [PATCH 185/343] Test adm group --- .github/workflows/manual_test.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index db8ffec9ff..259ba27933 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -56,18 +56,20 @@ jobs: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] index: [0] steps: - - run: lscpu - - run: df -h - - run: free -h - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR - uses: canonical/setup-lxd@v0.1.1 - run: lxd init --auto if: ${{ always() }} - run: lxc list if: ${{ always() }} + - run: time sudo snap install juju + if: ${{ always() }} - run: time juju bootstrap localhost localhost if: ${{ always() }} # - run: sudo snap install aproxy --edge From f715fca844910a250d81e56088a635d8f3875002 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 3 May 2024 18:45:49 +0800 Subject: [PATCH 186/343] SSH to medium runners --- .github/workflows/manual_test.yaml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 259ba27933..106b218ccf 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -56,6 +56,8 @@ jobs: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] index: [0] steps: + - name: tmate ssh + uses: canonical/action-tmate@debug # - run: lscpu # - run: df -h # - run: free -h @@ -63,15 +65,15 @@ jobs: # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY # - run: echo $DOCKERHUB_MIRROR - - uses: canonical/setup-lxd@v0.1.1 - - run: lxd init --auto - if: ${{ always() }} - - run: lxc list - if: ${{ always() }} - - run: time sudo snap install juju - if: ${{ always() }} - - run: time juju bootstrap localhost localhost - if: ${{ always() }} + # - uses: canonical/setup-lxd@v0.1.1 + # - run: lxd init --auto + # if: ${{ always() }} + # - run: lxc list + # if: ${{ always() }} + # - run: time sudo snap install juju + # if: ${{ always() }} + # - run: time juju bootstrap localhost localhost + # if: ${{ always() }} # - run: sudo snap install aproxy --edge # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 # - run: |- From 8f6a40c47c88dbde10d35fc5a5d931120d4200ec Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 18:30:04 +0800 Subject: [PATCH 187/343] Test microceph --- .github/workflows/manual_test.yaml | 23 +++++++++++------ scripts/test_microceph.py | 40 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 scripts/test_microceph.py diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 106b218ccf..7f7ea77de2 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -18,14 +18,21 @@ jobs: # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 - # self-host-x64: - # name: self-hosted x64 test - # runs-on: [self-hosted, X64, large, jammy] - # steps: - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: 3.8.10 + self-host-x64: + name: self-hosted x64 test + strategy: + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, X64, large, jammy] + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.10 + - run: pip install boto3 + - run: python scripts/test_microceph.py + # github-test: # name: GitHub x64 test # runs-on: ubuntu-latest diff --git a/scripts/test_microceph.py b/scripts/test_microceph.py new file mode 100644 index 0000000000..6b26cc11a3 --- /dev/null +++ b/scripts/test_microceph.py @@ -0,0 +1,40 @@ +import subprocess +import logging +import json + +import boto3 + +logger = logging.getLogger(__name__) + +if __name__ == "__main__": + logger.info("Setting up microceph") + subprocess.run(["sudo", "snap", "install", "microceph"], check=True) + subprocess.run(["sudo", "microceph", "cluster", "bootstrap"], check=True) + subprocess.run(["sudo", "microceph", "disk", "add", "loop,4G,3"], check=True) + subprocess.run(["sudo", "microceph", "enable", "rgw"], check=True) + output = subprocess.run( + [ + "sudo", + "microceph.radosgw-admin", + "user", + "create", + "--uid", + "test", + "--display-name", + "test", + ], + capture_output=True, + check=True, + encoding="utf-8", + ).stdout + key = json.loads(output)["keys"][0] + key_id = key["access_key"] + secret_key = key["secret_key"] + logger.info("Creating microceph bucket") + boto3.client( + "s3", + endpoint_url="http://localhost", + aws_access_key_id=key_id, + aws_secret_access_key=secret_key, + ).create_bucket(Bucket="test") + logger.info("Set up microceph") \ No newline at end of file From 73871ebea9e1356a7a348050c5b207b72a66ff32 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 18:34:05 +0800 Subject: [PATCH 188/343] Fix python version --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 7f7ea77de2..de8fd3e4f8 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -29,7 +29,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: 3.8.10 - run: pip install boto3 - run: python scripts/test_microceph.py From 06257016a4bc6cfbdbaff999f0e68ed927a57af1 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 18:37:22 +0800 Subject: [PATCH 189/343] Try another python version --- .github/workflows/manual_test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index de8fd3e4f8..7acae17647 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -22,14 +22,15 @@ jobs: name: self-hosted x64 test strategy: matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] runs-on: [self-hosted, X64, large, jammy] steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.8.10 + python-version: 3.10.8 - run: pip install boto3 - run: python scripts/test_microceph.py From 959606b46bd052dc9714c4ce765ae062f20c756c Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 18:42:29 +0800 Subject: [PATCH 190/343] Test on 10 runners --- .github/workflows/manual_test.yaml | 22 +++++++++++----------- scripts/test_microceph.py | 7 +++---- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 7acae17647..a803a6027c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -22,8 +22,8 @@ jobs: name: self-hosted x64 test strategy: matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - index: [0] + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0] runs-on: [self-hosted, X64, large, jammy] steps: - uses: actions/checkout@v4 @@ -56,16 +56,16 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - strategy: - matrix: + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # strategy: + # matrix: # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - index: [0] - steps: - - name: tmate ssh - uses: canonical/action-tmate@debug + # index: [0] + # steps: + # - name: tmate ssh + # uses: canonical/action-tmate@debug # - run: lscpu # - run: df -h # - run: free -h diff --git a/scripts/test_microceph.py b/scripts/test_microceph.py index 6b26cc11a3..5b0a113b8d 100644 --- a/scripts/test_microceph.py +++ b/scripts/test_microceph.py @@ -4,10 +4,9 @@ import boto3 -logger = logging.getLogger(__name__) if __name__ == "__main__": - logger.info("Setting up microceph") + print("Setting up microceph") subprocess.run(["sudo", "snap", "install", "microceph"], check=True) subprocess.run(["sudo", "microceph", "cluster", "bootstrap"], check=True) subprocess.run(["sudo", "microceph", "disk", "add", "loop,4G,3"], check=True) @@ -30,11 +29,11 @@ key = json.loads(output)["keys"][0] key_id = key["access_key"] secret_key = key["secret_key"] - logger.info("Creating microceph bucket") + print("Creating microceph bucket") boto3.client( "s3", endpoint_url="http://localhost", aws_access_key_id=key_id, aws_secret_access_key=secret_key, ).create_bucket(Bucket="test") - logger.info("Set up microceph") \ No newline at end of file + print("Set up microceph") \ No newline at end of file From 43a0601e6d2eac11217334d07a1345dfe1bf74d0 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 18:46:31 +0800 Subject: [PATCH 191/343] Test 10 runners --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a803a6027c..49d97a9b30 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -21,9 +21,9 @@ jobs: self-host-x64: name: self-hosted x64 test strategy: + fail-fast: false matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # index: [0] runs-on: [self-hosted, X64, large, jammy] steps: - uses: actions/checkout@v4 From b76423b58896efdf78c3afe66de9f20caf2c9631 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 18:51:00 +0800 Subject: [PATCH 192/343] Add 1 min sleep --- scripts/test_microceph.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test_microceph.py b/scripts/test_microceph.py index 5b0a113b8d..1ae43ee27b 100644 --- a/scripts/test_microceph.py +++ b/scripts/test_microceph.py @@ -1,6 +1,7 @@ import subprocess import logging import json +from time import sleep import boto3 @@ -29,6 +30,8 @@ key = json.loads(output)["keys"][0] key_id = key["access_key"] secret_key = key["secret_key"] + print("Wait for 60 seconds") + sleep(60) print("Creating microceph bucket") boto3.client( "s3", From a94a3029dd528613da7e30c4e0b5ecc39674774a Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 7 May 2024 19:00:15 +0800 Subject: [PATCH 193/343] Wait before --- scripts/test_microceph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test_microceph.py b/scripts/test_microceph.py index 1ae43ee27b..d9385fcf30 100644 --- a/scripts/test_microceph.py +++ b/scripts/test_microceph.py @@ -7,6 +7,8 @@ if __name__ == "__main__": + print("Wait for 60 seconds") + sleep(60) print("Setting up microceph") subprocess.run(["sudo", "snap", "install", "microceph"], check=True) subprocess.run(["sudo", "microceph", "cluster", "bootstrap"], check=True) @@ -30,8 +32,6 @@ key = json.loads(output)["keys"][0] key_id = key["access_key"] secret_key = key["secret_key"] - print("Wait for 60 seconds") - sleep(60) print("Creating microceph bucket") boto3.client( "s3", From 6c1a0aec4af47c88a71ad6ed22818025e60b7e95 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 8 May 2024 17:48:49 +0800 Subject: [PATCH 194/343] Test 10 arm runners --- .github/workflows/manual_test.yaml | 47 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 49d97a9b30..0479a7abd2 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -18,22 +18,21 @@ jobs: # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 - self-host-x64: - name: self-hosted x64 test - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, X64, large, jammy] - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.10.8 - - run: pip install boto3 - - run: python scripts/test_microceph.py - + # self-host-x64: + # name: self-hosted x64 test + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # runs-on: [self-hosted, X64, large, jammy] + # steps: + # - uses: actions/checkout@v4 + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: 3.10.8 + # - run: pip install boto3 + # - run: python scripts/test_microceph.py # github-test: # name: GitHub x64 test # runs-on: ubuntu-latest @@ -56,14 +55,14 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # strategy: - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # index: [0] - # steps: + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, medium] + strategy: + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - run: sleep 600 # - name: tmate ssh # uses: canonical/action-tmate@debug # - run: lscpu From 2af10e063ee1f7c203c502be25e0699bc411b22b Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 8 May 2024 17:52:59 +0800 Subject: [PATCH 195/343] Sleep 1 hour arm medium --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0479a7abd2..de8933506d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -62,7 +62,7 @@ jobs: matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - run: sleep 600 + - run: sleep 3600 # - name: tmate ssh # uses: canonical/action-tmate@debug # - run: lscpu From 23fc9a0aee87021e6b3c6ccbddc3154d99bf073b Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 8 May 2024 20:05:12 +0800 Subject: [PATCH 196/343] Test network of arm64 --- .github/workflows/manual_test.yaml | 3 +- is | 2434 ++++++++++++++++++++++++++++ 2 files changed, 2436 insertions(+), 1 deletion(-) create mode 100644 is diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index de8933506d..49c78b9dc9 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -59,10 +59,11 @@ jobs: name: self-hosted arm64 test runs-on: [self-hosted, ARM64, medium] strategy: + fail-fast: false matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - run: sleep 3600 + - run: watch -n 30 curl -sIXGET https://github.com # - name: tmate ssh # uses: canonical/action-tmate@debug # - run: lscpu diff --git a/is b/is new file mode 100644 index 0000000000..d019d0890e --- /dev/null +++ b/is @@ -0,0 +1,2434 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + GitHub: Let’s build from here · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ Skip to content + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + GitHub Universe is back: + + + Get tickets now for 35% off, only until July 8 + +
+ +
+ + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + +
+ + + + + + +
+ + + + + + + + + +
+
+ + + + + +
+
+ + +
+ +
+
+
+
+
+
+
+
+ +
+
+ + +
+
+ +
+
+ +
+
+
+ +

+ Let’s build from here +

+ +

+ The world’s leading AI-powered developer platform. +

+ +
+ +
+ + + Start a free enterprise trial + + + + +
+ + +
+

Trusted by the world’s leading organizations â†˜ï¸Ž

+ +
+ 3M logo + KPMG logo +
+ Mercedes-Benz logo + SAP logo +
+
+ P&G logo + Telus logo +
+
+
+
+
+
+ +
+
+
+ + + + + + +
+ + Start a free enterprise trial + + + + + + Sign up for GitHub + + + + +
+
+
+
+ +
+
+ +
+
+
+ + +
+
+
+ +
+

Productivity

+

+ Accelerate innovation + Our AI-powered platform increases the pace of software development. +

+
+ + +
+ +
+ +
+ +
+ + +
+
+ +
+ +
+
+
+
+ + + + + +

GitHub Copilot empowers developers to complete tasks 55% faster with contextualized AI coding assistance across workflows.

+ + + Explore GitHub Copilot + + + +
+
+ +
+
+ +
+
+ +
+ +
+ +
+ + +
+ +
+
+
+
+
+
+ Did you know? +
+ +

22% increase

+

in developer productivity after three years with GitHub

+ + Read the report + + +
+
+
+
+ +
+ +
+
+

GitHub Actions automates your build, test, and deployment workflow with simple and secure CI/CD.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+ +
+
+

GitHub Codespaces offers a complete dev environment in seconds. Code, build, test, and open pull requests from any repo.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+
+

GitHub Mobile and Copilot now let you manage projects and converse with your AI pair programmer on the go.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+
+
+ +
+
+
+
+
+ +
+
+
+ + +
+ +
+
+ +
+

Application security

+

+ Empower developers With GitHub, you can secure code in minutes. +

+
+ + +
+ + +
+
+
cmake.yml
+ on: push +
+ +
+
+
+
    +
  • +
    + + Build +
    + 1m 21s +
    • +
+
+
+ +
+
+
+
+
+ +
+
+ Steps +
+
+
    +
  • +
    + + Initialize CodeQL +
    + 1m 42s +
    • +
  • +
    + + Autobuild +
    + 1m 24s +
    • +
  • +
    + + Perform CodeQL Analyses +
    + 1m 36s +
    • +
+
+
+
+
+
+ + +
+
+
+ + +
+ +
+
+
+
+ + + + + +

GitHub Advanced Security enables you to find and fix vulnerabilities with ease and ship secure code quickly.

+ + + Dive into GitHub Advanced Security + + + +
+
+ +
+
+ +
+
+ +
+ + +
+ +
+
+
+
+
+
+ Did you know? +
+ +

7x faster

+

vulnerability fixes with GitHub1

+
+
+
+
+ +
+ +
+
+

Code scanning is our code analysis tool that helps you remediate issues in your code.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+ +
+
+

Dependabot makes it easy to find and fix vulnerable dependencies in your supply chain.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+
+

Secret scanning automatically looks for partner patterns and prevents fraudulent use of accidentally committed secrets.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+
+
+ +
+
+
+
+ +
+
+
+ + +
+ +
+
+ +
+

Collaboration

+

+ Supercharge collaboration + GitHub helps your teams work more efficiently together. +

+
+ + +
+ + + + + Illustration of project table view with cards grouped by 'Feature planning' phase. + + +
+
+
+ +
+ +
+
+
+
+ + + + + +

GitHub Issues and GitHub Projects supply project management tools that adapt to your team alongside your code.

+ + + Get started with GitHub Issues + + + +
+
+ +
+
+ +
+
+ +
+ + +
+ +
+
+
+
+
+
+ Did you know? +
+ +

80%

+

reduction in onboarding time with GitHub2

+
+
+
+
+ +
+ +
+ +
+
+

GitHub Discussions creates space to ask questions and have open-ended conversations.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ +
+
+

Pull requests allow real-time communication and collaboration about code changes.

+ + +
+
+ + + +
+ + +
+
 + +
+ +
+ + +
+
+

GitHub Sponsors lets you support your favorite open source maintainers and projects.

+ + +
+ +
+
+
+ + sindresorhus avatar +
sindresorhus
+ +
+
+ + ESLint avatar +
ESLint
+ +
+
+ + Directus avatar +
Directus
+ +
+
+ + chaynHQ avatar +
chaynHQ
+ +
+
+ + Samuel avatar +
Samuel
+ +
+
+ + Nick DeJesus avatar +
Nick DeJesus
+ +
+
+ + Nikema avatar +
Nikema
+ +
+
+ + Homebrew avatar +
Homebrew
+ +
+
+ + kazuya kawaguchi avatar +
kazuya kawaguchi
+ +
+
+
+ +
+
 + +
+
+ +
+ + +
+
+
+
+ +
+
+ + +
+
+
+ +
+
+

Over 100 million developers call GitHub home3

+

Whether you’re scaling your startup or just learning how to code, GitHub is your home. Join the world’s largest developer platform to build the innovations that empower humanity. Let’s build from here.

+
+ +
+
+ +
+ + + Start a free enterprise trial + + + + +
+ +
+
+
+
+ +
+
+
+ + + + + + +
+ + + + + + + + + + + + +
+ + +
+ + + +
+
+
    +
  • 1 This 7X times factor is based on data from the industry’s longest running analysis of fix rates +Veracode State of Software Security 2023, which cites the average time to fix 50% of flaws as 198 days vs. GitHub’s fix rates of 72% of flaws with in 28 days which is at a minimum of 7X faster when compared.
    • +
  • 2 The Total Economic Impactâ„¢ Of GitHub Enterprise Cloud and Advanced Security, a commissioned study conducted by Forrester Consulting, 2022. Results are for a composite organization based on interviewed customers.
    • +
  • 3 There are now 100 million developers around the world using GitHub. Read the blog post.
    • +
+
+ + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+
+ + + From bf9eed5f78b3fa7a4ccd3d5622a07587a61bd222 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 8 May 2024 21:10:42 +0800 Subject: [PATCH 197/343] Bash script to test hardware --- .github/workflows/manual_test.yaml | 2 +- scripts/test_hardware.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 scripts/test_hardware.sh diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 49c78b9dc9..634de54c3b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -63,7 +63,7 @@ jobs: matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - run: watch -n 30 curl -sIXGET https://github.com + - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug # - run: lscpu diff --git a/scripts/test_hardware.sh b/scripts/test_hardware.sh new file mode 100644 index 0000000000..c9810c3a3a --- /dev/null +++ b/scripts/test_hardware.sh @@ -0,0 +1,13 @@ + +while true +do + echo 'Start tests' + echo 'Disk' + df -h + echo 'Memory' + free -h + echo 'Network' + curl -sIXGET https://github.com + echo 'End tests' + sleep 60 +done \ No newline at end of file From fbde8df9cc8a99d686106e94ddd90658d72153ca Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 8 May 2024 21:18:49 +0800 Subject: [PATCH 198/343] Add checkout for downloading script --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 634de54c3b..fd4c887682 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -63,6 +63,7 @@ jobs: matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: + - uses: actions/checkout@v4 - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug From 143f85e07e7db37057bda3c4d1b291f2866ee1d3 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 8 May 2024 21:57:21 +0800 Subject: [PATCH 199/343] Test network with timestamp --- scripts/test_hardware.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/test_hardware.sh b/scripts/test_hardware.sh index c9810c3a3a..224f026bbc 100644 --- a/scripts/test_hardware.sh +++ b/scripts/test_hardware.sh @@ -1,13 +1,8 @@ while true do - echo 'Start tests' - echo 'Disk' - df -h - echo 'Memory' - free -h - echo 'Network' + echo 'Start end' + date curl -sIXGET https://github.com - echo 'End tests' sleep 60 done \ No newline at end of file From 71293cfcc9e8e2aa0d4b795740f794cff97fe573 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 9 May 2024 11:59:01 +0800 Subject: [PATCH 200/343] Test stg runner --- .github/workflows/manual_test.yaml | 43 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index fd4c887682..daaade604d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,18 +6,17 @@ on: workflow_dispatch: jobs: - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, test-arm] - # strategy: - # matrix: - # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # index: [0] - # steps: - # - run: groups - # - run: cat /etc/docker/daemon.json - # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, test-arm] + strategy: + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - run: groups + - run: cat /etc/docker/daemon.json + - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # self-host-x64: # name: self-hosted x64 test # strategy: @@ -55,16 +54,16 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - steps: - - uses: actions/checkout@v4 - - run: bash scripts/test_hardware.sh + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # steps: + # - uses: actions/checkout@v4 + # - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug # - run: lscpu From 959154cb60ecc07b3e8eeeba3127dec55659a7cd Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 9 May 2024 12:00:47 +0800 Subject: [PATCH 201/343] Add sleep --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index daaade604d..2535b53e22 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -17,6 +17,7 @@ jobs: - run: cat /etc/docker/daemon.json - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - run: docker pull jujusolutions/charm-base:ubuntu-20.04 + - run: sleep 3600 # self-host-x64: # name: self-hosted x64 test # strategy: From 796662697d56ef313f64202b50ee92c770a9770e Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 9 May 2024 12:10:48 +0800 Subject: [PATCH 202/343] Test pull rust image --- .github/workflows/manual_test.yaml | 42 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 2535b53e22..21aa749aca 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,18 +6,18 @@ on: workflow_dispatch: jobs: - self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] - strategy: - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - steps: - - run: groups - - run: cat /etc/docker/daemon.json - - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - run: docker pull jujusolutions/charm-base:ubuntu-20.04 - - run: sleep 3600 + # self-host-stg: + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, test-arm] + # strategy: + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # steps: + # - run: groups + # - run: cat /etc/docker/daemon.json + # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 + # - run: sleep 3600 # self-host-x64: # name: self-hosted x64 test # strategy: @@ -55,14 +55,16 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # steps: + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, medium] + strategy: + fail-fast: false + matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] + steps: + - run: docker pull rust # - uses: actions/checkout@v4 # - run: bash scripts/test_hardware.sh # - name: tmate ssh From 0f8b91e2c8b8d0384610cc64de18c1042df5144d Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 9 May 2024 12:53:57 +0800 Subject: [PATCH 203/343] Test again --- .github/workflows/manual_test.yaml | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 21aa749aca..b5b675ab24 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,18 +6,19 @@ on: workflow_dispatch: jobs: - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, test-arm] - # strategy: - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # steps: - # - run: groups - # - run: cat /etc/docker/daemon.json - # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 - # - run: sleep 3600 + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, test-arm] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - run: groups + - run: cat /etc/docker/daemon.json + - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + - run: docker pull jujusolutions/charm-base:ubuntu-20.04 + - run: sleep 3600 # self-host-x64: # name: self-hosted x64 test # strategy: @@ -55,16 +56,15 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - strategy: - fail-fast: false - matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - index: [0] - steps: - - run: docker pull rust + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # strategy: + # fail-fast: false + # matrix: + # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0] + # steps: # - uses: actions/checkout@v4 # - run: bash scripts/test_hardware.sh # - name: tmate ssh From 4ceae0fa0aa57b766181b5dc41c9348e42798e35 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 14 May 2024 09:23:36 +0200 Subject: [PATCH 204/343] Test networking of arm64 runners --- .github/workflows/manual_test.yaml | 44 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b5b675ab24..022fbba754 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,18 +7,18 @@ on: jobs: self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, test-arm] - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - steps: - - run: groups - - run: cat /etc/docker/daemon.json - - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - - run: docker pull jujusolutions/charm-base:ubuntu-20.04 - - run: sleep 3600 + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, test-arm] + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # steps: + # - run: groups + # - run: cat /etc/docker/daemon.json + # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 + # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 + # - run: sleep 3600 # self-host-x64: # name: self-hosted x64 test # strategy: @@ -56,17 +56,17 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # strategy: - # fail-fast: false - # matrix: - # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, medium] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # index: [0] - # steps: - # - uses: actions/checkout@v4 - # - run: bash scripts/test_hardware.sh + steps: + - uses: actions/checkout@v4 + - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug # - run: lscpu From da6c4b9119fab36c59bd5a578ba669e76242f9ed Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 14 May 2024 09:24:35 +0200 Subject: [PATCH 205/343] Fix format --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 022fbba754..ebf689b8ab 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - self-host-stg: + # self-host-stg: # name: self-hosted stg test # runs-on: [self-hosted, ARM64, test-arm] # strategy: From e80d941b440fd73f4ef70253a699ff607db95c46 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 14 May 2024 09:26:59 +0200 Subject: [PATCH 206/343] Improve the test logging --- scripts/test_hardware.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/test_hardware.sh b/scripts/test_hardware.sh index 224f026bbc..8b3a9f1941 100644 --- a/scripts/test_hardware.sh +++ b/scripts/test_hardware.sh @@ -1,8 +1,11 @@ while true do - echo 'Start end' + echo 'Start test' date + echo '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++' curl -sIXGET https://github.com + echo '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++' + echo 'End test' sleep 60 done \ No newline at end of file From ac26cadeef72d357192a99e968a301ee52dfa714 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 13 Jun 2024 17:04:51 +0800 Subject: [PATCH 207/343] Build arm charm --- .github/workflows/build_arm_charm.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/build_arm_charm.yaml diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml new file mode 100644 index 0000000000..da5cc71955 --- /dev/null +++ b/.github/workflows/build_arm_charm.yaml @@ -0,0 +1,26 @@ +name: build arm charm + +on: [push] + +jobs: + build: + runs-on: [self-hosted, linux, arm64] + steps: + - uses: actions/checkout@v4 + with: + repository: 'canonical/github-runner-operator' + ref: 'feat/openstack-integration' + - name: Install dependencies + run: | + sudo snap install charmcraft --classic + sudo lxd init --auto + sudo usermod -a -G lxd $USER + - name: Pack the charm + run: | + export http_proxy="" https_proxy="" HTTP_PROXY="" HTTPS_PROXY="" + sg lxd -c "charmcraft pack" + - name: upload artifact + uses: actions/upload-artifact@v4 + with: + name: charm + path: ./*.charm \ No newline at end of file From 144d9898491351d7ab42ab90d4bd17792a96d6f2 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 18 Jun 2024 09:16:30 +0800 Subject: [PATCH 208/343] Test uname --- .github/workflows/manual_test.yaml | 46 ++++++++++++++++-------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ebf689b8ab..00169be475 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -19,21 +19,22 @@ jobs: # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # - run: sleep 3600 - # self-host-x64: - # name: self-hosted x64 test - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # runs-on: [self-hosted, X64, large, jammy] - # steps: - # - uses: actions/checkout@v4 - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: 3.10.8 - # - run: pip install boto3 - # - run: python scripts/test_microceph.py + self-host-x64: + name: self-hosted x64 test + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, X64, large, jammy] + steps: + - run: uname -a + # - uses: actions/checkout@v4 + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: 3.10.8 + # - run: pip install boto3 + # - run: python scripts/test_microceph.py # github-test: # name: GitHub x64 test # runs-on: ubuntu-latest @@ -62,16 +63,17 @@ jobs: strategy: fail-fast: false matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # index: [0] + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] steps: - - uses: actions/checkout@v4 - - run: bash scripts/test_hardware.sh + # - uses: actions/checkout@v4 + # - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug - # - run: lscpu - # - run: df -h - # - run: free -h + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a # - run: echo $HTTP_PROXY # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY From d314eb84bfa8bf5205cc03edb50bcb8bc876c390 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 28 Jun 2024 12:21:02 +0800 Subject: [PATCH 209/343] Test noble runners --- .github/workflows/manual_test.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 00169be475..ed1506b40d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,12 @@ on: workflow_dispatch: jobs: + self-hosted-noble: + name: self-hosted noble test + runs-on: [self-hosted, X64, noble, noble-test] + steps: + - run: lsb_release -a + - run: groups # self-host-stg: # name: self-hosted stg test # runs-on: [self-hosted, ARM64, test-arm] From e192a4149ee470c4ae69775c0f11d7c5a482dc14 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 5 Jul 2024 14:51:45 +0800 Subject: [PATCH 210/343] Test amd64 edges runners --- .github/workflows/manual_test.yaml | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ed1506b40d..c754a78ec6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -27,11 +27,11 @@ jobs: # - run: sleep 3600 self-host-x64: name: self-hosted x64 test - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, X64, large, jammy] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, X64, edge, jammy] steps: - run: uname -a # - uses: actions/checkout@v4 @@ -63,23 +63,23 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - strategy: - fail-fast: false - matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - index: [0] - steps: + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # strategy: + # fail-fast: false + # matrix: + # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # index: [0] + # steps: # - uses: actions/checkout@v4 # - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a # - run: echo $HTTP_PROXY # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY From 47d1c44cfc0e5de08c17669b28a989eea10e9cfe Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 5 Jul 2024 15:13:19 +0800 Subject: [PATCH 211/343] test arm64 runners --- .github/workflows/manual_test.yaml | 48 +++++++++++++++--------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c754a78ec6..c98215a026 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - self-hosted-noble: - name: self-hosted noble test - runs-on: [self-hosted, X64, noble, noble-test] - steps: - - run: lsb_release -a - - run: groups + # self-hosted-noble: + # name: self-hosted noble test + # runs-on: [self-hosted, X64, noble, noble-test] + # steps: + # - run: lsb_release -a + # - run: groups # self-host-stg: # name: self-hosted stg test # runs-on: [self-hosted, ARM64, test-arm] @@ -25,15 +25,15 @@ jobs: # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 # - run: sleep 3600 - self-host-x64: - name: self-hosted x64 test - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, X64, edge, jammy] - steps: - - run: uname -a + # self-host-x64: + # name: self-hosted x64 test + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # runs-on: [self-hosted, X64, edge, jammy] + # steps: + # - run: uname -a # - uses: actions/checkout@v4 # - name: Set up Python # uses: actions/setup-python@v5 @@ -63,20 +63,20 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # strategy: - # fail-fast: false - # matrix: - # # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + self-host-test-arm64: + name: self-hosted arm64 test + runs-on: [self-hosted, ARM64, medium] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # index: [0] - # steps: + steps: # - uses: actions/checkout@v4 # - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug - # - run: lscpu + - run: lscpu # - run: df -h # - run: free -h # - run: uname -a From ce399f439e19a1bd3a2a965841678ba6c16cb9ed Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 29 Jul 2024 13:59:40 +0800 Subject: [PATCH 212/343] Build arm64 charm --- .github/workflows/build_arm_charm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml index da5cc71955..eb67193dab 100644 --- a/.github/workflows/build_arm_charm.yaml +++ b/.github/workflows/build_arm_charm.yaml @@ -9,7 +9,7 @@ jobs: - uses: actions/checkout@v4 with: repository: 'canonical/github-runner-operator' - ref: 'feat/openstack-integration' + ref: 'main' - name: Install dependencies run: | sudo snap install charmcraft --classic From 8b40a711c84a5a7de2c54c629bc5c62bab3b9d3d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:59:18 +0800 Subject: [PATCH 213/343] Test timeout --- .github/workflows/manual_test.yaml | 32 ++++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c98215a026..af3943adb2 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,14 @@ on: workflow_dispatch: jobs: - # self-hosted-noble: - # name: self-hosted noble test - # runs-on: [self-hosted, X64, noble, noble-test] - # steps: - # - run: lsb_release -a - # - run: groups + self-hosted-noble: + name: self-hosted noble test + runs-on: [self-hosted, X64, noble, noble-test] + # 24 hours + timeout-minutes: 1440 + steps: + - run: sleep 3d + timeout-minutes: 1440 # self-host-stg: # name: self-hosted stg test # runs-on: [self-hosted, ARM64, test-arm] @@ -63,20 +65,20 @@ jobs: # - name: rand read # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 # - run: sudo rm -f /write* /read* - self-host-test-arm64: - name: self-hosted arm64 test - runs-on: [self-hosted, ARM64, medium] - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # self-host-test-arm64: + # name: self-hosted arm64 test + # runs-on: [self-hosted, ARM64, medium] + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] # index: [0] - steps: + # steps: # - uses: actions/checkout@v4 # - run: bash scripts/test_hardware.sh # - name: tmate ssh # uses: canonical/action-tmate@debug - - run: lscpu + # - run: lscpu # - run: df -h # - run: free -h # - run: uname -a From f7a9475957f27bd3d168848e690b9a17a3a2e856 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:02:01 +0800 Subject: [PATCH 214/343] Test wait --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index af3943adb2..307519b4e6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,9 +6,9 @@ on: workflow_dispatch: jobs: - self-hosted-noble: - name: self-hosted noble test - runs-on: [self-hosted, X64, noble, noble-test] + self-hosted-wait-test: + name: self-hosted wait test + runs-on: [self-hosted, X64, large] # 24 hours timeout-minutes: 1440 steps: From ab54a0a5e83363292ede91e46037801b1595810a Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 6 Sep 2024 17:24:33 +0800 Subject: [PATCH 215/343] build arm charm --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 307519b4e6..b4ac93f424 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - pull_request: + # pull_request: workflow_dispatch: jobs: From 00fa4c1a6b7f56f8a1ca68820763b81909f29fb2 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Sep 2024 09:59:22 +0800 Subject: [PATCH 216/343] Build arm charm from main --- .github/workflows/build_arm_charm.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml index eb67193dab..57dbd5ce5b 100644 --- a/.github/workflows/build_arm_charm.yaml +++ b/.github/workflows/build_arm_charm.yaml @@ -1,6 +1,7 @@ name: build arm charm -on: [push] +on: + push: jobs: build: From 583195bbf682a8075f6dc414a83546b1a74ea6b4 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:54:29 +0800 Subject: [PATCH 217/343] test arm64 staging runners --- .github/workflows/build_arm_charm.yaml | 3 ++- .github/workflows/manual_test.yaml | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml index 57dbd5ce5b..0ae3687056 100644 --- a/.github/workflows/build_arm_charm.yaml +++ b/.github/workflows/build_arm_charm.yaml @@ -1,7 +1,8 @@ name: build arm charm on: - push: + # push: + workflow_dispatch: jobs: build: diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b4ac93f424..41c5cf634f 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,15 +14,16 @@ jobs: steps: - run: sleep 3d timeout-minutes: 1440 - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, test-arm] - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # steps: - # - run: groups + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, staging] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2] + steps: + - run: groups + - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From b7e458dcc60201c9459fd97cec5af6a38df8241d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:34:41 +0800 Subject: [PATCH 218/343] Test stg edge --- .github/workflows/manual_test.yaml | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 41c5cf634f..5877b4cbf9 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,24 +6,21 @@ on: workflow_dispatch: jobs: - self-hosted-wait-test: + self-hosted-edge: name: self-hosted wait test - runs-on: [self-hosted, X64, large] - # 24 hours - timeout-minutes: 1440 + runs-on: [self-hosted, X64, edge] steps: - - run: sleep 3d - timeout-minutes: 1440 - self-host-stg: - name: self-hosted stg test - runs-on: [self-hosted, ARM64, staging] - strategy: - fail-fast: false - matrix: - index: [0, 1, 2] - steps: - - run: groups - run: echo hello + # self-host-stg: + # name: self-hosted stg test + # runs-on: [self-hosted, ARM64, staging] + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2] + # steps: + # - run: groups + # - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From 5a3c9272d257827a9a2c650febe42211b5902c4b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:35:33 +0800 Subject: [PATCH 219/343] Fix workflow issue --- .github/workflows/integration_test.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index fa54c12b49..548fcb8388 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -8,10 +8,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - jobs: # test option values defined at test/conftest.py are passed on via repository secret # INTEGRATION_TEST_ARGS to operator-workflows automatically. From 87a7f7aad2ab29e45e1357af2588eeec6af09cb9 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:36:18 +0800 Subject: [PATCH 220/343] Add test for amd64 edge --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5877b4cbf9..863c34a5ac 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -2,7 +2,7 @@ name: Manual test on: # push: - # pull_request: + pull_request: workflow_dispatch: jobs: From 291f805a812cb5198d48351cc760c2289ad79bde Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:48:21 +0800 Subject: [PATCH 221/343] test arm64 stg runners --- .github/workflows/manual_test.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 863c34a5ac..a42d37d3b1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,21 +6,21 @@ on: workflow_dispatch: jobs: - self-hosted-edge: - name: self-hosted wait test - runs-on: [self-hosted, X64, edge] - steps: - - run: echo hello - # self-host-stg: - # name: self-hosted stg test - # runs-on: [self-hosted, ARM64, staging] - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2] + # self-hosted-edge: + # name: self-hosted wait test + # runs-on: [self-hosted, X64, edge] # steps: - # - run: groups # - run: echo hello + self-host-stg: + name: self-hosted stg test + runs-on: [self-hosted, ARM64, staging] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2] + steps: + - run: groups + - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From 058df0d51f356f8a72bfbd843020c8e073d7f11e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:33:02 +0800 Subject: [PATCH 222/343] Spawn test runners --- .github/workflows/manual_test.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a42d37d3b1..72d6c5686e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,14 +13,12 @@ jobs: # - run: echo hello self-host-stg: name: self-hosted stg test - runs-on: [self-hosted, ARM64, staging] - strategy: - fail-fast: false - matrix: - index: [0, 1, 2] + runs-on: [self-hosted, X64, large] steps: - run: groups - - run: echo hello + - name: tmate ssh + uses: canonical/action-tmate@debug + # - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From 8bc2efa1e4c65636b963dc0d906da14f1a6d40db Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 09:49:25 +0800 Subject: [PATCH 223/343] Test large runner --- .github/workflows/manual_test.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 72d6c5686e..01ca67aa1c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,11 +11,13 @@ jobs: # runs-on: [self-hosted, X64, edge] # steps: # - run: echo hello - self-host-stg: - name: self-hosted stg test + self-host: + name: self-hosted large runs-on: [self-hosted, X64, large] steps: - run: groups + - name: Setup operator environment + uses: charmed-kubernetes/actions-operator@main - name: tmate ssh uses: canonical/action-tmate@debug # - run: echo hello From 6994c86c206b9beb20724b858b33fd45ac73bb56 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 10:14:33 +0800 Subject: [PATCH 224/343] Fix operator setup?? --- .github/workflows/manual_test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 01ca67aa1c..828c68ce64 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -18,6 +18,9 @@ jobs: - run: groups - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main + with: + juju-channel: 3.1/stable + provider: lxd - name: tmate ssh uses: canonical/action-tmate@debug # - run: echo hello From ab96329fb0178175bb901f0ac6593715d7aa9daf Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 10:29:10 +0800 Subject: [PATCH 225/343] Another attempt to fix tox installation --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 828c68ce64..a0de813741 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,7 +15,7 @@ jobs: name: self-hosted large runs-on: [self-hosted, X64, large] steps: - - run: groups + - run: sudo apt install tox -yq - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: From a3d71c965459d300e729393e1383fed97fdf5c70 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 10:52:36 +0800 Subject: [PATCH 226/343] Test microk8s --- .github/workflows/manual_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a0de813741..6364402b11 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -19,8 +19,7 @@ jobs: - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: - juju-channel: 3.1/stable - provider: lxd + provider: microk8s - name: tmate ssh uses: canonical/action-tmate@debug # - run: echo hello From e181dc675d07382cdf939125b0e73a87c9ee59ef Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 11:59:11 +0800 Subject: [PATCH 227/343] Create juju env --- .github/workflows/manual_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6364402b11..5e6de669cf 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,13 +13,14 @@ jobs: # - run: echo hello self-host: name: self-hosted large - runs-on: [self-hosted, X64, large] + runs-on: [self-hosted, X64, large, jammy] steps: - run: sudo apt install tox -yq - name: Setup operator environment uses: charmed-kubernetes/actions-operator@main with: provider: microk8s + juju-channel: 3.4/stable - name: tmate ssh uses: canonical/action-tmate@debug # - run: echo hello From b120059941f59d706cc0fcade64496417697441c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 14:32:14 +0800 Subject: [PATCH 228/343] Test --- .github/workflows/manual_test.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5e6de669cf..5a5773bf2b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,14 +15,25 @@ jobs: name: self-hosted large runs-on: [self-hosted, X64, large, jammy] steps: - - run: sudo apt install tox -yq + # - name: Load netsuite client library + # uses: alnvdl-work/setup-private-repo@v2 + # with: + # repository: canonical/netsuite-client + # ssh-key: ${{ secrets.NETSUITE_CLIENT_SSH_KEY }} + # - name: Load decimal library + # uses: alnvdl-work/setup-private-repo@v2 + # with: + # repository: canonical/decimal + # ssh-key: ${{ secrets.DECIMAL_SSH_KEY }} - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@main + uses: charmed-kubernetes/actions-operator@1.1.0 with: provider: microk8s + channel: 1.25-strict/stable + microk8s-addons: "hostpath-storage rbac dns registry" juju-channel: 3.4/stable - - name: tmate ssh - uses: canonical/action-tmate@debug + - name: Setup tmate session + uses: canonical/action-tmate@main # - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 From ed8c3e248576388c57a926bb3bd0f909c75ff921 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:06:56 +0800 Subject: [PATCH 229/343] Debugging issue with strict microk8s --- .github/workflows/manual_test.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5a5773bf2b..33de12b53e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,16 +15,6 @@ jobs: name: self-hosted large runs-on: [self-hosted, X64, large, jammy] steps: - # - name: Load netsuite client library - # uses: alnvdl-work/setup-private-repo@v2 - # with: - # repository: canonical/netsuite-client - # ssh-key: ${{ secrets.NETSUITE_CLIENT_SSH_KEY }} - # - name: Load decimal library - # uses: alnvdl-work/setup-private-repo@v2 - # with: - # repository: canonical/decimal - # ssh-key: ${{ secrets.DECIMAL_SSH_KEY }} - name: Setup operator environment uses: charmed-kubernetes/actions-operator@1.1.0 with: From 20110d5150de891e1c6cc3298ca9e784598c0a04 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 23 Sep 2024 18:25:29 +0800 Subject: [PATCH 230/343] Test on 1.30 microk8s --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 33de12b53e..f9d5590a3f 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -19,7 +19,7 @@ jobs: uses: charmed-kubernetes/actions-operator@1.1.0 with: provider: microk8s - channel: 1.25-strict/stable + channel: 1.30-strict/stable microk8s-addons: "hostpath-storage rbac dns registry" juju-channel: 3.4/stable - name: Setup tmate session From 671ac1d8144b3c63fe7d84d209b5d8bb024e2296 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:22:27 +0800 Subject: [PATCH 231/343] Test microk8s --- .github/workflows/manual_test.yaml | 39 ++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f9d5590a3f..7cae42b29d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,14 +15,39 @@ jobs: name: self-hosted large runs-on: [self-hosted, X64, large, jammy] steps: - - name: Setup operator environment - uses: charmed-kubernetes/actions-operator@1.1.0 - with: - provider: microk8s - channel: 1.30-strict/stable - microk8s-addons: "hostpath-storage rbac dns registry" - juju-channel: 3.4/stable + - name: Set up microk8s + timeout-minutes: 5 + run: | + sudo apt-get update + sudo apt-get install retry -y + sudo snap install microk8s --channel='1.31-strict/stable' + sudo adduser "$USER" 'snap_microk8s' + - name: (IS hosted) Configure microk8s Docker Hub mirror + timeout-minutes: 5 + run: | + sudo tee /var/snap/microk8s/current/args/certs.d/docker.io/hosts.toml << EOF + server = "$DOCKERHUB_MIRROR" + [host."${DOCKERHUB_MIRROR#'https://'}"] + capabilities = ["pull", "resolve"] + EOF + sudo microk8s stop + sudo microk8s start + - name: Set up microk8s + id: microk8s-setup + timeout-minutes: 15 + run: | + # `newgrp` does not work in GitHub Actions; use `sg` instead + sg 'snap_microk8s' -c "microk8s status --wait-ready" + sg 'snap_microk8s' -c "retry --times 3 --delay 5 -- sudo microk8s enable dns" + sg 'snap_microk8s' -c "microk8s status --wait-ready" + sg 'snap_microk8s' -c "microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/coredns" + sg 'snap_microk8s' -c "retry --times 3 --delay 5 -- sudo microk8s enable hostpath-storage" + sg 'snap_microk8s' -c "microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/hostpath-provisioner" + mkdir ~/.kube/ + # Used by lightkube and kubernetes (Python package) + sg 'snap_microk8s' -c "microk8s config > ~/.kube/config" - name: Setup tmate session + if: always() uses: canonical/action-tmate@main # - run: echo hello # - run: cat /etc/docker/daemon.json From a99fbc01ca67fade5ca0f4bd494b1f906e46ecb0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:07:21 +0800 Subject: [PATCH 232/343] test microk8s dns error --- .github/workflows/manual_test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 7cae42b29d..ea75563c60 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -46,6 +46,10 @@ jobs: mkdir ~/.kube/ # Used by lightkube and kubernetes (Python package) sg 'snap_microk8s' -c "microk8s config > ~/.kube/config" + - run: sudo snap install juju + - run: sudo usermod -a -G snap_microk8s ubuntu + - run: juju add-model test + - run: juju deploy postgresql-k8s --channel 14/edge --trust - name: Setup tmate session if: always() uses: canonical/action-tmate@main From aab2296b2b0747361097be0fca7766aa67ce9a74 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:15:25 +0800 Subject: [PATCH 233/343] Add missing bootstrap --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ea75563c60..99b8fbeb48 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -48,6 +48,7 @@ jobs: sg 'snap_microk8s' -c "microk8s config > ~/.kube/config" - run: sudo snap install juju - run: sudo usermod -a -G snap_microk8s ubuntu + - run: juju bootstrap microk8s - run: juju add-model test - run: juju deploy postgresql-k8s --channel 14/edge --trust - name: Setup tmate session From a8aae17a2346f9e20d4f3f185260d96132d7e2ee Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:17:03 +0800 Subject: [PATCH 234/343] Fix group issue --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 99b8fbeb48..db58780f56 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -48,9 +48,9 @@ jobs: sg 'snap_microk8s' -c "microk8s config > ~/.kube/config" - run: sudo snap install juju - run: sudo usermod -a -G snap_microk8s ubuntu - - run: juju bootstrap microk8s + - run: sg 'snap_microk8s' -c "juju bootstrap microk8s" - run: juju add-model test - - run: juju deploy postgresql-k8s --channel 14/edge --trust + - run: juju deploy postgresql-k8s --channel 14/edge --trust - name: Setup tmate session if: always() uses: canonical/action-tmate@main From b371c75d03bac8e49603e6128538e91b395ea885 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:29:55 +0800 Subject: [PATCH 235/343] Trigger another workflow --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index db58780f56..84a89ed7c8 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -54,6 +54,7 @@ jobs: - name: Setup tmate session if: always() uses: canonical/action-tmate@main + # - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 From e2e0ac49c275cf3a036e51cb70c012c20d380af0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:37:44 +0800 Subject: [PATCH 236/343] Only tmate on error --- .github/workflows/manual_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 84a89ed7c8..f2cd54cf1d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -52,7 +52,6 @@ jobs: - run: juju add-model test - run: juju deploy postgresql-k8s --channel 14/edge --trust - name: Setup tmate session - if: always() uses: canonical/action-tmate@main # - run: echo hello From f49cdecbeea450107971287fd8940ad1b82778cc Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 26 Sep 2024 16:41:53 +0800 Subject: [PATCH 237/343] Matrix it --- .github/workflows/manual_test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index f2cd54cf1d..82340ea69b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,6 +14,10 @@ jobs: self-host: name: self-hosted large runs-on: [self-hosted, X64, large, jammy] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - name: Set up microk8s timeout-minutes: 5 @@ -52,6 +56,7 @@ jobs: - run: juju add-model test - run: juju deploy postgresql-k8s --channel 14/edge --trust - name: Setup tmate session + if: failure() uses: canonical/action-tmate@main # - run: echo hello From e16e9d10ed089070e77a7fa9dd9e8432a5b2a688 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 27 Sep 2024 09:33:49 +0800 Subject: [PATCH 238/343] Test 3 microk8s --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 82340ea69b..b0555da933 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0, 1, 2] steps: - name: Set up microk8s timeout-minutes: 5 @@ -56,7 +56,7 @@ jobs: - run: juju add-model test - run: juju deploy postgresql-k8s --channel 14/edge --trust - name: Setup tmate session - if: failure() + if: always() uses: canonical/action-tmate@main # - run: echo hello From 880bfbd9143b1fe838d52a09264786c8099bc35a Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:21:21 +0800 Subject: [PATCH 239/343] Use edge since we are out of large for 50 mins --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b0555da933..75cf844c4e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -13,7 +13,7 @@ jobs: # - run: echo hello self-host: name: self-hosted large - runs-on: [self-hosted, X64, large, jammy] + runs-on: [self-hosted, X64, edge, jammy] strategy: fail-fast: false matrix: From 28c4129bbd06c2c71493f3dfeeb9e7c33296a266 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:18:33 +0800 Subject: [PATCH 240/343] Test staging runners. --- .github/workflows/manual_test.yaml | 64 ++++++------------------------ 1 file changed, 12 insertions(+), 52 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 75cf844c4e..48f5908889 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,59 +6,19 @@ on: workflow_dispatch: jobs: - # self-hosted-edge: - # name: self-hosted wait test - # runs-on: [self-hosted, X64, edge] - # steps: - # - run: echo hello - self-host: - name: self-hosted large - runs-on: [self-hosted, X64, edge, jammy] - strategy: - fail-fast: false - matrix: - index: [0, 1, 2] + self-hosted-stg: + name: self-hosted staging + runs-on: [self-hosted, X64, staging] steps: - - name: Set up microk8s - timeout-minutes: 5 - run: | - sudo apt-get update - sudo apt-get install retry -y - sudo snap install microk8s --channel='1.31-strict/stable' - sudo adduser "$USER" 'snap_microk8s' - - name: (IS hosted) Configure microk8s Docker Hub mirror - timeout-minutes: 5 - run: | - sudo tee /var/snap/microk8s/current/args/certs.d/docker.io/hosts.toml << EOF - server = "$DOCKERHUB_MIRROR" - [host."${DOCKERHUB_MIRROR#'https://'}"] - capabilities = ["pull", "resolve"] - EOF - sudo microk8s stop - sudo microk8s start - - name: Set up microk8s - id: microk8s-setup - timeout-minutes: 15 - run: | - # `newgrp` does not work in GitHub Actions; use `sg` instead - sg 'snap_microk8s' -c "microk8s status --wait-ready" - sg 'snap_microk8s' -c "retry --times 3 --delay 5 -- sudo microk8s enable dns" - sg 'snap_microk8s' -c "microk8s status --wait-ready" - sg 'snap_microk8s' -c "microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/coredns" - sg 'snap_microk8s' -c "retry --times 3 --delay 5 -- sudo microk8s enable hostpath-storage" - sg 'snap_microk8s' -c "microk8s.kubectl rollout status --namespace kube-system --watch --timeout=5m deployments/hostpath-provisioner" - mkdir ~/.kube/ - # Used by lightkube and kubernetes (Python package) - sg 'snap_microk8s' -c "microk8s config > ~/.kube/config" - - run: sudo snap install juju - - run: sudo usermod -a -G snap_microk8s ubuntu - - run: sg 'snap_microk8s' -c "juju bootstrap microk8s" - - run: juju add-model test - - run: juju deploy postgresql-k8s --channel 14/edge --trust - - name: Setup tmate session - if: always() - uses: canonical/action-tmate@main - + - run: echo hello + # self-host: + # name: self-hosted large + # runs-on: [self-hosted, X64, edge, jammy] + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2] + # steps: # - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 From c9661f9911b3e61e315d4723dbef540cbc3f062b Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Nov 2024 12:33:04 +0800 Subject: [PATCH 241/343] Build arm64 charm for revision 293 --- .github/workflows/build_arm_charm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml index 0ae3687056..a664cb437e 100644 --- a/.github/workflows/build_arm_charm.yaml +++ b/.github/workflows/build_arm_charm.yaml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@v4 with: repository: 'canonical/github-runner-operator' - ref: 'main' + ref: 'b9dce2243310ef2c627fe7244b62824e5d44b541' - name: Install dependencies run: | sudo snap install charmcraft --classic From 53cdd3cd2234be2ea940472743ea00207370a6f2 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:45:49 +0800 Subject: [PATCH 242/343] Build arm charm --- .github/workflows/build_arm_charm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml index a664cb437e..56fb5ec785 100644 --- a/.github/workflows/build_arm_charm.yaml +++ b/.github/workflows/build_arm_charm.yaml @@ -1,7 +1,7 @@ name: build arm charm on: - # push: + push: workflow_dispatch: jobs: From 0809f1d20f62c11238cfd4d4d8ea48476723bc08 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:21:47 +0800 Subject: [PATCH 243/343] Test staging runners. --- .github/workflows/build_arm_charm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_arm_charm.yaml b/.github/workflows/build_arm_charm.yaml index 56fb5ec785..a664cb437e 100644 --- a/.github/workflows/build_arm_charm.yaml +++ b/.github/workflows/build_arm_charm.yaml @@ -1,7 +1,7 @@ name: build arm charm on: - push: + # push: workflow_dispatch: jobs: From 84052ccdd7323f10ef66fc59aa4c5a5279d22c8d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:50:46 +0800 Subject: [PATCH 244/343] Test prod private-endpoint --- .github/workflows/manual_test.yaml | 11 ++++++++--- src-docs/runner_manager.py.md | 10 +++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 48f5908889..a6d919bd78 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,11 +6,16 @@ on: workflow_dispatch: jobs: - self-hosted-stg: - name: self-hosted staging - runs-on: [self-hosted, X64, staging] + self-hosted-prod-private-endpoint: + name: self-hosted prod private endpoint + runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] steps: - run: echo hello + # self-hosted-stg: + # name: self-hosted staging + # runs-on: [self-hosted, X64, staging] + # steps: + # - run: echo hello # self-host: # name: self-hosted large # runs-on: [self-hosted, X64, edge, jammy] diff --git a/src-docs/runner_manager.py.md b/src-docs/runner_manager.py.md index 94f344c1bf..b1400ec1c5 100644 --- a/src-docs/runner_manager.py.md +++ b/src-docs/runner_manager.py.md @@ -50,7 +50,7 @@ Construct RunnerManager object for creating and managing runners. --- - + ### function `build_runner_image` @@ -87,7 +87,7 @@ Check if runner binary exists. --- - + ### function `flush` @@ -164,7 +164,7 @@ The runner binary URL changes when a new version is available. --- - + ### function `has_runner_image` @@ -181,7 +181,7 @@ Check if the runner image exists. --- - + ### function `reconcile` @@ -205,7 +205,7 @@ Bring runners in line with target. --- - + ### function `schedule_build_runner_image` From b33abd3548335cf6aa1dce8a8c3a45f59a6a0aa3 Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:17:38 +0800 Subject: [PATCH 245/343] Trigger a runner with tmate running --- .github/workflows/manual_test.yaml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a6d919bd78..10459b17f7 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,19 +6,22 @@ on: workflow_dispatch: jobs: - self-hosted-prod-private-endpoint: - name: self-hosted prod private endpoint - runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] - steps: - - run: echo hello + # self-hosted-prod-private-endpoint: + # name: self-hosted prod private endpoint + # runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] + # steps: + # - run: echo hello # self-hosted-stg: # name: self-hosted staging # runs-on: [self-hosted, X64, staging] # steps: # - run: echo hello - # self-host: - # name: self-hosted large - # runs-on: [self-hosted, X64, edge, jammy] + self-host: + name: self-hosted large + runs-on: [self-hosted, X64, edge, jammy] + steps: + - name: tmate ssh + uses: canonical/action-tmate@debug # strategy: # fail-fast: false # matrix: From 5e92cd5c042cf1f7e96f2fc0b3a90cbbb95ba75f Mon Sep 17 00:00:00 2001 From: Andrew Liaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:54:40 +0800 Subject: [PATCH 246/343] Start a github runner with tmate connection --- .github/workflows/manual_test.yaml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 10459b17f7..aa42abf066 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -16,12 +16,9 @@ jobs: # runs-on: [self-hosted, X64, staging] # steps: # - run: echo hello - self-host: - name: self-hosted large - runs-on: [self-hosted, X64, edge, jammy] - steps: - - name: tmate ssh - uses: canonical/action-tmate@debug + # self-host: + # name: self-hosted edge + # runs-on: [self-hosted, X64, edge, jammy] # strategy: # fail-fast: false # matrix: @@ -141,7 +138,11 @@ jobs: # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y # - run: $HOME/.cargo/bin/cargo install ripgrep - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # limit-access-to-actor: true + github-test: + name: GitHub x64 test + runs-on: ubuntu-latest + steps: + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true From 7f9ac29249b370256afa27fb452f19fbadfaaa4a Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 18 Feb 2025 17:40:58 +0800 Subject: [PATCH 247/343] Test canary arm64 jammy runners --- .github/workflows/manual_test.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index aa42abf066..ee9c196134 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,34 @@ on: workflow_dispatch: jobs: + self-hosted-medium-arm64-canary: + name: self-hosted-medium-arm64-canary + runs-on: [self-hosted, arm64, medium, canary] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR + - name: tmate ssh + uses: canonical/action-tmate@debug + self-hosted-large-arm64-canary: + name: self-hosted-large-arm64-canary + runs-on: [self-hosted, arm64, large, canary] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR + - name: tmate ssh + uses: canonical/action-tmate@debug # self-hosted-prod-private-endpoint: # name: self-hosted prod private endpoint # runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] From e8fdb4c358836a8a7839d43d981a9761cab72899 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 19 Feb 2025 09:22:40 +0800 Subject: [PATCH 248/343] Fix tmate branch --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ee9c196134..4332171579 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -19,7 +19,7 @@ jobs: - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR - name: tmate ssh - uses: canonical/action-tmate@debug + uses: canonical/action-tmate@main self-hosted-large-arm64-canary: name: self-hosted-large-arm64-canary runs-on: [self-hosted, arm64, large, canary] @@ -33,7 +33,7 @@ jobs: - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR - name: tmate ssh - uses: canonical/action-tmate@debug + uses: canonical/action-tmate@main # self-hosted-prod-private-endpoint: # name: self-hosted prod private endpoint # runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] From 80f7feb300813c9b2a3d0d97e4366aa2ca2f69ba Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 19 Feb 2025 16:19:41 +0800 Subject: [PATCH 249/343] Test noble canary --- .github/workflows/manual_test.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4332171579..dd42d4b444 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: self-hosted-medium-arm64-canary: name: self-hosted-medium-arm64-canary - runs-on: [self-hosted, arm64, medium, canary] + runs-on: [self-hosted, arm64, medium, canary, noble] steps: - run: lscpu - run: df -h @@ -22,7 +22,7 @@ jobs: uses: canonical/action-tmate@main self-hosted-large-arm64-canary: name: self-hosted-large-arm64-canary - runs-on: [self-hosted, arm64, large, canary] + runs-on: [self-hosted, arm64, large, canary, noble] steps: - run: lscpu - run: df -h @@ -166,11 +166,11 @@ jobs: # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh # - run: sh rustup.sh -y # - run: $HOME/.cargo/bin/cargo install ripgrep - github-test: - name: GitHub x64 test - runs-on: ubuntu-latest - steps: - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true + # github-test: + # name: GitHub x64 test + # runs-on: ubuntu-latest + # steps: + # - name: Setup tmate session + # uses: mxschmitt/action-tmate@v3 + # with: + # limit-access-to-actor: true From 1f9cdbe946c381d39c166ec7bea0ec2f0aa17b61 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 21 Feb 2025 10:17:04 +0800 Subject: [PATCH 250/343] Test a xlarge runner --- .github/workflows/manual_test.yaml | 48 +++++++++++++++++++----------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index dd42d4b444..a4af4b495c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,23 +6,9 @@ on: workflow_dispatch: jobs: - self-hosted-medium-arm64-canary: - name: self-hosted-medium-arm64-canary - runs-on: [self-hosted, arm64, medium, canary, noble] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - name: tmate ssh - uses: canonical/action-tmate@main - self-hosted-large-arm64-canary: - name: self-hosted-large-arm64-canary - runs-on: [self-hosted, arm64, large, canary, noble] + self-hosted-xlarge: + name: self-hosted-xlarge + runs-on: [self-hosted, xlarge, jammy, X64] steps: - run: lscpu - run: df -h @@ -34,6 +20,34 @@ jobs: - run: echo $DOCKERHUB_MIRROR - name: tmate ssh uses: canonical/action-tmate@main + # self-hosted-medium-arm64-canary: + # name: self-hosted-medium-arm64-canary + # runs-on: [self-hosted, arm64, medium, canary, noble] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - name: tmate ssh + # uses: canonical/action-tmate@main + # self-hosted-large-arm64-canary: + # name: self-hosted-large-arm64-canary + # runs-on: [self-hosted, arm64, large, canary, noble] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - name: tmate ssh + # uses: canonical/action-tmate@main # self-hosted-prod-private-endpoint: # name: self-hosted prod private endpoint # runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] From 7c2ed4668189ee6f55e91ff33f97726a3a97e671 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 21 Feb 2025 10:34:35 +0800 Subject: [PATCH 251/343] Test apt upgrade --- .github/workflows/manual_test.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a4af4b495c..8499d0dd5b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -10,16 +10,8 @@ jobs: name: self-hosted-xlarge runs-on: [self-hosted, xlarge, jammy, X64] steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - name: tmate ssh - uses: canonical/action-tmate@main + - run: sudo apt update + - run: sudo apt upgrade -y # self-hosted-medium-arm64-canary: # name: self-hosted-medium-arm64-canary # runs-on: [self-hosted, arm64, medium, canary, noble] From 5f5d426a492acec3ccf40ca276ad94aae3b53e44 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:41:26 +0800 Subject: [PATCH 252/343] Test reactive --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8499d0dd5b..0fda117b81 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,9 +6,9 @@ on: workflow_dispatch: jobs: - self-hosted-xlarge: - name: self-hosted-xlarge - runs-on: [self-hosted, xlarge, jammy, X64] + reactive-edge: + name: reactive-edge + runs-on: [self-hosted, reactive, edge, amd64, X64] steps: - run: sudo apt update - run: sudo apt upgrade -y From c17bd0d40bd20115dc57f68a970bb869799d8180 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:42:13 +0800 Subject: [PATCH 253/343] Disable a test --- .github/workflows/test_github_runner_manager.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_github_runner_manager.yaml b/.github/workflows/test_github_runner_manager.yaml index 3f7eff3758..2e0bc35c20 100644 --- a/.github/workflows/test_github_runner_manager.yaml +++ b/.github/workflows/test_github_runner_manager.yaml @@ -1,7 +1,8 @@ name: Tests for github-runner-manager on: - pull_request: + # pull_request: + workflow_dispatch: jobs: unit-tests-github-runner-manager: From c52410aee8932b6e399952db89cb57a6dcedff27 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:31:51 +0800 Subject: [PATCH 254/343] Test reactive --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0fda117b81..8cbf4feb79 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: reactive-edge: name: reactive-edge - runs-on: [self-hosted, reactive, edge, amd64, X64] + runs-on: [self-hosted, reactive] steps: - run: sudo apt update - run: sudo apt upgrade -y From 8ecc68b01f341ce1f70d23a59735801435be96a3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:52:48 +0800 Subject: [PATCH 255/343] Test tamte --- .github/workflows/manual_test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8cbf4feb79..4b613236ef 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - reactive-edge: - name: reactive-edge - runs-on: [self-hosted, reactive] + large-ps6: + name: large-ps6 + runs-on: [self-hosted, large, jammy] steps: - - run: sudo apt update - - run: sudo apt upgrade -y + - name: tmate ssh + uses: canonical/action-tmate@main # self-hosted-medium-arm64-canary: # name: self-hosted-medium-arm64-canary # runs-on: [self-hosted, arm64, medium, canary, noble] From 77c54248fa3d8d632e6316b879c65e3597de3a28 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:56:18 +0800 Subject: [PATCH 256/343] Test private endpoint --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4b613236ef..d2fb1de9ed 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,9 +6,9 @@ on: workflow_dispatch: jobs: - large-ps6: - name: large-ps6 - runs-on: [self-hosted, large, jammy] + private-ps6: + name: private + runs-on: [self-hosted, jammy, private-endpoint] steps: - name: tmate ssh uses: canonical/action-tmate@main From 138bd328160a034f4e910a6af5365ce8312a68f7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 11 Mar 2025 10:57:13 +0800 Subject: [PATCH 257/343] test private --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d2fb1de9ed..8d9b618c18 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -8,7 +8,7 @@ on: jobs: private-ps6: name: private - runs-on: [self-hosted, jammy, private-endpoint] + runs-on: [self-hosted-linux-amd64-jammy-private-endpoint-medium] steps: - name: tmate ssh uses: canonical/action-tmate@main From 55fc085f962c4389145d7f508978db4709c62619 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 13 Mar 2025 12:12:49 +0800 Subject: [PATCH 258/343] Test tiobe jammy --- .github/workflows/manual_test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8d9b618c18..afbc75ecff 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,9 +6,9 @@ on: workflow_dispatch: jobs: - private-ps6: - name: private - runs-on: [self-hosted-linux-amd64-jammy-private-endpoint-medium] + tiobe-jammy: + name: tiobe + runs-on: [self-hosted, linux, amd64, tiobe, jammy] steps: - name: tmate ssh uses: canonical/action-tmate@main From 9ffe65f11f0d10693101b005def3d247183554fe Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:34:51 +0800 Subject: [PATCH 259/343] test tiobe reactive runners --- .github/workflows/manual_test.yaml | 34 ++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index afbc75ecff..e600ae9f76 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,11 +7,37 @@ on: jobs: tiobe-jammy: - name: tiobe - runs-on: [self-hosted, linux, amd64, tiobe, jammy] + name: tiobe jammy + runs-on: [self-hosted, reactive, amd64, tiobe, jammy] steps: - - name: tmate ssh - uses: canonical/action-tmate@main + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + tiobe-noble: + name: tiobe noble + runs-on: [self-hosted, reactive, amd64, tiobe, noble] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + tiobe-jammy-large: + name: tiobe jammy large + runs-on: [self-hosted, reactive, amd64, large-tiobe, jammy] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + tiobe-noble-large: + name: tiobe noble large + runs-on: [self-hosted, reactive, amd64, large-tiobe, noble] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a # self-hosted-medium-arm64-canary: # name: self-hosted-medium-arm64-canary # runs-on: [self-hosted, arm64, medium, canary, noble] From 2785e7251f44f37a927b34d80c6b8d3675c05452 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:53:35 +0800 Subject: [PATCH 260/343] Test for release versin --- .github/workflows/manual_test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e600ae9f76..6991525975 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,6 +14,7 @@ jobs: - run: df -h - run: free -h - run: uname -a + - run: lsb_release -a tiobe-noble: name: tiobe noble runs-on: [self-hosted, reactive, amd64, tiobe, noble] @@ -22,6 +23,7 @@ jobs: - run: df -h - run: free -h - run: uname -a + - run: lsb_release -a tiobe-jammy-large: name: tiobe jammy large runs-on: [self-hosted, reactive, amd64, large-tiobe, jammy] @@ -30,6 +32,7 @@ jobs: - run: df -h - run: free -h - run: uname -a + - run: lsb_release -a tiobe-noble-large: name: tiobe noble large runs-on: [self-hosted, reactive, amd64, large-tiobe, noble] @@ -38,6 +41,7 @@ jobs: - run: df -h - run: free -h - run: uname -a + - run: lsb_release -a # self-hosted-medium-arm64-canary: # name: self-hosted-medium-arm64-canary # runs-on: [self-hosted, arm64, medium, canary, noble] From 73574d58944631a395959dd376c0cc2e3b001da8 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 14 Mar 2025 10:39:28 +0800 Subject: [PATCH 261/343] Test stg reactive --- .github/workflows/manual_test.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6991525975..92929379bd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,15 @@ on: workflow_dispatch: jobs: + stg-reactive: + name: staging reactive + runs-on: [self-hosted, stg-reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a tiobe-jammy: name: tiobe jammy runs-on: [self-hosted, reactive, amd64, tiobe, jammy] From e9fa56d20342b3f0197a2604d3281472c0af4fcb Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 18 Mar 2025 10:35:04 +0800 Subject: [PATCH 262/343] Test long label for reactive --- .github/workflows/manual_test.yaml | 33 +++++++++++------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 92929379bd..b3e97d4044 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,45 +6,36 @@ on: workflow_dispatch: jobs: - stg-reactive: - name: staging reactive - runs-on: [self-hosted, stg-reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - tiobe-jammy: - name: tiobe jammy - runs-on: [self-hosted, reactive, amd64, tiobe, jammy] + tiobe-noble: + name: tiobe noble + runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-tiobe] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - tiobe-noble: - name: tiobe noble - runs-on: [self-hosted, reactive, amd64, tiobe, noble] + tiobe: + name: tiobe + runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-tiobe] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - tiobe-jammy-large: - name: tiobe jammy large - runs-on: [self-hosted, reactive, amd64, large-tiobe, jammy] + tiobe-noble-large: + name: tiobe noble large + runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-large-tiobe] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - tiobe-noble-large: - name: tiobe noble large - runs-on: [self-hosted, reactive, amd64, large-tiobe, noble] + tiobe-large: + name: tiobe large + runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-large-tiobe] steps: - run: lscpu - run: df -h From f2af106d45e44b3bfad05868522631f6255c3433 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:25:01 +0800 Subject: [PATCH 263/343] Test LXD install --- .github/workflows/manual_test.yaml | 85 ++++++++++++++---------------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b3e97d4044..ba348f6e4f 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,42 +6,42 @@ on: workflow_dispatch: jobs: - tiobe-noble: - name: tiobe noble - runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-tiobe] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - tiobe: - name: tiobe - runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-tiobe] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - tiobe-noble-large: - name: tiobe noble large - runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-large-tiobe] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - tiobe-large: - name: tiobe large - runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-large-tiobe] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a + # tiobe-noble: + # name: tiobe noble + # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-tiobe] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # tiobe: + # name: tiobe + # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-tiobe] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # tiobe-noble-large: + # name: tiobe noble large + # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-large-tiobe] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # tiobe-large: + # name: tiobe large + # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-large-tiobe] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a # self-hosted-medium-arm64-canary: # name: self-hosted-medium-arm64-canary # runs-on: [self-hosted, arm64, medium, canary, noble] @@ -56,14 +56,11 @@ jobs: # - run: echo $DOCKERHUB_MIRROR # - name: tmate ssh # uses: canonical/action-tmate@main - # self-hosted-large-arm64-canary: - # name: self-hosted-large-arm64-canary - # runs-on: [self-hosted, arm64, large, canary, noble] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a + self-hosted-large: + name: self-hosted-large + runs-on: [self-hosted, amd64, large, jammy] + steps: + - run: sudo snap install --classic lxd # - run: echo $HTTP_PROXY # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY From 942eeb59161fe1df4f54e7c2fa71f1118272ebe1 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 27 Mar 2025 15:17:50 +0800 Subject: [PATCH 264/343] Test focal reactive runners --- .github/workflows/manual_test.yaml | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ba348f6e4f..10842f5282 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,44 @@ on: workflow_dispatch: jobs: + reactive-focal-large-long-label: + name: reactive focal large with long label + runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + reactive-focal-large: + name: reactive focal large + runs-on: [self-hosted, reactive, large, focal, amd64] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + reactive-focal-medium-long-label: + name: reactive focal medium with long label + runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-medium] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + reactive-focal-medium: + name: reactive focal medium + runs-on: [self-hosted, reactive, medium, focal, amd64] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + + # tiobe-noble: # name: tiobe noble # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-tiobe] From 58d731b0538a99358c0985df7ab1b1f4bd8848c7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 27 Mar 2025 15:59:07 +0800 Subject: [PATCH 265/343] Test again --- .github/workflows/manual_test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 10842f5282..e2ee415dbb 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -94,11 +94,11 @@ jobs: # - run: echo $DOCKERHUB_MIRROR # - name: tmate ssh # uses: canonical/action-tmate@main - self-hosted-large: - name: self-hosted-large - runs-on: [self-hosted, amd64, large, jammy] - steps: - - run: sudo snap install --classic lxd + # self-hosted-large: + # name: self-hosted-large + # runs-on: [self-hosted, amd64, large, jammy] + # steps: + # - run: sudo snap install --classic lxd # - run: echo $HTTP_PROXY # - run: echo $HTTPS_PROXY # - run: echo $NO_PROXY From d4076b9f039f9db99f3bff1e8091a46b05d0f6e6 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 23 Apr 2025 13:44:51 +0800 Subject: [PATCH 266/343] Test --- .github/workflows/manual_test.yaml | 92 +++++++++++++++--------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e2ee415dbb..edb024ea91 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,44 +6,42 @@ on: workflow_dispatch: jobs: - reactive-focal-large-long-label: - name: reactive focal large with long label - runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - reactive-focal-large: - name: reactive focal large - runs-on: [self-hosted, reactive, large, focal, amd64] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - reactive-focal-medium-long-label: - name: reactive focal medium with long label - runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-medium] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - reactive-focal-medium: - name: reactive focal medium - runs-on: [self-hosted, reactive, medium, focal, amd64] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - + # reactive-focal-large-long-label: + # name: reactive focal large with long label + # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # reactive-focal-large: + # name: reactive focal large + # runs-on: [self-hosted, reactive, large, focal, amd64] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # reactive-focal-medium-long-label: + # name: reactive focal medium with long label + # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-medium] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # reactive-focal-medium: + # name: reactive focal medium + # runs-on: [self-hosted, reactive, medium, focal, amd64] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a # tiobe-noble: # name: tiobe noble # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-tiobe] @@ -115,15 +113,15 @@ jobs: # runs-on: [self-hosted, X64, staging] # steps: # - run: echo hello - # self-host: - # name: self-hosted edge - # runs-on: [self-hosted, X64, edge, jammy] - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2] - # steps: - # - run: echo hello + self-host: + name: self-hosted edge + runs-on: [self-hosted, X64, edge, jammy] + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - run: echo hello # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From 98f7a07ea2e9fd163f70108f0540ce58dd14aee3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:44:00 +0800 Subject: [PATCH 267/343] test ps7 edge --- .github/workflows/manual_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index edb024ea91..37713c18b0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -119,9 +119,11 @@ jobs: strategy: fail-fast: false matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + index: [0] steps: - run: echo hello + - name: tmate ssh + uses: canonical/action-tmate@main # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From f5201ab59c8803c4672678f2f11b351de73c1d5c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 23 Apr 2025 14:47:49 +0800 Subject: [PATCH 268/343] Test large runner --- .github/workflows/manual_test.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 37713c18b0..4db9fef9eb 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -113,6 +113,17 @@ jobs: # runs-on: [self-hosted, X64, staging] # steps: # - run: echo hello + self-host-large: + name: self-hosted large + runs-on: [self-hosted, X64, large, jammy] + strategy: + fail-fast: false + matrix: + index: [0] + steps: + - run: echo hello + - name: tmate ssh + uses: canonical/action-tmate@main self-host: name: self-hosted edge runs-on: [self-hosted, X64, edge, jammy] From bdb7e17ef145d7170645fa2f1dc9750daaf8811e Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 15 May 2025 19:59:15 +0800 Subject: [PATCH 269/343] test tmate --- .github/workflows/manual_test.yaml | 50 +++++++++++++++++------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4db9fef9eb..1017d79ebb 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,12 @@ on: workflow_dispatch: jobs: + tmate-test: + runs-on: [self-hosted] + steps: + # Enable tmate debugging of manually-triggered workflows if the input option was provided + - name: Setup tmate session + uses: canonical/action-tmate@feat/ubuntu-arch # reactive-focal-large-long-label: # name: reactive focal large with long label # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] @@ -113,28 +119,28 @@ jobs: # runs-on: [self-hosted, X64, staging] # steps: # - run: echo hello - self-host-large: - name: self-hosted large - runs-on: [self-hosted, X64, large, jammy] - strategy: - fail-fast: false - matrix: - index: [0] - steps: - - run: echo hello - - name: tmate ssh - uses: canonical/action-tmate@main - self-host: - name: self-hosted edge - runs-on: [self-hosted, X64, edge, jammy] - strategy: - fail-fast: false - matrix: - index: [0] - steps: - - run: echo hello - - name: tmate ssh - uses: canonical/action-tmate@main + # self-host-large: + # name: self-hosted large + # runs-on: [self-hosted, X64, large, jammy] + # strategy: + # fail-fast: false + # matrix: + # index: [0] + # steps: + # - run: echo hello + # - name: tmate ssh + # uses: canonical/action-tmate@main + # self-host: + # name: self-hosted edge + # runs-on: [self-hosted, X64, edge, jammy] + # strategy: + # fail-fast: false + # matrix: + # index: [0] + # steps: + # - run: echo hello + # - name: tmate ssh + # uses: canonical/action-tmate@main # - run: cat /etc/docker/daemon.json # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 From b907ea49648239d76f2496e01a05ba254789f985 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 15 May 2025 20:41:46 +0800 Subject: [PATCH 270/343] Test powerpc tmate --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 1017d79ebb..67640716ba 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: tmate-test: - runs-on: [self-hosted] + runs-on: [self-hosted, ppc64el] steps: # Enable tmate debugging of manually-triggered workflows if the input option was provided - name: Setup tmate session From 7fb77668652ed735b4d9f039220ca32aaf7dc793 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:50:13 +0800 Subject: [PATCH 271/343] Test getting arm64 medium jammy runner --- .github/workflows/manual_test.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 67640716ba..78d3f93955 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,14 @@ on: workflow_dispatch: jobs: - tmate-test: - runs-on: [self-hosted, ppc64el] + test: + runs-on: [self-hosted, arm64, medium, jammy] steps: - # Enable tmate debugging of manually-triggered workflows if the input option was provided - - name: Setup tmate session - uses: canonical/action-tmate@feat/ubuntu-arch + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a # reactive-focal-large-long-label: # name: reactive focal large with long label # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] From 1076173e3fd2697449292a3b220972b28f14c26b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 3 Jun 2025 08:57:58 +0800 Subject: [PATCH 272/343] Test amd64 large noble --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 78d3f93955..b828710786 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: test: - runs-on: [self-hosted, arm64, medium, jammy] + runs-on: [self-hosted, amd64, large, noble] steps: - run: lscpu - run: df -h From e926ae2eccbacd9823e6e152d6203b280c25fcab Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:55:05 +0800 Subject: [PATCH 273/343] Test staging runners --- .github/workflows/manual_test.yaml | 43 ++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index b828710786..4bee21c80b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,14 +6,53 @@ on: workflow_dispatch: jobs: - test: - runs-on: [self-hosted, amd64, large, noble] + stg-reactive-test: + runs-on: [self-hosted,staging,amd64,self-hosted-linux-amd64-jammy-staging,stg-reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR + - uses: canonical/setup-lxd@v0.1.1 + - run: lxd init --auto + if: ${{ always() }} + - run: lxc list + if: ${{ always() }} + - run: time sudo snap install juju + if: ${{ always() }} + - run: time juju bootstrap localhost localhost + if: ${{ always() }} + self-host-x64: + name: self-hosted x64 test + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, staging,amd64,self-hosted-linux-amd64-jammy-staging] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR + - uses: canonical/setup-lxd@v0.1.1 + - run: lxd init --auto + if: ${{ always() }} + - run: lxc list + if: ${{ always() }} + - run: time sudo snap install juju + if: ${{ always() }} + - run: time juju bootstrap localhost localhost + if: ${{ always() }} # reactive-focal-large-long-label: # name: reactive focal large with long label # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] From b61a67ced2eebeafa989d0f59467b6dc08b5ff11 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:59:09 +0800 Subject: [PATCH 274/343] Test stg --- .github/workflows/manual_test.yaml | 47 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4bee21c80b..2ea225fc93 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,29 +6,28 @@ on: workflow_dispatch: jobs: - stg-reactive-test: - runs-on: [self-hosted,staging,amd64,self-hosted-linux-amd64-jammy-staging,stg-reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - uses: canonical/setup-lxd@v0.1.1 - - run: lxd init --auto - if: ${{ always() }} - - run: lxc list - if: ${{ always() }} - - run: time sudo snap install juju - if: ${{ always() }} - - run: time juju bootstrap localhost localhost - if: ${{ always() }} - self-host-x64: - name: self-hosted x64 test + # stg-reactive-test: + # runs-on: [self-hosted,staging,amd64,self-hosted-linux-amd64-jammy-staging,stg-reactive] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - uses: canonical/setup-lxd@v0.1.1 + # - run: lxd init --auto + # if: ${{ always() }} + # - run: lxc list + # if: ${{ always() }} + # - run: time sudo snap install juju + # if: ${{ always() }} + # - run: time juju bootstrap localhost localhost + # if: ${{ always() }} + stg-test: strategy: fail-fast: false matrix: @@ -45,7 +44,7 @@ jobs: - run: echo $NO_PROXY - run: echo $DOCKERHUB_MIRROR - uses: canonical/setup-lxd@v0.1.1 - - run: lxd init --auto + - run: sudo lxd init --auto if: ${{ always() }} - run: lxc list if: ${{ always() }} From 5b04526191374f867f656b3bd11c6598774a8640 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 10 Jun 2025 09:40:03 +0800 Subject: [PATCH 275/343] Test edge reactive --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 2ea225fc93..17978f798e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -27,12 +27,12 @@ jobs: # if: ${{ always() }} # - run: time juju bootstrap localhost localhost # if: ${{ always() }} - stg-test: + edge-test: strategy: fail-fast: false matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, staging,amd64,self-hosted-linux-amd64-jammy-staging] + runs-on: [self-hosted, edge,arm64,self-hosted-linux-arm64-noble-edge,reactive] steps: - run: lscpu - run: df -h From b545db5da8225a74aff11368b7b6c0d927c94863 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 4 Jun 2025 04:18:44 +0200 Subject: [PATCH 276/343] Redirect HKP protocol traffic (port 11371) to aproxy (#563) * Redirect HKP protocol traffic (port 11371) to aproxy * Fix linting issue and update changelog.md --- config.yaml | 2 +- docs/changelog.md | 4 ++++ docs/explanation/charm-architecture.md | 2 +- docs/local-lxd/explanation/charm-architecture.md | 2 +- .../github_runner_manager/templates/openstack-userdata.sh.j2 | 4 ++-- tests/integration/conftest.py | 3 +++ 6 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config.yaml b/config.yaml index cdcba8c077..902ea61019 100644 --- a/config.yaml +++ b/config.yaml @@ -38,7 +38,7 @@ options: description: >- (Experimental, may be removed) When set to true, aproxy (https://github.com/canonical/aproxy) will be installed within the runners. It will forward all HTTP(S) traffic to standard ports - (80, 443) to a proxy server configured by the juju model config 'juju-http-proxy' + (80, 443, 11371) to a proxy server configured by the juju model config 'juju-http-proxy' (or, if this is not set, 'juju-https-proxy' will be used). This is useful when the charm is deployed in a network that requires a proxy to access the internet. diff --git a/docs/changelog.md b/docs/changelog.md index 82575845da..c445e0480b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +### 2025-06-03 + +- Redirect HKP protocol traffic (port 11371) to aproxy + ### 2025-05-22 - Add possibility to run a script in the pre-job phase of a runner. This can be useful to setup diff --git a/docs/explanation/charm-architecture.md b/docs/explanation/charm-architecture.md index b915f69c74..f3b01b1796 100644 --- a/docs/explanation/charm-architecture.md +++ b/docs/explanation/charm-architecture.md @@ -90,7 +90,7 @@ This inconsistency can result in failed workflows, prompting the introduction of ### aproxy If the proxy configuration is utilised and [aproxy](https://github.com/canonical/aproxy) is specified through the charm's configuration option, -all HTTP(S) requests to standard ports (80, 443) within the GitHub workflow will be automatically directed +all HTTP(S) requests to standard ports (80, 443, 11371) within the GitHub workflow will be automatically directed to the specified HTTP(s) proxy. Network traffic destined for ports 80 and 443 is redirected to aproxy using iptables. aproxy then forwards received packets to the designated HTTP(S) proxy. Beyond that, the environment variables (`http_proxy`, `https_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`) diff --git a/docs/local-lxd/explanation/charm-architecture.md b/docs/local-lxd/explanation/charm-architecture.md index ee0795ee9c..b5f7a97d32 100644 --- a/docs/local-lxd/explanation/charm-architecture.md +++ b/docs/local-lxd/explanation/charm-architecture.md @@ -61,7 +61,7 @@ This inconsistency can result in failed workflows, prompting the introduction of ### aproxy If the proxy configuration is utilized and [aproxy](https://github.com/canonical/aproxy) is specified through the charm's configuration option, -all HTTP(S) requests to standard ports (80, 443) within the GitHub workflow will be automatically directed +all HTTP(S) requests to standard ports (80, 443, 11371) within the GitHub workflow will be automatically directed to the specified HTTP(s) proxy. Network traffic destined for ports 80 and 443 is redirected to aproxy using iptables. aproxy then forwards received packets to the designated HTTP(S) proxy. Beyond that, the environment variables (`http_proxy`, `https_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`) diff --git a/github-runner-manager/src/github_runner_manager/templates/openstack-userdata.sh.j2 b/github-runner-manager/src/github_runner_manager/templates/openstack-userdata.sh.j2 index 7f007b5920..f509b107af 100644 --- a/github-runner-manager/src/github_runner_manager/templates/openstack-userdata.sh.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/openstack-userdata.sh.j2 @@ -21,12 +21,12 @@ flush table ip aproxy table ip aproxy { chain prerouting { type nat hook prerouting priority dstnat; policy accept; - ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + ip daddr != \$private-ips tcp dport { 80, 443, 11371 } counter dnat to \$default-ip:54969 } chain output { type nat hook output priority -100; policy accept; - ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 + ip daddr != \$private-ips tcp dport { 80, 443, 11371 } counter dnat to \$default-ip:54969 } } EOF diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2a0386978d..7e3cc6d3e9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -185,6 +185,9 @@ def private_endpoint_config_fixture(pytestconfig: pytest.Config) -> PrivateEndpo auth_url = pytestconfig.getoption("--openstack-auth-url-amd64") password = pytestconfig.getoption("--openstack-password-amd64") password = password or os.environ.get("INTEGRATION_OPENSTACK_PASSWORD_AMD64") + assert ( + password + ), "Please specify the --openstack-password-amd64 option or INTEGRATION_OPENSTACK_PASSWORD_AMD64 environment variable" project_domain_name = pytestconfig.getoption("--openstack-project-domain-name-amd64") project_name = pytestconfig.getoption("--openstack-project-name-amd64") user_domain_name = pytestconfig.getoption("--openstack-user-domain-name-amd64") From 08ca88fd693b3a98d089035e6fcf66a542fb82ca Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Wed, 4 Jun 2025 14:50:44 +0200 Subject: [PATCH 277/343] pass base-channel 22.04 (#564) --- .github/workflows/promote_charm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml index ee2e39c2f2..d1a0ac0ffb 100644 --- a/.github/workflows/promote_charm.yaml +++ b/.github/workflows/promote_charm.yaml @@ -28,4 +28,5 @@ jobs: origin-channel: ${{ github.event.inputs.origin-channel }} destination-channel: ${{ github.event.inputs.destination-channel }} doc-automation-disabled: false + base-channel: '22.04' secrets: inherit From d4324a8eb322725d9e83d66ad9bb7dffe1c7759d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 9 Jun 2025 17:16:11 +0800 Subject: [PATCH 278/343] feat: github-runner-manager service for reconcilations (#561) --- actions.yaml | 2 - config.yaml | 8 +- docs/changelog.md | 8 + .../src/github_runner_manager/cli.py | 19 +- .../configuration/base.py | 2 + .../src/github_runner_manager/http_server.py | 24 +- .../manager/runner_scaler.py | 12 +- .../reactive/process_manager.py | 20 +- .../github_runner_manager/reactive/runner.py | 2 +- .../reactive/runner_manager.py | 3 + .../reconcile_service.py | 52 +++- .../tests/integration/conftest.py | 1 + .../tests/integration/test_cli.py | 3 +- github-runner-manager/tests/unit/conftest.py | 2 +- .../unit/reactive/test_runner_manager.py | 6 +- .../tests/unit/test_config.py | 2 + .../tests/unit/test_http_server.py | 2 +- .../tests/unit/test_runner_scaler.py | 3 +- src/charm.py | 239 ++++-------------- src/charm_state.py | 10 +- src/errors.py | 16 ++ src/event_timer.py | 161 ------------ src/factories.py | 1 + src/manager_client.py | 45 +++- src/manager_service.py | 38 ++- templates/dispatch-event.service.j2 | 10 - templates/dispatch-event.timer.j2 | 11 - tests/integration/conftest.py | 68 ++--- tests/integration/helpers/common.py | 43 +++- tests/integration/helpers/openstack.py | 4 +- .../test_charm_fork_path_change.py | 4 +- .../integration/test_charm_metrics_failure.py | 10 +- .../integration/test_charm_metrics_success.py | 6 +- tests/integration/test_charm_no_runner.py | 21 +- tests/integration/test_charm_runner.py | 12 +- tests/integration/test_debug_ssh.py | 3 +- tests/integration/test_jobmanager.py | 30 ++- tests/integration/test_reactive.py | 16 +- tests/unit/conftest.py | 15 -- tests/unit/test_charm.py | 211 +++------------- tests/unit/test_charm_state.py | 2 +- tests/unit/test_event_timer.py | 27 -- tests/unit/test_factories.py | 1 + tests/unit/test_manager_client.py | 1 + tests/unit/test_manager_service.py | 24 ++ 45 files changed, 463 insertions(+), 737 deletions(-) delete mode 100644 src/event_timer.py delete mode 100644 templates/dispatch-event.service.j2 delete mode 100644 templates/dispatch-event.timer.j2 delete mode 100644 tests/unit/test_event_timer.py diff --git a/actions.yaml b/actions.yaml index 00904bfffe..a8b1be89e7 100644 --- a/actions.yaml +++ b/actions.yaml @@ -3,7 +3,5 @@ check-runners: description: Get info on active and registered runners -reconcile-runners: - description: Remove offline runners and replace any missing runners flush-runners: description: Clear out all runners and start a new set diff --git a/config.yaml b/config.yaml index 902ea61019..30beec90b6 100644 --- a/config.yaml +++ b/config.yaml @@ -67,13 +67,11 @@ options: "{owner}/{repo}" format. reconcile-interval: type: int - default: 10 + default: 5 description: >- Minutes between each reconciliation of the current runners state and their targeted state. - On reconciliation, the charm polls the state of runners and see if actions are needed. A - random delay of 25% is added. Changes to this time interval will apply when next interval is - triggered. Prior to reconciliation, any packages, and services used by the software will be - updated. + On reconciliation, the charm polls the state of runners and see if actions are needed. The + value should be kept low, unless Github API rate limiting is encountered. repo-policy-compliance-token: type: string description: >- diff --git a/docs/changelog.md b/docs/changelog.md index c445e0480b..b4617d6104 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,12 @@ # Changelog +### 2025-06-04 + +- Reduce the reconcile-interval configuration from 10 minutes to 5 minutes. This is the interval +between reconciling the current and intended number of runners. The value should be kept low, +unless Github API rate limiting is encountered. +- Removed the reconcile-runners Juju action. + ### 2025-06-03 - Redirect HKP protocol traffic (port 11371) to aproxy @@ -9,6 +16,7 @@ - Add possibility to run a script in the pre-job phase of a runner. This can be useful to setup network/infrastructure specific things. + ### 2025-05-09 - The log rotation is done by copying the logs then truncating. There is a small chance some log might be lost during the log rotation. This affects the github-runner-manager service logs, and the reactive spawning logs. diff --git a/github-runner-manager/src/github_runner_manager/cli.py b/github-runner-manager/src/github_runner_manager/cli.py index 66dd957f73..f4b0f519e2 100644 --- a/github-runner-manager/src/github_runner_manager/cli.py +++ b/github-runner-manager/src/github_runner_manager/cli.py @@ -43,12 +43,19 @@ default=False, help="Debug mode for testing.", ) +@click.option( + "--python-path", + type=str, + default="", + help="The PYTHONPATH to the github-runner-manager library.", +) # The entry point for the CLI will be tested with integration test. def main( config_file: TextIO, host: str, port: int, debug: bool, + python_path: str, ) -> None: # pragma: no cover """Start the reconcile service. @@ -57,11 +64,17 @@ def main( host: The hostname to listen on for the HTTP server port: The port to listen on the HTTP server. debug: Whether to start the application in debug mode. + python_path: The PYTHONPATH to access the github-runner-manager library. """ + python_path_config = python_path if python_path else None log_level = logging.INFO if debug: log_level = logging.DEBUG - logging.basicConfig(level=log_level, stream=sys.stderr) + logging.basicConfig( + level=log_level, + stream=sys.stderr, + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + ) lock = Lock() config_str = config_file.read() @@ -70,7 +83,9 @@ def main( thread_manager = ThreadManager() thread_manager.add_thread(target=partial(start_http_server, config, lock, http_server_args)) - thread_manager.add_thread(target=partial(start_reconcile_service, config, lock)) + thread_manager.add_thread( + target=partial(start_reconcile_service, config, python_path_config, lock) + ) thread_manager.start() thread_manager.raise_on_error() diff --git a/github-runner-manager/src/github_runner_manager/configuration/base.py b/github-runner-manager/src/github_runner_manager/configuration/base.py index cdb29d1fd3..affa8c5a9a 100644 --- a/github-runner-manager/src/github_runner_manager/configuration/base.py +++ b/github-runner-manager/src/github_runner_manager/configuration/base.py @@ -45,6 +45,7 @@ class ApplicationConfiguration(BaseModel): non_reactive_configuration: Configuration for non-reactive mode. reactive_configuration: Configuration for reactive mode. openstack_configuration: Configuration for authorization to a OpenStack host. + reconcile_interval: Seconds to wait between reconciliation. """ name: str @@ -54,6 +55,7 @@ class ApplicationConfiguration(BaseModel): non_reactive_configuration: "NonReactiveConfiguration" reactive_configuration: "ReactiveConfiguration | None" openstack_configuration: OpenStackConfiguration + reconcile_interval: int @staticmethod def from_yaml_file(file: TextIO) -> "ApplicationConfiguration": diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index 5e04fb858e..be806a88fe 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -7,19 +7,16 @@ """ import dataclasses -import getpass -import grp import json -import os from dataclasses import dataclass from threading import Lock from flask import Flask, request -from github_runner_manager.configuration import ApplicationConfiguration, UserInfo +from github_runner_manager.configuration import ApplicationConfiguration from github_runner_manager.errors import CloudError, LockError from github_runner_manager.manager.runner_manager import FlushMode -from github_runner_manager.manager.runner_scaler import RunnerScaler +from github_runner_manager.reconcile_service import get_runner_scaler APP_CONFIG_NAME = "app_config" OPENSTACK_CONFIG_NAME = "openstack_config" @@ -49,7 +46,7 @@ def check_runner() -> tuple[str, int]: """ app_config = app.config[APP_CONFIG_NAME] app.logger.info("Checking runners...") - runner_scaler = _get_runner_scaler(app_config) + runner_scaler = get_runner_scaler(app_config) try: runner_info = runner_scaler.get_runner_info() except CloudError as err: @@ -78,7 +75,7 @@ def flush_runner() -> tuple[str, int]: lock = get_lock() with lock: app.logger.info("Flushing runners...") - runner_scaler = _get_runner_scaler(app_config) + runner_scaler = get_runner_scaler(app_config) app.logger.info("Flushing busy: %s", flush_busy) flush_mode = FlushMode.FLUSH_BUSY if flush_busy else FlushMode.FLUSH_IDLE try: @@ -121,19 +118,6 @@ class FlaskArgs: debug: bool -def _get_runner_scaler(app_config: ApplicationConfiguration) -> RunnerScaler: - """Get runner scaler. - - Args: - app_config: The configuration of github-runner-manager. - - Returns: - The RunnerScaler object. - """ - user = UserInfo(getpass.getuser(), grp.getgrgid(os.getgid())) - return RunnerScaler.build(app_config, user) - - def start_http_server( app_config: ApplicationConfiguration, lock: Lock, diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index 1f0ec2f37a..9cfa55f305 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -99,22 +99,25 @@ class _ReconcileMetricData: class RunnerScaler: """Manage the reconcile of runners.""" + # Disable too many locals due to this function is collecting and processing configurations. @classmethod - def build( + def build( # pylint: disable-msg=too-many-locals cls, application_configuration: ApplicationConfiguration, user: UserInfo, + python_path: str | None = None, ) -> "RunnerScaler": """Create a RunnerScaler from application and OpenStack configuration. Args: application_configuration: Main configuration for the application. user: The user to run reactive process. + python_path: The PYTHONPATH to access the github-runner-manager library. Returns: A new RunnerScaler. """ - labels = application_configuration.extra_labels + labels = list(application_configuration.extra_labels) server_config = None base_quantity = 0 if combinations := application_configuration.non_reactive_configuration.combinations: @@ -171,6 +174,7 @@ def build( user=user, base_quantity=base_quantity, max_quantity=max_quantity, + python_path=python_path, ) # The `user` argument will be removed once the charm no longer uses the github-runner-manager @@ -185,6 +189,7 @@ def __init__( # pylint: disable=too-many-arguments, too-many-positional-argumen user: UserInfo, base_quantity: int, max_quantity: int, + python_path: str | None = None, ): """Construct the object. @@ -194,12 +199,14 @@ def __init__( # pylint: disable=too-many-arguments, too-many-positional-argumen user: The user to run the reactive process. base_quantity: The number of intended non-reactive runners. max_quantity: The number of maximum runners for reactive. + python_path: The PYTHONPATH to access the github-runner-manager library. """ self._manager = runner_manager self._reactive_config = reactive_process_config self._user = user self._base_quantity = base_quantity self._max_quantity = max_quantity + self._python_path = python_path def get_runner_info(self) -> RunnerInfo: """Get information on the runners. @@ -283,6 +290,7 @@ def reconcile(self) -> int: runner_manager=self._manager, reactive_process_config=self._reactive_config, user=self._user, + python_path=self._python_path, ) reconcile_diff = reconcile_result.processes_diff metric_stats = reconcile_result.metric_stats diff --git a/github-runner-manager/src/github_runner_manager/reactive/process_manager.py b/github-runner-manager/src/github_runner_manager/reactive/process_manager.py index b05aa07242..0fd01b0c50 100644 --- a/github-runner-manager/src/github_runner_manager/reactive/process_manager.py +++ b/github-runner-manager/src/github_runner_manager/reactive/process_manager.py @@ -40,7 +40,10 @@ class ReactiveRunnerError(Exception): def reconcile( - quantity: int, reactive_process_config: ReactiveProcessConfig, user: UserInfo + quantity: int, + reactive_process_config: ReactiveProcessConfig, + user: UserInfo, + python_path: str | None = None, ) -> int: """Reconcile the number of reactive runner processes. @@ -48,6 +51,7 @@ def reconcile( quantity: The number of processes to spawn. reactive_process_config: The reactive runner configuration. user: The user to run the reactive process. + python_path: The PYTHONPATH to access the github-runner-manager library. Raises a ReactiveRunnerError if the runner fails to spawn. @@ -56,13 +60,17 @@ def reconcile( """ pids = _get_pids() current_quantity = len(pids) - logger.info("Current quantity of reactive runner processes: %s", current_quantity) + logger.info( + "Reactive runner processes: current quantity %s, expected quantity %s", + current_quantity, + quantity, + ) delta = quantity - current_quantity if delta > 0: logger.info("Will spawn %d new reactive runner process(es)", delta) _setup_logging_for_processes(user.user, user.group) for _ in range(delta): - _spawn_runner(reactive_process_config) + _spawn_runner(reactive_process_config, python_path) elif delta < 0: logger.info("Will kill %d process(es).", -delta) for pid in pids[:-delta]: @@ -126,16 +134,18 @@ def _setup_logging_for_processes(user: str, group: str) -> None: ) -def _spawn_runner(reactive_process_config: ReactiveProcessConfig) -> None: +def _spawn_runner(reactive_process_config: ReactiveProcessConfig, python_path: str | None) -> None: """Spawn a runner. Args: reactive_process_config: The runner configuration to pass to the spawned runner process. + python_path: The PYTHONPATH to access the github-runner-manager library. """ env = { - "PYTHONPATH": os.environ["PYTHONPATH"], RUNNER_CONFIG_ENV_VAR: reactive_process_config.json(), } + if python_path is not None: + env["PYTHONPATH"] = str(python_path) # We do not want to wait for the process to finish, so we do not use with statement. # We trust the command. command = " ".join( diff --git a/github-runner-manager/src/github_runner_manager/reactive/runner.py b/github-runner-manager/src/github_runner_manager/reactive/runner.py index c5a27cca09..6146f2e91d 100644 --- a/github-runner-manager/src/github_runner_manager/reactive/runner.py +++ b/github-runner-manager/src/github_runner_manager/reactive/runner.py @@ -47,7 +47,7 @@ def main() -> None: setup_root_logging() queue_config = runner_config.queue - user = UserInfo(getpass.getuser(), grp.getgrgid(os.getgid())) + user = UserInfo(getpass.getuser(), grp.getgrgid(os.getgid()).gr_name) openstack_runner_manager = OpenStackRunnerManager( config=runner_config.cloud_runner_manager, user=user ) diff --git a/github-runner-manager/src/github_runner_manager/reactive/runner_manager.py b/github-runner-manager/src/github_runner_manager/reactive/runner_manager.py index fe1217b063..38fcf52a95 100644 --- a/github-runner-manager/src/github_runner_manager/reactive/runner_manager.py +++ b/github-runner-manager/src/github_runner_manager/reactive/runner_manager.py @@ -37,6 +37,7 @@ def reconcile( runner_manager: RunnerManager, reactive_process_config: ReactiveProcessConfig, user: UserInfo, + python_path: str | None = None, ) -> ReconcileResult: """Reconcile runners reactively. @@ -80,6 +81,7 @@ def reconcile( runner_manager: The runner manager to interact with current running runners. reactive_process_config: The reactive runner config. user: The user to run the reactive process. + python_path: The PYTHONPATH to access the github-runner-manager library. Returns: The number of reactive processes created. If negative, its absolute value is equal @@ -122,6 +124,7 @@ def reconcile( quantity=process_quantity, reactive_process_config=reactive_process_config, user=user, + python_path=python_path, ) return ReconcileResult(processes_diff=processes_created, metric_stats=metric_stats) diff --git a/github-runner-manager/src/github_runner_manager/reconcile_service.py b/github-runner-manager/src/github_runner_manager/reconcile_service.py index 8c192fc7c0..f9ecd43c7f 100644 --- a/github-runner-manager/src/github_runner_manager/reconcile_service.py +++ b/github-runner-manager/src/github_runner_manager/reconcile_service.py @@ -3,24 +3,66 @@ """The reconcile service for managing the self-hosted runner.""" +import getpass +import grp import logging +import os +import uuid +from pathlib import Path from threading import Lock from time import sleep from github_runner_manager.configuration import ApplicationConfiguration +from github_runner_manager.configuration.base import UserInfo +from github_runner_manager.manager.runner_scaler import RunnerScaler logger = logging.getLogger(__name__) +RECONCILE_ID_FILE = Path("/var/log/reconcile.id") -def start_reconcile_service(_: ApplicationConfiguration, lock: Lock) -> None: +RECONCILE_SERVICE_START_MSG = "Starting the reconcile service..." +RECONCILE_START_MSG = "Start reconciliation" +RECONCILE_END_MSG = "End reconciliation" + + +def get_runner_scaler( + app_config: ApplicationConfiguration, python_path: str | None = None +) -> RunnerScaler: + """Get runner scaler. + + Args: + app_config: The configuration of github-runner-manager. + python_path: The PYTHONPATH to access the github-runner-manager library. + + Returns: + The RunnerScaler object. + """ + user = UserInfo(getpass.getuser(), grp.getgrgid(os.getgid()).gr_name) + return RunnerScaler.build(app_config, user, python_path) + + +def start_reconcile_service( + app_config: ApplicationConfiguration, python_path: str | None, lock: Lock +) -> None: """Start the reconcile server. Args: + app_config: The configuration of the application. + python_path: The PYTHONPATH to access the github-runner-manager library. lock: The lock representing modification access to the managed set of runners. """ - logger.info("Starting the reconcile_service...") + logger.info(RECONCILE_SERVICE_START_MSG) + + # This is used for in test to distinguish which reconcile run the unit is at. + RECONCILE_ID_FILE.write_text(str(uuid.uuid4()), encoding="utf-8") + while True: with lock: - logger.info("Acquired the lock for reconciling") - sleep(10) - logger.info("Released the lock") + logger.info(RECONCILE_START_MSG) + runner_scaler = get_runner_scaler(app_config, python_path=python_path) + delta = runner_scaler.reconcile() + logger.info("Change in number of runner after reconcile: %s", delta) + logger.info(RECONCILE_END_MSG) + RECONCILE_ID_FILE.write_text(str(uuid.uuid4()), encoding="utf-8") + + sleep(app_config.reconcile_interval * 60) diff --git a/github-runner-manager/tests/integration/conftest.py b/github-runner-manager/tests/integration/conftest.py index 74af5687c5..4f57fad230 100644 --- a/github-runner-manager/tests/integration/conftest.py +++ b/github-runner-manager/tests/integration/conftest.py @@ -77,6 +77,7 @@ def config_fixture() -> dict: "region_name": "test_region", }, }, + "reconcile_interval": 1, } diff --git a/github-runner-manager/tests/integration/test_cli.py b/github-runner-manager/tests/integration/test_cli.py index 086981e62f..0586973f51 100644 --- a/github-runner-manager/tests/integration/test_cli.py +++ b/github-runner-manager/tests/integration/test_cli.py @@ -6,6 +6,7 @@ import subprocess from time import sleep +from github_runner_manager.reconcile_service import RECONCILE_SERVICE_START_MSG from tests.integration.helper import get_app_log, health_check @@ -19,7 +20,7 @@ def test_app_reconcile(app: subprocess.Popen): app.terminate() logs = get_app_log(app) assert "Starting the server..." in logs - assert "Starting the reconcile_service..." in logs + assert RECONCILE_SERVICE_START_MSG in logs def test_app_http_server(app: subprocess.Popen): diff --git a/github-runner-manager/tests/unit/conftest.py b/github-runner-manager/tests/unit/conftest.py index 57e9a4c1c3..d276b286ad 100644 --- a/github-runner-manager/tests/unit/conftest.py +++ b/github-runner-manager/tests/unit/conftest.py @@ -14,4 +14,4 @@ @pytest.fixture(name="user_info", scope="module") def user_info_fixture(): - return UserInfo(getpass.getuser(), str(grp.getgrgid(os.getgid()))) + return UserInfo(getpass.getuser(), grp.getgrgid(os.getgid()).gr_name) diff --git a/github-runner-manager/tests/unit/reactive/test_runner_manager.py b/github-runner-manager/tests/unit/reactive/test_runner_manager.py index c14ab31a2b..da608e9cbc 100644 --- a/github-runner-manager/tests/unit/reactive/test_runner_manager.py +++ b/github-runner-manager/tests/unit/reactive/test_runner_manager.py @@ -97,6 +97,7 @@ def test_reconcile_positive_runner_diff( quantity=expected_process_quantity, reactive_process_config=reactive_process_config, user=user_info, + python_path=None, ) @@ -139,7 +140,10 @@ def test_reconcile_negative_runner_diff( runner_manager.cleanup.assert_called_once() runner_manager.delete_runners.assert_called_once_with(expected_number_of_runners_to_delete) reactive_process_manager.reconcile.assert_called_once_with( - quantity=0, reactive_process_config=reactive_process_config, user=user_info + quantity=0, + reactive_process_config=reactive_process_config, + user=user_info, + python_path=None, ) diff --git a/github-runner-manager/tests/unit/test_config.py b/github-runner-manager/tests/unit/test_config.py index 8794986ef1..595724f01f 100644 --- a/github-runner-manager/tests/unit/test_config.py +++ b/github-runner-manager/tests/unit/test_config.py @@ -98,6 +98,7 @@ username: test_username network: test_network vm_prefix: test_unit +reconcile_interval: 10 """ @@ -183,6 +184,7 @@ def app_config_fixture() -> ApplicationConfiguration: region_name="test_region", ), ), + reconcile_interval=10, ) diff --git a/github-runner-manager/tests/unit/test_http_server.py b/github-runner-manager/tests/unit/test_http_server.py index 97f190b5fe..4d3e0bf97e 100644 --- a/github-runner-manager/tests/unit/test_http_server.py +++ b/github-runner-manager/tests/unit/test_http_server.py @@ -38,7 +38,7 @@ def client_fixture(lock: Lock, monkeypatch) -> Iterator[FlaskClient]: def mock_runner_scaler_fixture(monkeypatch) -> MagicMock: mock = MagicMock(spec=RunnerScaler) monkeypatch.setattr( - "src.github_runner_manager.http_server.RunnerScaler.build", lambda x, z: mock + "src.github_runner_manager.reconcile_service.RunnerScaler.build", lambda x, y, z: mock ) return mock diff --git a/github-runner-manager/tests/unit/test_runner_scaler.py b/github-runner-manager/tests/unit/test_runner_scaler.py index 2cbb167bd7..376aa78d04 100644 --- a/github-runner-manager/tests/unit/test_runner_scaler.py +++ b/github-runner-manager/tests/unit/test_runner_scaler.py @@ -195,6 +195,7 @@ def application_configuration_fixture() -> ApplicationConfiguration: region_name="region", ), ), + reconcile_interval=10, ) @@ -439,7 +440,7 @@ def test_reconcile_runner_create_one_reactive( from github_runner_manager.reactive.runner_manager import ReconcileResult def _fake_reactive_reconcile( - expected_quantity: int, runner_manager, reactive_process_config, user + expected_quantity: int, runner_manager, reactive_process_config, user, python_path ): """Reactive reconcile fake.""" return ReconcileResult(processes_diff=expected_quantity, metric_stats={"event": ""}) diff --git a/src/charm.py b/src/charm.py index e880ee9d9b..c6ec75fb85 100755 --- a/src/charm.py +++ b/src/charm.py @@ -23,9 +23,6 @@ from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires from charms.grafana_agent.v0.cos_agent import COSAgentProvider from github_runner_manager import constants -from github_runner_manager.errors import ReconcileError -from github_runner_manager.manager.runner_manager import FlushMode -from github_runner_manager.manager.runner_scaler import RunnerScaler from github_runner_manager.platform.platform_provider import TokenError from github_runner_manager.utilities import set_env_var from ops.charm import ( @@ -49,7 +46,6 @@ IMAGE_INTEGRATION_NAME, LABELS_CONFIG_NAME, PATH_CONFIG_NAME, - RECONCILE_INTERVAL_CONFIG_NAME, TOKEN_CONFIG_NAME, CharmConfigInvalidError, CharmState, @@ -58,6 +54,8 @@ ) from errors import ( ConfigurationError, + ImageIntegrationMissingError, + ImageNotFoundError, LogrotateSetupError, MissingMongoDBError, RunnerManagerApplicationError, @@ -65,8 +63,6 @@ RunnerManagerServiceError, SubprocessError, ) -from event_timer import EventTimer, TimerStatusError -from factories import create_runner_scaler from manager_client import GitHubRunnerManagerClient # The reconcile loop can get stuck in a charm upgrade. Put a timeout so @@ -128,6 +124,14 @@ def func_with_catch_errors(self: "GithubRunnerCharm", event: EventT) -> None: except MissingMongoDBError as err: logger.exception("Missing integration data") self.unit.status = WaitingStatus(str(err)) + except ImageIntegrationMissingError: + logger.exception("Missing image integration.") + self.unit.status = BlockedStatus("Please provide image integration.") + manager_service.stop() + except ImageNotFoundError: + logger.exception("Missing image in image integration.") + self.unit.status = WaitingStatus("Waiting for image over integration.") + manager_service.stop() return func_with_catch_errors @@ -158,6 +162,9 @@ def func_with_catch_errors(self: "GithubRunnerCharm", event: ActionEvent) -> Non logger.exception("Issue with charm configuration") self.unit.status = BlockedStatus(str(err)) event.fail(str(err)) + except RunnerManagerServiceError as err: + logger.exception("Failed runner manager request") + event.fail(f"Failed runner manager request: {str(err)}") return func_with_catch_errors @@ -180,8 +187,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._grafana_agent = COSAgentProvider(self) - self._event_timer = EventTimer(self.unit.name) - self._stored.set_default( path=self.config[PATH_CONFIG_NAME], # for detecting changes token=self.config[TOKEN_CONFIG_NAME], # for detecting changes @@ -207,16 +212,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.on[IMAGE_INTEGRATION_NAME].relation_changed, self._on_image_relation_changed, ) - self.framework.observe(self.on.reconcile_runners, self._on_reconcile_runners) self.framework.observe(self.on.check_runners_action, self._on_check_runners_action) - self.framework.observe(self.on.reconcile_runners_action, self._on_reconcile_runners_action) self.framework.observe(self.on.flush_runners_action, self._on_flush_runners_action) self.framework.observe(self.on.update_status, self._on_update_status) self.database = DatabaseRequires( self, relation_name="mongodb", database_name=REACTIVE_MQ_DB_NAME ) - self.framework.observe(self.database.on.database_created, self._on_database_created) - self.framework.observe(self.database.on.endpoints_changed, self._on_endpoints_changed) self._manager_client = GitHubRunnerManagerClient( host=manager_service.GITHUB_RUNNER_MANAGER_ADDRESS, @@ -298,34 +299,8 @@ def _on_install(self, _: InstallEvent) -> None: @catch_charm_errors def _on_start(self, _: StartEvent) -> None: """Handle the start of the charm.""" - state = self._setup_state() - - self.unit.status = MaintenanceStatus("Starting runners") - if not self._get_set_image_ready_status(): - return - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - self._reconcile_openstack_runners( - runner_scaler, - ) - - def _set_reconcile_timer(self) -> None: - """Set the timer for regular reconciliation checks.""" - self._event_timer.ensure_event_timer( - event_name="reconcile-runners", - interval=int(self.config[RECONCILE_INTERVAL_CONFIG_NAME]), - timeout=RECONCILIATION_INTERVAL_TIMEOUT_SECONDS, - ) - - def _ensure_reconcile_timer_is_active(self) -> None: - """Ensure the timer for reconciliation event is active.""" - try: - reconcile_timer_is_active = self._event_timer.is_active(RECONCILE_RUNNERS_EVENT) - except TimerStatusError: - logger.exception("Failed to check the reconciliation event timer status") - else: - if not reconcile_timer_is_active: - logger.error("Reconciliation event timer is not activated") - self._set_reconcile_timer() + self._check_image_ready() + self.unit.status = ActiveStatus() @catch_charm_errors def _on_upgrade_charm(self, _: UpgradeCharmEvent) -> None: @@ -337,58 +312,26 @@ def _on_upgrade_charm(self, _: UpgradeCharmEvent) -> None: def _on_config_changed(self, _: ConfigChangedEvent) -> None: """Handle the configuration change.""" state = self._setup_state() - self._set_reconcile_timer() - self._setup_service(state) - flush_and_reconcile = False + flush_runners = False if state.charm_config.token != self._stored.token: self._stored.token = self.config[TOKEN_CONFIG_NAME] - flush_and_reconcile = True + flush_runners = True if self.config[PATH_CONFIG_NAME] != self._stored.path: self._stored.path = self.config[PATH_CONFIG_NAME] - flush_and_reconcile = True + flush_runners = True if self.config[LABELS_CONFIG_NAME] != self._stored.labels: self._stored.labels = self.config[LABELS_CONFIG_NAME] - flush_and_reconcile = True - - state = self._setup_state() - - if not self._get_set_image_ready_status(): - return - if flush_and_reconcile: - logger.info("Flush and reconcile on config-changed") - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - runner_scaler.flush(flush_mode=FlushMode.FLUSH_IDLE) - self._reconcile_openstack_runners( - runner_scaler, - ) - - @catch_charm_errors - def _on_reconcile_runners(self, _: ReconcileRunnersEvent) -> None: - """Event handler for reconciling runners.""" - self._trigger_reconciliation() + flush_runners = True - @catch_charm_errors - def _on_database_created(self, _: ops.RelationEvent) -> None: - """Handle the MongoDB database created event.""" - self._trigger_reconciliation() + self._check_image_ready() - @catch_charm_errors - def _on_endpoints_changed(self, _: ops.RelationEvent) -> None: - """Handle the MongoDB endpoints changed event.""" - self._trigger_reconciliation() - - def _trigger_reconciliation(self) -> None: - """Trigger the reconciliation of runners.""" - self.unit.status = MaintenanceStatus("Reconciling runners") - state = self._setup_state() + self._setup_service(state) - if not self._get_set_image_ready_status(): - return - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - self._reconcile_openstack_runners( - runner_scaler, - ) + if flush_runners: + logger.info("Flush runners on config-changed") + self._manager_client.flush_runner() + self.unit.status = ActiveStatus() @catch_action_errors def _on_check_runners_action(self, event: ActionEvent) -> None: @@ -397,69 +340,17 @@ def _on_check_runners_action(self, event: ActionEvent) -> None: Args: event: The event fired on check_runners action. """ - try: - info = self._manager_client.check_runner() - except RunnerManagerServiceError as err: - logger.exception("Failed check runner request") - event.fail(f"Failed check runner request: {str(err)}") - return + info = self._manager_client.check_runner() event.set_results(info) @catch_action_errors - def _on_reconcile_runners_action(self, event: ActionEvent) -> None: - """Handle the action of reconcile of runner state. - - Args: - event: Action event of reconciling the runner. - """ - self.unit.status = MaintenanceStatus("Reconciling runners") - state = self._setup_state() - - if not self._get_set_image_ready_status(): - event.fail("Openstack image not yet provided/ready.") - return - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - - self.unit.status = MaintenanceStatus("Reconciling runners") - try: - delta = runner_scaler.reconcile() - except ReconcileError: - logger.exception(FAILED_TO_RECONCILE_RUNNERS_MSG) - self.unit.status = ActiveStatus(ACTIVE_STATUS_RECONCILIATION_FAILED_MSG) - event.fail(FAILED_RECONCILE_ACTION_ERR_MSG) - return - - self.unit.status = ActiveStatus() - event.set_results({"delta": {"virtual-machines": delta}}) - - @catch_action_errors - def _on_flush_runners_action(self, event: ActionEvent) -> None: - """Handle the action of flushing all runner and reconciling afterwards. - - Args: - event: Action event of flushing all runners. - """ - state = self._setup_state() - - # Flushing mode not implemented for OpenStack yet. - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - flushed = runner_scaler.flush(flush_mode=FlushMode.FLUSH_IDLE) - logger.info("Flushed %s runners", flushed) - self.unit.status = MaintenanceStatus("Reconciling runners") - try: - delta = runner_scaler.reconcile() - except ReconcileError: - logger.exception(FAILED_TO_RECONCILE_RUNNERS_MSG) - self.unit.status = ActiveStatus(ACTIVE_STATUS_RECONCILIATION_FAILED_MSG) - event.fail(FAILED_RECONCILE_ACTION_ERR_MSG) - return - self.unit.status = ActiveStatus() - event.set_results({"delta": {"virtual-machines": delta}}) + def _on_flush_runners_action(self, _: ActionEvent) -> None: + """Handle the action of flushing all runner and reconciling afterwards.""" + self._manager_client.flush_runner() @catch_charm_errors def _on_update_status(self, _: UpdateStatusEvent) -> None: """Handle the update of charm status.""" - self._ensure_reconcile_timer_is_active() self._log_juju_processes() def _setup_service(self, state: CharmState) -> None: @@ -467,13 +358,17 @@ def _setup_service(self, state: CharmState) -> None: Args: state: The charm state. + + Raises: + RunnerManagerApplicationError: The runner manager service is not ready for requests or + has errors. """ try: manager_service.setup(state, self.app.name, self.unit.name) except RunnerManagerApplicationError: logging.exception("Unable to setup the github-runner-manager service") - # Not re-raising error for until the github-runner-manager service replaces the - # library. + raise + self._manager_client.wait_till_ready() @staticmethod def _log_juju_processes() -> None: @@ -513,25 +408,8 @@ def _log_charm_status(self) -> None: @catch_charm_errors def _on_stop(self, _: StopEvent) -> None: """Handle the stopping of the charm.""" - self._event_timer.disable_event_timer("reconcile-runners") - state = self._setup_state() - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - runner_scaler.flush(FlushMode.FLUSH_BUSY) - - def _reconcile_openstack_runners(self, runner_scaler: RunnerScaler) -> None: - """Reconcile the current runners state and intended runner state for OpenStack mode. - - Args: - runner_scaler: Scaler used to scale the amount of runners. - """ - self.unit.status = MaintenanceStatus("Reconciling runners") - try: - runner_scaler.reconcile() - except ReconcileError: - logger.exception(FAILED_TO_RECONCILE_RUNNERS_MSG) - self.unit.status = ActiveStatus(ACTIVE_STATUS_RECONCILIATION_FAILED_MSG) - else: - self.unit.status = ActiveStatus() + self._manager_client.flush_runner(busy=True) + manager_service.stop() def _install_deps(self) -> None: """Install dependences for the charm.""" @@ -558,15 +436,13 @@ def _apt_install(self, packages: Sequence[str]) -> None: def _on_debug_ssh_relation_changed(self, _: ops.RelationChangedEvent) -> None: """Handle debug ssh relation changed event.""" self.unit.status = MaintenanceStatus("Added debug-ssh relation") + self._check_image_ready() + state = self._setup_state() + self._setup_service(state) - if not self._get_set_image_ready_status(): - return - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - runner_scaler.flush() - self._reconcile_openstack_runners( - runner_scaler, - ) + self._manager_client.flush_runner() + self.unit.status = ActiveStatus() @catch_charm_errors def _on_image_relation_joined(self, _: ops.RelationJoinedEvent) -> None: @@ -582,32 +458,27 @@ def _on_image_relation_joined(self, _: ops.RelationJoinedEvent) -> None: @catch_charm_errors def _on_image_relation_changed(self, _: ops.RelationChangedEvent) -> None: """Handle image relation changed event.""" - state = self._setup_state() self.unit.status = MaintenanceStatus("Update image for runners") + self._check_image_ready() - if not self._get_set_image_ready_status(): - return + state = self._setup_state() + self._setup_service(state) - runner_scaler = create_runner_scaler(state, self.app.name, self.unit.name) - runner_scaler.flush(flush_mode=FlushMode.FLUSH_IDLE) - self._reconcile_openstack_runners( - runner_scaler, - ) + self._manager_client.flush_runner() + self.unit.status = ActiveStatus() - def _get_set_image_ready_status(self) -> bool: - """Check if image is ready for Openstack and charm status accordingly. + def _check_image_ready(self) -> None: + """Check if image is ready raises error if not. - Returns: - Whether the Openstack image is ready via image integration. + Raises: + ImageIntegrationMissingError: No image integration found. + ImageNotFoundError: No image found in the image integration. """ openstack_image = OpenstackImage.from_charm(self) if openstack_image is None: - self.unit.status = BlockedStatus("Please provide image integration.") - return False + raise ImageIntegrationMissingError("No image integration found") if not openstack_image.id: - self.unit.status = WaitingStatus("Waiting for image over integration.") - return False - return True + raise ImageNotFoundError("No image found in the image integration") @staticmethod def _create_labels(state: CharmState) -> list[str]: @@ -652,6 +523,8 @@ def _setup_runner_manager_user() -> None: ] ) execute_command(["/usr/bin/chmod", "700", f"/home/{constants.RUNNER_MANAGER_USER}/.ssh"]) + # Give the user access to write to /var/log + execute_command(["/usr/sbin/usermod", "-a", "-G", "syslog", constants.RUNNER_MANAGER_USER]) if __name__ == "__main__": diff --git a/src/charm_state.py b/src/charm_state.py index 1d78034297..fe3e4975d8 100644 --- a/src/charm_state.py +++ b/src/charm_state.py @@ -395,15 +395,17 @@ def check_reconcile_interval(cls, reconcile_interval: int) -> int: Returns: The validated reconcile_interval value. """ - # The EventTimer class sets a timeout of `reconcile_interval` - 1. - # Therefore the `reconcile_interval` must be at least 2. + # The reconcile_interval should be at least 2. + # Due to possible race condition with the message acknowledgement with job status checking + # in reactive process. if reconcile_interval < 2: logger.error( - "The %s configuration must be greater than 1", RECONCILE_INTERVAL_CONFIG_NAME + "The %s configuration must be greater than or equal to 1", + RECONCILE_INTERVAL_CONFIG_NAME, ) raise ValueError( f"The {RECONCILE_INTERVAL_CONFIG_NAME} configuration needs to be greater or equal" - " to 2" + " to 1" ) return reconcile_interval diff --git a/src/errors.py b/src/errors.py index db9d0c6868..ed1b651816 100644 --- a/src/errors.py +++ b/src/errors.py @@ -70,6 +70,10 @@ class RunnerManagerApplicationStartError(RunnerManagerApplicationError): """Represents an error raised when github-runner-manager application start failed.""" +class RunnerManagerApplicationStopError(RunnerManagerApplicationError): + """Represents an error raised when github-runner-manager application stop failed.""" + + class RunnerManagerServiceError(Exception): """Represents an error raised with request to github-runner-manager service.""" @@ -80,3 +84,15 @@ class RunnerManagerServiceConnectionError(RunnerManagerServiceError): class RunnerManagerServiceResponseError(RunnerManagerServiceError): """Represents a issue with the response to github-runner-manager service.""" + + +class RunnerManagerServiceNotReadyError(RunnerManagerServiceError): + """Represents the github-runner-manager service is not ready for requests.""" + + +class ImageIntegrationMissingError(Exception): + """Represents the missing image integration.""" + + +class ImageNotFoundError(Exception): + """Represents not founding the image in the image integration.""" diff --git a/src/event_timer.py b/src/event_timer.py deleted file mode 100644 index 660aeef91e..0000000000 --- a/src/event_timer.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -"""EventTimer for scheduling dispatch of juju event on regular intervals.""" -import logging -import subprocess # nosec B404 -from pathlib import Path -from typing import TypedDict - -import jinja2 - -from utilities import execute_command - -BIN_SYSTEMCTL = "/usr/bin/systemctl" - -logger = logging.getLogger(__name__) - - -class TimerError(Exception): - """Generic timer error as base exception.""" - - -class TimerEnableError(TimerError): - """Raised when unable to enable a event timer.""" - - -class TimerDisableError(TimerError): - """Raised when unable to disable a event timer.""" - - -class TimerStatusError(TimerError): - """Raised when unable to check status of a event timer.""" - - -class EventConfig(TypedDict): - """Configuration used by service and timer templates. - - Attributes: - event: Name of the event. - interval: Minutes between the event trigger. - random_delay: Minutes of random delay added between event trigger. - timeout: Minutes before the event handle is timeout. - unit: Name of the juju unit. - """ - - event: str - interval: int - random_delay: int - timeout: int - unit: str - - -class EventTimer: - """Manages the timer to emit juju events at regular intervals. - - Attributes: - unit_name (str): Name of the juju unit to emit events to. - """ - - _systemd_path = Path("/etc/systemd/system") - - def __init__(self, unit_name: str): - """Construct the timer manager. - - Args: - unit_name: Name of the juju unit to emit events to. - """ - self.unit_name = unit_name - self._jinja = jinja2.Environment( - loader=jinja2.FileSystemLoader("templates"), autoescape=True - ) - - def _render_event_template( - self, template_type: str, event_name: str, context: EventConfig - ) -> None: - """Write event configuration files to systemd path. - - Args: - template_type: Name of the template type to use. Can be 'service' or 'timer'. - event_name: Name of the event to schedule. - context: Addition configuration for the event to schedule. - """ - template = self._jinja.get_template(f"dispatch-event.{template_type}.j2") - dest = self._systemd_path / f"ghro.{event_name}.{template_type}" - dest.write_text(template.render(context)) - - def is_active(self, event_name: str) -> bool: - """Check if the systemd timer is active for the given event. - - Args: - event_name: Name of the juju event to check. - - Returns: - True if the timer is enabled, False otherwise. - - Raises: - TimerStatusError: Timer status cannot be determined. - """ - try: - # We choose status over is-active here to provide debug logs that show the output of - # the timer. - _, ret_code = execute_command( - [BIN_SYSTEMCTL, "status", f"ghro.{event_name}.timer"], check_exit=False - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as ex: - raise TimerStatusError from ex - - return ret_code == 0 - - def ensure_event_timer(self, event_name: str, interval: int, timeout: int) -> None: - """Ensure that a systemd service and timer are registered to dispatch the given event. - - The interval is how frequently, in minutes, the event should be dispatched. - - The timeout is the number of seconds before an event is timed out. If not set or 0, - it defaults to half the interval period. - - Args: - event_name: Name of the juju event to schedule. - interval: Number of minutes between emitting each event. - timeout: Timeout for each event handle in minutes. - - Raises: - TimerEnableError: Timer cannot be started. Events will be not emitted. - """ - timeout_in_secs = timeout * 60 - - context: EventConfig = { - "event": event_name, - "interval": interval, - "random_delay": interval // 4, - "timeout": timeout_in_secs, - "unit": self.unit_name, - } - self._render_event_template("service", event_name, context) - self._render_event_template("timer", event_name, context) - - systemd_timer = f"ghro.{event_name}.timer" - try: - execute_command([BIN_SYSTEMCTL, "daemon-reload"]) - execute_command([BIN_SYSTEMCTL, "enable", systemd_timer]) - execute_command([BIN_SYSTEMCTL, "start", systemd_timer]) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as ex: - raise TimerEnableError(f"Unable to enable systemd timer {systemd_timer}") from ex - - def disable_event_timer(self, event_name: str) -> None: - """Disable the systemd timer for the given event. - - Args: - event_name: Name of the juju event to disable. - - Raises: - TimerDisableError: Timer cannot be stopped. Events will be emitted continuously. - """ - systemd_timer = f"ghro.{event_name}.timer" - try: - # Don't check for errors in case the timer wasn't registered. - execute_command([BIN_SYSTEMCTL, "stop", systemd_timer], check_exit=False) - execute_command([BIN_SYSTEMCTL, "disable", systemd_timer], check_exit=False) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as ex: - raise TimerDisableError(f"Unable to disable systemd timer {systemd_timer}") from ex diff --git a/src/factories.py b/src/factories.py index 78947f8834..441dee8c65 100644 --- a/src/factories.py +++ b/src/factories.py @@ -93,6 +93,7 @@ def create_application_configuration( non_reactive_configuration=non_reactive_configuration, reactive_configuration=reactive_configuration, openstack_configuration=openstack_configuration, + reconcile_interval=state.charm_config.reconcile_interval, ) diff --git a/src/manager_client.py b/src/manager_client.py index 184f61bd2d..eb6b642e6c 100644 --- a/src/manager_client.py +++ b/src/manager_client.py @@ -7,17 +7,23 @@ import functools import json import logging +from time import sleep from typing import Any, Callable from urllib.parse import urljoin import requests -from errors import RunnerManagerServiceConnectionError, RunnerManagerServiceResponseError +from errors import ( + RunnerManagerServiceConnectionError, + RunnerManagerServiceNotReadyError, + RunnerManagerServiceResponseError, +) logger = logging.getLogger(__name__) NO_RESPONSE_ERROR_MESSAGE = "Failed request with no response" CONNECTION_ERROR_MESSAGE = "Failed request due to connection failure" +NOT_READY_ERROR_MESSAGE = "GitHub runner manager service not ready" def catch_requests_errors(func: Callable) -> Callable: @@ -115,6 +121,7 @@ def check_runner(self) -> dict[str, str]: Returns: The information on the runners. """ + self.wait_till_ready() response = self._request(_HTTPMethod.GET, "/runner/check") runner_info = json.loads(response.text) runner_info["runners"] = tuple(runner_info["runners"]) @@ -123,11 +130,45 @@ def check_runner(self) -> dict[str, str]: return runner_info @catch_requests_errors - def flush_runner(self, busy: bool = True) -> None: + def flush_runner(self, busy: bool = False) -> None: """Request to flush the runners. Args: busy: Whether to flush the busy runners. """ + self.wait_till_ready() params = {"flush-busy": str(busy)} self._request(_HTTPMethod.POST, "/runner/flush", params=params) + + def health_check(self) -> None: + """Request a health check on the runner manager service. + + This is used as a readiness check since the service does not have a dedicated readiness + endpoint. + + Raises: + RunnerManagerServiceNotReadyError: The runner manager service is not ready for + API requests. + """ + try: + response = self._request(_HTTPMethod.GET, "/health") + except requests.HTTPError as err: + raise RunnerManagerServiceNotReadyError(NOT_READY_ERROR_MESSAGE) from err + except requests.ConnectionError as err: + raise RunnerManagerServiceNotReadyError(NOT_READY_ERROR_MESSAGE) from err + + if response.status_code != 204: + raise RunnerManagerServiceNotReadyError(NOT_READY_ERROR_MESSAGE) + + def wait_till_ready(self) -> None: + """Wait till the runner manager service is ready for requests.""" + for _ in range(8): + try: + self.health_check() + except RunnerManagerServiceNotReadyError: + pass + else: + return + sleep(15) + # RunnerManagerServiceNotReadyError will be raised if the service is still not unhealthy. + self.health_check() diff --git a/src/manager_service.py b/src/manager_service.py index a26ad94e3f..5e655f74a8 100644 --- a/src/manager_service.py +++ b/src/manager_service.py @@ -5,6 +5,7 @@ import json import logging +import os import textwrap from pathlib import Path @@ -18,6 +19,7 @@ from errors import ( RunnerManagerApplicationInstallError, RunnerManagerApplicationStartError, + RunnerManagerApplicationStopError, SubprocessError, ) from factories import create_application_configuration @@ -51,7 +53,15 @@ def setup(state: CharmState, app_name: str, unit_name: str) -> None: state: The state of the charm. app_name: The Juju application name. unit_name: The Juju unit. + + Raises: + RunnerManagerApplicationStartError: Setup of the runner manager service has failed. """ + try: + if systemd.service_running(GITHUB_RUNNER_MANAGER_SERVICE_NAME): + systemd.service_stop(GITHUB_RUNNER_MANAGER_SERVICE_NAME) + except SystemdError as err: + raise RunnerManagerApplicationStartError(_SERVICE_SETUP_ERROR_MESSAGE) from err config = create_application_configuration(state, app_name, unit_name) config_file = _setup_config_file(config) GITHUB_RUNNER_MANAGER_SERVICE_LOG_DIR.mkdir(parents=True, exist_ok=True) @@ -67,11 +77,7 @@ def install_package() -> None: Raises: RunnerManagerApplicationInstallError: Unable to install the application. """ - try: - if systemd.service_running(GITHUB_RUNNER_MANAGER_SERVICE_NAME): - systemd.service_stop(GITHUB_RUNNER_MANAGER_SERVICE_NAME) - except SystemdError as err: - raise RunnerManagerApplicationInstallError(_SERVICE_STOP_ERROR_MESSAGE) from err + _stop() logger.info("Ensure pipx is at latest version") try: @@ -101,6 +107,24 @@ def install_package() -> None: raise RunnerManagerApplicationInstallError(_INSTALL_ERROR_MESSAGE) from err +def stop() -> None: + """Stop the GitHub runner manager service.""" + _stop() + + +def _stop() -> None: + """Stop the GitHub runner manager service. + + Raises: + RunnerManagerApplicationStopError: Failed to stop the service. + """ + try: + if systemd.service_running(GITHUB_RUNNER_MANAGER_SERVICE_NAME): + systemd.service_stop(GITHUB_RUNNER_MANAGER_SERVICE_NAME) + except SystemdError as err: + raise RunnerManagerApplicationStopError(_SERVICE_STOP_ERROR_MESSAGE) from err + + def _get_log_file_path(unit_name: str) -> Path: """Get the log file path. @@ -151,6 +175,7 @@ def _setup_service_file(config_file: Path, log_file: Path) -> None: config_file: The configuration file for the service. log_file: The file location to store the logs. """ + python_path = Path(os.getcwd()) / "venv" service_file_content = textwrap.dedent( f"""\ [Unit] @@ -161,8 +186,9 @@ def _setup_service_file(config_file: Path, log_file: Path) -> None: User={constants.RUNNER_MANAGER_USER} Group={constants.RUNNER_MANAGER_GROUP} ExecStart=github-runner-manager --config-file {str(config_file)} --host \ -{GITHUB_RUNNER_MANAGER_ADDRESS} --port {GITHUB_RUNNER_MANAGER_PORT} +{GITHUB_RUNNER_MANAGER_ADDRESS} --port {GITHUB_RUNNER_MANAGER_PORT} --python-path {str(python_path)} Restart=on-failure + KillMode=process StandardOutput=append:{log_file} StandardError=append:{log_file} diff --git a/templates/dispatch-event.service.j2 b/templates/dispatch-event.service.j2 deleted file mode 100644 index 8908f048fb..0000000000 --- a/templates/dispatch-event.service.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Dispatch the {{event}} event on {{unit}} - -[Service] -Type=oneshot -# For juju 3 and juju 2 compatibility. The juju-run binary was renamed to juju-exec for juju 3. -ExecStart=/usr/bin/timeout "{{timeout}}" /usr/bin/run-one /usr/bin/bash -c '/usr/bin/juju-exec "{{unit}}" "JUJU_DISPATCH_PATH={{event}} /usr/bin/timeout {{timeout}} ./dispatch" || /usr/bin/juju-run "{{unit}}" "JUJU_DISPATCH_PATH={{event}} /usr/bin/timeout {{timeout}} ./dispatch"' - -[Install] -WantedBy=multi-user.target diff --git a/templates/dispatch-event.timer.j2 b/templates/dispatch-event.timer.j2 deleted file mode 100644 index 5e1f01d731..0000000000 --- a/templates/dispatch-event.timer.j2 +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Timer to dispatch {{event}} event periodically -Requires=ghro.{{event}}.service - -[Timer] -Unit=ghro.{{event}}.service -OnUnitInactiveSec={{interval}}m -RandomizedDelaySec={{random_delay}}m - -[Install] -WantedBy=timers.target diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7e3cc6d3e9..f9af51f165 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -39,8 +39,8 @@ from tests.integration.helpers.common import ( MONGODB_APP_NAME, deploy_github_runner_charm, - reconcile, wait_for, + wait_for_reconcile, ) from tests.integration.helpers.openstack import OpenStackInstanceHelper, PrivateEndpointConfigs from tests.status_name import ACTIVE @@ -208,7 +208,7 @@ def private_endpoint_config_fixture(pytestconfig: pytest.Config) -> PrivateEndpo return None return { "auth_url": auth_url, - "password": password, + "password": str(password), "project_domain_name": project_domain_name, "project_name": project_name, "user_domain_name": user_domain_name, @@ -288,7 +288,9 @@ def openstack_test_flavor_fixture(pytestconfig: pytest.Config) -> str: @pytest.fixture(scope="module", name="openstack_connection") def openstack_connection_fixture( - clouds_yaml_contents: str, app_name: str + clouds_yaml_contents: str, + app_name: str, + existing_app_suffix: str, ) -> Generator[Connection, None, None]: """The openstack connection instance.""" clouds_yaml = yaml.safe_load(clouds_yaml_contents) @@ -298,23 +300,31 @@ def openstack_connection_fixture( with openstack.connect(first_cloud) as connection: yield connection - # servers, keys, security groups, security rules, images are created by the charm. - # don't remove security groups & rules since they are single instances. - # don't remove images since it will be moved to image-builder - for server in connection.list_servers(): - server_name: str = server.name - if server_name.startswith(app_name): - connection.delete_server(server_name) - for key in connection.list_keypairs(): - key_name: str = key.name - if key_name.startswith(app_name): - connection.delete_keypair(key_name) + if not existing_app_suffix: + # servers, keys, security groups, security rules, images are created by the charm. + # don't remove security groups & rules since they are single instances. + # don't remove images since it will be moved to image-builder + for server in connection.list_servers(): + server_name: str = server.name + if server_name.startswith(app_name): + connection.delete_server(server_name) + for key in connection.list_keypairs(): + key_name: str = key.name + if key_name.startswith(app_name): + connection.delete_keypair(key_name) -@pytest.fixture(scope="module") -def model(ops_test: OpsTest) -> Model: +@pytest_asyncio.fixture(scope="module") +async def model(ops_test: OpsTest, http_proxy: str, https_proxy: str, no_proxy: str) -> Model: """Juju model used in the test.""" assert ops_test.model is not None + await ops_test.model.set_config( + { + "juju-http-proxy": http_proxy, + "juju-https-proxy": https_proxy, + "juju-no-proxy": no_proxy, + } + ) return ops_test.model @@ -399,12 +409,14 @@ async def image_builder_fixture( else: app = model.applications[image_builder_app_name] yield app - # The github-image-builder does not clean keypairs. Until it does, - # we clean them manually here. - for key in openstack_connection.list_keypairs(): - key_name: str = key.name - if key_name.startswith(image_builder_app_name): - openstack_connection.delete_keypair(key_name) + + if not existing_app_suffix: + # The github-image-builder does not clean keypairs. Until it does, + # we clean them manually here. + for key in openstack_connection.list_keypairs(): + key_name: str = key.name + if key_name.startswith(image_builder_app_name): + openstack_connection.delete_keypair(key_name) @pytest_asyncio.fixture(scope="module", name="app_openstack_runner") @@ -436,7 +448,7 @@ async def app_openstack_runner_fixture( http_proxy=openstack_http_proxy, https_proxy=openstack_https_proxy, no_proxy=openstack_no_proxy, - reconcile_interval=60, + reconcile_interval=2, constraints={ "root-disk": 50 * 1024, "mem": 2 * 1024, @@ -470,7 +482,7 @@ async def app_scheduled_events_fixture( await application.set_config({"reconcile-interval": "8"}) await application.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "1"}) await model.wait_for_idle(apps=[application.name], status=ACTIVE, timeout=20 * 60) - await reconcile(app=application, model=model) + await wait_for_reconcile(app=application, model=model) return application @@ -481,9 +493,7 @@ async def app_no_wait_tmate_fixture( ): """Application to check tmate ssh with openstack without waiting for active.""" application = app_openstack_runner - await application.set_config( - {"reconcile-interval": "60", BASE_VIRTUAL_MACHINES_CONFIG_NAME: "1"} - ) + await application.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "1"}) return application @@ -509,7 +519,7 @@ async def app_runner( http_proxy=http_proxy, https_proxy=https_proxy, no_proxy=no_proxy, - reconcile_interval=60, + reconcile_interval=1, ) return application @@ -535,7 +545,7 @@ async def app_no_wait_fixture( http_proxy=http_proxy, https_proxy=https_proxy, no_proxy=no_proxy, - reconcile_interval=60, + reconcile_interval=1, wait_idle=False, ) await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "1"}) diff --git a/tests/integration/helpers/common.py b/tests/integration/helpers/common.py index be459a2185..7bba56e4db 100644 --- a/tests/integration/helpers/common.py +++ b/tests/integration/helpers/common.py @@ -8,6 +8,7 @@ import pathlib import time import typing +from asyncio import sleep from datetime import datetime, timezone from functools import partial from typing import Awaitable, Callable, ParamSpec, TypeVar, cast @@ -78,19 +79,47 @@ async def run_in_unit( return code, stdout, stderr -async def reconcile(app: Application, model: Model) -> None: - """Reconcile the runners. +async def get_reconcile_id(unit: Unit) -> str: + """Get reconcile UUID of the unit. - Uses the first unit found in the application for the reconciliation. + This is to distinguish whether reconcile has happened in a unit. Args: - app: The GitHub Runner Charm app to reconcile the runners for. + unit: The unit. + + Returns: + The UUID. + """ + _, stdout, _ = await run_in_unit( + unit, + "cat /var/log/reconcile.id", + assert_on_failure=True, + assert_msg="Unable to get reconcile ID", + ) + assert stdout is not None, "Got empty reconcile ID, this should be impossible" + return stdout + + +async def wait_for_reconcile(app: Application, model: Model) -> None: + """Wait until a reconcile has happened. + + Uses the first unit found in the application. + + Args: + app: The GitHub Runner Charm application. model: The machine charm model. """ - action = await app.units[0].run_action("reconcile-runners") - await action.wait() + # Wait the application is actively reconciling. Avoid waiting for image, etc. await model.wait_for_idle(apps=[app.name], status=ACTIVE) + unit = app.units[0] + base_id = await get_reconcile_id(unit) + for _ in range(10): + await sleep(60) + current_id = await get_reconcile_id(unit) + if base_id != current_id: + return + async def deploy_github_runner_charm( model: Model, @@ -438,6 +467,6 @@ async def get_github_runner_manager_service_log(unit: Unit) -> str: assert_msg="Failed to get the GitHub runner manager logs", ) - assert return_code == 0, f"Get log with cat failed with: {stderr}" + assert return_code == 0, f"Get log with cat {log_file_path} failed with: {stderr}" assert stdout is not None return stdout diff --git a/tests/integration/helpers/openstack.py b/tests/integration/helpers/openstack.py index 9cc19b6616..e32aabf18f 100644 --- a/tests/integration/helpers/openstack.py +++ b/tests/integration/helpers/openstack.py @@ -12,7 +12,7 @@ from openstack.compute.v2.server import Server from charm_state import BASE_VIRTUAL_MACHINES_CONFIG_NAME -from tests.integration.helpers.common import reconcile, run_in_unit, wait_for +from tests.integration.helpers.common import run_in_unit, wait_for, wait_for_reconcile logger = logging.getLogger(__name__) @@ -152,7 +152,7 @@ async def _set_app_runner_amount(app: Application, num_runners: int) -> None: num_runners: The number of runners. """ await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: f"{num_runners}"}) - await reconcile(app=app, model=app.model) + await wait_for_reconcile(app=app, model=app.model) async def get_runner_names(self, unit: Unit) -> list[str]: """Get the name of all the runners in the unit. diff --git a/tests/integration/test_charm_fork_path_change.py b/tests/integration/test_charm_fork_path_change.py index 0ee0b145fb..9bb1cb59b7 100644 --- a/tests/integration/test_charm_fork_path_change.py +++ b/tests/integration/test_charm_fork_path_change.py @@ -14,7 +14,7 @@ from ops.model import ActiveStatus from charm_state import PATH_CONFIG_NAME -from tests.integration.helpers.common import reconcile +from tests.integration.helpers.common import wait_for_reconcile from tests.integration.helpers.openstack import OpenStackInstanceHelper logger = logging.getLogger(__name__) @@ -48,7 +48,7 @@ async def test_path_config_change( await app_with_forked_repo.set_config({PATH_CONFIG_NAME: path}) logger.info("Reconciling (again)") - await reconcile(app=app_with_forked_repo, model=model) + await wait_for_reconcile(app=app_with_forked_repo, model=model) runner_names = await instance_helper.get_runner_names(unit) logger.info("runners: %s", runner_names) diff --git a/tests/integration/test_charm_metrics_failure.py b/tests/integration/test_charm_metrics_failure.py index 8269fb9d70..6f254b4eac 100644 --- a/tests/integration/test_charm_metrics_failure.py +++ b/tests/integration/test_charm_metrics_failure.py @@ -26,7 +26,7 @@ DISPATCH_CRASH_TEST_WORKFLOW_FILENAME, DISPATCH_FAILURE_TEST_WORKFLOW_FILENAME, dispatch_workflow, - reconcile, + wait_for_reconcile, ) from tests.integration.helpers.openstack import OpenStackInstanceHelper, setup_repo_policy @@ -46,7 +46,7 @@ async def app_fixture(model: Model, app_for_metric: Application) -> AsyncIterato "repo-policy-compliance-url": "", } ) - await reconcile(app=app_for_metric, model=model) + await wait_for_reconcile(app=app_for_metric, model=model) yield app_for_metric @@ -95,7 +95,7 @@ async def test_charm_issues_metrics_for_failed_repo_policy( BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0", } ) - await reconcile(app=app, model=model) + await wait_for_reconcile(app=app, model=model) await assert_events_after_reconciliation( app=app, @@ -146,7 +146,7 @@ async def test_charm_issues_metrics_for_abnormal_termination( # Make the runner terminate abnormally by killing run.sh runner_name = await instance_helper.get_runner_name(unit) kill_run_sh_cmd = "pkill -9 run.sh" - ret_code, stdout, stderr = await instance_helper.run_in_instance(unit, kill_run_sh_cmd) + ret_code, _, stderr = await instance_helper.run_in_instance(unit, kill_run_sh_cmd) assert ret_code == 0, f"Failed to kill run.sh with code {ret_code}: {stderr}" # Cancel workflow and wait that the runner is marked offline @@ -158,7 +158,7 @@ async def test_charm_issues_metrics_for_abnormal_termination( # Set the number of virtual machines to 0 to speedup reconciliation await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app=app, model=model) + await wait_for_reconcile(app=app, model=model) await assert_events_after_reconciliation( app=app, diff --git a/tests/integration/test_charm_metrics_success.py b/tests/integration/test_charm_metrics_success.py index a7987b8fb5..0443485ec0 100644 --- a/tests/integration/test_charm_metrics_success.py +++ b/tests/integration/test_charm_metrics_success.py @@ -23,7 +23,7 @@ from tests.integration.helpers.common import ( DISPATCH_TEST_WORKFLOW_FILENAME, dispatch_workflow, - reconcile, + wait_for_reconcile, ) from tests.integration.helpers.openstack import OpenStackInstanceHelper @@ -55,7 +55,7 @@ async def test_charm_issues_runner_installed_metric( # Set the number of virtual machines to 0 to speedup reconciliation await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app=app, model=model) + await wait_for_reconcile(app=app, model=model) metrics_log = await get_metrics_log(app.units[0]) log_lines = list(map(lambda line: json.loads(line), metrics_log.splitlines())) @@ -100,7 +100,7 @@ async def test_charm_issues_metrics_after_reconciliation( # Set the number of virtual machines to 0 to speedup reconciliation await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app=app, model=model) + await wait_for_reconcile(app=app, model=model) await assert_events_after_reconciliation( app=app, github_repository=github_repository, post_job_status=PostJobStatus.NORMAL diff --git a/tests/integration/test_charm_no_runner.py b/tests/integration/test_charm_no_runner.py index e715bff086..f2c9ad6f27 100644 --- a/tests/integration/test_charm_no_runner.py +++ b/tests/integration/test_charm_no_runner.py @@ -3,9 +3,12 @@ """Integration tests for github-runner charm with no runner.""" import logging -from asyncio import sleep import pytest +from github_runner_manager.reconcile_service import ( + RECONCILE_SERVICE_START_MSG, + RECONCILE_START_MSG, +) from juju.application import Application from juju.model import Model @@ -13,9 +16,9 @@ from manager_service import GITHUB_RUNNER_MANAGER_SERVICE_NAME from tests.integration.helpers.common import ( get_github_runner_manager_service_log, - reconcile, run_in_unit, wait_for, + wait_for_reconcile, ) from tests.integration.helpers.openstack import OpenStackInstanceHelper @@ -77,7 +80,7 @@ async def test_reconcile_runners( # 1. await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "1"}) - await reconcile(app=app, model=model) + await wait_for_reconcile(app=app, model=model) async def _runners_number(number) -> bool: """Check if there is the expected number of runners.""" @@ -88,7 +91,7 @@ async def _runners_number(number) -> bool: # 2. await app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app=app, model=model) + await wait_for_reconcile(app=app, model=model) await wait_for(lambda: _runners_number(0), timeout=10 * 60, check_interval=10) @@ -120,8 +123,7 @@ async def test_manager_service_started( ) log = await get_github_runner_manager_service_log(unit) - assert "Starting the server..." in log - assert "Starting the reconcile_service..." in log + assert RECONCILE_SERVICE_START_MSG in log # 2. return_code, _, _ = await run_in_unit( @@ -134,9 +136,8 @@ async def test_manager_service_started( assert return_code == 0 # Wait for more log lines. - await sleep(15) + await wait_for_reconcile(app, app.model) log = await get_github_runner_manager_service_log(unit) - assert "Starting the server..." not in log - assert "Starting the reconcile_service..." not in log - assert "Acquired the lock for reconciling" in log + assert RECONCILE_SERVICE_START_MSG not in log + assert RECONCILE_START_MSG in log diff --git a/tests/integration/test_charm_runner.py b/tests/integration/test_charm_runner.py index 7c31cf93cc..083ed04ee4 100644 --- a/tests/integration/test_charm_runner.py +++ b/tests/integration/test_charm_runner.py @@ -18,8 +18,8 @@ DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, dispatch_workflow, get_job_logs, - reconcile, wait_for, + wait_for_reconcile, ) from tests.integration.helpers.openstack import OpenStackInstanceHelper, setup_repo_policy @@ -36,7 +36,7 @@ async def app_fixture( yield basic_app await basic_app.set_config({BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(basic_app, basic_app.model) + await wait_for_reconcile(basic_app, basic_app.model) @pytest.mark.openstack @@ -102,10 +102,7 @@ async def test_flush_runner_and_resource_config( action = await app.units[0].run_action("flush-runners") await action.wait() - # There is a race condition in here. When deleting a runner in openstack, it can take - # a while to get the runner deleted and the "flush-runner" will not spawn a new runner. - # We may need to call flush-runners twice and wait in the middle until the openstack - # instance disappear. + await wait_for_reconcile(app, app.model) action = await app.units[0].run_action("check-runners") await action.wait() @@ -134,7 +131,6 @@ async def test_flush_runner_and_resource_config( await action.wait() assert action.status == "completed" - assert action.results["delta"]["virtual-machines"] == "0" @pytest.mark.openstack @@ -168,7 +164,7 @@ async def test_custom_pre_job_script( """, } ) - await reconcile(app, app.model) + await wait_for_reconcile(app, app.model) workflow_run = await dispatch_workflow( app=app, diff --git a/tests/integration/test_debug_ssh.py b/tests/integration/test_debug_ssh.py index 74459c7b0d..eb9484e376 100644 --- a/tests/integration/test_debug_ssh.py +++ b/tests/integration/test_debug_ssh.py @@ -12,7 +12,6 @@ from tests.integration.helpers.common import dispatch_workflow, get_job_logs from tests.integration.helpers.openstack import OpenStackInstanceHelper -from tests.status_name import ACTIVE logger = logging.getLogger(__name__) @@ -34,7 +33,7 @@ async def test_ssh_debug( act: when canonical/action-tmate is triggered. assert: the ssh connection info from action-log and tmate-ssh-server matches. """ - await model.wait_for_idle(status=ACTIVE, timeout=60 * 20) + await instance_helper.ensure_charm_has_runner(app_no_wait_tmate) unit = app_no_wait_tmate.units[0] # We need the runner to connect to the current machine, instead of the tmate_ssh_server unit, diff --git a/tests/integration/test_jobmanager.py b/tests/integration/test_jobmanager.py index c2aa66326e..c26abbe633 100644 --- a/tests/integration/test_jobmanager.py +++ b/tests/integration/test_jobmanager.py @@ -20,7 +20,6 @@ from jobmanager_client.models.v1_jobs_job_id_token_post200_response import ( V1JobsJobIdTokenPost200Response, ) -from juju.action import Action from juju.application import Application from pytest_httpserver import HTTPServer from pytest_operator.plugin import OpsTest @@ -30,10 +29,11 @@ EXPERIMENTAL_JOB_MANAGER_ONLY_TOKEN_VALUE, MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME, PATH_CONFIG_NAME, + RECONCILE_INTERVAL_CONFIG_NAME, TOKEN_CONFIG_NAME, ) from tests.integration.helpers.charm_metrics import clear_metrics_log -from tests.integration.helpers.common import reconcile, wait_for +from tests.integration.helpers.common import wait_for, wait_for_reconcile from tests.integration.helpers.openstack import OpenStackInstanceHelper, PrivateEndpointConfigs from tests.integration.utils_reactive import ( add_to_queue, @@ -41,7 +41,6 @@ clear_queue, get_mongodb_uri, ) -from tests.status_name import ACTIVE logger = logging.getLogger(__name__) pytestmark = pytest.mark.openstack @@ -102,16 +101,19 @@ async def app_fixture( PATH_CONFIG_NAME: "", BASE_VIRTUAL_MACHINES_CONFIG_NAME: "0", MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME: "1", + # Disable reconcile by setting it to a long time period since the test controls when + # the reconcile happens. + RECONCILE_INTERVAL_CONFIG_NAME: "60", } ) - await reconcile(app_for_jobmanager, app_for_jobmanager.model) + await wait_for_reconcile(app_for_jobmanager, app_for_jobmanager.model) await clear_metrics_log(app_for_jobmanager.units[0]) yield app_for_jobmanager # Call reconcile to enable cleanup of any runner spawned await app_for_jobmanager.set_config({MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app_for_jobmanager, app_for_jobmanager.model) + await wait_for_reconcile(app_for_jobmanager, app_for_jobmanager.model) @pytest.mark.abort_on_fail @@ -296,10 +298,10 @@ async def _prepare_runner() -> bool: # The health check is not returning deletable yet. Reconcile should not kill the runner. logger.info("First reconcile that should not delete the runner, as it is still healthy.") - action: Action = await app.units[0].run_action("reconcile-runners") - await action.wait() - await app.model.wait_for_idle(apps=[app.name], status=ACTIVE) - logger.info("First reconcile result %s %s", action.status, action.results) + # TMP: hack to trigger reconcile by changing the configuration, which cause config_changed hook + # to restart the reconcile service. + await app.set_config({RECONCILE_INTERVAL_CONFIG_NAME: "10"}) + await wait_for_reconcile(app, app.model) # At this point there should be a runner action = await app.units[0].run_action("check-runners") @@ -321,11 +323,11 @@ async def _prepare_runner() -> bool: logger.info("handler health %s", health_get_handler) logger.info("handlers %s", httpserver.format_matchers()) - logger.info("Second reconcile call: %s", action.results) - action = await app.units[0].run_action("reconcile-runners") - await action.wait() - await app.model.wait_for_idle(apps=[app.name], status=ACTIVE) - logger.info("Second reconcile result %s %s", action.status, action.results) + logger.info("Second reconcile call") + # TMP: hack to trigger reconcile by changing the configuration, which cause config_changed hook + # to restart the reconcile service. + await app.set_config({RECONCILE_INTERVAL_CONFIG_NAME: "5"}) + await wait_for_reconcile(app, app.model) action = await app.units[0].run_action("check-runners") await action.wait() diff --git a/tests/integration/test_reactive.py b/tests/integration/test_reactive.py index 5bbb31fde2..b8eafb5a15 100644 --- a/tests/integration/test_reactive.py +++ b/tests/integration/test_reactive.py @@ -23,9 +23,9 @@ DISPATCH_CRASH_TEST_WORKFLOW_FILENAME, DISPATCH_TEST_WORKFLOW_FILENAME, dispatch_workflow, - reconcile, wait_for, wait_for_completion, + wait_for_reconcile, wait_for_status, ) from tests.integration.utils_reactive import ( @@ -56,14 +56,14 @@ async def app_fixture( MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME: "1", } ) - await reconcile(app_for_reactive, app_for_reactive.model) + await wait_for_reconcile(app_for_reactive, app_for_reactive.model) await clear_metrics_log(app_for_reactive.units[0]) yield app_for_reactive # Call reconcile to enable cleanup of any runner spawned await app_for_reactive.set_config({MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app_for_reactive, app_for_reactive.model) + await wait_for_reconcile(app_for_reactive, app_for_reactive.model) @pytest.mark.abort_on_fail @@ -103,7 +103,7 @@ async def test_reactive_mode_spawns_runner( # There may be a race condition between getting the token and spawning the machine. await sleep(10) - await reconcile(app, app.model) + await wait_for_reconcile(app, app.model) try: await wait_for_completion(run, conclusion="success") @@ -122,7 +122,7 @@ async def _runner_installed_in_metrics_log() -> bool: True if the runner_installed event is logged, False otherwise. """ # trigger reconcile which extracts metrics - await reconcile(app, app.model) + await wait_for_reconcile(app, app.model) metrics_log = await get_metrics_log(app.units[0]) log_lines = list(map(lambda line: json.loads(line), metrics_log.splitlines())) events = set(map(lambda line: line.get("event"), log_lines)) @@ -193,7 +193,7 @@ async def test_reactive_mode_scale_down( mongodb_uri = await get_mongodb_uri(ops_test, app) await app.set_config({MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME: "2"}) - await reconcile(app, app.model) + await wait_for_reconcile(app, app.model) run = await dispatch_workflow( app=app, @@ -214,7 +214,7 @@ async def test_reactive_mode_scale_down( # 1. Scale down the number of virtual machines to 0 and call reconcile. await app.set_config({MAX_TOTAL_VIRTUAL_MACHINES_CONFIG_NAME: "0"}) - await reconcile(app, app.model) + await wait_for_reconcile(app, app.model) # we assume that the runner got deleted while running the job, so we expect a failed job await wait_for_completion(run, conclusion="failure") @@ -236,7 +236,7 @@ async def test_reactive_mode_scale_down( app.name, ) - await reconcile(app, app.model) + await wait_for_reconcile(app, app.model) run.update() assert run.status == "queued" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index dcaea25361..fd0f9251bf 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -9,11 +9,9 @@ import pytest from github_runner_manager.configuration.github import GitHubOrg -from github_runner_manager.manager.runner_scaler import RunnerScaler import charm_state import utilities -from tests.unit.mock import MockGhapiClient @pytest.fixture(name="exec_command") @@ -38,21 +36,8 @@ def disk_usage_mock(total_disk: int): @pytest.fixture(autouse=True) def mocks(monkeypatch, tmp_path, exec_command): - runner_scaler_mock = unittest.mock.MagicMock(spec=RunnerScaler) - - cron_path = tmp_path / "cron.d" - cron_path.mkdir() - - monkeypatch.setattr("charm.RunnerScaler", runner_scaler_mock) monkeypatch.setattr("charm.execute_command", exec_command) monkeypatch.setattr("charm_state.CHARM_STATE_PATH", Path(tmp_path / "charm_state.json")) - monkeypatch.setattr("event_timer.jinja2", unittest.mock.MagicMock()) - monkeypatch.setattr("event_timer.execute_command", exec_command) - monkeypatch.setattr( - "github_runner_manager.metrics.events.METRICS_LOG_PATH", Path(tmp_path / "metrics.log") - ) - monkeypatch.setattr("github_runner_manager.github_client.GhApi", MockGhapiClient) - monkeypatch.setattr("github_runner_manager.utilities.time", unittest.mock.MagicMock()) @pytest.fixture(autouse=True, name="cloud_name") diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 350cca31e2..9ec3c9b6c2 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -11,16 +11,11 @@ import pytest import yaml -from github_runner_manager.errors import ReconcileError -from github_runner_manager.manager.runner_manager import FlushMode -from github_runner_manager.manager.runner_scaler import RunnerScaler from github_runner_manager.platform.platform_provider import TokenError -from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus, StatusBase, WaitingStatus +from ops.model import BlockedStatus, StatusBase, WaitingStatus from ops.testing import Harness from charm import ( - ACTIVE_STATUS_RECONCILIATION_FAILED_MSG, - FAILED_RECONCILE_ACTION_ERR_MSG, GithubRunnerCharm, catch_action_errors, catch_charm_errors, @@ -39,12 +34,14 @@ ) from errors import ( ConfigurationError, + ImageIntegrationMissingError, + ImageNotFoundError, LogrotateSetupError, MissingMongoDBError, RunnerError, SubprocessError, ) -from event_timer import EventTimer, TimerEnableError +from manager_client import GitHubRunnerManagerClient TEST_PROXY_SERVER_URL = "http://proxy.server:1234" @@ -169,7 +166,6 @@ def setup_charm_harness(monkeypatch: pytest.MonkeyPatch) -> Harness: } ) harness.begin() - monkeypatch.setattr("charm.EventTimer.ensure_event_timer", MagicMock()) monkeypatch.setattr("charm.logrotate.setup", MagicMock()) return harness @@ -222,8 +218,6 @@ def test_common_install_code( harness.charm._setup_state = MagicMock(return_value=state_mock) monkeypatch.setattr("charm.logrotate.setup", setup_logrotate := MagicMock()) - event_timer_mock = MagicMock(spec=EventTimer) - harness.charm._event_timer = event_timer_mock getattr(harness.charm.on, hook).emit() @@ -242,80 +236,7 @@ def test_on_config_changed_failure(harness: Harness): assert "Invalid proxy configuration" in harness.charm.unit.status.message -def test_on_flush_runners_reconcile_error_fail(harness: Harness, monkeypatch: pytest.MonkeyPatch): - """ - arrange: Set up charm with Openstack mode and ReconcileError. - act: Run flush runner action. - assert: Action fails with generic message and goes in ActiveStatus. - """ - state_mock = MagicMock() - harness.charm._setup_state = MagicMock(return_value=state_mock) - - runner_scaler_mock = MagicMock(spec=RunnerScaler) - runner_scaler_mock.reconcile.side_effect = ReconcileError("mock error") - monkeypatch.setattr("charm.create_runner_scaler", MagicMock(return_value=runner_scaler_mock)) - - mock_event = MagicMock() - harness.charm._on_flush_runners_action(mock_event) - mock_event.fail.assert_called_with(FAILED_RECONCILE_ACTION_ERR_MSG) - assert harness.charm.unit.status.name == ActiveStatus.name - assert harness.charm.unit.status.message == ACTIVE_STATUS_RECONCILIATION_FAILED_MSG - - -def test_on_reconcile_runners_action_reconcile_error_fail( - harness: Harness, monkeypatch: pytest.MonkeyPatch -): - """ - arrange: Set up charm with Openstack mode, ready image, and ReconcileError. - act: Run reconcile runners action. - assert: Action fails with generic message and goes in ActiveStatus - """ - state_mock = MagicMock() - harness.charm._setup_state = MagicMock(return_value=state_mock) - - runner_scaler_mock = MagicMock(spec=RunnerScaler) - runner_scaler_mock.reconcile.side_effect = ReconcileError("mock error") - monkeypatch.setattr("charm.create_runner_scaler", MagicMock(return_value=runner_scaler_mock)) - monkeypatch.setattr( - OpenstackImage, - "from_charm", - MagicMock(return_value=OpenstackImage(id="test", tags=["test"])), - ) - - mock_event = MagicMock() - harness.charm._on_reconcile_runners_action(mock_event) - - mock_event.fail.assert_called_with(FAILED_RECONCILE_ACTION_ERR_MSG) - assert harness.charm.unit.status.name == ActiveStatus.name - assert harness.charm.unit.status.message == ACTIVE_STATUS_RECONCILIATION_FAILED_MSG - - -def test_on_reconcile_runners_reconcile_error(harness: Harness, monkeypatch: pytest.MonkeyPatch): - """ - arrange: Set up charm with Openstack mode, ready image, and ReconcileError. - act: Trigger reconcile_runners event. - assert: Unit goes into ActiveStatus with error message. - """ - state_mock = MagicMock() - harness.charm._setup_state = MagicMock(return_value=state_mock) - - runner_scaler_mock = MagicMock(spec=RunnerScaler) - runner_scaler_mock.reconcile.side_effect = ReconcileError("mock error") - monkeypatch.setattr("charm.create_runner_scaler", MagicMock(return_value=runner_scaler_mock)) - monkeypatch.setattr( - OpenstackImage, - "from_charm", - MagicMock(return_value=OpenstackImage(id="test", tags=["test"])), - ) - - mock_event = MagicMock() - harness.charm._on_reconcile_runners(mock_event) - - assert harness.charm.unit.status.name == ActiveStatus.name - assert harness.charm.unit.status.message == ACTIVE_STATUS_RECONCILIATION_FAILED_MSG - - -def test_on_stop_busy_flush(harness: Harness, monkeypatch: pytest.MonkeyPatch): +def test_on_stop_busy_flush_and_stop_service(harness: Harness, monkeypatch: pytest.MonkeyPatch): """ arrange: Set up charm with Openstack mode and runner scaler mock. act: Trigger stop event. @@ -323,13 +244,16 @@ def test_on_stop_busy_flush(harness: Harness, monkeypatch: pytest.MonkeyPatch): """ state_mock = MagicMock() harness.charm._setup_state = MagicMock(return_value=state_mock) - runner_scaler_mock = MagicMock(spec=RunnerScaler) - monkeypatch.setattr("charm.create_runner_scaler", MagicMock(return_value=runner_scaler_mock)) + manager_client_mock = MagicMock(spec=GitHubRunnerManagerClient) + harness.charm._manager_client = manager_client_mock + mock_manager_service = MagicMock() + monkeypatch.setattr("charm.manager_service", mock_manager_service) mock_event = MagicMock() harness.charm._on_stop(mock_event) - runner_scaler_mock.flush.assert_called_once_with(FlushMode.FLUSH_BUSY) + manager_client_mock.flush_runner.assert_called_once_with(busy=True) + mock_manager_service.stop.assert_called_once() @pytest.mark.parametrize( @@ -381,87 +305,17 @@ def test_charm_goes_into_waiting_state_on_missing_integration_data( assert "mock error" in harness.charm.unit.status.message -@pytest.mark.parametrize( - "hook", - [ - pytest.param("database_created", id="Database Created"), - pytest.param("endpoints_changed", id="Endpoints Changed"), - ], -) -def test_database_integration_events_trigger_reconciliation( - hook: str, monkeypatch: pytest.MonkeyPatch, harness: Harness -): - """ - arrange: Mock charm._trigger_reconciliation. - act: Fire mongodb relation events. - assert: _trigger_reconciliation has been called. - """ - reconciliation_mock = MagicMock() - relation_mock = MagicMock() - relation_mock.name = "mongodb" - relation_mock.id = 0 - monkeypatch.setattr("charm.GithubRunnerCharm._trigger_reconciliation", reconciliation_mock) - getattr(harness.charm.database.on, hook).emit(relation=relation_mock) - reconciliation_mock.assert_called_once() - - -# New tests should not be added here. This should be refactored to pytest over time. -# New test should be written with pytest, similar to the above tests. -# Consider to rewrite test with pytest if the tests below needs to be changed. -class TestCharm(unittest.TestCase): - """Test the GithubRunner charm.""" - - @patch("pathlib.Path.mkdir") - @patch("pathlib.Path.write_text") - @patch("subprocess.run") - def test_on_update_status(self, run, wt, mkdir): - """ - arrange: reconciliation event timer mocked to be \ - 1. active. \ - 2. inactive. \ - 3. inactive with error thrown for ensure_event_timer. - act: Emit update_status - assert: - 1. ensure_event_timer is not called. - 2. ensure_event_timer is called. - 3. Charm throws error. - """ - harness = Harness(GithubRunnerCharm) - - harness.update_config({PATH_CONFIG_NAME: "mockorg/repo", TOKEN_CONFIG_NAME: "mocktoken"}) - harness.begin() - - event_timer_mock = MagicMock(spec=EventTimer) - harness.charm._event_timer = event_timer_mock - event_timer_mock.is_active.return_value = True - - # 1. event timer is active - harness.charm.on.update_status.emit() - assert event_timer_mock.ensure_event_timer.call_count == 0 - assert not isinstance(harness.charm.unit.status, BlockedStatus) - - # 2. event timer is not active - event_timer_mock.is_active.return_value = False - harness.charm.on.update_status.emit() - event_timer_mock.ensure_event_timer.assert_called_once() - assert not isinstance(harness.charm.unit.status, BlockedStatus) - - # 3. ensure_event_timer throws error. - event_timer_mock.ensure_event_timer.side_effect = TimerEnableError("mock error") - with pytest.raises(TimerEnableError): - harness.charm.on.update_status.emit() - - def test_check_runners_action_with_errors(): mock_event = MagicMock() harness = Harness(GithubRunnerCharm) harness.begin() + harness.charm._manager_client.wait_till_ready = MagicMock() # No config harness.charm._on_check_runners_action(mock_event) mock_event.fail.assert_called_with( - "Failed check runner request: Failed request due to connection failure" + "Failed runner manager request: Failed request due to connection failure" ) @@ -550,16 +404,15 @@ def test_event_handler(self, _: typing.Any): @pytest.mark.parametrize( - "openstack_image, expected_status, expected_value", + "openstack_image, error", [ - pytest.param(None, BlockedStatus, False, id="Image integration missing."), + pytest.param(None, ImageIntegrationMissingError, id="Image integration missing."), pytest.param( - OpenstackImage(id=None, tags=None), WaitingStatus, False, id="Image not ready." + OpenstackImage(id=None, tags=None), ImageNotFoundError, id="Image not ready." ), pytest.param( OpenstackImage(id="test", tags=["test"]), - MaintenanceStatus, - True, + None, id="Valid image integration.", ), ], @@ -567,8 +420,7 @@ def test_event_handler(self, _: typing.Any): def test_openstack_image_ready_status( monkeypatch: pytest.MonkeyPatch, openstack_image: OpenstackImage | None, - expected_status: typing.Type[StatusBase], - expected_value: bool, + error: typing.Type[Exception] | None, ): """ arrange: given a monkeypatched OpenstackImage.from_charm that returns different values. @@ -579,16 +431,18 @@ def test_openstack_image_ready_status( harness = Harness(GithubRunnerCharm) harness.begin() - is_ready = harness.charm._get_set_image_ready_status() + if error is None: + harness.charm._check_image_ready() + return - assert isinstance(harness.charm.unit.status, expected_status) - assert is_ready == expected_value + with pytest.raises(error): + harness.charm._check_image_ready() -def test__on_image_relation_image_not_ready(): +def test__on_image_relation_image_not_ready(monkeypatch): """ arrange: given a charm with OpenStack instance type and a monkeypatched \ - _get_set_image_ready_status that returns False denoting image not ready. + with no image integration. act: when _on_image_relation_changed is called. assert: nothing happens since _get_set_image_ready_status should take care of status set. """ @@ -596,12 +450,11 @@ def test__on_image_relation_image_not_ready(): harness.begin() state_mock = MagicMock() harness.charm._setup_state = MagicMock(return_value=state_mock) - harness.charm._get_set_image_ready_status = MagicMock(return_value=False) harness.charm._on_image_relation_changed(MagicMock()) # the unit is in maintenance status since nothing has happened. - assert harness.charm.unit.status.name == MaintenanceStatus.name + assert harness.charm.unit.status.name == BlockedStatus.name def test__on_image_relation_image_ready(monkeypatch: pytest.MonkeyPatch): @@ -609,21 +462,19 @@ def test__on_image_relation_image_ready(monkeypatch: pytest.MonkeyPatch): arrange: given a charm with OpenStack instance type and a monkeypatched \ _get_set_image_ready_status that returns True denoting image ready. act: when _on_image_relation_changed is called. - assert: runner flush and reconcile is called. + assert: runner flush is called. """ harness = Harness(GithubRunnerCharm) harness.begin() state_mock = MagicMock() + monkeypatch.setattr("charm.manager_service", MagicMock()) + harness.charm._manager_client = MagicMock(spec=GitHubRunnerManagerClient) harness.charm._setup_state = MagicMock(return_value=state_mock) - harness.charm._get_set_image_ready_status = MagicMock(return_value=True) - runner_scaler_mock = MagicMock() - monkeypatch.setattr("charm.create_runner_scaler", MagicMock(return_value=runner_scaler_mock)) + harness.charm._check_image_ready = MagicMock(return_value=True) harness.charm._on_image_relation_changed(MagicMock()) - assert harness.charm.unit.status.name == ActiveStatus.name - runner_scaler_mock.flush.assert_called_once() - runner_scaler_mock.reconcile.assert_called_once() + harness.charm._manager_client.flush_runner.assert_called_once() def test__on_image_relation_joined(): diff --git a/tests/unit/test_charm_state.py b/tests/unit/test_charm_state.py index d0913afe62..b6294edc09 100644 --- a/tests/unit/test_charm_state.py +++ b/tests/unit/test_charm_state.py @@ -333,7 +333,7 @@ def test_check_reconcile_interval_invalid(reconcile_interval: int): CharmConfig.check_reconcile_interval(reconcile_interval) assert ( str(exc_info.value) - == "The reconcile-interval configuration needs to be greater or equal to 2" + == "The reconcile-interval configuration needs to be greater or equal to 1" ) diff --git a/tests/unit/test_event_timer.py b/tests/unit/test_event_timer.py deleted file mode 100644 index 062bb8407b..0000000000 --- a/tests/unit/test_event_timer.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. -import secrets - -from event_timer import EventTimer - - -def test_is_active_true(): - """ - arrange: create an EventTimer object and mock exec command to return zero exit code. - act: call is_active. - assert: return True. - """ - event = EventTimer(unit_name=secrets.token_hex(16)) - assert event.is_active(event_name=secrets.token_hex(16)) - - -def test_is_active_false(exec_command): - """ - arrange: create an EventTimer object and mock exec command to return non-zero exit code. - act: call is_active. - assert: return False. - """ - exec_command.return_value = ("", 1) - - event = EventTimer(unit_name=secrets.token_hex(16)) - assert not event.is_active(event_name=secrets.token_hex(16)) diff --git a/tests/unit/test_factories.py b/tests/unit/test_factories.py index 821a98afce..3f3554782c 100644 --- a/tests/unit/test_factories.py +++ b/tests/unit/test_factories.py @@ -130,6 +130,7 @@ def test_create_application_configuration( region_name="region", ), ), + reconcile_interval=5, ) diff --git a/tests/unit/test_manager_client.py b/tests/unit/test_manager_client.py index 5739f7a67a..f48582dfc8 100644 --- a/tests/unit/test_manager_client.py +++ b/tests/unit/test_manager_client.py @@ -25,6 +25,7 @@ def mock_requests_fixture() -> MagicMock: def client_fixture(mock__request: MagicMock): client = GitHubRunnerManagerClient("mock_ip", "mock_port") # Remove the request session as safe guard against issues real requests. + client.wait_till_ready = MagicMock() client._requests = None client._request = mock__request return client diff --git a/tests/unit/test_manager_service.py b/tests/unit/test_manager_service.py index 5568d8a808..b271e354eb 100644 --- a/tests/unit/test_manager_service.py +++ b/tests/unit/test_manager_service.py @@ -137,3 +137,27 @@ def test_install_package_failure(mock_execute_command: MagicMock): manager_service.install_package() assert manager_service._INSTALL_ERROR_MESSAGE in str(err.value) + + +def test_stop_with_running_service(mock_systemd: MagicMock): + """ + arrange: The service is running. + act: Run stop. + assert: There is a call for stopping service. + """ + mock_systemd.service_running.return_value = True + manager_service.stop() + mock_systemd.service_running.assert_called_once() + mock_systemd.service_stop.assert_called_once() + + +def test_stop_with_stopped_service(mock_systemd: MagicMock): + """ + arrange: The service is stopped. + act: Run stop. + assert: There is no call for stopping service. + """ + mock_systemd.service_running.return_value = False + manager_service.stop() + mock_systemd.service_running.assert_called_once() + mock_systemd.service_stop.assert_not_called() From 6835740d0923cae464ab8d9643e662cb5b19d7bc Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 11 Jun 2025 13:29:55 +0800 Subject: [PATCH 279/343] [ISD-3689] Fix: metric log owner on upgrade (#565) --- docs/changelog.md | 4 +++ .../reconcile_service.py | 3 +- src/charm.py | 12 ++++++++ tests/integration/helpers/common.py | 2 +- tests/unit/test_charm.py | 29 +++++++++++++++++++ 5 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b4617d6104..04dad3aa7b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,9 @@ # Changelog +### 2025-06-10 + +- Fix issue with upgraded charm unit unable to issue metrics due to metric log ownership issues. + ### 2025-06-04 - Reduce the reconcile-interval configuration from 10 minutes to 5 minutes. This is the interval diff --git a/github-runner-manager/src/github_runner_manager/reconcile_service.py b/github-runner-manager/src/github_runner_manager/reconcile_service.py index f9ecd43c7f..65ce52ec2c 100644 --- a/github-runner-manager/src/github_runner_manager/reconcile_service.py +++ b/github-runner-manager/src/github_runner_manager/reconcile_service.py @@ -18,8 +18,7 @@ logger = logging.getLogger(__name__) -RECONCILE_ID_FILE = Path("/var/log/reconcile.id") - +RECONCILE_ID_FILE = Path("~").expanduser() / "reconcile.id" RECONCILE_SERVICE_START_MSG = "Starting the reconcile service..." RECONCILE_START_MSG = "Start reconciliation" RECONCILE_END_MSG = "End reconciliation" diff --git a/src/charm.py b/src/charm.py index c6ec75fb85..ecf06b293a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -17,12 +17,14 @@ import json import logging import pathlib +import shutil from typing import Any, Callable, Sequence, TypeVar import ops from charms.data_platform_libs.v0.data_interfaces import DatabaseRequires from charms.grafana_agent.v0.cos_agent import COSAgentProvider from github_runner_manager import constants +from github_runner_manager.metrics.events import METRICS_LOG_PATH from github_runner_manager.platform.platform_provider import TokenError from github_runner_manager.utilities import set_env_var from ops.charm import ( @@ -525,6 +527,16 @@ def _setup_runner_manager_user() -> None: execute_command(["/usr/bin/chmod", "700", f"/home/{constants.RUNNER_MANAGER_USER}/.ssh"]) # Give the user access to write to /var/log execute_command(["/usr/sbin/usermod", "-a", "-G", "syslog", constants.RUNNER_MANAGER_USER]) + execute_command(["/usr/bin/chmod", "g+w", "/var/log"]) + + # For charm upgrade, previous revision root owns the metric logs, this is changed to runner + # manager. + if METRICS_LOG_PATH.exists(): + shutil.chown( + METRICS_LOG_PATH, + user=constants.RUNNER_MANAGER_USER, + group=constants.RUNNER_MANAGER_GROUP, + ) if __name__ == "__main__": diff --git a/tests/integration/helpers/common.py b/tests/integration/helpers/common.py index 7bba56e4db..85eaeb1c5a 100644 --- a/tests/integration/helpers/common.py +++ b/tests/integration/helpers/common.py @@ -92,7 +92,7 @@ async def get_reconcile_id(unit: Unit) -> str: """ _, stdout, _ = await run_in_unit( unit, - "cat /var/log/reconcile.id", + "cat /home/runner-manager/reconcile.id", assert_on_failure=True, assert_msg="Unable to get reconcile ID", ) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 9ec3c9b6c2..bdae8a1753 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -7,16 +7,19 @@ import typing import unittest import urllib.error +from pathlib import Path from unittest.mock import MagicMock, patch import pytest import yaml +from github_runner_manager import constants from github_runner_manager.platform.platform_provider import TokenError from ops.model import BlockedStatus, StatusBase, WaitingStatus from ops.testing import Harness from charm import ( GithubRunnerCharm, + _setup_runner_manager_user, catch_action_errors, catch_charm_errors, ) @@ -547,3 +550,29 @@ def test_on_config_changed_openstack_clouds_yaml(mock_side_effects): harness.charm.on.config_changed.emit() assert harness.charm.unit.status == BlockedStatus("Please provide image integration.") + + +def test_metric_log_ownership_for_upgrade(mock_side_effects, tmp_path: Path, monkeypatch): + """ + arrange: Metric log exists. + act: Upgrade charm. + assert: Call to change the metric ownership is called. + + For context, older revision the metric log is owned by root. + The current revision the metric log is owned by the runner manager. + This is to test on upgrade the charm ensures the log file has the correct owner. + """ + mock_metric_log_path = tmp_path + mock_metric_log_path.touch(exist_ok=True) + shutil_mock = MagicMock() + monkeypatch.setattr("charm.METRICS_LOG_PATH", mock_metric_log_path) + monkeypatch.setattr("charm.shutil", shutil_mock) + monkeypatch.setattr("charm.execute_command", MagicMock(return_value=(0, "Mock_stdout"))) + + _setup_runner_manager_user() + + shutil_mock.chown.assert_called_once_with( + mock_metric_log_path, + user=constants.RUNNER_MANAGER_USER, + group=constants.RUNNER_MANAGER_GROUP, + ) From 92bc51540326336ff5fd9cb9bb6909c972911723 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:46:14 +0800 Subject: [PATCH 280/343] Manual test --- .github/workflows/manual_test.yaml | 90 +++++++++++++++--------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 17978f798e..dab23225af 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,52 +6,52 @@ on: workflow_dispatch: jobs: - # stg-reactive-test: - # runs-on: [self-hosted,staging,amd64,self-hosted-linux-amd64-jammy-staging,stg-reactive] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - uses: canonical/setup-lxd@v0.1.1 - # - run: lxd init --auto - # if: ${{ always() }} - # - run: lxc list - # if: ${{ always() }} - # - run: time sudo snap install juju - # if: ${{ always() }} - # - run: time juju bootstrap localhost localhost - # if: ${{ always() }} - edge-test: - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, edge,arm64,self-hosted-linux-arm64-noble-edge,reactive] + stg-reactive-test: + runs-on: [self-hosted,test-yjpus3iu] steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR - - uses: canonical/setup-lxd@v0.1.1 - - run: sudo lxd init --auto - if: ${{ always() }} - - run: lxc list - if: ${{ always() }} - - run: time sudo snap install juju - if: ${{ always() }} - - run: time juju bootstrap localhost localhost - if: ${{ always() }} + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - run: echo $HTTP_PROXY + - run: echo $HTTPS_PROXY + - run: echo $NO_PROXY + - run: echo $DOCKERHUB_MIRROR + # - uses: canonical/setup-lxd@v0.1.1 + # - run: lxd init --auto + # if: ${{ always() }} + # - run: lxc list + # if: ${{ always() }} + # - run: time sudo snap install juju + # if: ${{ always() }} + # - run: time juju bootstrap localhost localhost + # if: ${{ always() }} + # edge-test: + # strategy: + # fail-fast: false + # matrix: + # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + # runs-on: [self-hosted, edge,arm64,self-hosted-linux-arm64-noble-edge,reactive] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR + # - uses: canonical/setup-lxd@v0.1.1 + # - run: sudo lxd init --auto + # if: ${{ always() }} + # - run: lxc list + # if: ${{ always() }} + # - run: time sudo snap install juju + # if: ${{ always() }} + # - run: time juju bootstrap localhost localhost + # if: ${{ always() }} # reactive-focal-large-long-label: # name: reactive focal large with long label # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] From 0d87517ea3efc7c9313b92d0ea1319d715da4941 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 13 Jun 2025 11:00:46 +0800 Subject: [PATCH 281/343] Get a edge with tmate --- .github/workflows/manual_test.yaml | 33 ++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index dab23225af..aa575d2974 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,18 +6,29 @@ on: workflow_dispatch: jobs: - stg-reactive-test: - runs-on: [self-hosted,test-yjpus3iu] + edge-tmate: + runs-on: [self-hosted, edge,arm64,self-hosted-linux-arm64-noble-edge,reactive] steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - run: echo $HTTP_PROXY - - run: echo $HTTPS_PROXY - - run: echo $NO_PROXY - - run: echo $DOCKERHUB_MIRROR + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - uses: actions/checkout@v4 + - name: tmate ssh + uses: canonical/action-tmate@main + # stg-reactive-test: + # runs-on: [self-hosted,test-yjpus3iu] + # steps: + # - run: lscpu + # - run: df -h + # - run: free -h + # - run: uname -a + # - run: lsb_release -a + # - run: echo $HTTP_PROXY + # - run: echo $HTTPS_PROXY + # - run: echo $NO_PROXY + # - run: echo $DOCKERHUB_MIRROR # - uses: canonical/setup-lxd@v0.1.1 # - run: lxd init --auto # if: ${{ always() }} From 70c4fef1eeaf67f589f27ca47b47e1826b9b1efb Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Jun 2025 09:45:41 +0800 Subject: [PATCH 282/343] Test staging reactive --- .github/workflows/manual_test.yaml | 302 +---------------------------- 1 file changed, 2 insertions(+), 300 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index aa575d2974..e333b1b166 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,309 +7,11 @@ on: jobs: edge-tmate: - runs-on: [self-hosted, edge,arm64,self-hosted-linux-arm64-noble-edge,reactive] + runs-on: [self-hosted, staging,amd64,self-hosted-linux-amd64-jammy-staging,stg-reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - - uses: actions/checkout@v4 - - name: tmate ssh - uses: canonical/action-tmate@main - # stg-reactive-test: - # runs-on: [self-hosted,test-yjpus3iu] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - uses: canonical/setup-lxd@v0.1.1 - # - run: lxd init --auto - # if: ${{ always() }} - # - run: lxc list - # if: ${{ always() }} - # - run: time sudo snap install juju - # if: ${{ always() }} - # - run: time juju bootstrap localhost localhost - # if: ${{ always() }} - # edge-test: - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # runs-on: [self-hosted, edge,arm64,self-hosted-linux-arm64-noble-edge,reactive] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - uses: canonical/setup-lxd@v0.1.1 - # - run: sudo lxd init --auto - # if: ${{ always() }} - # - run: lxc list - # if: ${{ always() }} - # - run: time sudo snap install juju - # if: ${{ always() }} - # - run: time juju bootstrap localhost localhost - # if: ${{ always() }} - # reactive-focal-large-long-label: - # name: reactive focal large with long label - # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-large] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # reactive-focal-large: - # name: reactive focal large - # runs-on: [self-hosted, reactive, large, focal, amd64] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # reactive-focal-medium-long-label: - # name: reactive focal medium with long label - # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-focal-medium] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # reactive-focal-medium: - # name: reactive focal medium - # runs-on: [self-hosted, reactive, medium, focal, amd64] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # tiobe-noble: - # name: tiobe noble - # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-tiobe] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # tiobe: - # name: tiobe - # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-tiobe] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # tiobe-noble-large: - # name: tiobe noble large - # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-noble-large-tiobe] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # tiobe-large: - # name: tiobe large - # runs-on: [self-hosted, reactive, self-hosted-linux-amd64-jammy-large-tiobe] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: lsb_release -a - # self-hosted-medium-arm64-canary: - # name: self-hosted-medium-arm64-canary - # runs-on: [self-hosted, arm64, medium, canary, noble] - # steps: - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - name: tmate ssh - # uses: canonical/action-tmate@main - # self-hosted-large: - # name: self-hosted-large - # runs-on: [self-hosted, amd64, large, jammy] - # steps: - # - run: sudo snap install --classic lxd - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - name: tmate ssh - # uses: canonical/action-tmate@main - # self-hosted-prod-private-endpoint: - # name: self-hosted prod private endpoint - # runs-on: [self-hosted, X64, self-hosted-linux-amd64-jammy-private-endpoint-medium] - # steps: - # - run: echo hello - # self-hosted-stg: - # name: self-hosted staging - # runs-on: [self-hosted, X64, staging] - # steps: - # - run: echo hello - # self-host-large: - # name: self-hosted large - # runs-on: [self-hosted, X64, large, jammy] - # strategy: - # fail-fast: false - # matrix: - # index: [0] - # steps: - # - run: echo hello - # - name: tmate ssh - # uses: canonical/action-tmate@main - # self-host: - # name: self-hosted edge - # runs-on: [self-hosted, X64, edge, jammy] - # strategy: - # fail-fast: false - # matrix: - # index: [0] - # steps: - # - run: echo hello - # - name: tmate ssh - # uses: canonical/action-tmate@main - # - run: cat /etc/docker/daemon.json - # - run: docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - run: docker pull jujusolutions/charm-base:ubuntu-20.04 - # - run: sleep 3600 - # self-host-x64: - # name: self-hosted x64 test - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # runs-on: [self-hosted, X64, edge, jammy] - # steps: - # - run: uname -a - # - uses: actions/checkout@v4 - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: 3.10.8 - # - run: pip install boto3 - # - run: python scripts/test_microceph.py - # github-test: - # name: GitHub x64 test - # runs-on: ubuntu-latest - # steps: - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: 3.8.10 - # - run: sudo apt-get update - # - run: sudo apt install -y fio - # - name: seq write - # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=write --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - # - run: sudo rm -f /write* /read* - # - name: rand write - # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randwrite --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - # - run: sudo rm -f /write* /read* - # - name: seq read - # run: sudo fio --name=write_throughput --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=1M --iodepth=64 --rw=read --group_reporting=1 --iodepth_batch_submit=64 --iodepth_batch_complete_max=64 - # - run: sudo rm -f /write* /read* - # - name: rand read - # run: sudo fio --name=write_iops --directory=/ --numjobs=4 --size=3G --time_based --runtime=60s --ramp_time=2s --ioengine=libaio --direct=1 --verify=0 --bs=4k --iodepth=256 --rw=randread --group_reporting=1 --iodepth_batch_submit=256 --iodepth_batch_complete_max=256 - # - run: sudo rm -f /write* /read* - # self-host-test-arm64: - # name: self-hosted arm64 test - # runs-on: [self-hosted, ARM64, medium] - # strategy: - # fail-fast: false - # matrix: - # index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - # index: [0] - # steps: - # - uses: actions/checkout@v4 - # - run: bash scripts/test_hardware.sh - # - name: tmate ssh - # uses: canonical/action-tmate@debug - # - run: lscpu - # - run: df -h - # - run: free -h - # - run: uname -a - # - run: echo $HTTP_PROXY - # - run: echo $HTTPS_PROXY - # - run: echo $NO_PROXY - # - run: echo $DOCKERHUB_MIRROR - # - uses: canonical/setup-lxd@v0.1.1 - # - run: lxd init --auto - # if: ${{ always() }} - # - run: lxc list - # if: ${{ always() }} - # - run: time sudo snap install juju - # if: ${{ always() }} - # - run: time juju bootstrap localhost localhost - # if: ${{ always() }} - # - run: sudo snap install aproxy --edge - # - run: sudo snap set aproxy proxy=squid.internal:3128 listen=:54969 - # - run: |- - # sudo nft -f - << EOF - # define default-ip = $(ip route get $(ip route show 0.0.0.0/0 | grep -oP 'via \K\S+') | grep -oP 'src \K\S+') - # define private-ips = { 10.0.0.0/8, 127.0.0.1/8, 172.16.0.0/12, 192.168.0.0/16 } - # table ip aproxy - # flush table ip aproxy - # table ip aproxy { - # chain prerouting { - # type nat hook prerouting priority dstnat; policy accept; - # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - # } - - # chain output { - # type nat hook output priority -100; policy accept; - # ip daddr != \$private-ips tcp dport { 80, 443 } counter dnat to \$default-ip:54969 - # } - # } - # EOF - # - run: nc -vz github-runner-dockerhub-cache.canonical.com 5000 - # - run: curl $DOCKERHUB_MIRROR -vvv - # - run: sudo apt install docker.io -yq - # - run : docker pull github-runner-dockerhub-cache.canonical.com:5000/jujusolutions/charm-base:ubuntu-20.04 - # - run : docker pull python - # - run: echo $TMATE_SERVER_HOST - # - run: echo $TMATE_SERVER_PORT - # - run: sudo apt install netcat - # - name: Test network connection to tmate ssh server - # run: nc -vz $TMATE_SERVER_HOST $TMATE_SERVER_PORT - # - name: tmate ssh - # uses: canonical/action-tmate@debug - # - uses: actions/checkout@v4.1.1 - # - uses: canonical/setup-lxd@v0.1.1 - # - name: Pack charm - # run: | - # sudo snap install charmcraft --classic --channel latest/stable - # charmcraft pack -v - # - run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs >> rustup.sh - # - run: sh rustup.sh -y - # - run: $HOME/.cargo/bin/cargo install ripgrep - # github-test: - # name: GitHub x64 test - # runs-on: ubuntu-latest - # steps: - # - name: Setup tmate session - # uses: mxschmitt/action-tmate@v3 - # with: - # limit-access-to-actor: true + - uses: actions/checkout@v4 \ No newline at end of file From 1e6a209e2de712cec0c6d27876d7a09c3ef2fbc2 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 16 Jun 2025 14:16:16 +0800 Subject: [PATCH 283/343] Test edge --- .github/workflows/manual_test.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e333b1b166..d3bcfd6ddd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,15 @@ on: workflow_dispatch: jobs: - edge-tmate: - runs-on: [self-hosted, staging,amd64,self-hosted-linux-amd64-jammy-staging,stg-reactive] + edge: + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, edge, amd64, reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - - run: lsb_release -a - - uses: actions/checkout@v4 \ No newline at end of file + - run: lsb_release -a \ No newline at end of file From 7888219c2a98e7cb5a081559dc6c4e2ffd89c4d9 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 17 Jun 2025 09:27:19 +0800 Subject: [PATCH 284/343] Hold a edge runner --- .github/workflows/manual_test.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d3bcfd6ddd..5cf104afa0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,14 +7,11 @@ on: jobs: edge: - strategy: - fail-fast: false - matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] runs-on: [self-hosted, edge, amd64, reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - - run: lsb_release -a \ No newline at end of file + - run: lsb_release -a + - run: sleep 100000 \ No newline at end of file From f2de54ecd85923b5123e93a28bf62af2f88c3556 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 17 Jun 2025 09:46:25 +0800 Subject: [PATCH 285/343] Edge tmate --- .github/workflows/manual_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 5cf104afa0..7a113bed36 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,4 +14,5 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a - - run: sleep 100000 \ No newline at end of file + - name: tmate ssh + uses: canonical/action-tmate@main \ No newline at end of file From 3e67743c906ccfeeb9001e86f14bc3b8b540ede6 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 19 Jun 2025 14:52:22 +0800 Subject: [PATCH 286/343] Test runners --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 7a113bed36..c2ecac53ca 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: edge: - runs-on: [self-hosted, edge, amd64, reactive] + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-large] steps: - run: lscpu - run: df -h From 0b5e33eb35651f0c27a34a9822c2f75de0d0463c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 19 Jun 2025 14:54:15 +0800 Subject: [PATCH 287/343] Test a reactive runner --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c2ecac53ca..aee3f53a1b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: edge: - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-large] + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-large,reactive] steps: - run: lscpu - run: df -h From bc5a298a9657bcf4c9fe8b459f124bf21bfe5dbf Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:50:58 +0800 Subject: [PATCH 288/343] Test nolbe large --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index aee3f53a1b..90ebb78b7a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - edge: - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-large,reactive] + test: + runs-on: [self-hosted, noble, large, jammy] steps: - run: lscpu - run: df -h From 690805d7bbba2150e7b82a16965cac9ccb5f1f45 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:51:03 +0800 Subject: [PATCH 289/343] Test nolbe large --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 90ebb78b7a..c1b08ad9be 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: test: - runs-on: [self-hosted, noble, large, jammy] + runs-on: [self-hosted, noble, large] steps: - run: lscpu - run: df -h From 254b3499ad740dcf12be2fef0644fade4fe683fb Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:58:43 +0800 Subject: [PATCH 290/343] test jammy --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c1b08ad9be..705a0ef6ef 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - test: - runs-on: [self-hosted, noble, large] + jammy-test: + runs-on: [self-hosted, jammy, large] steps: - run: lscpu - run: df -h From cb49ba0d34dad6f39ae7ddaa02a1e52db50b2850 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:13:26 +0800 Subject: [PATCH 291/343] Test stg --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 705a0ef6ef..10ba0f0c29 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - jammy-test: - runs-on: [self-hosted, jammy, large] + stg-test: + runs-on: [self-hosted, stg-private-endpoint] steps: - run: lscpu - run: df -h From 5b6d9ee086c2c31d6020adf05f1133790af2f0f3 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 3 Jul 2025 15:28:42 +0800 Subject: [PATCH 292/343] Run network tests --- .github/workflows/manual_test.yaml | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 10ba0f0c29..2258f80581 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,13 +6,21 @@ on: workflow_dispatch: jobs: - stg-test: - runs-on: [self-hosted, stg-private-endpoint] + network-speed-test-stg: + name: Network speed test AMD64 + runs-on: [self-hosted, stg-private-endpoint, X64] steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - name: tmate ssh - uses: canonical/action-tmate@main \ No newline at end of file + - name: Librespeed test + run: | + wget https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_amd64.tar.gz + tar -xvzf librespeed-cli_1.0.10_linux_amd64.tar.gz + ./librespeed-cli + network-speed-test-edge: + name: Network speed test AMD64 + runs-on: [self-hosted, edge, X64] + steps: + - name: Librespeed test + run: | + wget https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_amd64.tar.gz + tar -xvzf librespeed-cli_1.0.10_linux_amd64.tar.gz + ./librespeed-cli \ No newline at end of file From fde05e112b1921f72f842de65ce46f605e24f8a0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 25 Jul 2025 10:58:41 +0800 Subject: [PATCH 293/343] Test edge --- .github/workflows/manual_test.yaml | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 2258f80581..d3bcfd6ddd 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,21 +6,15 @@ on: workflow_dispatch: jobs: - network-speed-test-stg: - name: Network speed test AMD64 - runs-on: [self-hosted, stg-private-endpoint, X64] + edge: + strategy: + fail-fast: false + matrix: + index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + runs-on: [self-hosted, edge, amd64, reactive] steps: - - name: Librespeed test - run: | - wget https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_amd64.tar.gz - tar -xvzf librespeed-cli_1.0.10_linux_amd64.tar.gz - ./librespeed-cli - network-speed-test-edge: - name: Network speed test AMD64 - runs-on: [self-hosted, edge, X64] - steps: - - name: Librespeed test - run: | - wget https://github.com/librespeed/speedtest-cli/releases/download/v1.0.10/librespeed-cli_1.0.10_linux_amd64.tar.gz - tar -xvzf librespeed-cli_1.0.10_linux_amd64.tar.gz - ./librespeed-cli \ No newline at end of file + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a \ No newline at end of file From adf5d46144736b3893e225da74b30645efbc981d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 25 Jul 2025 11:02:40 +0800 Subject: [PATCH 294/343] Test arm64 --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d3bcfd6ddd..995d07b523 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, edge, amd64, reactive] + runs-on: [self-hosted, edge, arm64, reactive] steps: - run: lscpu - run: df -h From fbc9c71554771702aa58af59a3302cc55be0726b Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 25 Jul 2025 14:27:32 +0800 Subject: [PATCH 295/343] Test prod arm64 --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 995d07b523..071b7b34db 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -10,8 +10,8 @@ jobs: strategy: fail-fast: false matrix: - index: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - runs-on: [self-hosted, edge, arm64, reactive] + index: [0, 1] + runs-on: [self-hosted, large, arm64, reactive] steps: - run: lscpu - run: df -h From 266cd41ab114f8d2701f7716e92f115abf5a84cb Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 28 Jul 2025 09:56:30 +0800 Subject: [PATCH 296/343] Test a bunch of architectures --- .github/workflows/manual_test.yaml | 42 +++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 071b7b34db..419e5d51b0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,15 +6,51 @@ on: workflow_dispatch: jobs: - edge: + large-arm64: strategy: fail-fast: false matrix: - index: [0, 1] + index: [0] runs-on: [self-hosted, large, arm64, reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - - run: lsb_release -a \ No newline at end of file + - run: lsb_release -a + edge-arm64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, edge, arm64, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-s390x: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, large, s390x] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-ppc64le: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, large, ppc64le] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a From e28b8acaac3fd2882d155a10abcbd855a04c9bc5 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 28 Jul 2025 10:18:30 +0800 Subject: [PATCH 297/343] Fix the less used arch --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 419e5d51b0..0396238807 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -35,7 +35,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, large, s390x] + runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] steps: - run: lscpu - run: df -h @@ -47,7 +47,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, large, ppc64le] + runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] steps: - run: lscpu - run: df -h From 347502e3b03f0c98c06376340cef87d70056d929 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:38:34 +0800 Subject: [PATCH 298/343] Test all archs --- .github/workflows/manual_test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0396238807..8c6b83828e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,24 +6,24 @@ on: workflow_dispatch: jobs: - large-arm64: + medium-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, large, arm64, reactive] + runs-on: [self-hosted, medium, amd64, reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - edge-arm64: + medium-arm64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, edge, arm64, reactive] + runs-on: [self-hosted, medium, arm64, reactive] steps: - run: lscpu - run: df -h From 993d59d3e375bbf16668ab2d34ed8dc2a29acda4 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 5 Aug 2025 13:39:42 +0800 Subject: [PATCH 299/343] Test stg-private-endpoint --- .github/workflows/manual_test.yaml | 40 +++--------------------------- 1 file changed, 3 insertions(+), 37 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8c6b83828e..23761df587 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,46 +11,12 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, medium, amd64, reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - medium-arm64: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, medium, arm64, reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - edge-s390x: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - edge-ppc64le: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] + runs-on: [self-hosted, stg-private-endpoint] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main \ No newline at end of file From 1d2086c879b316194150f0e87eda07eb3510bd41 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 8 Aug 2025 14:23:07 +0800 Subject: [PATCH 300/343] Test staging private-endpoint on PS7 --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 23761df587..ff5b94806f 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - medium-amd64: + stg-private-endpoint: strategy: fail-fast: false matrix: From c57b29d95360c51937c74e7b6c14795486e49500 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Fri, 8 Aug 2025 15:00:52 +0800 Subject: [PATCH 301/343] Test private-endpoint ps7 --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ff5b94806f..d2aabb0387 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - stg-private-endpoint: + stg-private-endpoint-ps7: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, stg-private-endpoint] + runs-on: [self-hosted, stg-private-endpoint-ps7] steps: - run: lscpu - run: df -h From 0c8c9647b561ec96b564bb9be77764483b04aa27 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Mon, 11 Aug 2025 14:03:08 +0800 Subject: [PATCH 302/343] Test stg tmate --- .github/workflows/manual_test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d2aabb0387..bc204bde3d 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,6 +6,16 @@ on: workflow_dispatch: jobs: + stg-ps7: + runs-on: [staging,amd64,self-hosted-linux-amd64-jammy-staging,ps7] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main stg-private-endpoint-ps7: strategy: fail-fast: false From 6cf62ca7c7bab7c097713856f3473ed7122a92f7 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:29:16 +0800 Subject: [PATCH 303/343] get a edge with tmate --- .github/workflows/manual_test.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index bc204bde3d..ccb091106c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,22 +6,8 @@ on: workflow_dispatch: jobs: - stg-ps7: - runs-on: [staging,amd64,self-hosted-linux-amd64-jammy-staging,ps7] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - name: Setup tmate session - uses: canonical/action-tmate@main - stg-private-endpoint-ps7: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, stg-private-endpoint-ps7] + edge: + runs-on: [self-hosted, edge] steps: - run: lscpu - run: df -h From 29b39cec5d2355cd9a7dfca19ab0e7ea6ba640ca Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:35:51 +0800 Subject: [PATCH 304/343] Test checkout v5 --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ccb091106c..261f005b38 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -14,5 +14,6 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - uses: actions/checkout@v5 - name: Setup tmate session uses: canonical/action-tmate@main \ No newline at end of file From 291664b351720b3832fa92bf251bc2a305a11cb0 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:37:58 +0800 Subject: [PATCH 305/343] test on diff arch --- .github/workflows/manual_test.yaml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 261f005b38..e71b880f9a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,17 @@ on: workflow_dispatch: jobs: - edge: - runs-on: [self-hosted, edge] + amd64: + runs-on: [self-hosted, medium, amd64] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - uses: actions/checkout@v5 + s390x: + runs-on: [self-hosted, s390x] steps: - run: lscpu - run: df -h @@ -15,5 +24,3 @@ jobs: - run: uname -a - run: lsb_release -a - uses: actions/checkout@v5 - - name: Setup tmate session - uses: canonical/action-tmate@main \ No newline at end of file From edaa083e0e4803934ac7de430b9973448d901b42 Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:42:48 +0800 Subject: [PATCH 306/343] Tmate on s390x --- .github/workflows/manual_test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e71b880f9a..d4e2cca32a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -16,7 +16,7 @@ jobs: - run: lsb_release -a - uses: actions/checkout@v5 s390x: - runs-on: [self-hosted, s390x] + runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] steps: - run: lscpu - run: df -h @@ -24,3 +24,5 @@ jobs: - run: uname -a - run: lsb_release -a - uses: actions/checkout@v5 + - name: Setup tmate session + uses: canonical/action-tmate@main From 8b5b4a47012e6f44aa637a4ea880746737b5eb6c Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:19:35 +0800 Subject: [PATCH 307/343] Get a pfe-ci runner --- .github/workflows/manual_test.yaml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d4e2cca32a..69ab2f460c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,17 +6,8 @@ on: workflow_dispatch: jobs: - amd64: - runs-on: [self-hosted, medium, amd64] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - uses: actions/checkout@v5 - s390x: - runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] + pfe-ci: + runs-on: [self-hosted, self-hosted-linux-amd64-noble-pfe-ci] steps: - run: lscpu - run: df -h From fa539134e3a2e980273f945ed17c0033a251799d Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:35:08 +0800 Subject: [PATCH 308/343] Testing edge runners --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 69ab2f460c..cc0acbe721 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - pfe-ci: - runs-on: [self-hosted, self-hosted-linux-amd64-noble-pfe-ci] + edge: + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-edge] steps: - run: lscpu - run: df -h From 35bc7ee86c7b751e57a9042a446bf374b205b0ff Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:17:44 +0800 Subject: [PATCH 309/343] Debug pfe-ci --- .github/workflows/manual_test.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index cc0acbe721..4aa68b2ca8 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,19 @@ on: workflow_dispatch: jobs: - edge: - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-edge] + private: + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - uses: actions/checkout@v5 + - name: Setup tmate session + uses: canonical/action-tmate@main + pfe-ci: + runs-on: [self-hosted, self-hosted-linux-amd64-noble-pfe-ci] steps: - run: lscpu - run: df -h From 9afaee0f014a8e9ea7d99e97919998b023d57baa Mon Sep 17 00:00:00 2001 From: yhaliaw <43424755+yhaliaw@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:39:23 +0800 Subject: [PATCH 310/343] Test pfe --- .github/workflows/manual_test.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4aa68b2ca8..69ab2f460c 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,17 +6,6 @@ on: workflow_dispatch: jobs: - private: - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - uses: actions/checkout@v5 - - name: Setup tmate session - uses: canonical/action-tmate@main pfe-ci: runs-on: [self-hosted, self-hosted-linux-amd64-noble-pfe-ci] steps: From 3a5439ae61a0fbfa1855c8b1b33660af01a714c6 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 27 Aug 2025 11:10:31 +0800 Subject: [PATCH 311/343] Get a private-endpoint runner --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 69ab2f460c..0339fed045 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: pfe-ci: - runs-on: [self-hosted, self-hosted-linux-amd64-noble-pfe-ci] + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] steps: - run: lscpu - run: df -h From aeb218d4ce8be6f284122b64575e02669363b37e Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 2 Sep 2025 13:53:35 +0800 Subject: [PATCH 312/343] Get a edge runner --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0339fed045..0283fceede 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - pfe-ci: - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] + amd64-edge: + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-edge] steps: - run: lscpu - run: df -h From d14619ccea2a5b2f7c6c015c6223b03ff5c49431 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 2 Sep 2025 15:25:43 +0800 Subject: [PATCH 313/343] Get amd64-edge --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0283fceede..82105beaf1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: amd64-edge: - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-edge] + runs-on: [self-hosted, edge, amd64] steps: - run: lscpu - run: df -h From acc7c8ba35ea18399b298a9b2e2f125b9f6b48b6 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 4 Sep 2025 09:53:08 +0800 Subject: [PATCH 314/343] Test --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 82105beaf1..d4ec77c1c3 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,5 +15,6 @@ jobs: - run: uname -a - run: lsb_release -a - uses: actions/checkout@v5 + - run: ${{ secrets.INTEGRATION_TEST_ARGS }} > test.txt - name: Setup tmate session uses: canonical/action-tmate@main From 8297e74bf45af594646c1a7ffd8f8e473bf50581 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 4 Sep 2025 10:01:37 +0800 Subject: [PATCH 315/343] Test --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d4ec77c1c3..cdcbd22278 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -15,6 +15,6 @@ jobs: - run: uname -a - run: lsb_release -a - uses: actions/checkout@v5 - - run: ${{ secrets.INTEGRATION_TEST_ARGS }} > test.txt + - run: echo '${{ secrets.INTEGRATION_TEST_ARGS }}' > test.txt - name: Setup tmate session uses: canonical/action-tmate@main From dff945dd9bdfd2e352a05fcd95c7cb82eb014c38 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 4 Sep 2025 12:32:35 +0800 Subject: [PATCH 316/343] Get a pfe-ci runner --- .github/workflows/manual_test.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index cdcbd22278..da52736067 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,8 +6,8 @@ on: workflow_dispatch: jobs: - amd64-edge: - runs-on: [self-hosted, edge, amd64] + pfe-ci: + runs-on: [self-hosted, pfe-ci] steps: - run: lscpu - run: df -h @@ -15,6 +15,5 @@ jobs: - run: uname -a - run: lsb_release -a - uses: actions/checkout@v5 - - run: echo '${{ secrets.INTEGRATION_TEST_ARGS }}' > test.txt - name: Setup tmate session uses: canonical/action-tmate@main From b456559bd443ccc196c78af03d3a8b368b24e2bc Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 4 Sep 2025 14:52:11 +0800 Subject: [PATCH 317/343] Test labels for pfe-ci --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index da52736067..004d60e6e6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -7,7 +7,7 @@ on: jobs: pfe-ci: - runs-on: [self-hosted, pfe-ci] + runs-on: [self-hosted, x64, pfe-ci, noble] steps: - run: lscpu - run: df -h From c6a5850f20bb408df90170ed73e399b4f288c693 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 8 Sep 2025 08:16:16 +0800 Subject: [PATCH 318/343] Test runners --- .github/workflows/manual_test.yaml | 47 ++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 004d60e6e6..8c6b83828e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,14 +6,51 @@ on: workflow_dispatch: jobs: - pfe-ci: - runs-on: [self-hosted, x64, pfe-ci, noble] + medium-amd64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, medium, amd64, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + medium-arm64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, medium, arm64, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-s390x: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-ppc64le: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - - uses: actions/checkout@v5 - - name: Setup tmate session - uses: canonical/action-tmate@main From 32a0a86e76b4f960b79a9c3c6cc5f693fc66a2c1 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 10 Sep 2025 14:58:21 +0800 Subject: [PATCH 319/343] Test large amd64 --- .github/workflows/manual_test.yaml | 42 +++--------------------------- 1 file changed, 4 insertions(+), 38 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8c6b83828e..ed9a3c96d1 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,51 +6,17 @@ on: workflow_dispatch: jobs: - medium-amd64: + large-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, medium, amd64, reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - medium-arm64: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, medium, arm64, reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - edge-s390x: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - edge-ppc64le: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] + runs-on: [self-hosted, amd64, large] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main From 8bd5b24df8b706d9589c700f71e732de551121a1 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 12 Sep 2025 15:04:03 +0800 Subject: [PATCH 320/343] Get arm64 --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index ed9a3c96d1..c4f0e49aa9 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, amd64, large] + runs-on: [self-hosted, arm64, medium] steps: - run: lscpu - run: df -h From 27a15b72b75a940bc5f80945ef4412ebc5907621 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 19 Sep 2025 16:45:42 +0800 Subject: [PATCH 321/343] Get pfe-ci --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index c4f0e49aa9..a190900826 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - large-amd64: + pfe-ci: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, arm64, medium] + runs-on: [self-hosted, pfe-ci] steps: - run: lscpu - run: df -h From 78251bb08426f8221a5ba3e34ab9ac5336e7d3f5 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 22 Sep 2025 14:32:27 +0800 Subject: [PATCH 322/343] Get a PFE runner --- .github/workflows/manual_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index a190900826..cc041f8db0 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -16,7 +16,6 @@ jobs: - run: lscpu - run: df -h - run: free -h - - run: uname -a - run: lsb_release -a - name: Setup tmate session uses: canonical/action-tmate@main From 12352ee3b6740d35e8aa126f94226d2e3861ab38 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 25 Sep 2025 13:27:36 +0800 Subject: [PATCH 323/343] Test edge --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index cc041f8db0..9b72612576 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, pfe-ci] + runs-on: [self-hosted, edge, X64] steps: - run: lscpu - run: df -h From c37a120e87d251e64021f50517ae3735de27ffc9 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 25 Sep 2025 13:30:26 +0800 Subject: [PATCH 324/343] Test edge --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 9b72612576..821a5ad579 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - pfe-ci: + edge: strategy: fail-fast: false matrix: From 057e7e24bd27631aee3c79e69df9ad4b57c420ca Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 26 Sep 2025 14:29:02 +0800 Subject: [PATCH 325/343] Get pfe runner --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 821a5ad579..49d8d75f02 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - edge: + pfe-ci: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, edge, X64] + runs-on: [self-hosted, pfe-ci, X64] steps: - run: lscpu - run: df -h From 626e0d4a28814a8992dc2b2677a586b31ca01ee6 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 13 Oct 2025 15:31:01 +0800 Subject: [PATCH 326/343] Test the runners --- .github/workflows/manual_test.yaml | 43 +++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 49d8d75f02..8c6b83828e 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,16 +6,51 @@ on: workflow_dispatch: jobs: - pfe-ci: + medium-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, pfe-ci, X64] + runs-on: [self-hosted, medium, amd64, reactive] steps: - run: lscpu - run: df -h - run: free -h + - run: uname -a + - run: lsb_release -a + medium-arm64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, medium, arm64, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-s390x: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-ppc64le: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a - run: lsb_release -a - - name: Setup tmate session - uses: canonical/action-tmate@main From 0a551ef9f0c2959b5c65e94e47d1e7678d375d2c Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 13 Oct 2025 15:36:33 +0800 Subject: [PATCH 327/343] Test apt update --- .github/workflows/manual_test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 8c6b83828e..9c89c51eb5 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -18,6 +18,7 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - run: sudo apt update medium-arm64: strategy: fail-fast: false @@ -30,6 +31,7 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - run: sudo apt update edge-s390x: strategy: fail-fast: false @@ -42,6 +44,7 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - run: sudo apt update edge-ppc64le: strategy: fail-fast: false @@ -54,3 +57,4 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - run: sudo apt update From da920705f2f72aee632a493a77d7a1d0471d671b Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 13 Oct 2025 15:49:53 +0800 Subject: [PATCH 328/343] Test large runners --- .github/workflows/manual_test.yaml | 36 +++++------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 9c89c51eb5..821054bbfb 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - medium-amd64: + large-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, medium, amd64, reactive] + runs-on: [self-hosted, large, amd64, reactive] steps: - run: lscpu - run: df -h @@ -19,42 +19,16 @@ jobs: - run: uname -a - run: lsb_release -a - run: sudo apt update - medium-arm64: + large-arm64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, medium, arm64, reactive] + runs-on: [self-hosted, large, arm64, reactive] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - - run: sudo apt update - edge-s390x: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - run: sudo apt update - edge-ppc64le: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - run: sudo apt update + - run: sudo apt update \ No newline at end of file From 59a95881c769ef5731c6e6c216c0da09c201c6fc Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 13 Oct 2025 16:10:25 +0800 Subject: [PATCH 329/343] Test medium runners --- .github/workflows/manual_test.yaml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 821054bbfb..94676f68fe 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,25 +6,12 @@ on: workflow_dispatch: jobs: - large-amd64: + medium-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, large, amd64, reactive] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - run: sudo apt update - large-arm64: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, large, arm64, reactive] + runs-on: [self-hosted, medium, amd64, reactive] steps: - run: lscpu - run: df -h From 829ccb17cec33ead5faeaed10318143a00c98946 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 14 Oct 2025 08:20:16 +0800 Subject: [PATCH 330/343] medium runner debug --- .github/workflows/manual_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 94676f68fe..d9f85e4493 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -18,4 +18,5 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a - - run: sudo apt update \ No newline at end of file + - name: Setup tmate session + uses: canonical/action-tmate@main \ No newline at end of file From ddd12ae98661a6bc2ded789299de64ddcecf7727 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 14 Oct 2025 09:05:54 +0800 Subject: [PATCH 331/343] test pfe-ci --- .github/workflows/manual_test.yaml | 2 +- :w | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 :w diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index d9f85e4493..0c389dd788 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, medium, amd64, reactive] + runs-on: [self-hosted, X64, pfe-ci, noble, reactive] steps: - run: lscpu - run: df -h diff --git a/:w b/:w new file mode 100644 index 0000000000..141ae40629 --- /dev/null +++ b/:w @@ -0,0 +1,22 @@ +name: Manual test + +on: + # push: + pull_request: + workflow_dispatch: + +jobs: + medium-amd64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, X64, pfe-ci, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main \ No newline at end of file From ed46edaa2d35bd2d5fa67d8880b1dafd06e38eec Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 14 Oct 2025 09:29:39 +0800 Subject: [PATCH 332/343] Test pfe-ci runner --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 0c389dd788..661f65045b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - medium-amd64: + pfe-ci: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, X64, pfe-ci, noble, reactive] + runs-on: [self-hosted, X64, pfe-ci, noble] steps: - run: lscpu - run: df -h From 1e696798054879433fe4eef5c4d8d4cc2afcd72c Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 22 Oct 2025 11:48:30 +0800 Subject: [PATCH 333/343] test amd64 edge --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 661f65045b..9d2811c28b 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - pfe-ci: + amd64-edge: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, X64, pfe-ci, noble] + runs-on: [self-hosted, X64, edge] steps: - run: lscpu - run: df -h From d914593f6bd551b5209f9daa1cf6fd7910a8302a Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 23 Oct 2025 16:15:12 +0800 Subject: [PATCH 334/343] Test --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 9d2811c28b..6a00853a67 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - amd64-edge: + tiobe-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, X64, edge] + runs-on: [self-hosted, linux, amd64, tiobe, jammy] steps: - run: lscpu - run: df -h From 73f751545e6fca0a338ebd193af696a4bc581246 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 28 Oct 2025 09:56:15 +0800 Subject: [PATCH 335/343] get noble large --- .github/workflows/manual_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6a00853a67..6dad77cd8a 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,12 +6,12 @@ on: workflow_dispatch: jobs: - tiobe-amd64: + noble-large-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, linux, amd64, tiobe, jammy] + runs-on: [self-hosted, reactive-amd64-noble-large-ps7] steps: - run: lscpu - run: df -h From 44648e19775b7a35369625bb1a1042a1d2176df1 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 28 Oct 2025 10:19:11 +0800 Subject: [PATCH 336/343] Fix label --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 6dad77cd8a..87f2e2d2e5 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, reactive-amd64-noble-large-ps7] + runs-on: [self-hosted, self-hosted-linux-amd64-noble-large] steps: - run: lscpu - run: df -h From 26257a2414ff2de8e439f25a7064cc06461fc3f1 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 28 Oct 2025 10:37:43 +0800 Subject: [PATCH 337/343] Use reactive --- .github/workflows/manual_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 87f2e2d2e5..186a1690a6 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -11,7 +11,7 @@ jobs: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, self-hosted-linux-amd64-noble-large] + runs-on: [self-hosted, self-hosted-linux-amd64-noble-large, reactive] steps: - run: lscpu - run: df -h From 8a4480d1103c1aa0e19d77db8012ff4636447afb Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 28 Oct 2025 14:27:58 +0800 Subject: [PATCH 338/343] Use sleep instead --- .github/workflows/manual_test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 186a1690a6..db7aa5bb80 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -18,5 +18,6 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - run: sleep 60000000 - name: Setup tmate session uses: canonical/action-tmate@main \ No newline at end of file From 096cb1c8992209241b545ee60d8acb34e8ec1e69 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 31 Oct 2025 06:11:39 +0800 Subject: [PATCH 339/343] Test --- .github/workflows/manual_test.yaml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index db7aa5bb80..04360127aa 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,18 +6,27 @@ on: workflow_dispatch: jobs: - noble-large-amd64: + jammy-medium-amd64: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, self-hosted-linux-amd64-noble-large, reactive] + runs-on: [self-hosted, jammy, medium, amd64] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - - run: sleep 60000000 - - name: Setup tmate session - uses: canonical/action-tmate@main \ No newline at end of file + private-endpoint-amd64: + strategy: + fail-fast: false + matrix: + index: [0,1,2,3,4] + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a \ No newline at end of file From b4cad3db6a6fa5749555badbf46b64ba3d854986 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 13 Nov 2025 10:14:50 +0800 Subject: [PATCH 340/343] Test some private-endpoint runners --- .github/workflows/manual_test.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 04360127aa..4373e34804 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,27 +6,27 @@ on: workflow_dispatch: jobs: - jammy-medium-amd64: + pfe-ci: strategy: fail-fast: false matrix: index: [0] - runs-on: [self-hosted, jammy, medium, amd64] + runs-on: [self-hosted, X64, pfe-ci, noble] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - private-endpoint-amd64: - strategy: - fail-fast: false - matrix: - index: [0,1,2,3,4] - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] + - name: Setup tmate session + uses: canonical/action-tmate@main + private-endpoint-ps7: + runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - - run: lsb_release -a \ No newline at end of file + - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main \ No newline at end of file From 3109b52c6cb0ef25bed4b449ad81b526032dead7 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 13 Nov 2025 10:36:51 +0800 Subject: [PATCH 341/343] test xlarge ps7 private-endpoint --- .github/workflows/manual_test.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 4373e34804..01ad68f7ec 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -6,22 +6,8 @@ on: workflow_dispatch: jobs: - pfe-ci: - strategy: - fail-fast: false - matrix: - index: [0] - runs-on: [self-hosted, X64, pfe-ci, noble] - steps: - - run: lscpu - - run: df -h - - run: free -h - - run: uname -a - - run: lsb_release -a - - name: Setup tmate session - uses: canonical/action-tmate@main private-endpoint-ps7: - runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] + runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-xlarge] steps: - run: lscpu - run: df -h @@ -29,4 +15,4 @@ jobs: - run: uname -a - run: lsb_release -a - name: Setup tmate session - uses: canonical/action-tmate@main \ No newline at end of file + uses: canonical/action-tmate@main From 760c89b29c1767ca99f462f21d2f9616e71407fc Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 17 Nov 2025 10:17:47 +0800 Subject: [PATCH 342/343] Test runners --- .github/workflows/manual_test.yaml | 47 ++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index 01ad68f7ec..e9f952da47 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -4,15 +4,52 @@ on: # push: pull_request: workflow_dispatch: - jobs: - private-endpoint-ps7: - runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-xlarge] + medium-amd64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, medium, amd64, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + medium-arm64: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, medium, arm64, reactive] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-s390x: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, self-hosted-linux-s390x-noble-edge] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + edge-ppc64le: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, self-hosted-linux-ppc64el-noble-edge] steps: - run: lscpu - run: df -h - run: free -h - run: uname -a - run: lsb_release -a - - name: Setup tmate session - uses: canonical/action-tmate@main From abd70e80b2f0b3712bc6ebf0546d638dcfa7ab4c Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 18 Nov 2025 13:32:16 +0800 Subject: [PATCH 343/343] Test tmate --- .github/workflows/manual_test.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/manual_test.yaml b/.github/workflows/manual_test.yaml index e9f952da47..6a4023f386 100644 --- a/.github/workflows/manual_test.yaml +++ b/.github/workflows/manual_test.yaml @@ -17,6 +17,8 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main medium-arm64: strategy: fail-fast: false @@ -29,6 +31,8 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main edge-s390x: strategy: fail-fast: false @@ -41,6 +45,8 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main edge-ppc64le: strategy: fail-fast: false @@ -53,3 +59,19 @@ jobs: - run: free -h - run: uname -a - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main + pfe-ci: + strategy: + fail-fast: false + matrix: + index: [0] + runs-on: [self-hosted, X64, pfe-ci, noble] + steps: + - run: lscpu + - run: df -h + - run: free -h + - run: uname -a + - run: lsb_release -a + - name: Setup tmate session + uses: canonical/action-tmate@main