From 61cf61d64ccfe57e69891e0f9474e17da02fa375 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 2 Sep 2025 02:56:12 +0000 Subject: [PATCH 1/8] chore: add buckets to histogram metrics --- .../src/github_runner_manager/metrics/runner.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/github-runner-manager/src/github_runner_manager/metrics/runner.py b/github-runner-manager/src/github_runner_manager/metrics/runner.py index a8f8b9bd01..e027777021 100644 --- a/github-runner-manager/src/github_runner_manager/metrics/runner.py +++ b/github-runner-manager/src/github_runner_manager/metrics/runner.py @@ -50,21 +50,25 @@ name="runner_idle_duration_seconds", documentation="Time in seconds to runner waiting idle for the job to be picked up.", labelnames=[labels.FLAVOR], + buckets=[5, 10, 15, 30, 60, 60 * 2, 60 * 3, 60 * 5, 60 * 10, float("inf")], ) RUNNER_QUEUE_DURATION_SECONDS = Histogram( name="runner_queue_duration_seconds", documentation="Time taken in seconds for the job to be started.", labelnames=[labels.FLAVOR], + buckets=[5, 30, 60, 60 * 5, 60 * 10, 60 * 20, 60 * 30, 60 * 60, 60 * 60 * 2, float("inf")], ) EXTRACT_METRICS_DURATION_SECONDS = Histogram( name="extract_metrics_duration_seconds", documentation="Time taken in seconds for the metrics to be extracted.", labelnames=[labels.FLAVOR], + buckets=[5, 10, 15, 30, 60, 60 * 2, 60 * 3, 60 * 5, 60 * 10, float("inf")], ) JOB_DURATION_SECONDS = Histogram( name="job_duration_seconds", documentation="Time taken in seconds for the job to be completed.", labelnames=[labels.FLAVOR], + buckets=[60, 60 * 5, 60 * 10, 60 * 20, 60 * 30, 60 * 60, 60 * 60 * 2, float("inf")], ) JOB_REPOSITORY_COUNT = Gauge( name="job_repository_count", From a28cdea2add7ac9da54388bfda55c04665266636 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 2 Sep 2025 05:23:47 +0000 Subject: [PATCH 2/8] chore: define minute, hours, days in second constants --- .../github_runner_manager/metrics/runner.py | 60 +++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/metrics/runner.py b/github-runner-manager/src/github_runner_manager/metrics/runner.py index e027777021..8b37c84c55 100644 --- a/github-runner-manager/src/github_runner_manager/metrics/runner.py +++ b/github-runner-manager/src/github_runner_manager/metrics/runner.py @@ -39,24 +39,62 @@ logger = logging.getLogger(__name__) MAX_METRICS_FILE_SIZE = 1024 +MINUTE_IN_SECONDS = 60 +HOURS_IN_SECONDS = MINUTE_IN_SECONDS * 60 +DAYS_IN_SECONDS = HOURS_IN_SECONDS * 24 RUNNER_SPAWN_DURATION_SECONDS = Histogram( name="runner_spawn_duration_seconds", documentation="Time in seconds to initialize the VM and register the runner on GitHub.", labelnames=[labels.FLAVOR], - buckets=[5, 10, 15, 30, 60, 60 * 2, 60 * 3, 60 * 5, 60 * 10, float("inf")], + buckets=[ + 5, + 10, + 15, + 30, + MINUTE_IN_SECONDS, + MINUTE_IN_SECONDS * 2, + MINUTE_IN_SECONDS * 3, + MINUTE_IN_SECONDS * 5, + MINUTE_IN_SECONDS * 10, + float("inf"), + ], ) RUNNER_IDLE_DURATION_SECONDS = Histogram( name="runner_idle_duration_seconds", documentation="Time in seconds to runner waiting idle for the job to be picked up.", labelnames=[labels.FLAVOR], - buckets=[5, 10, 15, 30, 60, 60 * 2, 60 * 3, 60 * 5, 60 * 10, float("inf")], + buckets=[ + 5, + 10, + 15, + 30, + MINUTE_IN_SECONDS, + MINUTE_IN_SECONDS * 2, + MINUTE_IN_SECONDS * 3, + MINUTE_IN_SECONDS * 5, + MINUTE_IN_SECONDS * 10, + float("inf"), + ], ) RUNNER_QUEUE_DURATION_SECONDS = Histogram( name="runner_queue_duration_seconds", documentation="Time taken in seconds for the job to be started.", labelnames=[labels.FLAVOR], - buckets=[5, 30, 60, 60 * 5, 60 * 10, 60 * 20, 60 * 30, 60 * 60, 60 * 60 * 2, float("inf")], + buckets=[ + # seconds + 5, + 30, + MINUTE_IN_SECONDS, + MINUTE_IN_SECONDS * 5, + MINUTE_IN_SECONDS * 10, + MINUTE_IN_SECONDS * 20, + MINUTE_IN_SECONDS * 30, + HOURS_IN_SECONDS, + HOURS_IN_SECONDS * 2, + HOURS_IN_SECONDS * 5, + float("inf"), + ], ) EXTRACT_METRICS_DURATION_SECONDS = Histogram( name="extract_metrics_duration_seconds", @@ -68,7 +106,21 @@ name="job_duration_seconds", documentation="Time taken in seconds for the job to be completed.", labelnames=[labels.FLAVOR], - buckets=[60, 60 * 5, 60 * 10, 60 * 20, 60 * 30, 60 * 60, 60 * 60 * 2, float("inf")], + buckets=[ + MINUTE_IN_SECONDS, + MINUTE_IN_SECONDS * 5, + MINUTE_IN_SECONDS * 10, + MINUTE_IN_SECONDS * 20, + MINUTE_IN_SECONDS * 30, + MINUTE_IN_SECONDS * 60, + # hours + HOURS_IN_SECONDS * 2, + HOURS_IN_SECONDS * 4, + HOURS_IN_SECONDS * 6, + # days + DAYS_IN_SECONDS * 3, + float("inf"), + ], ) JOB_REPOSITORY_COUNT = Gauge( name="job_repository_count", From e28138d77fcfd7e4ec3378571a690ccf7f079c9a Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 2 Sep 2025 05:32:41 +0000 Subject: [PATCH 3/8] chore: add 5 days limit --- .../src/github_runner_manager/metrics/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/github-runner-manager/src/github_runner_manager/metrics/runner.py b/github-runner-manager/src/github_runner_manager/metrics/runner.py index 8b37c84c55..b3699fd282 100644 --- a/github-runner-manager/src/github_runner_manager/metrics/runner.py +++ b/github-runner-manager/src/github_runner_manager/metrics/runner.py @@ -119,6 +119,7 @@ HOURS_IN_SECONDS * 6, # days DAYS_IN_SECONDS * 3, + DAYS_IN_SECONDS * 5, float("inf"), ], ) From aae152b2bc3ace8f94f16baf23edac9de3193c9c Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 2 Sep 2025 06:08:41 +0000 Subject: [PATCH 4/8] chore: add comment on github run limit --- .../src/github_runner_manager/metrics/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/github-runner-manager/src/github_runner_manager/metrics/runner.py b/github-runner-manager/src/github_runner_manager/metrics/runner.py index b3699fd282..163ac95bad 100644 --- a/github-runner-manager/src/github_runner_manager/metrics/runner.py +++ b/github-runner-manager/src/github_runner_manager/metrics/runner.py @@ -119,6 +119,7 @@ HOURS_IN_SECONDS * 6, # days DAYS_IN_SECONDS * 3, + # Limit of the run is 5 days: https://docs.github.com/en/actions/reference/limits DAYS_IN_SECONDS * 5, float("inf"), ], From af89493054fffe018f8da6b6a49254e5619d64c8 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 3 Sep 2025 01:29:11 +0000 Subject: [PATCH 5/8] chore: trigger CI From b889d80eeafaaa818ff4983b534b563d0e7b93ec Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 3 Sep 2025 01:35:14 +0000 Subject: [PATCH 6/8] fix: shellcheck fixes --- jobmanager/client/git_push.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobmanager/client/git_push.sh b/jobmanager/client/git_push.sh index f53a75d4fa..26b9737a9c 100644 --- a/jobmanager/client/git_push.sh +++ b/jobmanager/client/git_push.sh @@ -43,9 +43,9 @@ if [ "$git_remote" = "" ]; then # git remote not defined if [ "$GIT_TOKEN" = "" ]; then echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment." - git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git + git remote add origin "https://${git_host}/${git_user_id}/${git_repo_id}.git" else - git remote add origin https://${git_user_id}:"${GIT_TOKEN}"@${git_host}/${git_user_id}/${git_repo_id}.git + git remote add origin "https://${git_user_id}:${GIT_TOKEN}@${git_host}/${git_user_id}/${git_repo_id}.git" fi fi From b93b61f0c2268adf553c7389473be3c34e85f49c Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 3 Sep 2025 02:23:42 +0000 Subject: [PATCH 7/8] ci: trigger CI From 8592a13f0f6d4f0f167f3b464768fd184581c14d Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 4 Sep 2025 17:30:51 +0000 Subject: [PATCH 8/8] fix: noble compatible microk8s --- tests/integration/setup-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index c905cf305b..fcf5cf54c8 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -12,7 +12,7 @@ ORIGINAL_CONTROLLER=$(juju controllers --format json | jq -r '.controllers | keys | .[0]') echo "bootstrapping microk8s juju controller" -sudo snap install microk8s --channel=1.32-strict/stable +sudo snap install microk8s --channel=1.34-strict/stable GROUP=snap_microk8s sudo usermod -a -G "$GROUP" "$USER" if [ "$(id -gn)" != "$GROUP" ]; then