From 2380399bc2f3d774ac6f4fc30f85a66f27e85f73 Mon Sep 17 00:00:00 2001 From: Maxim Rychkov Date: Mon, 27 Jul 2026 16:05:43 +0500 Subject: [PATCH 01/10] ci: write e2e logs to the constant file instead of /dev/null to be able to read logs --- .github/workflows/.reusable-e2e-tests-against-prod.yml | 6 +++--- .github/workflows/deploy-to-prod-from-default.yml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/.reusable-e2e-tests-against-prod.yml b/.github/workflows/.reusable-e2e-tests-against-prod.yml index 07ec1cc..c4a73d2 100644 --- a/.github/workflows/.reusable-e2e-tests-against-prod.yml +++ b/.github/workflows/.reusable-e2e-tests-against-prod.yml @@ -5,7 +5,7 @@ on: jobs: e2e-test-against-prod: - runs-on: ubuntu-24.04 + runs-on: self-hosted steps: - uses: actions/checkout@v4 @@ -21,9 +21,9 @@ jobs: - name: Run E2E Tests Against Local Env run: | - # Learn more about '> /dev/null 2>&1': https://stackoverflow.com/a/42919998 + # Redirecting logs to a file on the server # In essence it merges output and error streams and doesn't show errors in the terminal to avoid leakage of secrets in the pipeline - java -jar karate.jar . > /dev/null 2>&1 + java -jar karate.jar . > "/var/log/inner-circle/e2e/$(date +%F)-${{ github.event.repository.name }}-run-${{ github.run_id }}-job-${{ job.check_run_id }}.log" 2>&1 env: "AUTH_FIRST_TENANT_LOGIN_WITH_ALL_PERMISSIONS": ${{ secrets.INNER_CIRCLE_PROD_AUTH_FIRST_TENANT_LOGIN_WITH_ALL_PERMISSIONS }} "AUTH_FIRST_TENANT_PASSWORD_WITH_ALL_PERMISSIONS": ${{ secrets.INNER_CIRCLE_PROD_AUTH_FIRST_TENANT_PASSWORD_WITH_ALL_PERMISSIONS }} diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index 647ee4d..f9617fc 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -30,7 +30,8 @@ jobs: --state-values-set extraSecretEnvVars.InnerCircleServiceUrls__MailServiceUrl="${{ secrets.INNER_CIRCLE_PROD_MAIL_SERVICE_URL }}" \ --state-values-set extraSecretEnvVars.InnerCircleServiceUrls__EmployeesServiceUrl="${{ secrets.INNER_CIRCLE_PROD_EMPLOYEES_SERVICE_URL }}" \ --state-values-set extraSecretEnvVars.AuthenticationOptions__PublicSigningKey="${{ secrets.INNER_CIRCLE_PROD_PUBLIC_SIGNING_KEY }}" \ - --state-values-set extraSecretEnvVars.AuthenticationOptions__PrivateSigningKey="${{ secrets.INNER_CIRCLE_PROD_PRIVATE_SIGNING_KEY }}" > /dev/null 2>&1 + --state-values-set extraSecretEnvVars.AuthenticationOptions__PrivateSigningKey="${{ secrets.INNER_CIRCLE_PROD_PRIVATE_SIGNING_KEY }}" \ + > "/var/log/inner-circle/deploy/$(date +%F)-${{ github.event.repository.name }}-run-${{ github.run_id }}-job-${{ job.check_run_id }}.log" 2>&1 run-e2e-tests: uses: ./.github/workflows/.reusable-e2e-tests-against-prod.yml From fd2c7199ca13115bbe97301a4cc11cc88141e234 Mon Sep 17 00:00:00 2001 From: Maxim Rychkov Date: Mon, 27 Jul 2026 16:06:18 +0500 Subject: [PATCH 02/10] test: trigger workflows --- .github/workflows/deploy-to-prod-from-default.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index f9617fc..0acb271 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -2,8 +2,6 @@ name: Deploy to Prod on: push: - branches: - - master jobs: docker-build-and-push: From 562f52d1914fb6d6b33f3bab01d06e0eb1dd8f0d Mon Sep 17 00:00:00 2001 From: Maxim Rychkov <81160491+Yam1x@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:23:03 +0500 Subject: [PATCH 03/10] docs: add example of a logs filename --- .github/workflows/.reusable-e2e-tests-against-prod.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/.reusable-e2e-tests-against-prod.yml b/.github/workflows/.reusable-e2e-tests-against-prod.yml index c4a73d2..d8e6374 100644 --- a/.github/workflows/.reusable-e2e-tests-against-prod.yml +++ b/.github/workflows/.reusable-e2e-tests-against-prod.yml @@ -22,10 +22,11 @@ jobs: - name: Run E2E Tests Against Local Env run: | # Redirecting logs to a file on the server + # Example of a filename: "/var/log/inner-circle/e2e/2026-07-22-inner-circle-time-api-run-27459153975-job-81169369662.log" # In essence it merges output and error streams and doesn't show errors in the terminal to avoid leakage of secrets in the pipeline java -jar karate.jar . > "/var/log/inner-circle/e2e/$(date +%F)-${{ github.event.repository.name }}-run-${{ github.run_id }}-job-${{ job.check_run_id }}.log" 2>&1 env: "AUTH_FIRST_TENANT_LOGIN_WITH_ALL_PERMISSIONS": ${{ secrets.INNER_CIRCLE_PROD_AUTH_FIRST_TENANT_LOGIN_WITH_ALL_PERMISSIONS }} "AUTH_FIRST_TENANT_PASSWORD_WITH_ALL_PERMISSIONS": ${{ secrets.INNER_CIRCLE_PROD_AUTH_FIRST_TENANT_PASSWORD_WITH_ALL_PERMISSIONS }} "API_ROOT_URL": ${{ secrets.INNER_CIRCLE_PROD_AUTH_API_ROOT_URL }} - "SHOULD_USE_FAKE_EXTERNAL_DEPENDENCIES": "false" \ No newline at end of file + "SHOULD_USE_FAKE_EXTERNAL_DEPENDENCIES": "false" From 070ff1adbaf6b7178900f1018a60a1f90773a13e Mon Sep 17 00:00:00 2001 From: Maxim Rychkov <81160491+Yam1x@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:05:11 +0500 Subject: [PATCH 04/10] test: deploy without --suppress-diff flag --- .github/workflows/deploy-to-prod-from-default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index 0acb271..7cdfc17 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -19,7 +19,7 @@ jobs: # then they will break other vars # Also we use quotes because without them some vars (DB Connection) break down run: | - helmfile cache cleanup && helmfile apply --suppress-diff --namespace "${{ secrets.INNER_CIRCLE_PROD_NAMESPACE }}" -f Api/ci/helmfile.yaml \ + helmfile cache cleanup && helmfile apply --namespace "${{ secrets.INNER_CIRCLE_PROD_NAMESPACE }}" -f Api/ci/helmfile.yaml \ --state-values-set image.tag="sha-${{ github.sha }}" \ --state-values-set ingress.hostname="${{ secrets.INNER_CIRCLE_PROD_HOSTNAME }}" \ --state-values-set extraSecretEnvVars.ConnectionStrings__DefaultConnection="${{ secrets.INNER_CIRCLE_PROD_AUTH_DB_CONNECTION_STRING }}" \ From c24d40198d2191a6cf2111ca5c3447101971fe2f Mon Sep 17 00:00:00 2001 From: Maxim Rychkov Date: Tue, 28 Jul 2026 18:11:10 +0500 Subject: [PATCH 05/10] Revert "test: deploy without --suppress-diff flag" This reverts commit 070ff1adbaf6b7178900f1018a60a1f90773a13e. --- .github/workflows/deploy-to-prod-from-default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index 7cdfc17..0acb271 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -19,7 +19,7 @@ jobs: # then they will break other vars # Also we use quotes because without them some vars (DB Connection) break down run: | - helmfile cache cleanup && helmfile apply --namespace "${{ secrets.INNER_CIRCLE_PROD_NAMESPACE }}" -f Api/ci/helmfile.yaml \ + helmfile cache cleanup && helmfile apply --suppress-diff --namespace "${{ secrets.INNER_CIRCLE_PROD_NAMESPACE }}" -f Api/ci/helmfile.yaml \ --state-values-set image.tag="sha-${{ github.sha }}" \ --state-values-set ingress.hostname="${{ secrets.INNER_CIRCLE_PROD_HOSTNAME }}" \ --state-values-set extraSecretEnvVars.ConnectionStrings__DefaultConnection="${{ secrets.INNER_CIRCLE_PROD_AUTH_DB_CONNECTION_STRING }}" \ From fe2ff5401bc100aa5c943c2e6d991e60a71359ab Mon Sep 17 00:00:00 2001 From: Maxim Rychkov Date: Tue, 28 Jul 2026 18:11:10 +0500 Subject: [PATCH 06/10] Revert "test: trigger workflows" This reverts commit fd2c7199ca13115bbe97301a4cc11cc88141e234. --- .github/workflows/deploy-to-prod-from-default.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index 0acb271..f9617fc 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -2,6 +2,8 @@ name: Deploy to Prod on: push: + branches: + - master jobs: docker-build-and-push: From 69d783bd960cf38c019849073f97dc700ce08b92 Mon Sep 17 00:00:00 2001 From: Maxim Rychkov <81160491+Yam1x@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:44:03 +0500 Subject: [PATCH 07/10] cd: remove line break to write logs to the file and not leak a secrets instead of multi-line that is broken if there is an extra space in between the lines --- .github/workflows/deploy-to-prod-from-default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index f9617fc..231cb29 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -30,8 +30,8 @@ jobs: --state-values-set extraSecretEnvVars.InnerCircleServiceUrls__MailServiceUrl="${{ secrets.INNER_CIRCLE_PROD_MAIL_SERVICE_URL }}" \ --state-values-set extraSecretEnvVars.InnerCircleServiceUrls__EmployeesServiceUrl="${{ secrets.INNER_CIRCLE_PROD_EMPLOYEES_SERVICE_URL }}" \ --state-values-set extraSecretEnvVars.AuthenticationOptions__PublicSigningKey="${{ secrets.INNER_CIRCLE_PROD_PUBLIC_SIGNING_KEY }}" \ - --state-values-set extraSecretEnvVars.AuthenticationOptions__PrivateSigningKey="${{ secrets.INNER_CIRCLE_PROD_PRIVATE_SIGNING_KEY }}" \ - > "/var/log/inner-circle/deploy/$(date +%F)-${{ github.event.repository.name }}-run-${{ github.run_id }}-job-${{ job.check_run_id }}.log" 2>&1 + --state-values-set extraSecretEnvVars.AuthenticationOptions__PrivateSigningKey="${{ secrets.INNER_CIRCLE_PROD_PRIVATE_SIGNING_KEY }}" > "/var/log/inner-circle/deploy/$(date +%F)-${{ github.event.repository.name }}-run-${{ github.run_id }}-job-${{ job.check_run_id }}.log" 2>&1 + # on the last line where we redirect output we don't add a line break for convenience because it is easy to make it wrong having an extra space after \ symbol which leads to exposure of logs to the public pipeline logs instead of the needed private file on a runner run-e2e-tests: uses: ./.github/workflows/.reusable-e2e-tests-against-prod.yml From 7b9f014bab2c2d6f121f84d9824420ba2daecf3c Mon Sep 17 00:00:00 2001 From: Maxim Rychkov <81160491+Yam1x@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:46:45 +0500 Subject: [PATCH 08/10] docs: explain why we use self-hosted runner for e2e tests againt prod and add link for '2>&1' explanation --- .github/workflows/.reusable-e2e-tests-against-prod.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/.reusable-e2e-tests-against-prod.yml b/.github/workflows/.reusable-e2e-tests-against-prod.yml index d8e6374..399645b 100644 --- a/.github/workflows/.reusable-e2e-tests-against-prod.yml +++ b/.github/workflows/.reusable-e2e-tests-against-prod.yml @@ -5,6 +5,8 @@ on: jobs: e2e-test-against-prod: + # need to run on our own runners that we have access to the logs later on our servers + # otherwise, if we use shared runners they store log files in their VMs which we cannot access runs-on: self-hosted steps: - uses: actions/checkout@v4 @@ -23,6 +25,7 @@ jobs: run: | # Redirecting logs to a file on the server # Example of a filename: "/var/log/inner-circle/e2e/2026-07-22-inner-circle-time-api-run-27459153975-job-81169369662.log" + # Learn more about '2>&1': https://stackoverflow.com/a/42919998 # In essence it merges output and error streams and doesn't show errors in the terminal to avoid leakage of secrets in the pipeline java -jar karate.jar . > "/var/log/inner-circle/e2e/$(date +%F)-${{ github.event.repository.name }}-run-${{ github.run_id }}-job-${{ job.check_run_id }}.log" 2>&1 env: From a31e4dafd0581c74992b14dde3c7d5d86f15db88 Mon Sep 17 00:00:00 2001 From: Maxim Rychkov Date: Wed, 29 Jul 2026 09:48:14 +0500 Subject: [PATCH 09/10] format: remove extra empty line --- .github/workflows/.reusable-e2e-tests-against-prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/.reusable-e2e-tests-against-prod.yml b/.github/workflows/.reusable-e2e-tests-against-prod.yml index 399645b..d487f66 100644 --- a/.github/workflows/.reusable-e2e-tests-against-prod.yml +++ b/.github/workflows/.reusable-e2e-tests-against-prod.yml @@ -32,4 +32,4 @@ jobs: "AUTH_FIRST_TENANT_LOGIN_WITH_ALL_PERMISSIONS": ${{ secrets.INNER_CIRCLE_PROD_AUTH_FIRST_TENANT_LOGIN_WITH_ALL_PERMISSIONS }} "AUTH_FIRST_TENANT_PASSWORD_WITH_ALL_PERMISSIONS": ${{ secrets.INNER_CIRCLE_PROD_AUTH_FIRST_TENANT_PASSWORD_WITH_ALL_PERMISSIONS }} "API_ROOT_URL": ${{ secrets.INNER_CIRCLE_PROD_AUTH_API_ROOT_URL }} - "SHOULD_USE_FAKE_EXTERNAL_DEPENDENCIES": "false" + "SHOULD_USE_FAKE_EXTERNAL_DEPENDENCIES": "false" \ No newline at end of file From 544e8b0178b8febf801051ed33d8208ffc88ef99 Mon Sep 17 00:00:00 2001 From: Maxim Rychkov Date: Wed, 29 Jul 2026 09:49:53 +0500 Subject: [PATCH 10/10] cd: remove --suppress-diff flag to show helm diff in deploy logs --- .github/workflows/deploy-to-prod-from-default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-prod-from-default.yml b/.github/workflows/deploy-to-prod-from-default.yml index 231cb29..3a02cdd 100644 --- a/.github/workflows/deploy-to-prod-from-default.yml +++ b/.github/workflows/deploy-to-prod-from-default.yml @@ -21,7 +21,7 @@ jobs: # then they will break other vars # Also we use quotes because without them some vars (DB Connection) break down run: | - helmfile cache cleanup && helmfile apply --suppress-diff --namespace "${{ secrets.INNER_CIRCLE_PROD_NAMESPACE }}" -f Api/ci/helmfile.yaml \ + helmfile cache cleanup && helmfile apply --namespace "${{ secrets.INNER_CIRCLE_PROD_NAMESPACE }}" -f Api/ci/helmfile.yaml \ --state-values-set image.tag="sha-${{ github.sha }}" \ --state-values-set ingress.hostname="${{ secrets.INNER_CIRCLE_PROD_HOSTNAME }}" \ --state-values-set extraSecretEnvVars.ConnectionStrings__DefaultConnection="${{ secrets.INNER_CIRCLE_PROD_AUTH_DB_CONNECTION_STRING }}" \