@W-24278591 fix: Complete AuthFlowTester nightly coverage

.github/workflows/reusable-ui-workflow.yaml

@@ -228,13 +228,15 @@ jobs:
             --test-targets "class com.salesforce.samples.authflowtester.MultiUserLoginTests" \
             --timeout=45m \
             --num-flaky-test-attempts=1
-      - name: Run All Single User Tests
+      - name: Run Login Tests
         continue-on-error: true
         if: ${{ ! inputs.is_pr }}
         env:
           RUN_NUMBER: ${{ github.run_number }}
         run: |
-          GCLOUD_RESULTS_DIR="authflowtester-single-user-build-${RUN_NUMBER}"
+          # Keep the same disjoint groups as a full PR run. The previous single-user
+          # invocation reached Firebase's 60-minute limit and silently omitted late tests.
+          GCLOUD_RESULTS_DIR="authflowtester-nightly-login-build-${RUN_NUMBER}"
           DEVICE_ARGS=()
           for LEVEL in $FULL_API_RANGE; do
             DEVICE_MODEL="MediumPhone.arm"
@@ -251,7 +253,69 @@ jobs:
             --environment-variables clearPackageData=true \
             --app "native/NativeSampleApps/AuthFlowTester/build/outputs/apk/debug/AuthFlowTester-debug.apk" \
             --test "native/NativeSampleApps/AuthFlowTester/build/outputs/apk/androidTest/debug/AuthFlowTester-debug-androidTest.apk" \
-            --test-targets "notClass com.salesforce.samples.authflowtester.MultiUserLoginTests" \
+            --test-targets "notClass com.salesforce.samples.authflowtester.MultiUserLoginTests,notClass com.salesforce.samples.authflowtester.DPoPLoginTests,notClass com.salesforce.samples.authflowtester.LoginWithRestartTests,notClass com.salesforce.samples.authflowtester.RTRLoginTests,notClass com.salesforce.samples.authflowtester.RefreshTokenMigrationTests,notClass com.salesforce.samples.authflowtester.WelcomeLoginTests" \
+            "${DEVICE_ARGS[@]}" \
+            --directories-to-pull=/sdcard \
+            --results-dir="${GCLOUD_RESULTS_DIR}" \
+            --results-history-name=AuthFlowTester \
+            --no-performance-metrics \
+            --num-flaky-test-attempts=1 \
+            --timeout=60m || true
+      - name: Run Welcome Discovery Tests
+        continue-on-error: true
+        if: ${{ ! inputs.is_pr }}
+        env:
+          RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          GCLOUD_RESULTS_DIR="authflowtester-nightly-welcome-build-${RUN_NUMBER}"
+          DEVICE_ARGS=()
+          for LEVEL in $FULL_API_RANGE; do
+            DEVICE_MODEL="MediumPhone.arm"
+            if [ "$PS16K_MIN_API_VERSION" -le "$LEVEL" ]; then
+              DEVICE_MODEL="MediumPhone_ps16k.arm"
+            fi
+            DEVICE_ARGS+=(--device "model=${DEVICE_MODEL},version=${LEVEL},locale=en,orientation=portrait")
+          done
+
+          gcloud firebase test android run \
+            --project mobile-apps-firebase-test \
+            --type instrumentation \
+            --use-orchestrator \
+            --environment-variables clearPackageData=true \
+            --app "native/NativeSampleApps/AuthFlowTester/build/outputs/apk/debug/AuthFlowTester-debug.apk" \
+            --test "native/NativeSampleApps/AuthFlowTester/build/outputs/apk/androidTest/debug/AuthFlowTester-debug-androidTest.apk" \
+            --test-targets "class com.salesforce.samples.authflowtester.WelcomeLoginTests" \
+            "${DEVICE_ARGS[@]}" \
+            --directories-to-pull=/sdcard \
+            --results-dir="${GCLOUD_RESULTS_DIR}" \
+            --results-history-name=AuthFlowTester \
+            --no-performance-metrics \
+            --num-flaky-test-attempts=1 \
+            --timeout=10m || true
+      - name: Run Token Lifecycle Tests
+        continue-on-error: true
+        if: ${{ ! inputs.is_pr }}
+        env:
+          RUN_NUMBER: ${{ github.run_number }}
+        run: |
+          GCLOUD_RESULTS_DIR="authflowtester-nightly-token-lifecycle-build-${RUN_NUMBER}"
+          DEVICE_ARGS=()
+          for LEVEL in $FULL_API_RANGE; do
+            DEVICE_MODEL="MediumPhone.arm"
+            if [ "$PS16K_MIN_API_VERSION" -le "$LEVEL" ]; then
+              DEVICE_MODEL="MediumPhone_ps16k.arm"
+            fi
+            DEVICE_ARGS+=(--device "model=${DEVICE_MODEL},version=${LEVEL},locale=en,orientation=portrait")
+          done
+
+          gcloud firebase test android run \
+            --project mobile-apps-firebase-test \
+            --type instrumentation \
+            --use-orchestrator \
+            --environment-variables clearPackageData=true \
+            --app "native/NativeSampleApps/AuthFlowTester/build/outputs/apk/debug/AuthFlowTester-debug.apk" \
+            --test "native/NativeSampleApps/AuthFlowTester/build/outputs/apk/androidTest/debug/AuthFlowTester-debug-androidTest.apk" \
+            --test-targets "class com.salesforce.samples.authflowtester.DPoPLoginTests,class com.salesforce.samples.authflowtester.LoginWithRestartTests,class com.salesforce.samples.authflowtester.RTRLoginTests,class com.salesforce.samples.authflowtester.RefreshTokenMigrationTests" \
             "${DEVICE_ARGS[@]}" \
             --directories-to-pull=/sdcard \
             --results-dir="${GCLOUD_RESULTS_DIR}" \
@@ -265,7 +329,7 @@ jobs:
         env:
           RUN_NUMBER: ${{ github.run_number }}
         run: |
-          GCLOUD_RESULTS_DIR="authflowtester-multi-user-build-${RUN_NUMBER}"
+          GCLOUD_RESULTS_DIR="authflowtester-nightly-multi-user-build-${RUN_NUMBER}"
           DEVICE_ARGS=()
           for LEVEL in $FULL_API_RANGE; do
             DEVICE_MODEL="MediumPhone.arm"
@@ -289,7 +353,7 @@ jobs:
             --results-history-name=AuthFlowTester \
             --no-performance-metrics \
             --num-flaky-test-attempts=1 \
-            --timeout=30m || true
+            --timeout=60m || true
       - name: Copy Test Results
         continue-on-error: true
         if: success() || failure()
@@ -314,40 +378,69 @@ jobs:
               gsutil cp "${EXECUTION_DIR}/instrumentation.results" \
                 "firebase_results/${OUTPUT_PREFIX}_api_${API_LEVEL}_instrumentation.results"
             done
-            # Pass 2: merge failed and missing rerun cases. A crashed orchestrator process can
-            # omit a test from the original XML entirely even though AndroidJUnitRunner found it.
+            # Pass 2: replace original cases with their final retry outcome. A crashed
+            # orchestrator process can also omit a test from the original XML entirely.
             for RESULT_FILE in $(gsutil ls "${BUCKET_PATH}/*/test_result_1.xml" 2>/dev/null | grep "rerun"); do
               DEVICE_DIR=$(echo "${RESULT_FILE}" | sed 's|.*/\([^/]*\)/test_result_1.xml|\1|')
               API_LEVEL=$(echo "${DEVICE_DIR}" | sed 's/.*-\([0-9]*\)-.*/\1/')
               RERUN_TMP="firebase_results/${OUTPUT_PREFIX}_api_${API_LEVEL}_rerun_tmp.xml"
               ORIG_FILE="firebase_results/${OUTPUT_PREFIX}_api_${API_LEVEL}_test_result.xml"
               gsutil cp "${RESULT_FILE}" "${RERUN_TMP}"
-              python3 - "${ORIG_FILE}" "${RERUN_TMP}" "${ORIG_FILE}" << 'PYEOF'
-          import sys, xml.etree.ElementTree as ET
-          orig = ET.parse(sys.argv[1])
-          rerun = ET.parse(sys.argv[2])
-          def suite(t):
-            r = t.getroot()
-            return r if r.tag == 'testsuite' else r.find('testsuite')
-          os_el, rs_el = suite(orig), suite(rerun)
-          failed_keys = set()
-          existing_keys = set()
-          for tc in os_el.findall('testcase'):
-            key = f"{tc.get('name','')}|{tc.get('classname','')}|{tc.get('file','')}"
-            existing_keys.add(key)
-            if tc.find('failure') is not None or tc.find('error') is not None:
-              failed_keys.add(key)
-          added = 0
-          for tc in rs_el.findall('testcase'):
-            key = f"{tc.get('name','')}|{tc.get('classname','')}|{tc.get('file','')}"
-            if key in failed_keys or key not in existing_keys:
-              os_el.append(tc)
-              existing_keys.add(key)
-              added += 1
-          os_el.set('tests', str(int(os_el.get('tests','0')) + added))
-          with open(sys.argv[3], 'w') as f:
-            f.write(ET.tostring(orig.getroot(), encoding='unicode'))
+              if ! python3 - "${ORIG_FILE}" "${RERUN_TMP}" "${ORIG_FILE}" << 'PYEOF'
+          import sys
+          import xml.etree.ElementTree as ET
+          from pathlib import Path
+
+          MAX_JUNIT_BYTES = 25 * 1024 * 1024
+
+          def parse_bounded(path_value):
+            path = Path(path_value)
+            if path.stat().st_size > MAX_JUNIT_BYTES:
+              raise ValueError(f"JUnit result exceeds {MAX_JUNIT_BYTES} bytes: {path.name}")
+            data = path.read_bytes()
+            declaration_sample = data[:256 * 1024].upper()
+            if b'<!DOCTYPE' in declaration_sample or b'<!ENTITY' in declaration_sample:
+              raise ValueError(f"unsafe XML declaration in {path.name}")
+            return ET.ElementTree(ET.fromstring(data))
+
+          def suite(tree):
+            root = tree.getroot()
+            return root if root.tag == 'testsuite' else root.find('testsuite')
+
+          def key(testcase):
+            return testcase.get('classname', ''), testcase.get('name', '')
+
+          orig = parse_bounded(sys.argv[1])
+          rerun = parse_bounded(sys.argv[2])
+          original_suite, rerun_suite = suite(orig), suite(rerun)
+          if original_suite is None or rerun_suite is None:
+            raise ValueError('missing testsuite element')
+
+          positions = {
+            key(testcase): index
+            for index, testcase in enumerate(list(original_suite))
+            if testcase.tag == 'testcase'
+          }
+          for testcase in rerun_suite.findall('testcase'):
+            test_key = key(testcase)
+            if test_key in positions:
+              child_index = positions[test_key]
+              original_suite.remove(list(original_suite)[child_index])
+              original_suite.insert(child_index, testcase)
+            else:
+              positions[test_key] = len(list(original_suite))
+              original_suite.append(testcase)
+
+          testcases = original_suite.findall('testcase')
+          original_suite.set('tests', str(len(testcases)))
+          original_suite.set('failures', str(sum(tc.find('failure') is not None for tc in testcases)))
+          original_suite.set('errors', str(sum(tc.find('error') is not None for tc in testcases)))
+          original_suite.set('skipped', str(sum(tc.find('skipped') is not None for tc in testcases)))
+          Path(sys.argv[3]).write_bytes(ET.tostring(orig.getroot(), encoding='utf-8', xml_declaration=True))
           PYEOF
+              then
+                echo "::warning title=Retry result merge failed::${OUTPUT_PREFIX} API ${API_LEVEL}: keeping the original Firebase result; see the Python error above"
+              fi
               rm "${RERUN_TMP}"
             done
           }
@@ -380,35 +473,64 @@ jobs:
               fi
             fi
           else
-            SINGLE_PATH="${BUCKET}/authflowtester-single-user-build-${RUN_NUMBER}"
-            if gsutil ls "${SINGLE_PATH}" > /dev/null 2>&1; then
-              copy_results_by_api_level "${SINGLE_PATH}" "single-user"
-            fi
-
-            MULTI_PATH="${BUCKET}/authflowtester-multi-user-build-${RUN_NUMBER}"
-            if gsutil ls "${MULTI_PATH}" > /dev/null 2>&1; then
-              copy_results_by_api_level "${MULTI_PATH}" "multi-user"
-            fi
+            for GROUP in login welcome token-lifecycle multi-user; do
+              NIGHTLY_PATH="${BUCKET}/authflowtester-nightly-${GROUP}-build-${RUN_NUMBER}"
+              if gsutil ls "${NIGHTLY_PATH}" > /dev/null 2>&1; then
+                copy_results_by_api_level "${NIGHTLY_PATH}" "nightly-${GROUP}"
+              fi
+            done
           fi
-      - name: Validate Complete PR Test Results
-        if: ${{ !cancelled() && inputs.is_pr && inputs.run_all_ui_tests }}
+      - name: Validate Complete Test Results
+        if: ${{ !cancelled() && (!inputs.is_pr || inputs.run_all_ui_tests) }}
+        env:
+          IS_PR: ${{ inputs.is_pr }}
         run: |
           python3 - << 'PYEOF'
           import glob
+          import os
           import re
           import sys
           import xml.etree.ElementTree as ET
+          from pathlib import Path
+
+          MAX_JUNIT_BYTES = 25 * 1024 * 1024
 
-          expected_prefixes = {"pr-login", "pr-welcome", "pr-token-lifecycle", "pr-multi-user"}
+          def parse_bounded(path_value):
+              path = Path(path_value)
+              if path.stat().st_size > MAX_JUNIT_BYTES:
+                  raise ValueError(f"JUnit result exceeds {MAX_JUNIT_BYTES} bytes: {path.name}")
+              data = path.read_bytes()
+              declaration_sample = data[:256 * 1024].upper()
+              if b"<!DOCTYPE" in declaration_sample or b"<!ENTITY" in declaration_sample:
+                  raise ValueError(f"unsafe XML declaration in {path.name}")
+              return ET.fromstring(data)
+
+          is_pr = os.environ["IS_PR"] == "true"
+          prefix_root = "pr" if is_pr else "nightly"
+          expected_prefixes = {
+              f"{prefix_root}-login",
+              f"{prefix_root}-welcome",
+              f"{prefix_root}-token-lifecycle",
+              f"{prefix_root}-multi-user",
+          }
+          expected_api_levels = (
+              {os.environ["PR_API_VERSION"]}
+              if is_pr
+              else set(os.environ["FULL_API_RANGE"].split())
+          )
           instrumentation_files = glob.glob("firebase_results/*_instrumentation.results")
-          found_prefixes = set()
+          found_groups = set()
+          keys_by_api = {api_level: set() for api_level in expected_api_levels}
           errors = []
 
           for instrumentation_file in instrumentation_files:
-              output_prefix = re.sub(r"_api_\d+_instrumentation\.results$", "", instrumentation_file.split("/")[-1])
+              match = re.match(r"(.+)_api_(\d+)_instrumentation\.results$", instrumentation_file.split("/")[-1])
+              if not match:
+                  continue
+              output_prefix, api_level = match.groups()
               if output_prefix not in expected_prefixes:
                   continue
-              found_prefixes.add(output_prefix)
+              found_groups.add((output_prefix, api_level))
 
               xml_file = instrumentation_file.replace("_instrumentation.results", "_test_result.xml")
               try:
@@ -417,31 +539,50 @@ jobs:
                   if not expected_counts:
                       raise ValueError("missing AndroidJUnitRunner numtests value")
 
-                  root = ET.parse(xml_file).getroot()
+                  root = parse_bounded(xml_file)
                   test_keys = {
-                      (tc.get("classname", ""), tc.get("name", ""), tc.get("file", ""))
+                      (tc.get("classname", ""), tc.get("name", ""))
                       for tc in root.iter("testcase")
                   }
                   expected_count = max(expected_counts)
-                  if len(test_keys) < expected_count:
+                  if len(test_keys) != expected_count:
                       errors.append(
                           f"{output_prefix}: AndroidJUnitRunner found {expected_count} tests, "
-                          f"but Firebase reported only {len(test_keys)} unique test cases"
+                          f"but Firebase reported {len(test_keys)} unique test cases"
+                      )
+                  duplicate_keys = keys_by_api.setdefault(api_level, set()).intersection(test_keys)
+                  if duplicate_keys:
+                      errors.append(
+                          f"API {api_level}: {len(duplicate_keys)} logical tests appeared in multiple groups"
                       )
+                  keys_by_api[api_level].update(test_keys)
               except (OSError, ET.ParseError, ValueError) as error:
                   errors.append(f"{output_prefix}: could not validate complete results: {error}")
 
-          missing_prefixes = expected_prefixes - found_prefixes
-          if missing_prefixes:
-              errors.append("missing result groups: " + ", ".join(sorted(missing_prefixes)))
+          expected_groups = {
+              (prefix, api_level)
+              for prefix in expected_prefixes
+              for api_level in expected_api_levels
+          }
+          missing_groups = expected_groups - found_groups
+          if missing_groups:
+              errors.append(
+                  "missing result groups: "
+                  + ", ".join(f"{prefix} API {api}" for prefix, api in sorted(missing_groups))
+              )
+
+          for api_level in sorted(expected_api_levels):
+              actual_count = len(keys_by_api[api_level])
+              if actual_count != 110:
+                  errors.append(f"API {api_level}: expected 110 logical tests, found {actual_count}")
 
           if errors:
               print("Incomplete Firebase Test Lab results:", file=sys.stderr)
               for error in errors:
                   print(f"- {error}", file=sys.stderr)
               sys.exit(1)
 
-          print("All Firebase Test Lab result groups contain every discovered test case.")
+          print("All Firebase Test Lab result groups contain 110 unique logical tests per API level.")
           PYEOF
       - name: Test Report
         uses: mikepenz/action-junit-report@3a81627bfac62268172037048872e8ebd4207e6d  # v6.4.1