diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b5ab5d..503efca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,6 +186,11 @@ jobs: udid=$(scripts/prepare_simulator.sh '${{ matrix.family }}') test -n "$udid" echo "udid=$udid" >> "$GITHUB_OUTPUT" + # Retried, not because a flaky test is acceptable but because one blocking an unrelated + # pull request is worse. Only the tests that failed are retried, a test that is actually + # broken fails all three attempts, and the result bundle records which ones needed a retry + # — so this reports flakiness rather than hiding it. Check that record before trusting a + # green run that took three goes. - name: Test app lifecycle and UI on the ${{ matrix.family }} run: | trap 'xcrun simctl shutdown all >/dev/null 2>&1 || true' EXIT @@ -193,7 +198,9 @@ jobs: xcodebuild -xctestrun "$xctestrun" \ -destination "platform=iOS Simulator,id=${{ steps.simulator.outputs.udid }}" \ -resultBundlePath "build/CI-${{ matrix.family }}.xcresult" \ - -parallel-testing-enabled NO test-without-building + -parallel-testing-enabled NO \ + -retry-tests-on-failure -test-iterations 3 \ + test-without-building - name: Upload test results and screenshots if: always() uses: actions/upload-artifact@v7 diff --git a/UITests/KeyboardResizeUITests.swift b/UITests/KeyboardResizeUITests.swift index 34c9f4a..de2873c 100644 --- a/UITests/KeyboardResizeUITests.swift +++ b/UITests/KeyboardResizeUITests.swift @@ -8,19 +8,26 @@ final class KeyboardResizeUITests: XCTestCase { let app = XCUIApplication() app.launchArguments = ["--ui-testing"] app.launch() - XCTAssertTrue(app.buttons["addHost"].waitForExistence(timeout: 5)) - app.buttons["addHost"].tap() - app.textFields["hostName"].tap() - app.textFields["hostName"].typeText("Keyboard resize") - app.textFields["hostAddress"].tap() - app.textFields["hostAddress"].typeText("192.0.2.1") - app.textFields["hostUsername"].tap() - app.textFields["hostUsername"].typeText("tester") - app.buttons["saveHost"].tap() - app.buttons["host-Keyboard resize"].tap() + app.buttons["addHost"].tapWhenReady() + app.textFields["hostName"].typeWhenReady("Keyboard resize") + app.textFields["hostAddress"].typeWhenReady("192.0.2.1") + app.textFields["hostUsername"].typeWhenReady("tester") + app.buttons["saveHost"].tapWhenReady() + app.buttons["host-Keyboard resize"].tapWhenReady() XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10)) - app.buttons["Cancel"].tap() - XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5)) + // #35 landed here. A tap delivered while the credential sheet was still animating in + // was dropped — worse on an iPad, where it is a form sheet that travels further and + // settles later — and the sheet then stayed up with the connection still `.connecting`, + // so the button waited for below, which `TerminalScreen` only draws once the phase + // leaves `.connecting`, never appeared at all. `tapWhenReady` is what stops that. + app.buttons["Cancel"].tapWhenReady() + // Through `waitUntil` rather than `waitForExistence` so a failure here keeps a + // screenshot: it is what would say whether the sheet is still up or the phase never + // moved. Cancelling resumes the credential continuation with nil and the connection + // reports `.disconnected`, so this button is the visible end of that path. + try await waitUntil("Cancelling the credential prompt must offer Reconnect") { + app.buttons["Reconnect"].exists + } let terminal = app.descendants(matching: .any).matching(identifier: "terminal").firstMatch let control = app.buttons["terminalAccessoryControl"] var expandedHeight: CGFloat? @@ -111,7 +118,10 @@ final class KeyboardResizeUITests: XCTestCase { } @MainActor private func waitUntil(_ message: @autoclosure () -> String, condition: () -> Bool) async throws { - let deadline = ContinuousClock.now.advanced(by: .seconds(5)) + // Twenty seconds, not five. This is how long the test waits before giving up, not a + // claim about how fast anything has to be, and five is short on a runner that has been + // building for half an hour and is on its second simulator. + let deadline = ContinuousClock.now.advanced(by: .seconds(20)) while !condition() { if ContinuousClock.now >= deadline { let screenshot = XCTAttachment(screenshot: XCUIScreen.main.screenshot()) diff --git a/UITests/TailscaleImportUITests.swift b/UITests/TailscaleImportUITests.swift index edbdaff..f231443 100644 --- a/UITests/TailscaleImportUITests.swift +++ b/UITests/TailscaleImportUITests.swift @@ -118,7 +118,7 @@ final class TailscaleImportUITests: XCTestCase { let settings = UIDevice.current.userInterfaceIdiom == .pad ? app.buttons["workspaceSettings"] : app.buttons["Settings"] - settings.tap() + settings.tapWhenReady() XCTAssertTrue(app.staticTexts["Terminal font preview"].waitForExistence(timeout: 5)) app.navigationBars["Settings"].buttons["Done"].tap() XCTAssertTrue(app.buttons["importTailscaleHosts"].waitForExistence(timeout: 5)) diff --git a/UITests/UITestInteraction.swift b/UITests/UITestInteraction.swift new file mode 100644 index 0000000..397691a --- /dev/null +++ b/UITests/UITestInteraction.swift @@ -0,0 +1,50 @@ +import XCTest + +/// Interacting with an element that exists but is not ready yet is the single most common way a +/// UI test in this suite fails for a reason that has nothing to do with what it is testing. +/// +/// Three of them were observed in one day: a tap delivered while a form sheet was still animating +/// in was dropped and the test waited out a button that would never come (#35); a text field was +/// tapped before its sheet had presented and reported "No matches found"; and a tap on Settings +/// reported "Timed out while synthesizing event" after the app spent half a minute not idle. +/// +/// `waitForExistence` does not cover any of those. An element exists as soon as it is in the +/// accessibility tree, which is before it has arrived where it is going and before it can be +/// touched. `isHittable` is the question worth asking, and asking it is what these do. +extension XCUIElement { + /// Waits for the element to be there and touchable, then taps it. + /// + /// The timeout is patience, not a deadline anything has to meet: reaching it means the test + /// was going to fail regardless, and twenty seconds is not long on a CI runner that has been + /// building for half an hour and is on its second simulator. + @MainActor + func tapWhenReady(timeout: TimeInterval = 20, file: StaticString = #filePath, line: UInt = #line) { + guard waitUntilHittable(timeout: timeout) else { + XCTFail("\(self) never became hittable", file: file, line: line) + return + } + tap() + } + + /// Taps and types in one step, which is the shape every form in these tests uses. + @MainActor + func typeWhenReady(_ text: String, timeout: TimeInterval = 20, + file: StaticString = #filePath, line: UInt = #line) { + tapWhenReady(timeout: timeout, file: file, line: line) + typeText(text) + } + + /// True once the element is present and can receive a touch. Polls rather than using a + /// predicate expectation so a caller can decide what a timeout means. + @MainActor + @discardableResult + func waitUntilHittable(timeout: TimeInterval = 20) -> Bool { + if exists, isHittable { return true } + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if exists, isHittable { return true } + _ = waitForExistence(timeout: 0.1) + } + return exists && isHittable + } +} diff --git a/UITests/iOSSHUITests.swift b/UITests/iOSSHUITests.swift index 3766d1a..9dc6888 100644 --- a/UITests/iOSSHUITests.swift +++ b/UITests/iOSSHUITests.swift @@ -6,17 +6,14 @@ final class iOSSHUITests: XCTestCase { let app = XCUIApplication() app.launchArguments = ["--ui-testing"] app.launch() - app.buttons["addFirstHost"].tap() - app.textFields["hostName"].tap() - app.textFields["hostName"].typeText("Japanese input") - app.textFields["hostAddress"].tap() - app.textFields["hostAddress"].typeText("192.0.2.1") - app.textFields["hostUsername"].tap() - app.textFields["hostUsername"].typeText("tester") - app.buttons["saveHost"].tap() - app.buttons["host-Japanese input"].tap() + app.buttons["addFirstHost"].tapWhenReady() + app.textFields["hostName"].typeWhenReady("Japanese input") + app.textFields["hostAddress"].typeWhenReady("192.0.2.1") + app.textFields["hostUsername"].typeWhenReady("tester") + app.buttons["saveHost"].tapWhenReady() + app.buttons["host-Japanese input"].tapWhenReady() XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10)) - app.buttons["Cancel"].tap() + app.buttons["Cancel"].tapWhenReady() XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5)) let terminal = app.descendants(matching: .any).matching(identifier: "terminal").firstMatch terminal.tap() @@ -73,17 +70,14 @@ final class iOSSHUITests: XCTestCase { let app = XCUIApplication() app.launchArguments = ["--ui-testing"] app.launch() - app.buttons["addFirstHost"].tap() - app.textFields["hostName"].tap() - app.textFields["hostName"].typeText("Viewport") - app.textFields["hostAddress"].tap() - app.textFields["hostAddress"].typeText("192.0.2.1") - app.textFields["hostUsername"].tap() - app.textFields["hostUsername"].typeText("tester") - app.buttons["saveHost"].tap() + app.buttons["addFirstHost"].tapWhenReady() + app.textFields["hostName"].typeWhenReady("Viewport") + app.textFields["hostAddress"].typeWhenReady("192.0.2.1") + app.textFields["hostUsername"].typeWhenReady("tester") + app.buttons["saveHost"].tapWhenReady() app.buttons["host-Viewport"].tap() XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10)) - app.buttons["Cancel"].tap() + app.buttons["Cancel"].tapWhenReady() XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5)) let terminal = app.descendants(matching: .any).matching(identifier: "terminal").firstMatch terminal.tap() @@ -96,7 +90,7 @@ final class iOSSHUITests: XCTestCase { app.buttons["Reconnect"].tap() XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10)) - app.buttons["Cancel"].tap() + app.buttons["Cancel"].tapWhenReady() terminal.tap() try await assertTerminalIsAboveAccessory(app) @@ -136,17 +130,14 @@ final class iOSSHUITests: XCTestCase { let app = XCUIApplication() app.launchArguments = ["--ui-testing"] app.launch() - app.buttons["addFirstHost"].tap() - app.textFields["hostName"].tap() - app.textFields["hostName"].typeText("Terminal test") - app.textFields["hostAddress"].tap() - app.textFields["hostAddress"].typeText("192.0.2.1") - app.textFields["hostUsername"].tap() - app.textFields["hostUsername"].typeText("tester") - app.buttons["saveHost"].tap() + app.buttons["addFirstHost"].tapWhenReady() + app.textFields["hostName"].typeWhenReady("Terminal test") + app.textFields["hostAddress"].typeWhenReady("192.0.2.1") + app.textFields["hostUsername"].typeWhenReady("tester") + app.buttons["saveHost"].tapWhenReady() app.buttons["host-Terminal test"].tap() XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10)) - app.buttons["Cancel"].tap() + app.buttons["Cancel"].tapWhenReady() XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5)) XCTAssertTrue(app.descendants(matching: .any).matching(identifier: "terminal").firstMatch.exists) XCTAssertEqual(app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "rendering could not start")).count, 0) @@ -164,22 +155,19 @@ final class iOSSHUITests: XCTestCase { app.launchArguments = ["--ui-testing"] app.launch() XCTAssertTrue(app.buttons["addFirstHost"].waitForExistence(timeout: 10)) - app.buttons["addFirstHost"].tap() + app.buttons["addFirstHost"].tapWhenReady() XCTAssertFalse(app.buttons["saveHost"].isEnabled) - app.textFields["hostName"].tap() - app.textFields["hostName"].typeText("Development") - app.textFields["hostAddress"].tap() - app.textFields["hostAddress"].typeText("192.0.2.1") - app.textFields["hostUsername"].tap() - app.textFields["hostUsername"].typeText("developer") - app.buttons["saveHost"].tap() + app.textFields["hostName"].typeWhenReady("Development") + app.textFields["hostAddress"].typeWhenReady("192.0.2.1") + app.textFields["hostUsername"].typeWhenReady("developer") + app.buttons["saveHost"].tapWhenReady() let host = app.buttons["host-Development"] XCTAssertTrue(host.waitForExistence(timeout: 5)) host.swipeRight() app.buttons["Edit"].tap() XCTAssertEqual(app.textFields["hostAddress"].value as? String, "192.0.2.1") XCTAssertEqual(app.textFields["hostUsername"].value as? String, "developer") - app.buttons["Cancel"].tap() + app.buttons["Cancel"].tapWhenReady() host.swipeLeft() app.buttons["Delete"].tap() XCTAssertTrue(app.buttons["addFirstHost"].waitForExistence(timeout: 5)) @@ -199,13 +187,10 @@ final class iOSSHUITests: XCTestCase { XCTAssertTrue(privacyPolicyLink.waitForExistence(timeout: 5)) XCTAssertTrue(app.descendants(matching: .any)["supportLink"].waitForExistence(timeout: 5)) app.buttons["Done"].tap() - app.buttons["addFirstHost"].tap() - app.textFields["hostName"].tap() - app.textFields["hostName"].typeText("Bad host") - app.textFields["hostAddress"].tap() - app.textFields["hostAddress"].typeText("bad host") - app.textFields["hostUsername"].tap() - app.textFields["hostUsername"].typeText("root") + app.buttons["addFirstHost"].tapWhenReady() + app.textFields["hostName"].typeWhenReady("Bad host") + app.textFields["hostAddress"].typeWhenReady("bad host") + app.textFields["hostUsername"].typeWhenReady("root") XCTAssertFalse(app.buttons["saveHost"].isEnabled) } } diff --git a/iOSSH.xcodeproj/project.pbxproj b/iOSSH.xcodeproj/project.pbxproj index 6f39d51..e3d5733 100644 --- a/iOSSH.xcodeproj/project.pbxproj +++ b/iOSSH.xcodeproj/project.pbxproj @@ -13,6 +13,7 @@ 126F1F2E6A74A9F1358D33A7 /* TailscaleHostImport.swift in Sources */ = {isa = PBXBuildFile; fileRef = A53EAFCE919B8B842360534F /* TailscaleHostImport.swift */; }; 1A27F80BD8BA8845F5431492 /* TailscaleImportInbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF3D87F64507E681986AEBE8 /* TailscaleImportInbox.swift */; }; 280E96841D6B5FB2DA1950EA /* TerminalRender in Frameworks */ = {isa = PBXBuildFile; productRef = 0130968861023AF2207E184C /* TerminalRender */; }; + 2F4E7D3547D91377083B60D7 /* UITestInteraction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DA05394A523D1CB258AF61B /* UITestInteraction.swift */; }; 34789C86E04E9350FF66A3F2 /* GhosttyEngine in Frameworks */ = {isa = PBXBuildFile; productRef = 33892EBDFF6FA3FEA57EB471 /* GhosttyEngine */; }; 3492C275BD7A4FCD99475FF6 /* ThirdPartyNotices.txt in Resources */ = {isa = PBXBuildFile; fileRef = ABDE0513C54E763730592D8B /* ThirdPartyNotices.txt */; }; 375F741B0F46FCFBDD506F79 /* TerminalViewportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C11265FA4D2A5C381E689173 /* TerminalViewportTests.swift */; }; @@ -89,6 +90,7 @@ 731526C7B108797D5F45E403 /* HostRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HostRecord.swift; sourceTree = ""; }; 79832E5C775352D87B6AA169 /* TerminalOutputTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalOutputTests.swift; sourceTree = ""; }; 8CB0BC27787CF8C4490CD014 /* TerminalFontLibraryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalFontLibraryTests.swift; sourceTree = ""; }; + 8DA05394A523D1CB258AF61B /* UITestInteraction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UITestInteraction.swift; sourceTree = ""; }; 911D7DEF8CB2D674031B953C /* GhosttyEngine */ = {isa = PBXFileReference; lastKnownFileType = folder; name = GhosttyEngine; path = spike/GhosttyEngine; sourceTree = SOURCE_ROOT; }; 9DD4F05CEEF4A486E17BCFC1 /* TerminalCore */ = {isa = PBXFileReference; lastKnownFileType = folder; name = TerminalCore; path = Packages/TerminalCore; sourceTree = SOURCE_ROOT; }; A53EAFCE919B8B842360534F /* TailscaleHostImport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TailscaleHostImport.swift; sourceTree = ""; }; @@ -145,6 +147,7 @@ 6D1E0C43310C3EB044EB52DB /* iPadWorkspaceUITests.swift */, DC54B35541FEAEB0228D113C /* KeyboardResizeUITests.swift */, 27587AAB792BED1360F24EF3 /* TailscaleImportUITests.swift */, + 8DA05394A523D1CB258AF61B /* UITestInteraction.swift */, ); path = UITests; sourceTree = ""; @@ -365,6 +368,7 @@ files = ( 434804C23161B0F2B6237EDD /* KeyboardResizeUITests.swift in Sources */, EFBBA1911C95BC26AB6B7E43 /* TailscaleImportUITests.swift in Sources */, + 2F4E7D3547D91377083B60D7 /* UITestInteraction.swift in Sources */, 903CE9BF45D214388BF6D148 /* iOSSHUITests.swift in Sources */, 62771FFD9ACCA67D5D9B5D86 /* iPadWorkspaceUITests.swift in Sources */, );