Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,14 +186,21 @@ jobs:
udid=$(scripts/prepare_simulator.sh '${{ matrix.family }}')
test -n "$udid"
echo "udid=$udid" >> "$GITHUB_OUTPUT"
# Retried, not because a flaky test is acceptable but because one blocking an unrelated
# pull request is worse. Only the tests that failed are retried, a test that is actually
# broken fails all three attempts, and the result bundle records which ones needed a retry
# — so this reports flakiness rather than hiding it. Check that record before trusting a
# green run that took three goes.
- name: Test app lifecycle and UI on the ${{ matrix.family }}
run: |
trap 'xcrun simctl shutdown all >/dev/null 2>&1 || true' EXIT
xctestrun=$(echo build/DerivedData/Build/Products/*.xctestrun)
xcodebuild -xctestrun "$xctestrun" \
-destination "platform=iOS Simulator,id=${{ steps.simulator.outputs.udid }}" \
-resultBundlePath "build/CI-${{ matrix.family }}.xcresult" \
-parallel-testing-enabled NO test-without-building
-parallel-testing-enabled NO \
-retry-tests-on-failure -test-iterations 3 \
test-without-building
- name: Upload test results and screenshots
if: always()
uses: actions/upload-artifact@v7
Expand Down
36 changes: 23 additions & 13 deletions UITests/KeyboardResizeUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,26 @@ final class KeyboardResizeUITests: XCTestCase {
let app = XCUIApplication()
app.launchArguments = ["--ui-testing"]
app.launch()
XCTAssertTrue(app.buttons["addHost"].waitForExistence(timeout: 5))
app.buttons["addHost"].tap()
app.textFields["hostName"].tap()
app.textFields["hostName"].typeText("Keyboard resize")
app.textFields["hostAddress"].tap()
app.textFields["hostAddress"].typeText("192.0.2.1")
app.textFields["hostUsername"].tap()
app.textFields["hostUsername"].typeText("tester")
app.buttons["saveHost"].tap()
app.buttons["host-Keyboard resize"].tap()
app.buttons["addHost"].tapWhenReady()
app.textFields["hostName"].typeWhenReady("Keyboard resize")
app.textFields["hostAddress"].typeWhenReady("192.0.2.1")
app.textFields["hostUsername"].typeWhenReady("tester")
app.buttons["saveHost"].tapWhenReady()
app.buttons["host-Keyboard resize"].tapWhenReady()
XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10))
app.buttons["Cancel"].tap()
XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5))
// #35 landed here. A tap delivered while the credential sheet was still animating in
// was dropped — worse on an iPad, where it is a form sheet that travels further and
// settles later — and the sheet then stayed up with the connection still `.connecting`,
// so the button waited for below, which `TerminalScreen` only draws once the phase
// leaves `.connecting`, never appeared at all. `tapWhenReady` is what stops that.
app.buttons["Cancel"].tapWhenReady()
// Through `waitUntil` rather than `waitForExistence` so a failure here keeps a
// screenshot: it is what would say whether the sheet is still up or the phase never
// moved. Cancelling resumes the credential continuation with nil and the connection
// reports `.disconnected`, so this button is the visible end of that path.
try await waitUntil("Cancelling the credential prompt must offer Reconnect") {
app.buttons["Reconnect"].exists
}
let terminal = app.descendants(matching: .any).matching(identifier: "terminal").firstMatch
let control = app.buttons["terminalAccessoryControl"]
var expandedHeight: CGFloat?
Expand Down Expand Up @@ -111,7 +118,10 @@ final class KeyboardResizeUITests: XCTestCase {
}

@MainActor private func waitUntil(_ message: @autoclosure () -> String, condition: () -> Bool) async throws {
let deadline = ContinuousClock.now.advanced(by: .seconds(5))
// Twenty seconds, not five. This is how long the test waits before giving up, not a
// claim about how fast anything has to be, and five is short on a runner that has been
// building for half an hour and is on its second simulator.
let deadline = ContinuousClock.now.advanced(by: .seconds(20))
while !condition() {
if ContinuousClock.now >= deadline {
let screenshot = XCTAttachment(screenshot: XCUIScreen.main.screenshot())
Expand Down
2 changes: 1 addition & 1 deletion UITests/TailscaleImportUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ final class TailscaleImportUITests: XCTestCase {

let settings = UIDevice.current.userInterfaceIdiom == .pad
? app.buttons["workspaceSettings"] : app.buttons["Settings"]
settings.tap()
settings.tapWhenReady()
XCTAssertTrue(app.staticTexts["Terminal font preview"].waitForExistence(timeout: 5))
app.navigationBars["Settings"].buttons["Done"].tap()
XCTAssertTrue(app.buttons["importTailscaleHosts"].waitForExistence(timeout: 5))
Expand Down
50 changes: 50 additions & 0 deletions UITests/UITestInteraction.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import XCTest

/// Interacting with an element that exists but is not ready yet is the single most common way a
/// UI test in this suite fails for a reason that has nothing to do with what it is testing.
///
/// Three of them were observed in one day: a tap delivered while a form sheet was still animating
/// in was dropped and the test waited out a button that would never come (#35); a text field was
/// tapped before its sheet had presented and reported "No matches found"; and a tap on Settings
/// reported "Timed out while synthesizing event" after the app spent half a minute not idle.
///
/// `waitForExistence` does not cover any of those. An element exists as soon as it is in the
/// accessibility tree, which is before it has arrived where it is going and before it can be
/// touched. `isHittable` is the question worth asking, and asking it is what these do.
extension XCUIElement {
/// Waits for the element to be there and touchable, then taps it.
///
/// The timeout is patience, not a deadline anything has to meet: reaching it means the test
/// was going to fail regardless, and twenty seconds is not long on a CI runner that has been
/// building for half an hour and is on its second simulator.
@MainActor
func tapWhenReady(timeout: TimeInterval = 20, file: StaticString = #filePath, line: UInt = #line) {
guard waitUntilHittable(timeout: timeout) else {
XCTFail("\(self) never became hittable", file: file, line: line)
return
}
tap()
}

/// Taps and types in one step, which is the shape every form in these tests uses.
@MainActor
func typeWhenReady(_ text: String, timeout: TimeInterval = 20,
file: StaticString = #filePath, line: UInt = #line) {
tapWhenReady(timeout: timeout, file: file, line: line)
typeText(text)
}

/// True once the element is present and can receive a touch. Polls rather than using a
/// predicate expectation so a caller can decide what a timeout means.
@MainActor
@discardableResult
func waitUntilHittable(timeout: TimeInterval = 20) -> Bool {
if exists, isHittable { return true }
let deadline = Date().addingTimeInterval(timeout)
while Date() < deadline {
if exists, isHittable { return true }
_ = waitForExistence(timeout: 0.1)
}
return exists && isHittable
}
}
75 changes: 30 additions & 45 deletions UITests/iOSSHUITests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,14 @@ final class iOSSHUITests: XCTestCase {
let app = XCUIApplication()
app.launchArguments = ["--ui-testing"]
app.launch()
app.buttons["addFirstHost"].tap()
app.textFields["hostName"].tap()
app.textFields["hostName"].typeText("Japanese input")
app.textFields["hostAddress"].tap()
app.textFields["hostAddress"].typeText("192.0.2.1")
app.textFields["hostUsername"].tap()
app.textFields["hostUsername"].typeText("tester")
app.buttons["saveHost"].tap()
app.buttons["host-Japanese input"].tap()
app.buttons["addFirstHost"].tapWhenReady()
app.textFields["hostName"].typeWhenReady("Japanese input")
app.textFields["hostAddress"].typeWhenReady("192.0.2.1")
app.textFields["hostUsername"].typeWhenReady("tester")
app.buttons["saveHost"].tapWhenReady()
app.buttons["host-Japanese input"].tapWhenReady()
XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10))
app.buttons["Cancel"].tap()
app.buttons["Cancel"].tapWhenReady()
XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5))
let terminal = app.descendants(matching: .any).matching(identifier: "terminal").firstMatch
terminal.tap()
Expand Down Expand Up @@ -73,17 +70,14 @@ final class iOSSHUITests: XCTestCase {
let app = XCUIApplication()
app.launchArguments = ["--ui-testing"]
app.launch()
app.buttons["addFirstHost"].tap()
app.textFields["hostName"].tap()
app.textFields["hostName"].typeText("Viewport")
app.textFields["hostAddress"].tap()
app.textFields["hostAddress"].typeText("192.0.2.1")
app.textFields["hostUsername"].tap()
app.textFields["hostUsername"].typeText("tester")
app.buttons["saveHost"].tap()
app.buttons["addFirstHost"].tapWhenReady()
app.textFields["hostName"].typeWhenReady("Viewport")
app.textFields["hostAddress"].typeWhenReady("192.0.2.1")
app.textFields["hostUsername"].typeWhenReady("tester")
app.buttons["saveHost"].tapWhenReady()
app.buttons["host-Viewport"].tap()
XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10))
app.buttons["Cancel"].tap()
app.buttons["Cancel"].tapWhenReady()
XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5))
let terminal = app.descendants(matching: .any).matching(identifier: "terminal").firstMatch
terminal.tap()
Expand All @@ -96,7 +90,7 @@ final class iOSSHUITests: XCTestCase {

app.buttons["Reconnect"].tap()
XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10))
app.buttons["Cancel"].tap()
app.buttons["Cancel"].tapWhenReady()
terminal.tap()
try await assertTerminalIsAboveAccessory(app)

Expand Down Expand Up @@ -136,17 +130,14 @@ final class iOSSHUITests: XCTestCase {
let app = XCUIApplication()
app.launchArguments = ["--ui-testing"]
app.launch()
app.buttons["addFirstHost"].tap()
app.textFields["hostName"].tap()
app.textFields["hostName"].typeText("Terminal test")
app.textFields["hostAddress"].tap()
app.textFields["hostAddress"].typeText("192.0.2.1")
app.textFields["hostUsername"].tap()
app.textFields["hostUsername"].typeText("tester")
app.buttons["saveHost"].tap()
app.buttons["addFirstHost"].tapWhenReady()
app.textFields["hostName"].typeWhenReady("Terminal test")
app.textFields["hostAddress"].typeWhenReady("192.0.2.1")
app.textFields["hostUsername"].typeWhenReady("tester")
app.buttons["saveHost"].tapWhenReady()
app.buttons["host-Terminal test"].tap()
XCTAssertTrue(app.secureTextFields["Password"].waitForExistence(timeout: 10))
app.buttons["Cancel"].tap()
app.buttons["Cancel"].tapWhenReady()
XCTAssertTrue(app.buttons["Reconnect"].waitForExistence(timeout: 5))
XCTAssertTrue(app.descendants(matching: .any).matching(identifier: "terminal").firstMatch.exists)
XCTAssertEqual(app.staticTexts.matching(NSPredicate(format: "label CONTAINS %@", "rendering could not start")).count, 0)
Expand All @@ -164,22 +155,19 @@ final class iOSSHUITests: XCTestCase {
app.launchArguments = ["--ui-testing"]
app.launch()
XCTAssertTrue(app.buttons["addFirstHost"].waitForExistence(timeout: 10))
app.buttons["addFirstHost"].tap()
app.buttons["addFirstHost"].tapWhenReady()
XCTAssertFalse(app.buttons["saveHost"].isEnabled)
app.textFields["hostName"].tap()
app.textFields["hostName"].typeText("Development")
app.textFields["hostAddress"].tap()
app.textFields["hostAddress"].typeText("192.0.2.1")
app.textFields["hostUsername"].tap()
app.textFields["hostUsername"].typeText("developer")
app.buttons["saveHost"].tap()
app.textFields["hostName"].typeWhenReady("Development")
app.textFields["hostAddress"].typeWhenReady("192.0.2.1")
app.textFields["hostUsername"].typeWhenReady("developer")
app.buttons["saveHost"].tapWhenReady()
let host = app.buttons["host-Development"]
XCTAssertTrue(host.waitForExistence(timeout: 5))
host.swipeRight()
app.buttons["Edit"].tap()
XCTAssertEqual(app.textFields["hostAddress"].value as? String, "192.0.2.1")
XCTAssertEqual(app.textFields["hostUsername"].value as? String, "developer")
app.buttons["Cancel"].tap()
app.buttons["Cancel"].tapWhenReady()
host.swipeLeft()
app.buttons["Delete"].tap()
XCTAssertTrue(app.buttons["addFirstHost"].waitForExistence(timeout: 5))
Expand All @@ -199,13 +187,10 @@ final class iOSSHUITests: XCTestCase {
XCTAssertTrue(privacyPolicyLink.waitForExistence(timeout: 5))
XCTAssertTrue(app.descendants(matching: .any)["supportLink"].waitForExistence(timeout: 5))
app.buttons["Done"].tap()
app.buttons["addFirstHost"].tap()
app.textFields["hostName"].tap()
app.textFields["hostName"].typeText("Bad host")
app.textFields["hostAddress"].tap()
app.textFields["hostAddress"].typeText("bad host")
app.textFields["hostUsername"].tap()
app.textFields["hostUsername"].typeText("root")
app.buttons["addFirstHost"].tapWhenReady()
app.textFields["hostName"].typeWhenReady("Bad host")
app.textFields["hostAddress"].typeWhenReady("bad host")
app.textFields["hostUsername"].typeWhenReady("root")
XCTAssertFalse(app.buttons["saveHost"].isEnabled)
}
}
4 changes: 4 additions & 0 deletions iOSSH.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
126F1F2E6A74A9F1358D33A7 /* TailscaleHostImport.swift in Sources */ = {isa = PBXBuildFile; fileRef = A53EAFCE919B8B842360534F /* TailscaleHostImport.swift */; };
1A27F80BD8BA8845F5431492 /* TailscaleImportInbox.swift in Sources */ = {isa = PBXBuildFile; fileRef = BF3D87F64507E681986AEBE8 /* TailscaleImportInbox.swift */; };
280E96841D6B5FB2DA1950EA /* TerminalRender in Frameworks */ = {isa = PBXBuildFile; productRef = 0130968861023AF2207E184C /* TerminalRender */; };
2F4E7D3547D91377083B60D7 /* UITestInteraction.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8DA05394A523D1CB258AF61B /* UITestInteraction.swift */; };
34789C86E04E9350FF66A3F2 /* GhosttyEngine in Frameworks */ = {isa = PBXBuildFile; productRef = 33892EBDFF6FA3FEA57EB471 /* GhosttyEngine */; };
3492C275BD7A4FCD99475FF6 /* ThirdPartyNotices.txt in Resources */ = {isa = PBXBuildFile; fileRef = ABDE0513C54E763730592D8B /* ThirdPartyNotices.txt */; };
375F741B0F46FCFBDD506F79 /* TerminalViewportTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C11265FA4D2A5C381E689173 /* TerminalViewportTests.swift */; };
Expand Down Expand Up @@ -89,6 +90,7 @@
731526C7B108797D5F45E403 /* HostRecord.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HostRecord.swift; sourceTree = "<group>"; };
79832E5C775352D87B6AA169 /* TerminalOutputTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalOutputTests.swift; sourceTree = "<group>"; };
8CB0BC27787CF8C4490CD014 /* TerminalFontLibraryTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TerminalFontLibraryTests.swift; sourceTree = "<group>"; };
8DA05394A523D1CB258AF61B /* UITestInteraction.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UITestInteraction.swift; sourceTree = "<group>"; };
911D7DEF8CB2D674031B953C /* GhosttyEngine */ = {isa = PBXFileReference; lastKnownFileType = folder; name = GhosttyEngine; path = spike/GhosttyEngine; sourceTree = SOURCE_ROOT; };
9DD4F05CEEF4A486E17BCFC1 /* TerminalCore */ = {isa = PBXFileReference; lastKnownFileType = folder; name = TerminalCore; path = Packages/TerminalCore; sourceTree = SOURCE_ROOT; };
A53EAFCE919B8B842360534F /* TailscaleHostImport.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TailscaleHostImport.swift; sourceTree = "<group>"; };
Expand Down Expand Up @@ -145,6 +147,7 @@
6D1E0C43310C3EB044EB52DB /* iPadWorkspaceUITests.swift */,
DC54B35541FEAEB0228D113C /* KeyboardResizeUITests.swift */,
27587AAB792BED1360F24EF3 /* TailscaleImportUITests.swift */,
8DA05394A523D1CB258AF61B /* UITestInteraction.swift */,
);
path = UITests;
sourceTree = "<group>";
Expand Down Expand Up @@ -365,6 +368,7 @@
files = (
434804C23161B0F2B6237EDD /* KeyboardResizeUITests.swift in Sources */,
EFBBA1911C95BC26AB6B7E43 /* TailscaleImportUITests.swift in Sources */,
2F4E7D3547D91377083B60D7 /* UITestInteraction.swift in Sources */,
903CE9BF45D214388BF6D148 /* iOSSHUITests.swift in Sources */,
62771FFD9ACCA67D5D9B5D86 /* iPadWorkspaceUITests.swift in Sources */,
);
Expand Down
Loading