ci: Run UI Tests in SauceLab (#1464)

Instead of AppCenter, run the UI tests in SauceLab. I split up the tests and relaunch the app for every screen to make the tests more stable.
getsentry · Nov 22, 2021 · 3c106ee · 3c106ee
1 parent 73b8114
commit 3c106ee
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 33 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -75,14 +75,15 @@ jobs:
         run: fastlane ui_tests_${{matrix.target}}
         shell: sh
 
-  ios-swift-appcenter:
-    name: AppCenter for iOS Swift
+  ios-swift-saucelabs:
+    name: SauceLabs for iOS Swift
     runs-on: macos-11
     steps:
       - uses: actions/checkout@v2
       - run: ./scripts/ci-select-xcode.sh 
+      - run: npm install -g saucectl
 
-      - name: Run Fastlane
+      - run: fastlane build_ios_swift_ui_test
         env:
           APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
           APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
@@ -91,22 +92,10 @@ jobs:
           MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
           MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
           MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
-        run: fastlane build_ios_swift_ui_test
-        shell: sh 
-
-      ## AppCenter CLI can't handle multiple apps within DerivedData
-      ## Therefore we remove the AppClip
-      - name: Remove AppClip
-        run: rm -r DerivedData/Build/Products/Debug-iphoneos/iOS-SwiftClip.app
 
-      - name: Run in AppCenter
-        shell: sh
-        run: >-
-          appcenter test run xcuitest 
-          --app "philipphofmann/iOS-Swift-Test"
-          --devices "philipphofmann/ios-swift-pr" 
-          --test-series "master" 
-          --locale "en_US" 
-          --build-dir DerivedData/Build/Products/Debug-iphoneos/ 
-          --token ${{ secrets.APP_CENTER_TOKEN }}
+      - name: Run Tests in Saucelab
+        env:
+          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
+          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
+        run: saucectl run
 
diff --git a/.sauce/config.yml b/.sauce/config.yml
@@ -0,0 +1,34 @@
+apiVersion: v1alpha
+kind: xcuitest
+sauce:
+  region: us-west-1
+  concurrency: 2
+
+defaults:
+  timeout: 5m
+
+retries: 2
+
+xcuitest:
+  app: ./DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
+  testApp: ./DerivedData/Build/Products/Debug-iphoneos/iOS-SwiftUITests-Runner.app
+
+suites:
+  - name: "iOS-Swift"
+    devices:
+      - name: "iPhone.*"
+        orientation: "portrait"
+        platformVersion: "15.1"
+
+      - name: "iPhone.*"
+        orientation: "portrait"
+        platformVersion: "14.8"
+
+      # Tests don't work currently on iOS 13.7, because Start of XCUITest-Runner times out.
+
+artifacts:
+  download:
+    when: always
+    match:
+      - "*.junit.xml"
+    directory: ./artifacts/
diff --git a/Samples/iOS-Swift/iOS-SwiftUITests/LaunchUITests.swift b/Samples/iOS-Swift/iOS-SwiftUITests/LaunchUITests.swift
@@ -1,24 +1,46 @@
 import XCTest
 
 class LaunchUITests: XCTestCase {
+
+    private let timeout: TimeInterval = 10
+    private let app: XCUIApplication = XCUIApplication()
 
     override func setUpWithError() throws {
         try super.setUpWithError()
         continueAfterFailure = false
-    }
-
-    func testLaunch() {
-        let app = XCUIApplication()
-        app.launch()
 
-        func visitScreen(buttonText: String) {
-            app.buttons[buttonText].tap()
-            app.swipeDown(velocity: .fast)
-        }
+        app.launch()
+        XCUIDevice.shared.orientation = .portrait
 
-        visitScreen(buttonText: "Lorem Ipsum")
-        visitScreen(buttonText: "Test Navigation Transaction")
-        visitScreen(buttonText: "Show Nib")
-        visitScreen(buttonText: "Show SwiftUI")
+        waitForExistenseOfMainScreen()
+    }
+
+    override func tearDownWithError() throws {
+        try super.tearDownWithError()
+        app.terminate()
+    }
+
+    func testLoremIpsum() {
+        app.buttons["Lorem Ipsum"].tap()
+        XCTAssertTrue(app.textViews.firstMatch.waitForExistence(timeout: timeout), "Lorem Ipsum not loaded.")
+    }
+
+    func testNavigationTransaction() {
+        app.buttons["Test Navigation Transaction"].tap()
+        XCTAssertTrue(app.images.firstMatch.waitForExistence(timeout: timeout), "Navigation transaction not loaded.")
+    }
+
+    func testShowNib() {
+        app.buttons["Show Nib"].tap()
+        XCTAssertTrue(app.buttons["Button"].waitForExistence(timeout: timeout), "Show Nib not loaded.")
+    }
+
+    func testShowSwiftUI() {
+        app.buttons["Show SwiftUI"].tap()
+        XCTAssertTrue(app.staticTexts["SwiftUI!"].waitForExistence(timeout: timeout), "SwiftUI not loaded.")
+    }
+
+    private func waitForExistenseOfMainScreen() {
+        XCTAssertTrue(app.buttons["captureMessage"].waitForExistence(timeout: timeout), "Home Screen doesn't exist.")
     }
 }