From de17547c4c02f3b3f0fcdb625d59ac4c944dae89 Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Thu, 10 Feb 2022 15:35:53 -0300
Subject: [PATCH 01/10] Run iOS 16 tests continuously and upload results to
 observability server

---
 .github/workflows/check-pod.yaml              |  58 ---------
 .github/workflows/docs.yml                    |  52 --------
 .github/workflows/features.yml                |  14 ---
 .../workflows/integration-test-iOS16_2.yaml   |  69 +---------
 .github/workflows/integration-test-macOS.yaml | 118 ------------------
 .../workflows/integration-test-tvOS16_1.yaml  | 118 ------------------
 ...ntinuously-run-tests-and-upload-results.sh |  62 +++++++++
 fastlane/Scanfile                             |   2 +-
 8 files changed, 67 insertions(+), 426 deletions(-)
 delete mode 100644 .github/workflows/check-pod.yaml
 delete mode 100644 .github/workflows/docs.yml
 delete mode 100644 .github/workflows/features.yml
 delete mode 100644 .github/workflows/integration-test-macOS.yaml
 delete mode 100644 .github/workflows/integration-test-tvOS16_1.yaml
 create mode 100755 Scripts/continuously-run-tests-and-upload-results.sh

diff --git a/.github/workflows/check-pod.yaml b/.github/workflows/check-pod.yaml
deleted file mode 100644
index 41666c435..000000000
--- a/.github/workflows/check-pod.yaml
+++ /dev/null
@@ -1,58 +0,0 @@
-name: Check Pod
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-jobs:
-  check:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Select Specific Xcode Version (14.2)
-        run: |
-          sudo xcode-select -s /Applications/Xcode_14.2.app
-          echo "Selected Xcode version:"
-          xcodebuild -version
-
-      # Run the steps we document in the Release Process.
-      # unzip commands included as proof-of-life for the Carthage output.
-      - name: Print Ruby version
-        run: ruby --version
-      - name: Print Carthage version
-        run: 'echo -n "carthage version: " && carthage version'
-      - name: Print CocoaPods version
-        run: 'echo -n "pod version: " && pod --version --verbose'
-      - name: Print Make version
-        run: make --version
-      - name: Build Carthage dependencies
-        run: make update
-      - name: Build Ably framework
-        run: make carthage_package
-      - name: Print contents of generated ZIP file
-        run: |
-          unzip -l Ably.framework.zip
-          unzip -l Ably.framework.zip | grep 'Mac/Ably.framework'
-          unzip -l Ably.framework.zip | grep 'tvOS/Ably.framework'
-          unzip -l Ably.framework.zip | grep 'iOS/Ably.framework'
-      - name: Validate pod
-        run: pod lib lint
-        # We move Ably.framework.zip into a directory. This is because, by
-        # default, macOS’s Archive Utility unzips directly-nested zip files, so
-        # if Ably.framework.zip were at the top level of the zip file that
-        # actions/upload-artifact creates, then Archive Utility would unzip
-        # Ably.framework.zip too, which we don’t want, since we want this file
-        # to be kept intact so that we can upload it to GitHub releases as
-        # described in CONTRIBUTING.md.
-      - name: Prepare built framework for archiving
-        run: |
-          mkdir -p carthage-built-framework-artifact-contents/carthage-built-framework
-          mv Ably.framework.zip carthage-built-framework-artifact-contents/carthage-built-framework
-      - name: Archive built framework
-        uses: actions/upload-artifact@v3
-        with:
-          name: carthage-built-framework
-          path: carthage-built-framework-artifact-contents
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
deleted file mode 100644
index 59cdd0f05..000000000
--- a/.github/workflows/docs.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-name: Docs Generation
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-    tags:
-      - '*'
-
-jobs:
-  build:
-    runs-on: macos-latest
-    
-    permissions:
-      deployments: write
-      id-token: write
-      
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Select Specific Xcode Version (14.2)
-        run: |
-          sudo xcode-select -s /Applications/Xcode_14.2.app
-          echo "Selected Xcode version:"
-          xcodebuild -version
-
-      - name: Install Dependencies
-        run: |
-          make submodules
-          bundle install
-          make update_carthage_dependencies_macos
-
-      - name: Build Documentation
-        run: |
-          ./Scripts/jazzy.sh
-          ls -al Docs/jazzy
-
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-region: eu-west-2
-          role-to-assume: arn:aws:iam::${{ secrets.ABLY_AWS_ACCOUNT_ID_SDK }}:role/ably-sdk-builds-ably-cocoa
-          role-session-name: "${{ github.run_id }}-${{ github.run_number }}"
-
-      - name: Upload Documentation
-        uses: ably/sdk-upload-action@v1
-        with:
-          sourcePath: Docs/jazzy
-          githubToken: ${{ secrets.GITHUB_TOKEN }}
-          artifactName: jazzydoc
-          
diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml
deleted file mode 100644
index d1c123738..000000000
--- a/.github/workflows/features.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-name: Features
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-jobs:
-  build:
-    uses: ably/features/.github/workflows/sdk-features.yml@main
-    with:
-      repository-name: ably-cocoa
-    secrets: inherit
diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml
index ef3f45e55..d3975db96 100644
--- a/.github/workflows/integration-test-iOS16_2.yaml
+++ b/.github/workflows/integration-test-iOS16_2.yaml
@@ -51,73 +51,12 @@ jobs:
           path: xcparse/.build/debug/xcparse
           key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
 
-      - name: Reset Simulators
-        run: xcrun simctl erase all
-
-      - name: Install Dependencies and Run Tests
+      - name: Install Dependencies and Run Tests Continuously
+        env:
+          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
         run: |
           brew install xcbeautify
           make submodules
           bundle install
           make update_carthage_dependencies_ios
-          bundle exec fastlane test_iOS16_2
-
-      - name: Check Static Analyzer Output
-        id: analyzer-output
-        run: |
-          if [[ -z $(find ./derived_data -name "report-*.html") ]]; then
-            echo "Static Analyzer found no issues."
-          else
-            echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build."
-            exit 1
-          fi
-      
-      - name: Static Analyzer Reports Uploading
-        if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: static-analyzer-reports-test_iOS16_2
-          path: ./derived_data/**/report-*.html
-
-      - name: Run Examples Tests
-        working-directory: ./Examples/Tests
-        run: |
-          pod repo update
-          pod install
-          bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_iOS16_2"
-
-      - name: Build APNS Example Project
-        working-directory: ./Examples/AblyPush
-        run: |
-          xcodebuild build -scheme "AblyPushExample" -destination "platform=iOS Simulator,name=iPhone 14" -configuration "Debug"
-      
-      - name: Xcodebuild Logs Artifact
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: xcodebuild-logs
-          path: ~/Library/Developer/Xcode/DerivedData/*/Logs
-
-      - name: Upload test output artifact
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: test-output
-          path: fastlane/test_output
-
-      - name: Upload test results to observability server
-        if: always()
-        env:
-          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
-        run: Scripts/upload_test_results.sh
-      
-      - name: Swift Package Manager - Installation Test
-        working-directory: ./
-        run:  |
-          echo 'Current Branch: ' $GITHUB_HEAD_REF
-          echo 'Current Revision (SHA):' $GITHUB_SHA
-          echo Current Path: $(pwd)
-          export PACKAGE_URL=file://$(pwd)
-          export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF
-          export PACKAGE_REVISION=$GITHUB_SHA
-          swift test --package-path Examples/SPM -v
+          Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2
diff --git a/.github/workflows/integration-test-macOS.yaml b/.github/workflows/integration-test-macOS.yaml
deleted file mode 100644
index 2cf7ff101..000000000
--- a/.github/workflows/integration-test-macOS.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-name: "Integration Test: macOS Latest"
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-# IMPORTANT NOTES:
-# - Changes made to this file needs to replicated across other integration-test-*.yaml files.
-# - The Fastlane lane name is duplicated in more than one place within this workflow.
-
-jobs:
-  check:
-    runs-on: macos-latest
-
-    env:
-      LC_CTYPE: en_US.UTF-8
-      LANG: en_US.UTF-8
-      ABLY_ENV: sandbox
-
-    steps:
-      - name: Check out SDK repo
-        uses: actions/checkout@v2
-        
-      - name: Select Specific Xcode Version (14.2)
-        run: |
-          sudo xcode-select -s /Applications/Xcode_14.2.app
-          echo "Selected Xcode version:"
-          xcodebuild -version
-
-      - name: Log environment information
-        run: ./Scripts/log-environment-information.sh
-
-      - name: Check out xcparse repo
-        uses: actions/checkout@v3
-        with:
-          repository: ably-forks/xcparse
-          ref: emit-test-case-info
-          path: xcparse
-
-      - id: get-xcparse-commit-sha
-        name: Get xcparse commit SHA
-        run: |
-          cd xcparse
-          echo "::set-output name=sha::$(git rev-parse HEAD)"
-
-      - name: "actions/cache@v3 (xcparse binary)"
-        uses: actions/cache@v3
-        with:
-          path: xcparse/.build/debug/xcparse
-          key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
-
-      - name: Reset Simulators
-        run: xcrun simctl erase all
-
-      - name: Install Dependencies and Run Tests
-        run: |
-          brew install xcbeautify
-          make submodules
-          bundle install
-          make update_carthage_dependencies_macos
-          bundle exec fastlane test_macOS
-
-      - name: Check Static Analyzer Output
-        id: analyzer-output
-        run: |
-          if [[ -z $(find ./derived_data -name "report-*.html") ]]; then
-            echo "Static Analyzer found no issues."
-          else
-            echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build."
-            exit 1
-          fi
-      
-      - name: Static Analyzer Reports Uploading
-        if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: static-analyzer-reports-test_macOS
-          path: ./derived_data/**/report-*.html
-
-      - name: Run Examples Tests
-        working-directory: ./Examples/Tests
-        run: |
-          pod repo update
-          pod install
-          bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_macOS"
-      
-      - name: Xcodebuild Logs Artifact
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: xcodebuild-logs
-          path: ~/Library/Developer/Xcode/DerivedData/*/Logs
-
-      - name: Upload test output artifact
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: test-output
-          path: fastlane/test_output
-
-      - name: Upload test results to observability server
-        if: always()
-        env:
-          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
-        run: Scripts/upload_test_results.sh
-      
-      - name: Swift Package Manager - Installation Test
-        working-directory: ./
-        run:  |
-          echo 'Current Branch: ' $GITHUB_HEAD_REF
-          echo 'Current Revision (SHA):' $GITHUB_SHA
-          echo Current Path: $(pwd)
-          export PACKAGE_URL=file://$(pwd)
-          export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF
-          export PACKAGE_REVISION=$GITHUB_SHA
-          swift test --package-path Examples/SPM -v
diff --git a/.github/workflows/integration-test-tvOS16_1.yaml b/.github/workflows/integration-test-tvOS16_1.yaml
deleted file mode 100644
index d259d0856..000000000
--- a/.github/workflows/integration-test-tvOS16_1.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-name: "Integration Test: tvOS 16.1"
-
-on:
-  pull_request:
-  push:
-    branches:
-      - main
-
-# IMPORTANT NOTES:
-# - Changes made to this file needs to replicated across other integration-test-*.yaml files.
-# - The Fastlane lane name is duplicated in more than one place within this workflow.
-
-jobs:
-  check:
-    runs-on: macos-latest
-
-    env:
-      LC_CTYPE: en_US.UTF-8
-      LANG: en_US.UTF-8
-      ABLY_ENV: sandbox
-
-    steps:
-      - name: Check out SDK repo
-        uses: actions/checkout@v2
-        
-      - name: Select Specific Xcode Version (14.2)
-        run: |
-          sudo xcode-select -s /Applications/Xcode_14.2.app
-          echo "Selected Xcode version:"
-          xcodebuild -version
-
-      - name: Log environment information
-        run: ./Scripts/log-environment-information.sh
-
-      - name: Check out xcparse repo
-        uses: actions/checkout@v3
-        with:
-          repository: ably-forks/xcparse
-          ref: emit-test-case-info
-          path: xcparse
-
-      - id: get-xcparse-commit-sha
-        name: Get xcparse commit SHA
-        run: |
-          cd xcparse
-          echo "::set-output name=sha::$(git rev-parse HEAD)"
-
-      - name: "actions/cache@v3 (xcparse binary)"
-        uses: actions/cache@v3
-        with:
-          path: xcparse/.build/debug/xcparse
-          key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
-
-      - name: Reset Simulators
-        run: xcrun simctl erase all
-
-      - name: Install Dependencies and Run Tests
-        run: |
-          brew install xcbeautify
-          make submodules
-          bundle install
-          make update_carthage_dependencies_tvos
-          bundle exec fastlane test_tvOS16_1
-
-      - name: Check Static Analyzer Output
-        id: analyzer-output
-        run: |
-          if [[ -z $(find ./derived_data -name "report-*.html") ]]; then
-            echo "Static Analyzer found no issues."
-          else
-            echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build."
-            exit 1
-          fi
-      
-      - name: Static Analyzer Reports Uploading
-        if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: static-analyzer-reports-test_tvOS16_1
-          path: ./derived_data/**/report-*.html
-
-      - name: Run Examples Tests
-        working-directory: ./Examples/Tests
-        run: |
-          pod repo update
-          pod install
-          bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_tvOS_16_1"
-      
-      - name: Xcodebuild Logs Artifact
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: xcodebuild-logs
-          path: ~/Library/Developer/Xcode/DerivedData/*/Logs
-
-      - name: Upload test output artifact
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: test-output
-          path: fastlane/test_output
-
-      - name: Upload test results to observability server
-        if: always()
-        env:
-          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
-        run: Scripts/upload_test_results.sh
-      
-      - name: Swift Package Manager - Installation Test
-        working-directory: ./
-        run:  |
-          echo 'Current Branch: ' $GITHUB_HEAD_REF
-          echo 'Current Revision (SHA):' $GITHUB_SHA
-          echo Current Path: $(pwd)
-          export PACKAGE_URL=file://$(pwd)
-          export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF
-          export PACKAGE_REVISION=$GITHUB_SHA
-          swift test --package-path Examples/SPM -v
diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh
new file mode 100755
index 000000000..4d7206abf
--- /dev/null
+++ b/Scripts/continuously-run-tests-and-upload-results.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -e
+
+# 1. Grab command-line options.
+
+# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -l|--lane) lane="$2"; shift ;;
+        -u|--upload-server-base-url) upload_server_base_url="$2"; shift ;;
+        *) echo "Unknown parameter passed: $1"; exit 1 ;;
+    esac
+    shift
+done
+
+if [[ -z $lane ]]
+then
+  echo "You need to specify the Fastlane lane to run (-l / --lane)." 2>&1
+  exit 1
+fi
+
+# 2. Run the tests in a loop and report the results.
+
+declare -i iteration=1
+while true
+do
+  echo "BEGIN ITERATION ${iteration}" 2>&1
+
+  rm -rf fastlane/test_output
+  xcrun simctl erase all
+
+  set +e
+  bundle exec fastlane --verbose $lane
+  tests_exit_value=$?
+  set -e
+
+  if [[ tests_exit_value -eq 0 ]]
+  then
+    echo "ITERATION ${iteration}: Tests passed."
+  else
+    echo "ITERATION ${iteration}: Tests failed (exit value ${tests_exit_value})."
+  fi
+
+  echo "ITERATION ${iteration}: Uploading results to observability server."
+
+  # https://unix.stackexchange.com/questions/446847/conditionally-pass-params-to-a-script
+  optional_params=()
+
+  if [[ ! -z $upload_server_base_url ]]
+  then
+    optional_params+=(--upload-server-base-url "${upload_server_base_url}")
+  fi
+
+  ./Scripts/upload_test_results.sh \
+    --iteration $iteration \
+    "${optional_params[@]}"
+
+  echo "END ITERATION ${iteration}" 2>&1
+
+  iteration+=1
+done
diff --git a/fastlane/Scanfile b/fastlane/Scanfile
index fea35296b..c0f23f385 100644
--- a/fastlane/Scanfile
+++ b/fastlane/Scanfile
@@ -1,5 +1,5 @@
 open_report false
-clean true
+clean false
 skip_slack true
 ensure_devices_found true
 output_types "junit"

From 5d2f6629282929302ecfbebe7593cad9ef634619 Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 1 Mar 2022 16:45:06 -0300
Subject: [PATCH 02/10] Print xcodebuild raw output as part of loop

This is so that we can see things like library logs, or any additional
logging we might add to help debug a test case.
---
 Scripts/continuously-run-tests-and-upload-results.sh | 6 ++++++
 fastlane/Scanfile                                    | 2 ++
 2 files changed, 8 insertions(+)

diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh
index 4d7206abf..95b1a5ff0 100755
--- a/Scripts/continuously-run-tests-and-upload-results.sh
+++ b/Scripts/continuously-run-tests-and-upload-results.sh
@@ -28,6 +28,7 @@ do
   echo "BEGIN ITERATION ${iteration}" 2>&1
 
   rm -rf fastlane/test_output
+  rm -rf xcodebuild_output
   xcrun simctl erase all
 
   set +e
@@ -42,6 +43,11 @@ do
     echo "ITERATION ${iteration}: Tests failed (exit value ${tests_exit_value})."
   fi
 
+  echo "ITERATION ${iteration}: BEGIN xcodebuild raw output."
+  ls xcodebuild_output
+  cat xcodebuild_output/**
+  echo "ITERATION ${iteration}: END xcodebuild raw output."
+
   echo "ITERATION ${iteration}: Uploading results to observability server."
 
   # https://unix.stackexchange.com/questions/446847/conditionally-pass-params-to-a-script
diff --git a/fastlane/Scanfile b/fastlane/Scanfile
index c0f23f385..da002c455 100644
--- a/fastlane/Scanfile
+++ b/fastlane/Scanfile
@@ -6,3 +6,5 @@ output_types "junit"
 # I'm being explicit about this because I want to make sure it's being used, to make sure that trainer is used to generate the JUnit report
 xcodebuild_formatter "xcbeautify"
 result_bundle true
+# Just for printing inside these loop jobs
+buildlog_path "xcodebuild_output"

From 373201abae77139a4c225af6fad2aeb13fecac3a Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 29 Mar 2022 16:25:47 -0300
Subject: [PATCH 03/10] Add script for fetching test case logs from GitHub

---
 Scripts/fetch-test-logs.sh | 493 +++++++++++++++++++++++++++++++++++++
 1 file changed, 493 insertions(+)
 create mode 100755 Scripts/fetch-test-logs.sh

diff --git a/Scripts/fetch-test-logs.sh b/Scripts/fetch-test-logs.sh
new file mode 100755
index 000000000..60caa5e4b
--- /dev/null
+++ b/Scripts/fetch-test-logs.sh
@@ -0,0 +1,493 @@
+#!/bin/bash
+
+# Retrieves the raw xcodebuild output for one or more test observability server
+# uploads.
+#
+# Only works for tests that were run using the
+# continuously-run-tests-and-upload-results script in this directory.
+
+# Usage:
+# ./fetch-test-logs.sh --repo ably/ably-cocoa --test-case-id <id> --filter [filter]
+#
+# or
+#
+# ./fetch-test-logs.sh --repo ably/ably-cocoa --upload-id <id>
+
+# Options:
+#
+# -r / --repo <name>: The 'org/name'-formatted name of the GitHub repo, for
+# example 'ably/ably-cocoa'.
+#
+# -t / --test-case-id <id>: The ID of a test case saved on the test
+# observability server. Will fetch all uploads that match the filter specified
+# using the --filter option, and then save the results inside the directory
+# specified by the --output-directory option, in the following hierarchy, where
+# uploads are split into those where the test case failed and those where it
+# didn’t (which doesn’t necessarily imply that the test case passed; it may not
+# have run at all):
+#
+# <output directory>
+# ├── info.json (contains metadata about the results in this directory)
+# ├── upload_logs
+# │   ├── failed
+# │   │   └── xcodebuild-logs-upload-<upload ID>.txt, ...
+# │   └── not_failed
+# │       └── xcodebuild-logs-upload-<upload ID>.txt, ...
+# └── test-case-logs (unless --no-extract-test-case-logs specified)
+#     ├── failed
+#     │   └── xcodebuild-logs-upload-<upload ID>.txt, ...
+#     └── not_failed
+#         └── xcodebuild-logs-upload-<upload ID>.txt, ...
+#
+# The upload_logs directory contains the full logs for that upload, and the
+# test_case_logs directory contains just the segments of the logs that
+# correspond to the specific test case.
+#
+# -d / --output-directory <dir>: Where to output the logs generated by the
+# --test-case-id option to. Defaults to ./xcodebuild-logs-test-case-<test case ID>-<current datetime>.
+#
+# -f / --filter <filter>: A URL query string describing a filter to be applied
+# to the uploads fetched when using the --test-case-id option. For example,
+# "branches[]=main&createdBefore=2022-02-20".
+#
+# -n / --no-extract-test-case-logs: Will cause the --test-case-id option to not
+# attempt to extract the segment of the upload log that corresponds to the test
+# case.
+#
+# -i / --upload-id <id>: The ID of a upload saved on the test observability
+# server.
+#
+# -u / --upload-server-base-url <url>: Allows you to specify a URL to use as
+# the upload server base URL. Defaults to https://test-observability.herokuapp.com.
+#
+# -o / --output-file <file>: Where to output the logs generated by the
+# --upload-id option to. Defaults to ./xcodebuild-logs-upload-<upload ID>.txt.
+#
+# -c / --cache-directory <dir>: Where to cache the GitHub logs. Defaults to
+# ~/Library/Caches/com.ably.testObservabilityLogs. Will be created if doesn’t
+# exist.
+#
+# -a / --no-use-github-auth: Will not prompt the user for an access token to be
+# used for making requests to the GitHub API. Useful if all the required GitHub
+# job logs are already cached locally.
+
+set -e
+
+check_dependencies() {
+  if ! which jq >/dev/null; then
+    echo "You need to install jq." 2>&1
+    exit 1
+  fi
+}
+
+get_github_access_token() {
+  # https://stackoverflow.com/questions/3980668/how-to-get-a-password-from-a-shell-script-without-echoing#comment4260181_3980904
+  read -s -p "Enter your GitHub access token (this will be used to fetch logs from the GitHub API): " github_access_token
+
+  echo
+
+  if [[ -z $github_access_token ]]; then
+    echo "You need to specify a GitHub access token." 2>&1
+    exit 1
+  fi
+
+  echo
+}
+
+# Args:
+# $1: JSON representation of the test observability server upload
+# $2: Path to write the logs to
+fetch_and_write_logs_for_upload() {
+  upload_json=$1
+  output_file=$2
+
+  # (TIL I learned that `echo` will interpret backslash sequences, which we
+  # don’t want.  Appparently in general printf is recommended over echo.)
+  # https://stackoverflow.com/questions/43528202/prevent-echo-from-interpreting-backslash-escapes
+  github_repository=$(printf '%s' $upload_json | jq --raw-output '.githubRepository')
+  github_run_id=$(printf '%s' $upload_json | jq --raw-output '.githubRunId')
+  github_run_attempt=$(printf '%s' $upload_json | jq --raw-output '.githubRunAttempt')
+  github_job=$(printf '%s' $upload_json | jq --raw-output '.githubJob')
+  iteration=$(printf '%s' $upload_json | jq --raw-output '.iteration')
+
+  echo "Upload comes from GitHub repository ${github_repository}. It has GitHub run ID ${github_run_id}, run attempt number ${github_run_attempt}, and job name ${github_job}. It corresponds to loop iteration ${iteration}."
+
+  # Check whether we have a cached log for this job.
+  # (We cache the job logs because when running the tests continuously, with
+  # verbose logging enabled, a job log can be ~1.5GB.)
+
+  log_file_name="github-log-${github_repository//\//-}-run-${github_run_id}-attempt-${github_run_attempt}-job-${github_job}"
+  log_file_path="${cache_directory}/${log_file_name}"
+
+  if [[ -f "${log_file_path}" ]]; then
+    echo "GitHub job log file already exists at ${log_file_path}. Skipping download." 2>&1
+  else
+    echo "GitHub job log file not yet downloaded." 2>&1
+
+    # (I wonder if this information that I’m fetching from GitHub is stuff that
+    # I should have just had in the upload in the first place? Not that
+    # important right now.)
+
+    github_api_base_url="https://api.github.com"
+
+    # From the GitHub API, fetch the jobs for this workflow run attempt.
+    # https://docs.github.com/en/rest/reference/actions#list-jobs-for-a-workflow-run-attempt
+    github_jobs_json=$(curl \
+      --fail \
+      -H "Accept: application/vnd.github.v3+json" \
+      "${github_auth_curl_args[@]}" \
+      "${github_api_base_url}/repos/${github_repository}/actions/runs/${github_run_id}/attempts/${github_run_attempt}/jobs")
+
+    # From this list of jobs, find the one that corresponds to our upload.
+    github_job_id=$(printf "%s" $github_jobs_json | jq \
+      --arg jobName "${github_job}" \
+      '.jobs[] | select(.name == $jobName) | .id')
+
+    if [[ -z $github_job_id ]]; then
+      echo "Could not find job with name ${github_job} in attempt ${github_run_attempt} of run ${github_run_id} in GitHub repository ${github_repository}." 2>&1
+      exit 1
+    fi
+
+    echo "Upload corresponds to GitHub job ID ${github_job_id}. Downloading logs. This may take a while."
+
+    # From the GitHub API, fetch the logs for this job and cache them.
+    # https://docs.github.com/en/rest/reference/actions#download-job-logs-for-a-workflow-run
+
+    if [[ ! -d "${cache_directory}" ]]; then
+      mkdir -p "${cache_directory}"
+    fi
+
+    curl \
+      --fail \
+      --location \
+      -H "Accept: application/vnd.github.v3+json" \
+      "${github_auth_curl_args[@]}" \
+      "${github_api_base_url}/repos/${github_repository}/actions/jobs/${github_job_id}/logs" >"${log_file_path}.partial"
+
+    mv "${log_file_path}.partial" "${log_file_path}"
+
+    echo "Saved GitHub job logs to ${log_file_path}."
+  fi
+
+  # Extract the part of the logs that corresponds to the raw xcodebuild output for this iteration.
+  # https://stackoverflow.com/a/18870500
+
+  echo "Finding xcodebuild output for iteration ${iteration}."
+
+  xcodebuild_output_start_marker="ITERATION ${iteration}: BEGIN xcodebuild raw output"
+  xcodebuild_output_start_line_number=$(sed -n "/${xcodebuild_output_start_marker}/=" "${log_file_path}")
+
+  if [[ -z "${xcodebuild_output_start_line_number}" ]]; then
+    echo "Couldn’t find start of xcodebuild raw output (couldn’t find marker \"${xcodebuild_output_start_marker}\")." 2>&1
+    echo "This may be because the GitHub job hasn’t finished yet, or because the tests are not being run in a loop, or it may be an upload created before this functionality was implemented." 2>&1
+    echo "You may need to delete the cached log file ${log_file_path}." 2>&1
+    exit 1
+  fi
+
+  xcodebuild_output_end_marker="ITERATION ${iteration}: END xcodebuild raw output"
+  xcodebuild_output_end_line_number=$(sed -n "/${xcodebuild_output_end_marker}/=" "${log_file_path}")
+
+  if [[ -z "${xcodebuild_output_end_line_number}" ]]; then
+    echo "Couldn’t find end of xcodebuild raw output (couldn’t find marker \"${xcodebuild_output_end_marker}\")." 2>&1
+    exit 1
+  fi
+
+  # Strip the GitHub-added timestamps (which just correspond to the time that `cat` was executed on the log file, and hence aren’t of any use) from the start of each line.
+
+  echo "Stripping GitHub timestamps."
+
+  # https://arkit.co.in/print-given-range-of-lines-using-awk-perl-head-tail-and-python/
+  sed -n "${xcodebuild_output_start_line_number},${xcodebuild_output_end_line_number} p" "${log_file_path}" | sed -e 's/^[^ ]* //' >"${output_file}"
+
+  echo "Wrote xcodebuild output to ${output_file}." 2>&1
+}
+
+default_output_file_for_upload_id() {
+  echo "xcodebuild-logs-upload-$1.txt"
+}
+
+run_for_test_case() {
+  # From the test observability server API, fetch the test case and extract its
+  # properties.
+
+  echo "Fetching test case ${test_case_id} from ${upload_server_base_url}." 2>&1
+
+  test_case_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/test_cases/${test_case_id}")
+
+  test_class_name=$(printf '%s' $test_case_json | jq --raw-output '.testClassName')
+  test_case_name=$(printf '%s' $test_case_json | jq --raw-output '.testCaseName')
+
+  printf "Test case ${test_case_id} has test class name ${test_class_name} and test case name ${test_case_name}.\n\n"
+
+  # From the test observability server API, fetch the filtered uploads.
+
+  if [[ -z $filter ]]; then
+    filter_description="no filter"
+    filter_query=""
+  else
+    filter_description="filter ${filter}"
+    filter_query="?${filter}"
+  fi
+
+  echo "Fetching uploads for test case ${test_case_id}, with ${filter_description}, from ${upload_server_base_url}." 2>&1
+
+  uploads_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/test_cases/${test_case_id}/uploads${filter_query}")
+
+  number_of_uploads=$(printf '%s' $uploads_json | jq '. | length')
+
+  if [[ ${number_of_uploads} -eq 1 ]]; then
+    echo "There is 1 upload". 2>&1
+  else
+    echo "There are ${number_of_uploads} uploads". 2>&1
+  fi
+
+  echo
+
+  mkdir "${output_directory}"
+  mkdir "${output_directory}/upload_logs"
+  mkdir "${output_directory}/test_case_logs"
+
+  failed_upload_logs_output_directory="${output_directory}/upload_logs/failed"
+  mkdir "${failed_upload_logs_output_directory}"
+  not_failed_upload_logs_output_directory="${output_directory}/upload_logs/not_failed"
+  mkdir "${not_failed_upload_logs_output_directory}"
+
+  failed_test_case_logs_output_directory="${output_directory}/test_case_logs/failed"
+  mkdir "${failed_test_case_logs_output_directory}"
+  not_failed_test_case_logs_output_directory="${output_directory}/test_case_logs/not_failed"
+  mkdir "${not_failed_test_case_logs_output_directory}"
+
+  jq -n \
+    --arg testCaseId "${test_case_id}" \
+    --arg filter "${filter}" \
+    --arg uploadServerBaseUrl "${upload_server_base_url}" \
+    '{ fetchedAt: (now | todateiso8601), testCaseId: $testCaseId, filter: $filter, uploadServerBaseUrl: $uploadServerBaseUrl }' \
+    >"${output_directory}/info.json"
+
+  for ((i = 0; i < number_of_uploads; i += 1)); do
+    failed=$(printf '%s' $uploads_json | jq ".[${i}].failed")
+    upload_json=$(printf '%s' $uploads_json | jq ".[${i}].upload")
+
+    upload_id=$(printf '%s' $upload_json | jq --raw-output '.id')
+
+    echo "[$((i + 1)) of ${number_of_uploads}] Processing upload ${upload_id}." 2>&1
+
+    output_file_without_directory=$(default_output_file_for_upload_id "${upload_id}")
+
+    if [[ $failed == "true" ]]; then
+      upload_log_output_file="${failed_upload_logs_output_directory}/${output_file_without_directory}"
+      test_case_log_output_file="${failed_test_case_logs_output_directory}/${output_file_without_directory}"
+    else
+      upload_log_output_file="${not_failed_upload_logs_output_directory}/${output_file_without_directory}"
+      test_case_log_output_file="${not_failed_test_case_logs_output_directory}/${output_file_without_directory}"
+    fi
+
+    fetch_and_write_logs_for_upload "${upload_json}" "${upload_log_output_file}"
+
+    if [[ -z "${no_extract_test_case_logs}" ]]; then
+      extract_logs_for_test_case "${test_class_name}" "${test_case_name}" "${upload_log_output_file}" "${test_case_log_output_file}"
+    fi
+
+    echo
+  done
+}
+
+# Args:
+# $1: Test class name e.g. RealtimeClientPresenceTests
+# $2: Test case name e.g. test__037__Presence__update__should_update_the_data_for_the_present_member_with_a_value()
+# $3: Path of the xcodebuild logs for the entire test suite run
+# $4: Path of where to write the test case logs to.
+extract_logs_for_test_case() {
+  # Extract the part of the logs that corresponds to the raw xcodebuild output for this iteration. (We have similar code in fetch_and_write_logs_for_upload.)
+
+  test_class_name=$1
+  test_case_name=$2
+  upload_log_file=$3
+  output_file=$4
+
+  # (For some reason, the test case name in the observability server has
+  # trailing (), but in the xcodebuild logs it doesn’t. So strip them.)
+  sanitised_test_case_name="${test_case_name//[()]/}"
+
+  echo "Finding logs for test class ${test_class_name}, test case ${test_case_name} in ${upload_log_file}." 2>&1
+
+  test_case_log_start_marker="Test Case.*${test_class_name} ${sanitised_test_case_name}.*started"
+  test_case_log_start_line_number=$(sed -n "/${test_case_log_start_marker}/=" "${upload_log_file}")
+
+  if [[ -z "${test_case_log_start_line_number}" ]]; then
+    echo "Couldn’t find start of test case output (couldn’t find marker \"${test_case_log_start_marker}\")." 2>&1
+    exit 1
+  fi
+
+  test_case_log_end_marker="Test Case.*${test_class_name} ${sanitised_test_case_name}.*(passed|failed)|Restarting after unexpected exit, crash, or test timeout in ${test_class_name}\/${sanitised_test_case_name}\(\)"
+  test_case_log_end_line_number=$(sed -En "/${test_case_log_end_marker}/=" "${upload_log_file}")
+
+  if [[ -z "${test_case_log_end_line_number}" ]]; then
+    echo "Couldn’t find end of test case output (couldn’t find marker \"${test_case_log_end_marker}\")." 2>&1
+    exit 1
+  fi
+
+  sed -n "${test_case_log_start_line_number},${test_case_log_end_line_number} p" "${upload_log_file}" >"${output_file}"
+
+  echo "Wrote test case log to ${output_file}." 2>&1
+}
+
+run_for_upload() {
+  # From the test observability server API, fetch the upload, to find the
+  # GitHub run ID, attempt number, job name, and iteration.
+
+  echo "Fetching upload ${upload_id} from ${upload_server_base_url}." 2>&1
+
+  upload_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/uploads/${upload_id}")
+
+  fetch_and_write_logs_for_upload "${upload_json}" "${output_file}"
+}
+
+check_dependencies
+
+# Grab and validate command-line options, and apply defaults.
+
+# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    -r | --repo)
+      repo="$2"
+      shift
+      ;;
+    -t | --test-case-id)
+      if [[ -z "$2" ]]; then
+        echo "You must specify a test case ID when using the --test-case-id option." 2>&1
+        exit 1
+      fi
+      test_case_id="$2"
+      shift
+      ;;
+    -d | --output-directory)
+      if [[ -z "$2" ]]; then
+        echo "You must specify an output directory when using the --output-directory option." 2>&1
+        exit 1
+      fi
+      output_directory="$2"
+      shift
+      ;;
+    -f | --filter)
+      if [[ -z "$2" ]]; then
+        echo "You must specify a filter when using the --filter option." 2>&1
+        exit 1
+      fi
+      filter="$2"
+      shift
+      ;;
+    -n | --no-extract-test-case-logs) no_extract_test_case_logs="1" ;;
+    -i | --upload-id)
+      if [[ -z "$2" ]]; then
+        echo "You must specify an upload ID when using the --upload-id option." 2>&1
+        exit 1
+      fi
+      upload_id="$2"
+      shift
+      ;;
+    -u | --upload-server-base-url)
+      if [[ -z "$2" ]]; then
+        echo "You must specify a base URL when using the --upload-server-base-url option." 2>&1
+        exit 1
+      fi
+      upload_server_base_url="$2"
+      shift
+      ;;
+    -o | --output-file)
+      if [[ -z "$2" ]]; then
+        echo "You must specify an output file when using the --output-file option." 2>&1
+        exit 1
+      fi
+      output_file="$2"
+      shift
+      ;;
+    -c | --cache-directory)
+      if [[ -z "$2" ]]; then
+        echo "You must specify a cache directory when using the --cache-directory option." 2>&1
+        exit 1
+      fi
+      cache_directory="$2"
+      shift
+      ;;
+    -a | --no-use-github-auth) no_use_github_auth="1" ;;
+    *)
+      echo "Unknown parameter passed: $1" 2>&1
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [[ -z $repo ]]; then
+  echo "You need to specify a repo (-r / --repo)." 2>&1
+  exit 1
+fi
+
+if [[ -z $test_case_id && -z $upload_id ]]; then
+  echo "You need to specify the test case ID (-t / --test-case-id) or upload ID (-i / --upload-id)." 2>&1
+  exit 1
+fi
+
+if [[ -n $test_case_id && -n $upload_id ]]; then
+  echo "You cannot specify both a test case ID and an upload ID." 2>&1
+  exit 1
+fi
+
+if [[ -n $test_case_id && -n $upload_id ]]; then
+  echo "You cannot specify both a test case ID and an upload ID." 2>&1
+  exit 1
+fi
+
+if [[ -z $test_case_id && -n $output_directory ]]; then
+  echo "You can only specify an output directory with a test case ID (-t / --test-case-id)." 2>&1
+  exit 1
+fi
+
+if [[ -z $output_directory ]]; then
+  output_directory="xcodebuild-logs-test-case-${test_case_id}-$(date -Iseconds)"
+fi
+
+if [[ -z $test_case_id && -n $filter ]]; then
+  echo "You can only specify a filter with a test case ID (-t / --test-case-id)." 2>&1
+  exit 1
+fi
+
+if [[ -z $test_case_id && -n $no_extract_test_case_logs ]]; then
+  echo "You can only specify the --no-extract-test-case-logs option with a test case ID (-t / --test-case-id)." 2>&1
+  exit 1
+fi
+
+if [[ -z $upload_server_base_url ]]; then
+  upload_server_base_url="https://test-observability.herokuapp.com"
+fi
+
+if [[ -z $upload_id && -n $output_file ]]; then
+  echo "You can only specify an output file with an upload ID (-i / --upload-id)." 2>&1
+  exit 1
+fi
+
+if [[ -z $output_file ]]; then
+  output_file=$(default_output_file_for_upload_id "${upload_id}")
+fi
+
+if [[ -z $cache_directory ]]; then
+  cache_directory="${HOME}/Library/Caches/com.ably.testObservabilityLogs"
+fi
+
+github_auth_curl_args=()
+if [[ -z $no_use_github_auth ]]; then
+  # Get the GitHub access token from the user. We don’t allow them to specify it on the command line.
+  github_access_token=""
+  get_github_access_token
+  github_auth_curl_args+=(-H "Authorization: token ${github_access_token}")
+fi
+
+# Run the appropriate function based on arguments.
+
+if [[ -n $test_case_id ]]; then
+  run_for_test_case
+elif [[ -n $upload_id ]]; then
+  run_for_upload
+fi

From 2984c8c12bfafa74203be1bec32de15aedf49748 Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Mon, 11 Apr 2022 15:08:47 -0300
Subject: [PATCH 04/10] Upload .xcresult bundles as an artifact

---
 .../workflows/integration-test-iOS16_2.yaml   |  7 +++
 ...ntinuously-run-tests-and-upload-results.sh | 50 +++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml
index d3975db96..f6524c211 100644
--- a/.github/workflows/integration-test-iOS16_2.yaml
+++ b/.github/workflows/integration-test-iOS16_2.yaml
@@ -60,3 +60,10 @@ jobs:
           bundle install
           make update_carthage_dependencies_ios
           Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2
+
+      - name: Upload .xcresult bundles
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: xcresult-bundles
+          path: xcresult-bundles
diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh
index 95b1a5ff0..20a02ad5c 100755
--- a/Scripts/continuously-run-tests-and-upload-results.sh
+++ b/Scripts/continuously-run-tests-and-upload-results.sh
@@ -58,9 +58,59 @@ do
     optional_params+=(--upload-server-base-url "${upload_server_base_url}")
   fi
 
+  set +e
   ./Scripts/upload_test_results.sh \
     --iteration $iteration \
     "${optional_params[@]}"
+  # We defer failing the script until after copying the .xcresult bundle.
+  upload_exit_value=$?
+  set -e
+
+  if [[ upload_exit_value -eq 0 ]]
+  then
+    echo "ITERATION ${iteration}: Upload succeeded."
+  else
+    echo "ITERATION ${iteration}: Upload failed (exit value ${upload_exit_value}). Will exit after copying result bundle."
+  fi
+
+  # Find the .xcresult bundle and copy it to the directory that will eventually be saved as an artifact.
+
+  result_bundles=$(find fastlane/test_output/sdk -name '*.xcresult')
+  if [[ -z $result_bundles ]]
+  then
+    number_of_result_bundles=0
+  else
+    number_of_result_bundles=$(echo "${result_bundles}" | wc -l)
+  fi
+
+  if [[ $number_of_result_bundles -eq 0 ]]
+  then
+    echo "ITERATION ${iteration}: No result bundles found."
+    exit 1
+  fi
+
+  if [[ $number_of_result_bundles -gt 1 ]]
+  then
+    echo -e "ITERATION ${iteration}: Multiple result bundles found:\n${result_bundles}"
+    exit 1
+  fi
+
+  echo "ITERATION ${iteration}: Report bundle found: ${result_bundles}"
+
+  if [[ ! -d xcresult-bundles ]]; then
+    mkdir xcresult-bundles
+  fi
+
+  mkdir "xcresult-bundles/${iteration}"
+  cp -r "${result_bundles}" "xcresult-bundles/${iteration}"
+
+  echo "ITERATION ${iteration}: Copied result bundle to xcresult-bundles/${iteration}."
+
+  if [[ upload_exit_value -ne 0 ]]
+  then
+    echo "ITERATION ${iteration}: Terminating due to failed upload."
+    exit $upload_exit_value
+  fi
 
   echo "END ITERATION ${iteration}" 2>&1
 

From 39ab78725ae235df8cffa7ac635b843b70b3df62 Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 12 Apr 2022 10:32:23 -0300
Subject: [PATCH 05/10] =?UTF-8?q?Make=20sure=20continuously-run-tests-and-?=
 =?UTF-8?q?upload-results.sh=20doesn=E2=80=99t=20exceed=20GitHub=20job=20r?=
 =?UTF-8?q?unning=20time=20limit?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I’m trying to understand the reason that the upload artifacts step is
hung here [1], and I’m wondering if it’s because the job execution limit had
already been reached by my script.

[1] https://github.com/ably/ably-cocoa/runs/5979297645?check_suite_focus=true
---
 .../workflows/integration-test-iOS16_2.yaml   |  1 +
 ...ntinuously-run-tests-and-upload-results.sh | 41 +++++++++++++++++--
 2 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml
index f6524c211..937995a43 100644
--- a/.github/workflows/integration-test-iOS16_2.yaml
+++ b/.github/workflows/integration-test-iOS16_2.yaml
@@ -56,6 +56,7 @@ jobs:
           TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
         run: |
           brew install xcbeautify
+          brew install coreutils # for `timeout`
           make submodules
           bundle install
           make update_carthage_dependencies_ios
diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh
index 20a02ad5c..f58f30f78 100755
--- a/Scripts/continuously-run-tests-and-upload-results.sh
+++ b/Scripts/continuously-run-tests-and-upload-results.sh
@@ -2,7 +2,15 @@
 
 set -e
 
-# 1. Grab command-line options.
+# 1. Check dependencies.
+
+if ! which timeout > /dev/null
+then
+  echo "You need to install timeout (\`brew install coreutils\` on macOS)." 2>&1
+  exit 1
+fi
+
+# 2. Grab command-line options.
 
 # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
 while [[ "$#" -gt 0 ]]; do
@@ -20,7 +28,17 @@ then
   exit 1
 fi
 
-# 2. Run the tests in a loop and report the results.
+# 3. Capture the time at which we started, to make sure we don’t exceed the
+# maximum job running time.
+started_at=`date +%s`
+# https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration
+let github_job_maximum_execution_seconds=6*60*60
+# We assume that the part of the job that ran before this script took at most 10 minutes, and that uploading the artifacts will take 30 minutes.
+let must_end_by=$((started_at + github_job_maximum_execution_seconds - (10 + 30) * 60))
+
+echo "We’ll make sure this script ends by `date -r${must_end_by}`." 2>&1
+
+# 4. Run the tests in a loop and report the results.
 
 declare -i iteration=1
 while true
@@ -32,10 +50,27 @@ do
   xcrun simctl erase all
 
   set +e
-  bundle exec fastlane --verbose $lane
+  let allowed_execution_time=$must_end_by-`date +%s`
+  set -e
+
+  if [[ $allowed_execution_time -le 0 ]]; then
+    echo "ITERATION ${iteration}: Allowed execution time reached. Exiting." 2>&1
+    exit 0
+  fi
+
+  echo "ITERATION ${iteration}: Running fastlane with a timeout of ${allowed_execution_time} seconds." 2>&1
+
+  set +e
+  timeout --kill-after=20 ${allowed_execution_time} bundle exec fastlane --verbose $lane
   tests_exit_value=$?
   set -e
 
+  if [[ tests_exit_value -eq 124 || tests_exit_value -eq 137 ]]; then
+    # Execution timed out.
+    echo "ITERATION ${iteration}: Cancelled the execution of fastlane since it exceeded timeout imposed by maximum GitHub running time. Terminating this script."
+    exit 0
+  fi
+
   if [[ tests_exit_value -eq 0 ]]
   then
     echo "ITERATION ${iteration}: Tests passed."

From ff9efde727ea1cbdef0de9b3a0dbc52049dd09fa Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 12 Apr 2022 10:39:02 -0300
Subject: [PATCH 06/10] Log the size of .xcresult bundles waiting to be
 uploaded

To help with understanding issue described in 39ab787.
---
 ...ntinuously-run-tests-and-upload-results.sh | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh
index f58f30f78..c64f1a6a3 100755
--- a/Scripts/continuously-run-tests-and-upload-results.sh
+++ b/Scripts/continuously-run-tests-and-upload-results.sh
@@ -40,6 +40,17 @@ echo "We’ll make sure this script ends by `date -r${must_end_by}`." 2>&1
 
 # 4. Run the tests in a loop and report the results.
 
+end_iteration_with_exit_value() {
+  if [[ -e xcresult-bundles ]]
+  then
+    echo "There are `du -d0 -h xcresult-bundles | awk -F '\t' '{print $1}'` of xcresult bundles to be uploaded."
+  else
+    echo "There are no xcresult bundles to be uploaded."
+  fi
+
+  exit $1
+}
+
 declare -i iteration=1
 while true
 do
@@ -55,7 +66,7 @@ do
 
   if [[ $allowed_execution_time -le 0 ]]; then
     echo "ITERATION ${iteration}: Allowed execution time reached. Exiting." 2>&1
-    exit 0
+    end_iteration_with_exit_value 0
   fi
 
   echo "ITERATION ${iteration}: Running fastlane with a timeout of ${allowed_execution_time} seconds." 2>&1
@@ -68,7 +79,7 @@ do
   if [[ tests_exit_value -eq 124 || tests_exit_value -eq 137 ]]; then
     # Execution timed out.
     echo "ITERATION ${iteration}: Cancelled the execution of fastlane since it exceeded timeout imposed by maximum GitHub running time. Terminating this script."
-    exit 0
+    end_iteration_with_exit_value 0
   fi
 
   if [[ tests_exit_value -eq 0 ]]
@@ -121,13 +132,13 @@ do
   if [[ $number_of_result_bundles -eq 0 ]]
   then
     echo "ITERATION ${iteration}: No result bundles found."
-    exit 1
+    end_iteration_with_exit_value 1
   fi
 
   if [[ $number_of_result_bundles -gt 1 ]]
   then
     echo -e "ITERATION ${iteration}: Multiple result bundles found:\n${result_bundles}"
-    exit 1
+    end_iteration_with_exit_value 1
   fi
 
   echo "ITERATION ${iteration}: Report bundle found: ${result_bundles}"
@@ -144,7 +155,7 @@ do
   if [[ upload_exit_value -ne 0 ]]
   then
     echo "ITERATION ${iteration}: Terminating due to failed upload."
-    exit $upload_exit_value
+    end_iteration_with_exit_value $upload_exit_value
   fi
 
   echo "END ITERATION ${iteration}" 2>&1

From 4374542396ba67007b06e17fa28e93eca1a63b0c Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 12 Apr 2022 10:58:21 -0300
Subject: [PATCH 07/10] Tar and zip the xcresult bundles
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I’m hoping this will reduce the upload time (more by reducing the number
of files than the size).
---
 .github/workflows/integration-test-iOS16_2.yaml      | 4 ++--
 Scripts/continuously-run-tests-and-upload-results.sh | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml
index 937995a43..febfcec50 100644
--- a/.github/workflows/integration-test-iOS16_2.yaml
+++ b/.github/workflows/integration-test-iOS16_2.yaml
@@ -66,5 +66,5 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: xcresult-bundles
-          path: xcresult-bundles
+          name: xcresult-bundles.tar.gz
+          path: xcresult-bundles.tar.gz
diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh
index c64f1a6a3..2b9346109 100755
--- a/Scripts/continuously-run-tests-and-upload-results.sh
+++ b/Scripts/continuously-run-tests-and-upload-results.sh
@@ -44,6 +44,8 @@ end_iteration_with_exit_value() {
   if [[ -e xcresult-bundles ]]
   then
     echo "There are `du -d0 -h xcresult-bundles | awk -F '\t' '{print $1}'` of xcresult bundles to be uploaded."
+    tar --create --gzip xcresult-bundles > xcresult-bundles.tar.gz
+    echo "The file xcresult-bundles.tar.gz that will be uploaded as an artifact is `du -d0 -h xcresult-bundles.tar.gz | awk -F '\t' '{print $1}'`."
   else
     echo "There are no xcresult bundles to be uploaded."
   fi

From c9c00d2357533cdb0782ace33fce96c2231d1cb6 Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Thu, 12 May 2022 17:18:00 -0300
Subject: [PATCH 08/10] Add script for generating multiple jobs and workflows

This lets us choose the length and parallelism of our test runs.
---
 Scripts/set-ci-length-and-parallelism.sh | 104 +++++++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100755 Scripts/set-ci-length-and-parallelism.sh

diff --git a/Scripts/set-ci-length-and-parallelism.sh b/Scripts/set-ci-length-and-parallelism.sh
new file mode 100755
index 000000000..efd705441
--- /dev/null
+++ b/Scripts/set-ci-length-and-parallelism.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -e
+
+# Usage:
+# ./set-ci-length-and-parallelism.sh --workflows <num> --jobs-per-workflow <num>
+
+# Check dependencies.
+if ! which yq > /dev/null; then
+  echo "You need to install yq." 2>&1
+  exit 1
+fi
+
+# Grab and validate command-line options.
+
+# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    --workflows)
+      if [[ -z "$2" ]]; then
+        echo "You must specify the number of workflows." 2>&1
+        exit 1
+      fi
+      num_workflows="$2"
+      shift
+      ;;
+    --jobs-per-workflow)
+      if [[ -z "$2" ]]; then
+        echo "You must specify the number of jobs per workflow." 2>&1
+        exit 1
+      fi
+      jobs_per_workflow="$2"
+      shift
+      ;;
+    *)
+      echo "Unknown parameter passed: $1" 2>&1
+      exit 1
+      ;;
+  esac
+  shift
+done
+
+if [[ -z $num_workflows ]]; then
+  echo "You need to specify the number of workflows (--workflows)." 2>&1
+  exit 1
+fi
+
+if [[ ! $num_workflows =~ ^-?[0-9]+$ ]]; then
+  echo "The number of workflows must be a number." 2>&1
+  exit 1
+fi
+
+if [[ $num_workflows -lt 1 ]]; then
+  echo "The number of workflows must be 1 or more." 2>&1
+  exit 1
+fi
+
+if [[ -z $jobs_per_workflow ]]; then
+  echo "You need to specify the number of jobs per workflow (--jobs-per-workflow)." 2>&1
+  exit 1
+fi
+
+if [[ ! $jobs_per_workflow =~ ^-?[0-9]+$ ]]; then
+  echo "The number of jobs per workflow must be a number." 2>&1
+  exit 1
+fi
+
+if [[ $jobs_per_workflow -lt 1 ]]; then
+  echo "The number of jobs per workflow must be 1 or more." 2>&1
+  exit 1
+fi
+
+workflow_file_without_extension=".github/workflows/integration-test-iOS16_2"
+workflow_file_extension=".yaml"
+
+workflow_file="${workflow_file_without_extension}${workflow_file_extension}"
+workflow_name=$(yq .name $workflow_file)
+
+# First, we apply the number of jobs per workflow.
+
+yq -i '(.jobs.check | key) = "check-1"' $workflow_file
+yq -i "(.jobs.check-1.steps[] | select(.with.path == \"xcresult-bundles.tar.gz\")).with.name = \"xcresult-bundles-1.tar.gz\"" $workflow_file
+
+for ((i=2; i <= $jobs_per_workflow; i += 1))
+do
+  yq -i ".jobs.check-${i} = .jobs.check-$(($i-1))" $workflow_file
+  yq -i ".jobs.check-${i}.needs = [\"check-$(($i-1))\"]" $workflow_file
+  yq -i "(.jobs.check-${i}.steps[] | select(.with.path == \"xcresult-bundles.tar.gz\")).with.name = \"xcresult-bundles-${i}.tar.gz\"" $workflow_file
+done
+
+# Now, we duplicate the workflow file the requested number of times.
+
+mv $workflow_file "${workflow_file_without_extension}-1${workflow_file_extension}"
+
+for ((i=1; i <= $num_workflows; i += 1))
+do
+  new_workflow_file="${workflow_file_without_extension}-${i}${workflow_file_extension}"
+
+  if [[ $i -gt 1 ]]; then
+    cp "${workflow_file_without_extension}-$((i-1))${workflow_file_extension}" $new_workflow_file
+  fi
+
+  yq -i ".name = \"${workflow_name} (workflow ${i})\"" $new_workflow_file
+done

From 79fa00f14e8e4163183f03c1753e9e7edf8c6a9f Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 7 Nov 2023 14:44:23 -0300
Subject: [PATCH 09/10] Try to reduce failures in some tests that exercise
 token expiry
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

I’ve noticed these tests intermittently failing on CI. They all wait for
the Ably service to consider a token as having expired by waiting for
its `ttl` to elapse. However, clock discrepancies between Ably servers
might mean that this doesn’t always work. So let’s try adding a bit of a
tolerance to the wait and see if it helps.

There are probably other tests that are affected similarly; these are
just the ones that caught my eye.
---
 Test/Test Utilities/TestUtilities.swift        |  5 +++++
 Test/Tests/AuthTests.swift                     |  5 +++--
 Test/Tests/RealtimeClientConnectionTests.swift |  2 +-
 Test/Tests/RestClientTests.swift               | 15 +++++++++------
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/Test/Test Utilities/TestUtilities.swift b/Test/Test Utilities/TestUtilities.swift
index 92aa7e809..403203dbd 100644
--- a/Test/Test Utilities/TestUtilities.swift	
+++ b/Test/Test Utilities/TestUtilities.swift	
@@ -346,6 +346,11 @@ class AblyTests {
 
         return .init(sequence)
     }
+
+    /**
+     Tests that wait for the Ably service to consider a token as having expired by waiting for its `ttl` to elapse should wait this additional tolerance (arbitarily chosen) to compensate for clock differences between different Ably servers.
+     */
+    static let tokenExpiryTolerance: TimeInterval = 2
 }
 
 /// A helper class for performing HTTP requests synchronously in tests.
diff --git a/Test/Tests/AuthTests.swift b/Test/Tests/AuthTests.swift
index 1b1d96817..149f79c48 100644
--- a/Test/Tests/AuthTests.swift
+++ b/Test/Tests/AuthTests.swift
@@ -262,13 +262,14 @@ class AuthTests: XCTestCase {
     func test__021__Token__authentication_method__should_transition_the_connection_to_the_FAILED_state_when_the_server_responds_with_a_token_error_and_there_is_no_way_to_renew_the_token() throws {
         let test = Test()
         let options = try AblyTests.clientOptions(for: test)
-        options.tokenDetails = try getTestTokenDetails(for: test, ttl: 0.1)
+        let tokenTtl = 0.1
+        options.tokenDetails = try getTestTokenDetails(for: test, ttl: tokenTtl)
         options.autoConnect = false
         options.testOptions.transportFactory = TestProxyTransportFactory()
 
         // Token will expire, expecting 40142
         waitUntil(timeout: testTimeout) { done in
-            delay(0.2) { done() }
+            delay(tokenTtl + AblyTests.tokenExpiryTolerance) { done() }
         }
 
         let realtime = ARTRealtime(options: options)
diff --git a/Test/Tests/RealtimeClientConnectionTests.swift b/Test/Tests/RealtimeClientConnectionTests.swift
index 346b4689d..f58dd1e07 100644
--- a/Test/Tests/RealtimeClientConnectionTests.swift
+++ b/Test/Tests/RealtimeClientConnectionTests.swift
@@ -2184,7 +2184,7 @@ class RealtimeClientConnectionTests: XCTestCase {
 
         // Let the token expire
         waitUntil(timeout: testTimeout) { done in
-            delay(tokenTtl) {
+            delay(tokenTtl + AblyTests.tokenExpiryTolerance) {
                 done()
             }
         }
diff --git a/Test/Tests/RestClientTests.swift b/Test/Tests/RestClientTests.swift
index 923dab460..af7a470a4 100644
--- a/Test/Tests/RestClientTests.swift
+++ b/Test/Tests/RestClientTests.swift
@@ -689,7 +689,8 @@ class RestClientTests: XCTestCase {
         let auth = client.auth
 
         let tokenParams = ARTTokenParams()
-        tokenParams.ttl = 3.0 // Seconds
+        let tokenTtl = 3.0
+        tokenParams.ttl = NSNumber(value: tokenTtl) // Seconds
 
         let options: ARTClientOptions = try AblyTests.waitFor(timeout: testTimeout) { value in
             auth.requestToken(tokenParams, with: nil) { tokenDetails, error in
@@ -735,7 +736,7 @@ class RestClientTests: XCTestCase {
 
         waitUntil(timeout: testTimeout) { done in
             // Delay for token expiration
-            delay(TimeInterval(truncating: tokenParams.ttl!)) {
+            delay(tokenTtl + AblyTests.tokenExpiryTolerance) {
                 // [40140, 40150) - token expired and will not recover because authUrl is invalid
                 publishTestMessage(rest, channelName: test.uniqueChannelName()) { error in
                     guard let errorCode = testHTTPExecutor.responses.first?.value(forHTTPHeaderField: "X-Ably-Errorcode") else {
@@ -758,7 +759,8 @@ class RestClientTests: XCTestCase {
         let auth = client.auth
 
         let tokenParams = ARTTokenParams()
-        tokenParams.ttl = 3.0 // Seconds
+        let tokenTtl = 3.0
+        tokenParams.ttl = NSNumber(value: tokenTtl) // Seconds
 
         waitUntil(timeout: testTimeout) { done in
             auth.requestToken(tokenParams, with: nil) { tokenDetails, error in
@@ -798,7 +800,7 @@ class RestClientTests: XCTestCase {
                 rest.internal.httpExecutor = testHTTPExecutor
 
                 // Delay for token expiration
-                delay(TimeInterval(truncating: tokenParams.ttl!)) {
+                delay(tokenTtl + AblyTests.tokenExpiryTolerance) {
                     // [40140, 40150) - token expired and will not recover because authUrl is invalid
                     publishTestMessage(rest, channelName: test.uniqueChannelName()) { error in
                         guard let errorCode = testHTTPExecutor.responses.first?.value(forHTTPHeaderField: "X-Ably-Errorcode") else {
@@ -1744,10 +1746,11 @@ class RestClientTests: XCTestCase {
     func test__012__RestClient__should_indicate_an_error_if_there_is_no_way_to_renew_the_token() throws {
         let test = Test()
         let options = try AblyTests.clientOptions(for: test)
-        options.token = try getTestToken(for: test, ttl: 0.1)
+        let tokenTtl = 0.1
+        options.token = try getTestToken(for: test, ttl: tokenTtl)
         let client = ARTRest(options: options)
         waitUntil(timeout: testTimeout) { done in
-            delay(0.1) {
+            delay(tokenTtl + AblyTests.tokenExpiryTolerance) {
                 client.channels.get(test.uniqueChannelName()).publish(nil, data: "message") { error in
                     guard let error = error else {
                         fail("Error is empty"); done()

From 0b38683b747b131c60c4c75255890a2bcd42a5c7 Mon Sep 17 00:00:00 2001
From: Lawrence Forooghian <lawrence@forooghian.com>
Date: Tue, 7 Nov 2023 15:17:23 -0300
Subject: [PATCH 10/10] ./Scripts/set-ci-length-and-parallelism.sh --workflows
 2 --jobs-per-workflow 2

---
 ...2.yaml => integration-test-iOS16_2-1.yaml} |  68 ++++++++---
 .../workflows/integration-test-iOS16_2-2.yaml | 108 ++++++++++++++++++
 2 files changed, 161 insertions(+), 15 deletions(-)
 rename .github/workflows/{integration-test-iOS16_2.yaml => integration-test-iOS16_2-1.yaml} (51%)
 create mode 100644 .github/workflows/integration-test-iOS16_2-2.yaml

diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2-1.yaml
similarity index 51%
rename from .github/workflows/integration-test-iOS16_2.yaml
rename to .github/workflows/integration-test-iOS16_2-1.yaml
index febfcec50..61cb89c5e 100644
--- a/.github/workflows/integration-test-iOS16_2.yaml
+++ b/.github/workflows/integration-test-iOS16_2-1.yaml
@@ -1,56 +1,45 @@
-name: "Integration Test: iOS 16.2"
-
+name: "Integration Test: iOS 16.2 (workflow 1)"
 on:
   pull_request:
   push:
     branches:
       - main
-
 # IMPORTANT NOTES:
 # - Changes made to this file needs to replicated across other integration-test-*.yaml files.
 # - The Fastlane lane name is duplicated in more than one place within this workflow.
-
 jobs:
-  check:
+  check-1:
     runs-on: macos-latest
-
     env:
       LC_CTYPE: en_US.UTF-8
       LANG: en_US.UTF-8
       ABLY_ENV: sandbox
-
     steps:
       - name: Check out SDK repo
         uses: actions/checkout@v2
-        
       - name: Select Specific Xcode Version (14.2)
         run: |
           sudo xcode-select -s /Applications/Xcode_14.2.app
           echo "Selected Xcode version:"
           xcodebuild -version
-
       - name: Log environment information
         run: ./Scripts/log-environment-information.sh
-
       - name: Check out xcparse repo
         uses: actions/checkout@v3
         with:
           repository: ably-forks/xcparse
           ref: emit-test-case-info
           path: xcparse
-
       - id: get-xcparse-commit-sha
         name: Get xcparse commit SHA
         run: |
           cd xcparse
           echo "::set-output name=sha::$(git rev-parse HEAD)"
-
       - name: "actions/cache@v3 (xcparse binary)"
         uses: actions/cache@v3
         with:
           path: xcparse/.build/debug/xcparse
           key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
-
       - name: Install Dependencies and Run Tests Continuously
         env:
           TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
@@ -61,10 +50,59 @@ jobs:
           bundle install
           make update_carthage_dependencies_ios
           Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2
-
       - name: Upload .xcresult bundles
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: xcresult-bundles.tar.gz
+          name: xcresult-bundles-1.tar.gz
           path: xcresult-bundles.tar.gz
+  check-2:
+    runs-on: macos-latest
+    env:
+      LC_CTYPE: en_US.UTF-8
+      LANG: en_US.UTF-8
+      ABLY_ENV: sandbox
+    steps:
+      - name: Check out SDK repo
+        uses: actions/checkout@v2
+      - name: Select Specific Xcode Version (14.2)
+        run: |
+          sudo xcode-select -s /Applications/Xcode_14.2.app
+          echo "Selected Xcode version:"
+          xcodebuild -version
+      - name: Log environment information
+        run: ./Scripts/log-environment-information.sh
+      - name: Check out xcparse repo
+        uses: actions/checkout@v3
+        with:
+          repository: ably-forks/xcparse
+          ref: emit-test-case-info
+          path: xcparse
+      - id: get-xcparse-commit-sha
+        name: Get xcparse commit SHA
+        run: |
+          cd xcparse
+          echo "::set-output name=sha::$(git rev-parse HEAD)"
+      - name: "actions/cache@v3 (xcparse binary)"
+        uses: actions/cache@v3
+        with:
+          path: xcparse/.build/debug/xcparse
+          key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
+      - name: Install Dependencies and Run Tests Continuously
+        env:
+          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
+        run: |
+          brew install xcbeautify
+          brew install coreutils # for `timeout`
+          make submodules
+          bundle install
+          make update_carthage_dependencies_ios
+          Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2
+      - name: Upload .xcresult bundles
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: xcresult-bundles-2.tar.gz
+          path: xcresult-bundles.tar.gz
+    needs:
+      - check-1
diff --git a/.github/workflows/integration-test-iOS16_2-2.yaml b/.github/workflows/integration-test-iOS16_2-2.yaml
new file mode 100644
index 000000000..333674e85
--- /dev/null
+++ b/.github/workflows/integration-test-iOS16_2-2.yaml
@@ -0,0 +1,108 @@
+name: "Integration Test: iOS 16.2 (workflow 2)"
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+# IMPORTANT NOTES:
+# - Changes made to this file needs to replicated across other integration-test-*.yaml files.
+# - The Fastlane lane name is duplicated in more than one place within this workflow.
+jobs:
+  check-1:
+    runs-on: macos-latest
+    env:
+      LC_CTYPE: en_US.UTF-8
+      LANG: en_US.UTF-8
+      ABLY_ENV: sandbox
+    steps:
+      - name: Check out SDK repo
+        uses: actions/checkout@v2
+      - name: Select Specific Xcode Version (14.2)
+        run: |
+          sudo xcode-select -s /Applications/Xcode_14.2.app
+          echo "Selected Xcode version:"
+          xcodebuild -version
+      - name: Log environment information
+        run: ./Scripts/log-environment-information.sh
+      - name: Check out xcparse repo
+        uses: actions/checkout@v3
+        with:
+          repository: ably-forks/xcparse
+          ref: emit-test-case-info
+          path: xcparse
+      - id: get-xcparse-commit-sha
+        name: Get xcparse commit SHA
+        run: |
+          cd xcparse
+          echo "::set-output name=sha::$(git rev-parse HEAD)"
+      - name: "actions/cache@v3 (xcparse binary)"
+        uses: actions/cache@v3
+        with:
+          path: xcparse/.build/debug/xcparse
+          key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
+      - name: Install Dependencies and Run Tests Continuously
+        env:
+          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
+        run: |
+          brew install xcbeautify
+          brew install coreutils # for `timeout`
+          make submodules
+          bundle install
+          make update_carthage_dependencies_ios
+          Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2
+      - name: Upload .xcresult bundles
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: xcresult-bundles-1.tar.gz
+          path: xcresult-bundles.tar.gz
+  check-2:
+    runs-on: macos-latest
+    env:
+      LC_CTYPE: en_US.UTF-8
+      LANG: en_US.UTF-8
+      ABLY_ENV: sandbox
+    steps:
+      - name: Check out SDK repo
+        uses: actions/checkout@v2
+      - name: Select Specific Xcode Version (14.2)
+        run: |
+          sudo xcode-select -s /Applications/Xcode_14.2.app
+          echo "Selected Xcode version:"
+          xcodebuild -version
+      - name: Log environment information
+        run: ./Scripts/log-environment-information.sh
+      - name: Check out xcparse repo
+        uses: actions/checkout@v3
+        with:
+          repository: ably-forks/xcparse
+          ref: emit-test-case-info
+          path: xcparse
+      - id: get-xcparse-commit-sha
+        name: Get xcparse commit SHA
+        run: |
+          cd xcparse
+          echo "::set-output name=sha::$(git rev-parse HEAD)"
+      - name: "actions/cache@v3 (xcparse binary)"
+        uses: actions/cache@v3
+        with:
+          path: xcparse/.build/debug/xcparse
+          key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }}
+      - name: Install Dependencies and Run Tests Continuously
+        env:
+          TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }}
+        run: |
+          brew install xcbeautify
+          brew install coreutils # for `timeout`
+          make submodules
+          bundle install
+          make update_carthage_dependencies_ios
+          Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2
+      - name: Upload .xcresult bundles
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: xcresult-bundles-2.tar.gz
+          path: xcresult-bundles.tar.gz
+    needs:
+      - check-1