From aaf6b69d42621fd51114313e6d457611e13054f2 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Fri, 3 Nov 2023 14:50:40 -0300 Subject: [PATCH 01/14] Make creation of ARTRealtimeTransportError a bit more readable --- Source/ARTWebSocketTransport.m | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Source/ARTWebSocketTransport.m b/Source/ARTWebSocketTransport.m index 3ac74e466..f0394e87f 100644 --- a/Source/ARTWebSocketTransport.m +++ b/Source/ARTWebSocketTransport.m @@ -282,9 +282,14 @@ - (void)webSocket:(id)webSocket didCloseWithCode:(NSInteger)code r [_delegate realtimeTransportDisconnected:self withError:nil]; break; case ARTWsRefuse: - case ARTWsPolicyValidation: - [_delegate realtimeTransportRefused:self withError:[[ARTRealtimeTransportError alloc] initWithError:[ARTErrorInfo createWithCode:code message:reason] type:ARTRealtimeTransportErrorTypeRefused url:self.websocketURL]]; + case ARTWsPolicyValidation: { + ARTErrorInfo *const errorInfo = [ARTErrorInfo createWithCode:code message:reason]; + ARTRealtimeTransportError *const error = [[ARTRealtimeTransportError alloc] initWithError:errorInfo + type:ARTRealtimeTransportErrorTypeRefused + url:self.websocketURL]; + [_delegate realtimeTransportRefused:self withError:error]; break; + } case ARTWsTooBig: [_delegate realtimeTransportTooBig:self]; break; @@ -292,10 +297,15 @@ - (void)webSocket:(id)webSocket didCloseWithCode:(NSInteger)code r case ARTWsCloseProtocolError: case ARTWsUnexpectedCondition: case ARTWsExtension: - case ARTWsTlsError: + case ARTWsTlsError: { // Failed - [_delegate realtimeTransportFailed:self withError:[[ARTRealtimeTransportError alloc] initWithError:[ARTErrorInfo createWithCode:code message:reason] type:ARTRealtimeTransportErrorTypeOther url:self.websocketURL]]; + ARTErrorInfo *const errorInfo = [ARTErrorInfo createWithCode:code message:reason]; + ARTRealtimeTransportError *const error = [[ARTRealtimeTransportError alloc] initWithError:errorInfo + type:ARTRealtimeTransportErrorTypeOther + url:self.websocketURL]; + [_delegate realtimeTransportFailed:self withError:error]; break; + } default: NSAssert(true, @"WebSocket close: unknown code"); break; From 3c0875a5add89ded160001312fd1a164017ee107 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Fri, 3 Nov 2023 14:55:47 -0300 Subject: [PATCH 02/14] Document validity of ARTRealtimeTransportError badResponseCode --- Source/PrivateHeaders/Ably/ARTRealtimeTransport.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/PrivateHeaders/Ably/ARTRealtimeTransport.h b/Source/PrivateHeaders/Ably/ARTRealtimeTransport.h index 46d02d544..96dc62310 100644 --- a/Source/PrivateHeaders/Ably/ARTRealtimeTransport.h +++ b/Source/PrivateHeaders/Ably/ARTRealtimeTransport.h @@ -32,6 +32,9 @@ typedef NS_ENUM(NSUInteger, ARTRealtimeTransportState) { @property (nonatomic) NSError *error; @property (nonatomic) ARTRealtimeTransportErrorType type; +/** + This meaning of this property is only defined if the error is of type `ARTRealtimeTransportErrorTypeBadResponse`. + */ @property (nonatomic) NSInteger badResponseCode; @property (nonatomic) NSURL *url; From e3604ea7b24beb11aa08f460038c4de745ecc906 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Mon, 6 Nov 2023 09:34:23 -0300 Subject: [PATCH 03/14] Explain approach of fallback host test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’m going to need to change this test in an upcoming commit and want the original intent to be clear before I do. --- Test/Tests/RealtimeClientConnectionTests.swift | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Test/Tests/RealtimeClientConnectionTests.swift b/Test/Tests/RealtimeClientConnectionTests.swift index 734aa8845..ad6e39514 100644 --- a/Test/Tests/RealtimeClientConnectionTests.swift +++ b/Test/Tests/RealtimeClientConnectionTests.swift @@ -3708,12 +3708,16 @@ class RealtimeClientConnectionTests: XCTestCase { client.connect() defer { client.dispose(); client.close() } + // We expect the first connection attempt to fail due to the .fakeNetworkResponse configured above. This error does not meet the criteria for trying a fallback host, and so should not provoke the use of a fallback host. Hence the connection should give up and transition to the FAILED state (which causes the publish to fail). We should see that there was only one connection attempt, to the primary host. + waitUntil(timeout: testTimeout) { done in - channel.publish(nil, data: "message") { _ in + channel.publish(nil, data: "message") { error in + XCTAssertNotNil(error) done() } } + XCTAssertEqual(client.connection.state, .failed) XCTAssertEqual(urlConnections.count, 1) XCTAssertTrue(NSRegularExpression.match(urlConnections[0].absoluteString, pattern: "//realtime.ably.io")) } From 0f1a91292109b744b0925e85ca3d70e974029b53 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Mon, 6 Nov 2023 11:52:10 -0300 Subject: [PATCH 04/14] Remove use of non-routable IP address in RTN14d test The use of the non-completing authCallback already ensures a timeout. --- Test/Tests/RealtimeClientConnectionTests.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Test/Tests/RealtimeClientConnectionTests.swift b/Test/Tests/RealtimeClientConnectionTests.swift index ad6e39514..bf7c1c632 100644 --- a/Test/Tests/RealtimeClientConnectionTests.swift +++ b/Test/Tests/RealtimeClientConnectionTests.swift @@ -2262,7 +2262,6 @@ class RealtimeClientConnectionTests: XCTestCase { func test__059__Connection__connection_request_fails__connection_attempt_fails_for_any_recoverable_reason() throws { let test = Test() let options = try AblyTests.commonAppSetup(for: test) - options.realtimeHost = "10.255.255.1" // non-routable IP address options.disconnectedRetryTimeout = 1.0 options.autoConnect = false options.testOptions.realtimeRequestTimeout = 0.1 From 30a09795b97948d41b0959ee37428955171c77b4 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Mon, 6 Nov 2023 12:00:55 -0300 Subject: [PATCH 05/14] Refactor RTN14d test to allow configuring failure reason Want to add a new test with another failure reason. --- .../Tests/RealtimeClientConnectionTests.swift | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/Test/Tests/RealtimeClientConnectionTests.swift b/Test/Tests/RealtimeClientConnectionTests.swift index bf7c1c632..4524c05e1 100644 --- a/Test/Tests/RealtimeClientConnectionTests.swift +++ b/Test/Tests/RealtimeClientConnectionTests.swift @@ -2258,21 +2258,21 @@ class RealtimeClientConnectionTests: XCTestCase { } } - // RTN14d, RTB1 - func test__059__Connection__connection_request_fails__connection_attempt_fails_for_any_recoverable_reason() throws { - let test = Test() + func testRTN14dAndRTB1( + test: Test, + extraTimeNeededToObserveEachRetry: TimeInterval = 0, + modifyOptions: (ARTClientOptions) -> Void, + checkError: (ARTErrorInfo) -> Void + ) throws { let options = try AblyTests.commonAppSetup(for: test) options.disconnectedRetryTimeout = 1.0 options.autoConnect = false - options.testOptions.realtimeRequestTimeout = 0.1 let jitterCoefficients = StaticJitterCoefficients() let mockJitterCoefficientGenerator = MockJitterCoefficientGenerator(coefficients: jitterCoefficients) options.testOptions.jitterCoefficientGenerator = mockJitterCoefficientGenerator - options.authCallback = { _, _ in - // Ignore `completion` closure to force a time out - } + modifyOptions(options) let numberOfRetriesToWaitFor = 5 // arbitrarily chosen, large enough for us to have confidence that a sequence of retries is occurring, with the correct retry delays @@ -2283,7 +2283,7 @@ class RealtimeClientConnectionTests: XCTestCase { ).prefix(numberOfRetriesToWaitFor + 1 /* The +1 can be removed after #1782 is fixed; see note below */) ) let timesNeededToObserveRetries = expectedRetryDelays.map { retryDelay in - options.testOptions.realtimeRequestTimeout // waiting for connect to time out + extraTimeNeededToObserveEachRetry + retryDelay // waiting for retry to occur + 0.2 // some extra tolerance, arbitrarily chosen } @@ -2341,7 +2341,7 @@ class RealtimeClientConnectionTests: XCTestCase { let firstObservedStateChange = observedStateChanges[observedStateChangesStartIndexForThisRetry] XCTAssertEqual(firstObservedStateChange.stateChange.previous, .connecting) XCTAssertEqual(firstObservedStateChange.stateChange.current, .disconnected) - XCTAssertTrue(firstObservedStateChange.stateChange.reason!.message.contains("timed out")) + checkError(firstObservedStateChange.stateChange.reason!) XCTAssertEqual(firstObservedStateChange.stateChange.retryIn, expectedRetryDelay) if (firstObservedStateChangeToDisconnected == nil) { @@ -2370,6 +2370,25 @@ class RealtimeClientConnectionTests: XCTestCase { .to(beLessThan(connectionStateTtl + timeTakenByPotentialExcessRetry + tolerance)) } + // RTN14d, RTB1 + func test__059__Connection__connection_request_fails__connection_attempt_fails_for_any_recoverable_reason__for_example_a_timeout() throws { + let test = Test() + + let realtimeRequestTimeout = 0.1 + + try testRTN14dAndRTB1(test: test, + extraTimeNeededToObserveEachRetry: realtimeRequestTimeout, // waiting for connect to time out + modifyOptions: { options in + options.testOptions.realtimeRequestTimeout = realtimeRequestTimeout + options.authCallback = { _, _ in + // Ignore `completion` closure to force a time out + } + }, + checkError: { error in + XCTAssertTrue(error.message.contains("timed out")) + }) + } + // RTN14e func test__060__Connection__connection_request_fails__connection_state_has_been_in_the_DISCONNECTED_state_for_more_than_the_default_connectionStateTtl_should_change_the_state_to_SUSPENDED() throws { let test = Test() From 86b7cc72d1cf593bdd81d1a62021f7c0212c3210 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Fri, 3 Nov 2023 15:24:49 -0300 Subject: [PATCH 06/14] Treat all transport errors as recoverable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RTN14d says that a transport error should be considered recoverable if it is > a network failure, a timeout such as RTN14c, or a disconnected > response, other than a token failure RTN14b) However, it does not define what it means by a "network failure", leading to each platform having to provide its own interpretation. In particular, a client recently told us that they’ve been seeing lots of OSStatus 9806 (errSSLClosedAbort) errors. This appears to indicate some sort of failure to perform an SSL handshake. We don’t understand the cause of this issue but we noticed that it was causing the client to transition to the FAILED state. Speaking to Paddy, he said that this error should be provoking a connection retry, not failure. And more broadly, he indicated that, basically, _all_ transport errors should be considered recoverable. So, that’s what we do here. He’s raised a specification issue [1] for us to properly specify this and to decide if there are any nuances to consider, but is keen for us to implement this broad behaviour in ably-cocoa ASAP to help the customer experiencing the errSSLClosedAbort errors. Resolves #1817. [1] https://github.com/ably/specification/issues/171 --- Source/ARTRealtime.m | 30 ++++------ Test/Test Utilities/TestUtilities.swift | 18 ++++-- .../Tests/RealtimeClientConnectionTests.swift | 56 +++++++++++++------ 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/Source/ARTRealtime.m b/Source/ARTRealtime.m index ac1b0af9e..d207b27c8 100644 --- a/Source/ARTRealtime.m +++ b/Source/ARTRealtime.m @@ -1629,21 +1629,10 @@ - (void)realtimeTransportFailed:(id)transport withError:(A return; } } - - switch (transportError.type) { - case ARTRealtimeTransportErrorTypeBadResponse: - case ARTRealtimeTransportErrorTypeOther: { - ARTErrorInfo *const errorInfo = [ARTErrorInfo createFromNSError:transportError.error]; - ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:errorInfo]; - [self transition:ARTRealtimeFailed withMetadata:metadata]; - break; - } - default: { - ARTErrorInfo *error = [ARTErrorInfo createFromNSError:transportError.error]; - ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:error]; - [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; - } - } + + ARTErrorInfo *const errorInfo = [ARTErrorInfo createFromNSError:transportError.error]; + ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:errorInfo]; + [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; } - (void)realtimeTransportNeverConnected:(id)transport { @@ -1654,7 +1643,7 @@ - (void)realtimeTransportNeverConnected:(id)transport { ARTErrorInfo *const errorInfo = [ARTErrorInfo createWithCode:ARTClientCodeErrorTransport message:@"Transport never connected"]; ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:errorInfo]; - [self transition:ARTRealtimeFailed withMetadata:metadata]; + [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; } - (void)realtimeTransportRefused:(id)transport withError:(ARTRealtimeTransportError *)error { @@ -1666,15 +1655,16 @@ - (void)realtimeTransportRefused:(id)transport withError:( if (error && error.type == ARTRealtimeTransportErrorTypeRefused) { ARTErrorInfo *const errorInfo = [ARTErrorInfo createWithCode:ARTClientCodeErrorTransport message:[NSString stringWithFormat:@"Connection refused using %@", error.url]]; ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:errorInfo]; - [self transition:ARTRealtimeFailed withMetadata:metadata]; + [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; } else if (error) { ARTErrorInfo *const errorInfo = [ARTErrorInfo createFromNSError:error.error]; ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:errorInfo]; - [self transition:ARTRealtimeFailed withMetadata:metadata]; + [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; } else { - [self transition:ARTRealtimeFailed withMetadata:[[ARTConnectionStateChangeMetadata alloc] init]]; + ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] init]; + [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; } } @@ -1686,7 +1676,7 @@ - (void)realtimeTransportTooBig:(id)transport { ARTErrorInfo *const errorInfo = [ARTErrorInfo createWithCode:ARTClientCodeErrorTransport message:@"Transport too big"]; ARTConnectionStateChangeMetadata *const metadata = [[ARTConnectionStateChangeMetadata alloc] initWithErrorInfo:errorInfo]; - [self transition:ARTRealtimeFailed withMetadata:metadata]; + [self transitionToDisconnectedOrSuspendedWithMetadata:metadata]; } - (void)realtimeTransportSetMsgSerial:(id)transport msgSerial:(int64_t)msgSerial { diff --git a/Test/Test Utilities/TestUtilities.swift b/Test/Test Utilities/TestUtilities.swift index ef07b6ccf..92aa7e809 100644 --- a/Test/Test Utilities/TestUtilities.swift +++ b/Test/Test Utilities/TestUtilities.swift @@ -751,6 +751,7 @@ enum FakeNetworkResponse { case requestTimeout(timeout: TimeInterval) case hostInternalError(code: Int) case host400BadRequest + case arbitraryError var error: NSError { switch self { @@ -764,6 +765,8 @@ enum FakeNetworkResponse { return NSError(domain: AblyTestsErrorDomain, code: code, userInfo: [NSLocalizedDescriptionKey: "internal error", NSLocalizedFailureReasonErrorKey: AblyTestsErrorDomain + ".FakeNetworkResponse"]) case .host400BadRequest: return NSError(domain: AblyTestsErrorDomain, code: 400, userInfo: [NSLocalizedDescriptionKey: "bad request", NSLocalizedFailureReasonErrorKey: AblyTestsErrorDomain + ".FakeNetworkResponse"]) + case .arbitraryError: + return NSError(domain: AblyTestsErrorDomain, code: 1, userInfo: [NSLocalizedDescriptionKey: "error from FakeNetworkResponse.arbitraryError"]) } } @@ -779,6 +782,8 @@ enum FakeNetworkResponse { return ARTRealtimeTransportError(error: error, badResponseCode: code, url: url) case .host400BadRequest: return ARTRealtimeTransportError(error: error, badResponseCode: 400, url: url) + case .arbitraryError: + return ARTRealtimeTransportError(error: error, type: .other, url: url) } } } @@ -864,6 +869,8 @@ class MockHTTP: ARTHttp { requestCallback?(HTTPURLResponse(url: URL(string: "http://cocoa.test.suite")!, statusCode: code, httpVersion: nil, headerFields: nil), nil, nil) case .host400BadRequest: requestCallback?(HTTPURLResponse(url: URL(string: "http://cocoa.test.suite")!, statusCode: 400, httpVersion: nil, headerFields: nil), nil, nil) + case .arbitraryError: + requestCallback?(nil, nil, NSError(domain: AblyTestsErrorDomain, code: 1, userInfo: [NSLocalizedDescriptionKey: "error from FakeNetworkResponse.arbitraryError"])) } } @@ -1289,7 +1296,8 @@ class TestProxyTransport: ARTWebSocketTransport { case .noInternet, .hostUnreachable, .hostInternalError, - .host400BadRequest: + .host400BadRequest, + .arbitraryError: performFakeConnectionError(0.1, error: networkResponse.transportError(for: url)) case .requestTimeout(let timeout): performFakeConnectionError(0.1 + timeout, error: networkResponse.transportError(for: url)) @@ -1658,9 +1666,11 @@ extension ARTWebSocketTransport { } func simulateIncomingError() { - let error = NSError(domain: ARTAblyErrorDomain, code: 0, userInfo: [NSLocalizedDescriptionKey:"Fail test"]) - let webSocketDelegate = self as ARTWebSocketDelegate - webSocketDelegate.webSocket?(self.websocket!, didFailWithError: error) + // Simulate receiving an ERROR ProtocolMessage, which should put a client into the FAILED state (per RTN15i) + let protocolMessage = ARTProtocolMessage() + protocolMessage.action = .error + protocolMessage.error = ARTErrorInfo.create(withCode: 50000 /* arbitrarily chosen */, message: "Fail test") + receive(protocolMessage) } } diff --git a/Test/Tests/RealtimeClientConnectionTests.swift b/Test/Tests/RealtimeClientConnectionTests.swift index 4524c05e1..346b4689d 100644 --- a/Test/Tests/RealtimeClientConnectionTests.swift +++ b/Test/Tests/RealtimeClientConnectionTests.swift @@ -2389,6 +2389,21 @@ class RealtimeClientConnectionTests: XCTestCase { }) } + // RTN14d, RTB1 + func test__059b__Connection__connection_request_fails__connection_attempt_fails_for_any_recoverable_reason__for_example_an_arbitrary_transport_error() throws { + let test = Test() + + try testRTN14dAndRTB1(test: test, + modifyOptions: { options in + let transportFactory = TestProxyTransportFactory() + transportFactory.fakeNetworkResponse = .arbitraryError + options.testOptions.transportFactory = transportFactory + }, + checkError: { error in + XCTAssertTrue(error.message.contains("error from FakeNetworkResponse.arbitraryError")) + }) + } + // RTN14e func test__060__Connection__connection_request_fails__connection_state_has_been_in_the_DISCONNECTED_state_for_more_than_the_default_connectionStateTtl_should_change_the_state_to_SUSPENDED() throws { let test = Test() @@ -3703,41 +3718,46 @@ class RealtimeClientConnectionTests: XCTestCase { try testMovesToDisconnectedWithNetworkingError(NSError(domain: "kCFErrorDomainCFNetwork", code: 1337, userInfo: [NSLocalizedDescriptionKey: "shouldn't matter"]), for: test) } - func test__090__Connection__Host_Fallback__should_not_use_an_alternative_host_when_the_client_receives_a_bad_request() { - let test = Test() + func test__090__Connection__Host_Fallback__should_not_use_an_alternative_host_when_the_client_receives_a_bad_request() throws { let options = ARTClientOptions(key: "xxxx:xxxx") options.autoConnect = false + options.disconnectedRetryTimeout = 1.0 // so that the test doesn't have to wait a long time to observe a retry options.testOptions.realtimeRequestTimeout = 1.0 let transportFactory = TestProxyTransportFactory() options.testOptions.transportFactory = transportFactory let client = ARTRealtime(options: options) - let channel = client.channels.get(test.uniqueChannelName()) transportFactory.fakeNetworkResponse = .host400BadRequest - var urlConnections = [URL]() - transportFactory.networkConnectEvent = { transport, url in - if client.internal.transport !== transport { - return + let dataGatherer = DataGatherer(description: "Observe emitted state changes and transport connection attempts") { submit in + var stateChanges: [ARTConnectionStateChange] = [] + var urlConnections = [URL]() + + client.connection.on { stateChange in + stateChanges.append(stateChange) + if (stateChanges.count == 3) { + submit((stateChanges: stateChanges, urlConnections: urlConnections)) + } + } + + transportFactory.networkConnectEvent = { transport, url in + if client.internal.transport !== transport { + return + } + urlConnections.append(url) } - urlConnections.append(url) } client.connect() defer { client.dispose(); client.close() } - // We expect the first connection attempt to fail due to the .fakeNetworkResponse configured above. This error does not meet the criteria for trying a fallback host, and so should not provoke the use of a fallback host. Hence the connection should give up and transition to the FAILED state (which causes the publish to fail). We should see that there was only one connection attempt, to the primary host. + let data = try dataGatherer.waitForData(timeout: testTimeout) - waitUntil(timeout: testTimeout) { done in - channel.publish(nil, data: "message") { error in - XCTAssertNotNil(error) - done() - } - } + // We expect the first connection attempt to fail due to the .fakeNetworkResponse configured above. This error does not meet the criteria for trying a fallback host, and so should not provoke the use of a fallback host. Hence the connection should transition to DISCONNECTED, and then subsequently retry, transitioning back to CONNECTING. We should see that there were two connection attempts, both to the primary host. - XCTAssertEqual(client.connection.state, .failed) - XCTAssertEqual(urlConnections.count, 1) - XCTAssertTrue(NSRegularExpression.match(urlConnections[0].absoluteString, pattern: "//realtime.ably.io")) + XCTAssertEqual(data.stateChanges.map(\.current), [.connecting, .disconnected, .connecting]) + XCTAssertEqual(data.urlConnections.count, 2) + XCTAssertTrue(data.urlConnections.allSatisfy { url in NSRegularExpression.match(url.absoluteString, pattern: "//realtime.ably.io") }) } // RTN17a From de17547c4c02f3b3f0fcdb625d59ac4c944dae89 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Thu, 10 Feb 2022 15:35:53 -0300 Subject: [PATCH 07/14] Run iOS 16 tests continuously and upload results to observability server --- .github/workflows/check-pod.yaml | 58 --------- .github/workflows/docs.yml | 52 -------- .github/workflows/features.yml | 14 --- .../workflows/integration-test-iOS16_2.yaml | 69 +--------- .github/workflows/integration-test-macOS.yaml | 118 ------------------ .../workflows/integration-test-tvOS16_1.yaml | 118 ------------------ ...ntinuously-run-tests-and-upload-results.sh | 62 +++++++++ fastlane/Scanfile | 2 +- 8 files changed, 67 insertions(+), 426 deletions(-) delete mode 100644 .github/workflows/check-pod.yaml delete mode 100644 .github/workflows/docs.yml delete mode 100644 .github/workflows/features.yml delete mode 100644 .github/workflows/integration-test-macOS.yaml delete mode 100644 .github/workflows/integration-test-tvOS16_1.yaml create mode 100755 Scripts/continuously-run-tests-and-upload-results.sh diff --git a/.github/workflows/check-pod.yaml b/.github/workflows/check-pod.yaml deleted file mode 100644 index 41666c435..000000000 --- a/.github/workflows/check-pod.yaml +++ /dev/null @@ -1,58 +0,0 @@ -name: Check Pod - -on: - pull_request: - push: - branches: - - main - -jobs: - check: - runs-on: macos-latest - steps: - - uses: actions/checkout@v2 - - - name: Select Specific Xcode Version (14.2) - run: | - sudo xcode-select -s /Applications/Xcode_14.2.app - echo "Selected Xcode version:" - xcodebuild -version - - # Run the steps we document in the Release Process. - # unzip commands included as proof-of-life for the Carthage output. - - name: Print Ruby version - run: ruby --version - - name: Print Carthage version - run: 'echo -n "carthage version: " && carthage version' - - name: Print CocoaPods version - run: 'echo -n "pod version: " && pod --version --verbose' - - name: Print Make version - run: make --version - - name: Build Carthage dependencies - run: make update - - name: Build Ably framework - run: make carthage_package - - name: Print contents of generated ZIP file - run: | - unzip -l Ably.framework.zip - unzip -l Ably.framework.zip | grep 'Mac/Ably.framework' - unzip -l Ably.framework.zip | grep 'tvOS/Ably.framework' - unzip -l Ably.framework.zip | grep 'iOS/Ably.framework' - - name: Validate pod - run: pod lib lint - # We move Ably.framework.zip into a directory. This is because, by - # default, macOS’s Archive Utility unzips directly-nested zip files, so - # if Ably.framework.zip were at the top level of the zip file that - # actions/upload-artifact creates, then Archive Utility would unzip - # Ably.framework.zip too, which we don’t want, since we want this file - # to be kept intact so that we can upload it to GitHub releases as - # described in CONTRIBUTING.md. - - name: Prepare built framework for archiving - run: | - mkdir -p carthage-built-framework-artifact-contents/carthage-built-framework - mv Ably.framework.zip carthage-built-framework-artifact-contents/carthage-built-framework - - name: Archive built framework - uses: actions/upload-artifact@v3 - with: - name: carthage-built-framework - path: carthage-built-framework-artifact-contents diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 59cdd0f05..000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Docs Generation - -on: - pull_request: - push: - branches: - - main - tags: - - '*' - -jobs: - build: - runs-on: macos-latest - - permissions: - deployments: write - id-token: write - - steps: - - uses: actions/checkout@v2 - - - name: Select Specific Xcode Version (14.2) - run: | - sudo xcode-select -s /Applications/Xcode_14.2.app - echo "Selected Xcode version:" - xcodebuild -version - - - name: Install Dependencies - run: | - make submodules - bundle install - make update_carthage_dependencies_macos - - - name: Build Documentation - run: | - ./Scripts/jazzy.sh - ls -al Docs/jazzy - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: eu-west-2 - role-to-assume: arn:aws:iam::${{ secrets.ABLY_AWS_ACCOUNT_ID_SDK }}:role/ably-sdk-builds-ably-cocoa - role-session-name: "${{ github.run_id }}-${{ github.run_number }}" - - - name: Upload Documentation - uses: ably/sdk-upload-action@v1 - with: - sourcePath: Docs/jazzy - githubToken: ${{ secrets.GITHUB_TOKEN }} - artifactName: jazzydoc - diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml deleted file mode 100644 index d1c123738..000000000 --- a/.github/workflows/features.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Features - -on: - pull_request: - push: - branches: - - main - -jobs: - build: - uses: ably/features/.github/workflows/sdk-features.yml@main - with: - repository-name: ably-cocoa - secrets: inherit diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml index ef3f45e55..d3975db96 100644 --- a/.github/workflows/integration-test-iOS16_2.yaml +++ b/.github/workflows/integration-test-iOS16_2.yaml @@ -51,73 +51,12 @@ jobs: path: xcparse/.build/debug/xcparse key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }} - - name: Reset Simulators - run: xcrun simctl erase all - - - name: Install Dependencies and Run Tests + - name: Install Dependencies and Run Tests Continuously + env: + TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} run: | brew install xcbeautify make submodules bundle install make update_carthage_dependencies_ios - bundle exec fastlane test_iOS16_2 - - - name: Check Static Analyzer Output - id: analyzer-output - run: | - if [[ -z $(find ./derived_data -name "report-*.html") ]]; then - echo "Static Analyzer found no issues." - else - echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build." - exit 1 - fi - - - name: Static Analyzer Reports Uploading - if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }} - uses: actions/upload-artifact@v2 - with: - name: static-analyzer-reports-test_iOS16_2 - path: ./derived_data/**/report-*.html - - - name: Run Examples Tests - working-directory: ./Examples/Tests - run: | - pod repo update - pod install - bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_iOS16_2" - - - name: Build APNS Example Project - working-directory: ./Examples/AblyPush - run: | - xcodebuild build -scheme "AblyPushExample" -destination "platform=iOS Simulator,name=iPhone 14" -configuration "Debug" - - - name: Xcodebuild Logs Artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: xcodebuild-logs - path: ~/Library/Developer/Xcode/DerivedData/*/Logs - - - name: Upload test output artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: fastlane/test_output - - - name: Upload test results to observability server - if: always() - env: - TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} - run: Scripts/upload_test_results.sh - - - name: Swift Package Manager - Installation Test - working-directory: ./ - run: | - echo 'Current Branch: ' $GITHUB_HEAD_REF - echo 'Current Revision (SHA):' $GITHUB_SHA - echo Current Path: $(pwd) - export PACKAGE_URL=file://$(pwd) - export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF - export PACKAGE_REVISION=$GITHUB_SHA - swift test --package-path Examples/SPM -v + Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2 diff --git a/.github/workflows/integration-test-macOS.yaml b/.github/workflows/integration-test-macOS.yaml deleted file mode 100644 index 2cf7ff101..000000000 --- a/.github/workflows/integration-test-macOS.yaml +++ /dev/null @@ -1,118 +0,0 @@ -name: "Integration Test: macOS Latest" - -on: - pull_request: - push: - branches: - - main - -# IMPORTANT NOTES: -# - Changes made to this file needs to replicated across other integration-test-*.yaml files. -# - The Fastlane lane name is duplicated in more than one place within this workflow. - -jobs: - check: - runs-on: macos-latest - - env: - LC_CTYPE: en_US.UTF-8 - LANG: en_US.UTF-8 - ABLY_ENV: sandbox - - steps: - - name: Check out SDK repo - uses: actions/checkout@v2 - - - name: Select Specific Xcode Version (14.2) - run: | - sudo xcode-select -s /Applications/Xcode_14.2.app - echo "Selected Xcode version:" - xcodebuild -version - - - name: Log environment information - run: ./Scripts/log-environment-information.sh - - - name: Check out xcparse repo - uses: actions/checkout@v3 - with: - repository: ably-forks/xcparse - ref: emit-test-case-info - path: xcparse - - - id: get-xcparse-commit-sha - name: Get xcparse commit SHA - run: | - cd xcparse - echo "::set-output name=sha::$(git rev-parse HEAD)" - - - name: "actions/cache@v3 (xcparse binary)" - uses: actions/cache@v3 - with: - path: xcparse/.build/debug/xcparse - key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }} - - - name: Reset Simulators - run: xcrun simctl erase all - - - name: Install Dependencies and Run Tests - run: | - brew install xcbeautify - make submodules - bundle install - make update_carthage_dependencies_macos - bundle exec fastlane test_macOS - - - name: Check Static Analyzer Output - id: analyzer-output - run: | - if [[ -z $(find ./derived_data -name "report-*.html") ]]; then - echo "Static Analyzer found no issues." - else - echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build." - exit 1 - fi - - - name: Static Analyzer Reports Uploading - if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }} - uses: actions/upload-artifact@v2 - with: - name: static-analyzer-reports-test_macOS - path: ./derived_data/**/report-*.html - - - name: Run Examples Tests - working-directory: ./Examples/Tests - run: | - pod repo update - pod install - bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_macOS" - - - name: Xcodebuild Logs Artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: xcodebuild-logs - path: ~/Library/Developer/Xcode/DerivedData/*/Logs - - - name: Upload test output artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: fastlane/test_output - - - name: Upload test results to observability server - if: always() - env: - TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} - run: Scripts/upload_test_results.sh - - - name: Swift Package Manager - Installation Test - working-directory: ./ - run: | - echo 'Current Branch: ' $GITHUB_HEAD_REF - echo 'Current Revision (SHA):' $GITHUB_SHA - echo Current Path: $(pwd) - export PACKAGE_URL=file://$(pwd) - export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF - export PACKAGE_REVISION=$GITHUB_SHA - swift test --package-path Examples/SPM -v diff --git a/.github/workflows/integration-test-tvOS16_1.yaml b/.github/workflows/integration-test-tvOS16_1.yaml deleted file mode 100644 index d259d0856..000000000 --- a/.github/workflows/integration-test-tvOS16_1.yaml +++ /dev/null @@ -1,118 +0,0 @@ -name: "Integration Test: tvOS 16.1" - -on: - pull_request: - push: - branches: - - main - -# IMPORTANT NOTES: -# - Changes made to this file needs to replicated across other integration-test-*.yaml files. -# - The Fastlane lane name is duplicated in more than one place within this workflow. - -jobs: - check: - runs-on: macos-latest - - env: - LC_CTYPE: en_US.UTF-8 - LANG: en_US.UTF-8 - ABLY_ENV: sandbox - - steps: - - name: Check out SDK repo - uses: actions/checkout@v2 - - - name: Select Specific Xcode Version (14.2) - run: | - sudo xcode-select -s /Applications/Xcode_14.2.app - echo "Selected Xcode version:" - xcodebuild -version - - - name: Log environment information - run: ./Scripts/log-environment-information.sh - - - name: Check out xcparse repo - uses: actions/checkout@v3 - with: - repository: ably-forks/xcparse - ref: emit-test-case-info - path: xcparse - - - id: get-xcparse-commit-sha - name: Get xcparse commit SHA - run: | - cd xcparse - echo "::set-output name=sha::$(git rev-parse HEAD)" - - - name: "actions/cache@v3 (xcparse binary)" - uses: actions/cache@v3 - with: - path: xcparse/.build/debug/xcparse - key: ${{ runner.os }}-xcparse-${{ steps.get-xcparse-commit-sha.outputs.sha }} - - - name: Reset Simulators - run: xcrun simctl erase all - - - name: Install Dependencies and Run Tests - run: | - brew install xcbeautify - make submodules - bundle install - make update_carthage_dependencies_tvos - bundle exec fastlane test_tvOS16_1 - - - name: Check Static Analyzer Output - id: analyzer-output - run: | - if [[ -z $(find ./derived_data -name "report-*.html") ]]; then - echo "Static Analyzer found no issues." - else - echo "Static Analyzer found some issues. HTML report will be available in Artifacts section. Failing build." - exit 1 - fi - - - name: Static Analyzer Reports Uploading - if: ${{ failure() && steps.analyzer-output.outcome == 'failure' }} - uses: actions/upload-artifact@v2 - with: - name: static-analyzer-reports-test_tvOS16_1 - path: ./derived_data/**/report-*.html - - - name: Run Examples Tests - working-directory: ./Examples/Tests - run: | - pod repo update - pod install - bundle exec fastlane scan -s Tests --output-directory "fastlane/test_output/examples/test_tvOS_16_1" - - - name: Xcodebuild Logs Artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: xcodebuild-logs - path: ~/Library/Developer/Xcode/DerivedData/*/Logs - - - name: Upload test output artifact - if: always() - uses: actions/upload-artifact@v2 - with: - name: test-output - path: fastlane/test_output - - - name: Upload test results to observability server - if: always() - env: - TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} - run: Scripts/upload_test_results.sh - - - name: Swift Package Manager - Installation Test - working-directory: ./ - run: | - echo 'Current Branch: ' $GITHUB_HEAD_REF - echo 'Current Revision (SHA):' $GITHUB_SHA - echo Current Path: $(pwd) - export PACKAGE_URL=file://$(pwd) - export PACKAGE_BRANCH_NAME=$GITHUB_HEAD_REF - export PACKAGE_REVISION=$GITHUB_SHA - swift test --package-path Examples/SPM -v diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh new file mode 100755 index 000000000..4d7206abf --- /dev/null +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +# 1. Grab command-line options. + +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +while [[ "$#" -gt 0 ]]; do + case $1 in + -l|--lane) lane="$2"; shift ;; + -u|--upload-server-base-url) upload_server_base_url="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +if [[ -z $lane ]] +then + echo "You need to specify the Fastlane lane to run (-l / --lane)." 2>&1 + exit 1 +fi + +# 2. Run the tests in a loop and report the results. + +declare -i iteration=1 +while true +do + echo "BEGIN ITERATION ${iteration}" 2>&1 + + rm -rf fastlane/test_output + xcrun simctl erase all + + set +e + bundle exec fastlane --verbose $lane + tests_exit_value=$? + set -e + + if [[ tests_exit_value -eq 0 ]] + then + echo "ITERATION ${iteration}: Tests passed." + else + echo "ITERATION ${iteration}: Tests failed (exit value ${tests_exit_value})." + fi + + echo "ITERATION ${iteration}: Uploading results to observability server." + + # https://unix.stackexchange.com/questions/446847/conditionally-pass-params-to-a-script + optional_params=() + + if [[ ! -z $upload_server_base_url ]] + then + optional_params+=(--upload-server-base-url "${upload_server_base_url}") + fi + + ./Scripts/upload_test_results.sh \ + --iteration $iteration \ + "${optional_params[@]}" + + echo "END ITERATION ${iteration}" 2>&1 + + iteration+=1 +done diff --git a/fastlane/Scanfile b/fastlane/Scanfile index fea35296b..c0f23f385 100644 --- a/fastlane/Scanfile +++ b/fastlane/Scanfile @@ -1,5 +1,5 @@ open_report false -clean true +clean false skip_slack true ensure_devices_found true output_types "junit" From 5d2f6629282929302ecfbebe7593cad9ef634619 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Tue, 1 Mar 2022 16:45:06 -0300 Subject: [PATCH 08/14] Print xcodebuild raw output as part of loop This is so that we can see things like library logs, or any additional logging we might add to help debug a test case. --- Scripts/continuously-run-tests-and-upload-results.sh | 6 ++++++ fastlane/Scanfile | 2 ++ 2 files changed, 8 insertions(+) diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh index 4d7206abf..95b1a5ff0 100755 --- a/Scripts/continuously-run-tests-and-upload-results.sh +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -28,6 +28,7 @@ do echo "BEGIN ITERATION ${iteration}" 2>&1 rm -rf fastlane/test_output + rm -rf xcodebuild_output xcrun simctl erase all set +e @@ -42,6 +43,11 @@ do echo "ITERATION ${iteration}: Tests failed (exit value ${tests_exit_value})." fi + echo "ITERATION ${iteration}: BEGIN xcodebuild raw output." + ls xcodebuild_output + cat xcodebuild_output/** + echo "ITERATION ${iteration}: END xcodebuild raw output." + echo "ITERATION ${iteration}: Uploading results to observability server." # https://unix.stackexchange.com/questions/446847/conditionally-pass-params-to-a-script diff --git a/fastlane/Scanfile b/fastlane/Scanfile index c0f23f385..da002c455 100644 --- a/fastlane/Scanfile +++ b/fastlane/Scanfile @@ -6,3 +6,5 @@ output_types "junit" # I'm being explicit about this because I want to make sure it's being used, to make sure that trainer is used to generate the JUnit report xcodebuild_formatter "xcbeautify" result_bundle true +# Just for printing inside these loop jobs +buildlog_path "xcodebuild_output" From 373201abae77139a4c225af6fad2aeb13fecac3a Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Tue, 29 Mar 2022 16:25:47 -0300 Subject: [PATCH 09/14] Add script for fetching test case logs from GitHub --- Scripts/fetch-test-logs.sh | 493 +++++++++++++++++++++++++++++++++++++ 1 file changed, 493 insertions(+) create mode 100755 Scripts/fetch-test-logs.sh diff --git a/Scripts/fetch-test-logs.sh b/Scripts/fetch-test-logs.sh new file mode 100755 index 000000000..60caa5e4b --- /dev/null +++ b/Scripts/fetch-test-logs.sh @@ -0,0 +1,493 @@ +#!/bin/bash + +# Retrieves the raw xcodebuild output for one or more test observability server +# uploads. +# +# Only works for tests that were run using the +# continuously-run-tests-and-upload-results script in this directory. + +# Usage: +# ./fetch-test-logs.sh --repo ably/ably-cocoa --test-case-id --filter [filter] +# +# or +# +# ./fetch-test-logs.sh --repo ably/ably-cocoa --upload-id + +# Options: +# +# -r / --repo : The 'org/name'-formatted name of the GitHub repo, for +# example 'ably/ably-cocoa'. +# +# -t / --test-case-id : The ID of a test case saved on the test +# observability server. Will fetch all uploads that match the filter specified +# using the --filter option, and then save the results inside the directory +# specified by the --output-directory option, in the following hierarchy, where +# uploads are split into those where the test case failed and those where it +# didn’t (which doesn’t necessarily imply that the test case passed; it may not +# have run at all): +# +# +# ├── info.json (contains metadata about the results in this directory) +# ├── upload_logs +# │   ├── failed +# │ │   └── xcodebuild-logs-upload-.txt, ... +# │   └── not_failed +# │ └── xcodebuild-logs-upload-.txt, ... +# └── test-case-logs (unless --no-extract-test-case-logs specified) +# ├── failed +# │   └── xcodebuild-logs-upload-.txt, ... +# └── not_failed +# └── xcodebuild-logs-upload-.txt, ... +# +# The upload_logs directory contains the full logs for that upload, and the +# test_case_logs directory contains just the segments of the logs that +# correspond to the specific test case. +# +# -d / --output-directory : Where to output the logs generated by the +# --test-case-id option to. Defaults to ./xcodebuild-logs-test-case--. +# +# -f / --filter : A URL query string describing a filter to be applied +# to the uploads fetched when using the --test-case-id option. For example, +# "branches[]=main&createdBefore=2022-02-20". +# +# -n / --no-extract-test-case-logs: Will cause the --test-case-id option to not +# attempt to extract the segment of the upload log that corresponds to the test +# case. +# +# -i / --upload-id : The ID of a upload saved on the test observability +# server. +# +# -u / --upload-server-base-url : Allows you to specify a URL to use as +# the upload server base URL. Defaults to https://test-observability.herokuapp.com. +# +# -o / --output-file : Where to output the logs generated by the +# --upload-id option to. Defaults to ./xcodebuild-logs-upload-.txt. +# +# -c / --cache-directory : Where to cache the GitHub logs. Defaults to +# ~/Library/Caches/com.ably.testObservabilityLogs. Will be created if doesn’t +# exist. +# +# -a / --no-use-github-auth: Will not prompt the user for an access token to be +# used for making requests to the GitHub API. Useful if all the required GitHub +# job logs are already cached locally. + +set -e + +check_dependencies() { + if ! which jq >/dev/null; then + echo "You need to install jq." 2>&1 + exit 1 + fi +} + +get_github_access_token() { + # https://stackoverflow.com/questions/3980668/how-to-get-a-password-from-a-shell-script-without-echoing#comment4260181_3980904 + read -s -p "Enter your GitHub access token (this will be used to fetch logs from the GitHub API): " github_access_token + + echo + + if [[ -z $github_access_token ]]; then + echo "You need to specify a GitHub access token." 2>&1 + exit 1 + fi + + echo +} + +# Args: +# $1: JSON representation of the test observability server upload +# $2: Path to write the logs to +fetch_and_write_logs_for_upload() { + upload_json=$1 + output_file=$2 + + # (TIL I learned that `echo` will interpret backslash sequences, which we + # don’t want. Appparently in general printf is recommended over echo.) + # https://stackoverflow.com/questions/43528202/prevent-echo-from-interpreting-backslash-escapes + github_repository=$(printf '%s' $upload_json | jq --raw-output '.githubRepository') + github_run_id=$(printf '%s' $upload_json | jq --raw-output '.githubRunId') + github_run_attempt=$(printf '%s' $upload_json | jq --raw-output '.githubRunAttempt') + github_job=$(printf '%s' $upload_json | jq --raw-output '.githubJob') + iteration=$(printf '%s' $upload_json | jq --raw-output '.iteration') + + echo "Upload comes from GitHub repository ${github_repository}. It has GitHub run ID ${github_run_id}, run attempt number ${github_run_attempt}, and job name ${github_job}. It corresponds to loop iteration ${iteration}." + + # Check whether we have a cached log for this job. + # (We cache the job logs because when running the tests continuously, with + # verbose logging enabled, a job log can be ~1.5GB.) + + log_file_name="github-log-${github_repository//\//-}-run-${github_run_id}-attempt-${github_run_attempt}-job-${github_job}" + log_file_path="${cache_directory}/${log_file_name}" + + if [[ -f "${log_file_path}" ]]; then + echo "GitHub job log file already exists at ${log_file_path}. Skipping download." 2>&1 + else + echo "GitHub job log file not yet downloaded." 2>&1 + + # (I wonder if this information that I’m fetching from GitHub is stuff that + # I should have just had in the upload in the first place? Not that + # important right now.) + + github_api_base_url="https://api.github.com" + + # From the GitHub API, fetch the jobs for this workflow run attempt. + # https://docs.github.com/en/rest/reference/actions#list-jobs-for-a-workflow-run-attempt + github_jobs_json=$(curl \ + --fail \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_auth_curl_args[@]}" \ + "${github_api_base_url}/repos/${github_repository}/actions/runs/${github_run_id}/attempts/${github_run_attempt}/jobs") + + # From this list of jobs, find the one that corresponds to our upload. + github_job_id=$(printf "%s" $github_jobs_json | jq \ + --arg jobName "${github_job}" \ + '.jobs[] | select(.name == $jobName) | .id') + + if [[ -z $github_job_id ]]; then + echo "Could not find job with name ${github_job} in attempt ${github_run_attempt} of run ${github_run_id} in GitHub repository ${github_repository}." 2>&1 + exit 1 + fi + + echo "Upload corresponds to GitHub job ID ${github_job_id}. Downloading logs. This may take a while." + + # From the GitHub API, fetch the logs for this job and cache them. + # https://docs.github.com/en/rest/reference/actions#download-job-logs-for-a-workflow-run + + if [[ ! -d "${cache_directory}" ]]; then + mkdir -p "${cache_directory}" + fi + + curl \ + --fail \ + --location \ + -H "Accept: application/vnd.github.v3+json" \ + "${github_auth_curl_args[@]}" \ + "${github_api_base_url}/repos/${github_repository}/actions/jobs/${github_job_id}/logs" >"${log_file_path}.partial" + + mv "${log_file_path}.partial" "${log_file_path}" + + echo "Saved GitHub job logs to ${log_file_path}." + fi + + # Extract the part of the logs that corresponds to the raw xcodebuild output for this iteration. + # https://stackoverflow.com/a/18870500 + + echo "Finding xcodebuild output for iteration ${iteration}." + + xcodebuild_output_start_marker="ITERATION ${iteration}: BEGIN xcodebuild raw output" + xcodebuild_output_start_line_number=$(sed -n "/${xcodebuild_output_start_marker}/=" "${log_file_path}") + + if [[ -z "${xcodebuild_output_start_line_number}" ]]; then + echo "Couldn’t find start of xcodebuild raw output (couldn’t find marker \"${xcodebuild_output_start_marker}\")." 2>&1 + echo "This may be because the GitHub job hasn’t finished yet, or because the tests are not being run in a loop, or it may be an upload created before this functionality was implemented." 2>&1 + echo "You may need to delete the cached log file ${log_file_path}." 2>&1 + exit 1 + fi + + xcodebuild_output_end_marker="ITERATION ${iteration}: END xcodebuild raw output" + xcodebuild_output_end_line_number=$(sed -n "/${xcodebuild_output_end_marker}/=" "${log_file_path}") + + if [[ -z "${xcodebuild_output_end_line_number}" ]]; then + echo "Couldn’t find end of xcodebuild raw output (couldn’t find marker \"${xcodebuild_output_end_marker}\")." 2>&1 + exit 1 + fi + + # Strip the GitHub-added timestamps (which just correspond to the time that `cat` was executed on the log file, and hence aren’t of any use) from the start of each line. + + echo "Stripping GitHub timestamps." + + # https://arkit.co.in/print-given-range-of-lines-using-awk-perl-head-tail-and-python/ + sed -n "${xcodebuild_output_start_line_number},${xcodebuild_output_end_line_number} p" "${log_file_path}" | sed -e 's/^[^ ]* //' >"${output_file}" + + echo "Wrote xcodebuild output to ${output_file}." 2>&1 +} + +default_output_file_for_upload_id() { + echo "xcodebuild-logs-upload-$1.txt" +} + +run_for_test_case() { + # From the test observability server API, fetch the test case and extract its + # properties. + + echo "Fetching test case ${test_case_id} from ${upload_server_base_url}." 2>&1 + + test_case_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/test_cases/${test_case_id}") + + test_class_name=$(printf '%s' $test_case_json | jq --raw-output '.testClassName') + test_case_name=$(printf '%s' $test_case_json | jq --raw-output '.testCaseName') + + printf "Test case ${test_case_id} has test class name ${test_class_name} and test case name ${test_case_name}.\n\n" + + # From the test observability server API, fetch the filtered uploads. + + if [[ -z $filter ]]; then + filter_description="no filter" + filter_query="" + else + filter_description="filter ${filter}" + filter_query="?${filter}" + fi + + echo "Fetching uploads for test case ${test_case_id}, with ${filter_description}, from ${upload_server_base_url}." 2>&1 + + uploads_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/test_cases/${test_case_id}/uploads${filter_query}") + + number_of_uploads=$(printf '%s' $uploads_json | jq '. | length') + + if [[ ${number_of_uploads} -eq 1 ]]; then + echo "There is 1 upload". 2>&1 + else + echo "There are ${number_of_uploads} uploads". 2>&1 + fi + + echo + + mkdir "${output_directory}" + mkdir "${output_directory}/upload_logs" + mkdir "${output_directory}/test_case_logs" + + failed_upload_logs_output_directory="${output_directory}/upload_logs/failed" + mkdir "${failed_upload_logs_output_directory}" + not_failed_upload_logs_output_directory="${output_directory}/upload_logs/not_failed" + mkdir "${not_failed_upload_logs_output_directory}" + + failed_test_case_logs_output_directory="${output_directory}/test_case_logs/failed" + mkdir "${failed_test_case_logs_output_directory}" + not_failed_test_case_logs_output_directory="${output_directory}/test_case_logs/not_failed" + mkdir "${not_failed_test_case_logs_output_directory}" + + jq -n \ + --arg testCaseId "${test_case_id}" \ + --arg filter "${filter}" \ + --arg uploadServerBaseUrl "${upload_server_base_url}" \ + '{ fetchedAt: (now | todateiso8601), testCaseId: $testCaseId, filter: $filter, uploadServerBaseUrl: $uploadServerBaseUrl }' \ + >"${output_directory}/info.json" + + for ((i = 0; i < number_of_uploads; i += 1)); do + failed=$(printf '%s' $uploads_json | jq ".[${i}].failed") + upload_json=$(printf '%s' $uploads_json | jq ".[${i}].upload") + + upload_id=$(printf '%s' $upload_json | jq --raw-output '.id') + + echo "[$((i + 1)) of ${number_of_uploads}] Processing upload ${upload_id}." 2>&1 + + output_file_without_directory=$(default_output_file_for_upload_id "${upload_id}") + + if [[ $failed == "true" ]]; then + upload_log_output_file="${failed_upload_logs_output_directory}/${output_file_without_directory}" + test_case_log_output_file="${failed_test_case_logs_output_directory}/${output_file_without_directory}" + else + upload_log_output_file="${not_failed_upload_logs_output_directory}/${output_file_without_directory}" + test_case_log_output_file="${not_failed_test_case_logs_output_directory}/${output_file_without_directory}" + fi + + fetch_and_write_logs_for_upload "${upload_json}" "${upload_log_output_file}" + + if [[ -z "${no_extract_test_case_logs}" ]]; then + extract_logs_for_test_case "${test_class_name}" "${test_case_name}" "${upload_log_output_file}" "${test_case_log_output_file}" + fi + + echo + done +} + +# Args: +# $1: Test class name e.g. RealtimeClientPresenceTests +# $2: Test case name e.g. test__037__Presence__update__should_update_the_data_for_the_present_member_with_a_value() +# $3: Path of the xcodebuild logs for the entire test suite run +# $4: Path of where to write the test case logs to. +extract_logs_for_test_case() { + # Extract the part of the logs that corresponds to the raw xcodebuild output for this iteration. (We have similar code in fetch_and_write_logs_for_upload.) + + test_class_name=$1 + test_case_name=$2 + upload_log_file=$3 + output_file=$4 + + # (For some reason, the test case name in the observability server has + # trailing (), but in the xcodebuild logs it doesn’t. So strip them.) + sanitised_test_case_name="${test_case_name//[()]/}" + + echo "Finding logs for test class ${test_class_name}, test case ${test_case_name} in ${upload_log_file}." 2>&1 + + test_case_log_start_marker="Test Case.*${test_class_name} ${sanitised_test_case_name}.*started" + test_case_log_start_line_number=$(sed -n "/${test_case_log_start_marker}/=" "${upload_log_file}") + + if [[ -z "${test_case_log_start_line_number}" ]]; then + echo "Couldn’t find start of test case output (couldn’t find marker \"${test_case_log_start_marker}\")." 2>&1 + exit 1 + fi + + test_case_log_end_marker="Test Case.*${test_class_name} ${sanitised_test_case_name}.*(passed|failed)|Restarting after unexpected exit, crash, or test timeout in ${test_class_name}\/${sanitised_test_case_name}\(\)" + test_case_log_end_line_number=$(sed -En "/${test_case_log_end_marker}/=" "${upload_log_file}") + + if [[ -z "${test_case_log_end_line_number}" ]]; then + echo "Couldn’t find end of test case output (couldn’t find marker \"${test_case_log_end_marker}\")." 2>&1 + exit 1 + fi + + sed -n "${test_case_log_start_line_number},${test_case_log_end_line_number} p" "${upload_log_file}" >"${output_file}" + + echo "Wrote test case log to ${output_file}." 2>&1 +} + +run_for_upload() { + # From the test observability server API, fetch the upload, to find the + # GitHub run ID, attempt number, job name, and iteration. + + echo "Fetching upload ${upload_id} from ${upload_server_base_url}." 2>&1 + + upload_json=$(curl --fail --header "Accept: application/json" "${upload_server_base_url}/repos/${repo}/uploads/${upload_id}") + + fetch_and_write_logs_for_upload "${upload_json}" "${output_file}" +} + +check_dependencies + +# Grab and validate command-line options, and apply defaults. + +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +while [[ "$#" -gt 0 ]]; do + case $1 in + -r | --repo) + repo="$2" + shift + ;; + -t | --test-case-id) + if [[ -z "$2" ]]; then + echo "You must specify a test case ID when using the --test-case-id option." 2>&1 + exit 1 + fi + test_case_id="$2" + shift + ;; + -d | --output-directory) + if [[ -z "$2" ]]; then + echo "You must specify an output directory when using the --output-directory option." 2>&1 + exit 1 + fi + output_directory="$2" + shift + ;; + -f | --filter) + if [[ -z "$2" ]]; then + echo "You must specify a filter when using the --filter option." 2>&1 + exit 1 + fi + filter="$2" + shift + ;; + -n | --no-extract-test-case-logs) no_extract_test_case_logs="1" ;; + -i | --upload-id) + if [[ -z "$2" ]]; then + echo "You must specify an upload ID when using the --upload-id option." 2>&1 + exit 1 + fi + upload_id="$2" + shift + ;; + -u | --upload-server-base-url) + if [[ -z "$2" ]]; then + echo "You must specify a base URL when using the --upload-server-base-url option." 2>&1 + exit 1 + fi + upload_server_base_url="$2" + shift + ;; + -o | --output-file) + if [[ -z "$2" ]]; then + echo "You must specify an output file when using the --output-file option." 2>&1 + exit 1 + fi + output_file="$2" + shift + ;; + -c | --cache-directory) + if [[ -z "$2" ]]; then + echo "You must specify a cache directory when using the --cache-directory option." 2>&1 + exit 1 + fi + cache_directory="$2" + shift + ;; + -a | --no-use-github-auth) no_use_github_auth="1" ;; + *) + echo "Unknown parameter passed: $1" 2>&1 + exit 1 + ;; + esac + shift +done + +if [[ -z $repo ]]; then + echo "You need to specify a repo (-r / --repo)." 2>&1 + exit 1 +fi + +if [[ -z $test_case_id && -z $upload_id ]]; then + echo "You need to specify the test case ID (-t / --test-case-id) or upload ID (-i / --upload-id)." 2>&1 + exit 1 +fi + +if [[ -n $test_case_id && -n $upload_id ]]; then + echo "You cannot specify both a test case ID and an upload ID." 2>&1 + exit 1 +fi + +if [[ -n $test_case_id && -n $upload_id ]]; then + echo "You cannot specify both a test case ID and an upload ID." 2>&1 + exit 1 +fi + +if [[ -z $test_case_id && -n $output_directory ]]; then + echo "You can only specify an output directory with a test case ID (-t / --test-case-id)." 2>&1 + exit 1 +fi + +if [[ -z $output_directory ]]; then + output_directory="xcodebuild-logs-test-case-${test_case_id}-$(date -Iseconds)" +fi + +if [[ -z $test_case_id && -n $filter ]]; then + echo "You can only specify a filter with a test case ID (-t / --test-case-id)." 2>&1 + exit 1 +fi + +if [[ -z $test_case_id && -n $no_extract_test_case_logs ]]; then + echo "You can only specify the --no-extract-test-case-logs option with a test case ID (-t / --test-case-id)." 2>&1 + exit 1 +fi + +if [[ -z $upload_server_base_url ]]; then + upload_server_base_url="https://test-observability.herokuapp.com" +fi + +if [[ -z $upload_id && -n $output_file ]]; then + echo "You can only specify an output file with an upload ID (-i / --upload-id)." 2>&1 + exit 1 +fi + +if [[ -z $output_file ]]; then + output_file=$(default_output_file_for_upload_id "${upload_id}") +fi + +if [[ -z $cache_directory ]]; then + cache_directory="${HOME}/Library/Caches/com.ably.testObservabilityLogs" +fi + +github_auth_curl_args=() +if [[ -z $no_use_github_auth ]]; then + # Get the GitHub access token from the user. We don’t allow them to specify it on the command line. + github_access_token="" + get_github_access_token + github_auth_curl_args+=(-H "Authorization: token ${github_access_token}") +fi + +# Run the appropriate function based on arguments. + +if [[ -n $test_case_id ]]; then + run_for_test_case +elif [[ -n $upload_id ]]; then + run_for_upload +fi From 2984c8c12bfafa74203be1bec32de15aedf49748 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Mon, 11 Apr 2022 15:08:47 -0300 Subject: [PATCH 10/14] Upload .xcresult bundles as an artifact --- .../workflows/integration-test-iOS16_2.yaml | 7 +++ ...ntinuously-run-tests-and-upload-results.sh | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml index d3975db96..f6524c211 100644 --- a/.github/workflows/integration-test-iOS16_2.yaml +++ b/.github/workflows/integration-test-iOS16_2.yaml @@ -60,3 +60,10 @@ jobs: bundle install make update_carthage_dependencies_ios Scripts/continuously-run-tests-and-upload-results.sh --lane test_iOS16_2 + + - name: Upload .xcresult bundles + uses: actions/upload-artifact@v3 + if: always() + with: + name: xcresult-bundles + path: xcresult-bundles diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh index 95b1a5ff0..20a02ad5c 100755 --- a/Scripts/continuously-run-tests-and-upload-results.sh +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -58,9 +58,59 @@ do optional_params+=(--upload-server-base-url "${upload_server_base_url}") fi + set +e ./Scripts/upload_test_results.sh \ --iteration $iteration \ "${optional_params[@]}" + # We defer failing the script until after copying the .xcresult bundle. + upload_exit_value=$? + set -e + + if [[ upload_exit_value -eq 0 ]] + then + echo "ITERATION ${iteration}: Upload succeeded." + else + echo "ITERATION ${iteration}: Upload failed (exit value ${upload_exit_value}). Will exit after copying result bundle." + fi + + # Find the .xcresult bundle and copy it to the directory that will eventually be saved as an artifact. + + result_bundles=$(find fastlane/test_output/sdk -name '*.xcresult') + if [[ -z $result_bundles ]] + then + number_of_result_bundles=0 + else + number_of_result_bundles=$(echo "${result_bundles}" | wc -l) + fi + + if [[ $number_of_result_bundles -eq 0 ]] + then + echo "ITERATION ${iteration}: No result bundles found." + exit 1 + fi + + if [[ $number_of_result_bundles -gt 1 ]] + then + echo -e "ITERATION ${iteration}: Multiple result bundles found:\n${result_bundles}" + exit 1 + fi + + echo "ITERATION ${iteration}: Report bundle found: ${result_bundles}" + + if [[ ! -d xcresult-bundles ]]; then + mkdir xcresult-bundles + fi + + mkdir "xcresult-bundles/${iteration}" + cp -r "${result_bundles}" "xcresult-bundles/${iteration}" + + echo "ITERATION ${iteration}: Copied result bundle to xcresult-bundles/${iteration}." + + if [[ upload_exit_value -ne 0 ]] + then + echo "ITERATION ${iteration}: Terminating due to failed upload." + exit $upload_exit_value + fi echo "END ITERATION ${iteration}" 2>&1 From 39ab78725ae235df8cffa7ac635b843b70b3df62 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Tue, 12 Apr 2022 10:32:23 -0300 Subject: [PATCH 11/14] =?UTF-8?q?Make=20sure=20continuously-run-tests-and-?= =?UTF-8?q?upload-results.sh=20doesn=E2=80=99t=20exceed=20GitHub=20job=20r?= =?UTF-8?q?unning=20time=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’m trying to understand the reason that the upload artifacts step is hung here [1], and I’m wondering if it’s because the job execution limit had already been reached by my script. [1] https://github.com/ably/ably-cocoa/runs/5979297645?check_suite_focus=true --- .../workflows/integration-test-iOS16_2.yaml | 1 + ...ntinuously-run-tests-and-upload-results.sh | 41 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml index f6524c211..937995a43 100644 --- a/.github/workflows/integration-test-iOS16_2.yaml +++ b/.github/workflows/integration-test-iOS16_2.yaml @@ -56,6 +56,7 @@ jobs: TEST_OBSERVABILITY_SERVER_AUTH_KEY: ${{ secrets.TEST_OBSERVABILITY_SERVER_AUTH_KEY }} run: | brew install xcbeautify + brew install coreutils # for `timeout` make submodules bundle install make update_carthage_dependencies_ios diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh index 20a02ad5c..f58f30f78 100755 --- a/Scripts/continuously-run-tests-and-upload-results.sh +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -2,7 +2,15 @@ set -e -# 1. Grab command-line options. +# 1. Check dependencies. + +if ! which timeout > /dev/null +then + echo "You need to install timeout (\`brew install coreutils\` on macOS)." 2>&1 + exit 1 +fi + +# 2. Grab command-line options. # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash while [[ "$#" -gt 0 ]]; do @@ -20,7 +28,17 @@ then exit 1 fi -# 2. Run the tests in a loop and report the results. +# 3. Capture the time at which we started, to make sure we don’t exceed the +# maximum job running time. +started_at=`date +%s` +# https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration +let github_job_maximum_execution_seconds=6*60*60 +# We assume that the part of the job that ran before this script took at most 10 minutes, and that uploading the artifacts will take 30 minutes. +let must_end_by=$((started_at + github_job_maximum_execution_seconds - (10 + 30) * 60)) + +echo "We’ll make sure this script ends by `date -r${must_end_by}`." 2>&1 + +# 4. Run the tests in a loop and report the results. declare -i iteration=1 while true @@ -32,10 +50,27 @@ do xcrun simctl erase all set +e - bundle exec fastlane --verbose $lane + let allowed_execution_time=$must_end_by-`date +%s` + set -e + + if [[ $allowed_execution_time -le 0 ]]; then + echo "ITERATION ${iteration}: Allowed execution time reached. Exiting." 2>&1 + exit 0 + fi + + echo "ITERATION ${iteration}: Running fastlane with a timeout of ${allowed_execution_time} seconds." 2>&1 + + set +e + timeout --kill-after=20 ${allowed_execution_time} bundle exec fastlane --verbose $lane tests_exit_value=$? set -e + if [[ tests_exit_value -eq 124 || tests_exit_value -eq 137 ]]; then + # Execution timed out. + echo "ITERATION ${iteration}: Cancelled the execution of fastlane since it exceeded timeout imposed by maximum GitHub running time. Terminating this script." + exit 0 + fi + if [[ tests_exit_value -eq 0 ]] then echo "ITERATION ${iteration}: Tests passed." From ff9efde727ea1cbdef0de9b3a0dbc52049dd09fa Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Tue, 12 Apr 2022 10:39:02 -0300 Subject: [PATCH 12/14] Log the size of .xcresult bundles waiting to be uploaded To help with understanding issue described in 39ab787. --- ...ntinuously-run-tests-and-upload-results.sh | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh index f58f30f78..c64f1a6a3 100755 --- a/Scripts/continuously-run-tests-and-upload-results.sh +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -40,6 +40,17 @@ echo "We’ll make sure this script ends by `date -r${must_end_by}`." 2>&1 # 4. Run the tests in a loop and report the results. +end_iteration_with_exit_value() { + if [[ -e xcresult-bundles ]] + then + echo "There are `du -d0 -h xcresult-bundles | awk -F '\t' '{print $1}'` of xcresult bundles to be uploaded." + else + echo "There are no xcresult bundles to be uploaded." + fi + + exit $1 +} + declare -i iteration=1 while true do @@ -55,7 +66,7 @@ do if [[ $allowed_execution_time -le 0 ]]; then echo "ITERATION ${iteration}: Allowed execution time reached. Exiting." 2>&1 - exit 0 + end_iteration_with_exit_value 0 fi echo "ITERATION ${iteration}: Running fastlane with a timeout of ${allowed_execution_time} seconds." 2>&1 @@ -68,7 +79,7 @@ do if [[ tests_exit_value -eq 124 || tests_exit_value -eq 137 ]]; then # Execution timed out. echo "ITERATION ${iteration}: Cancelled the execution of fastlane since it exceeded timeout imposed by maximum GitHub running time. Terminating this script." - exit 0 + end_iteration_with_exit_value 0 fi if [[ tests_exit_value -eq 0 ]] @@ -121,13 +132,13 @@ do if [[ $number_of_result_bundles -eq 0 ]] then echo "ITERATION ${iteration}: No result bundles found." - exit 1 + end_iteration_with_exit_value 1 fi if [[ $number_of_result_bundles -gt 1 ]] then echo -e "ITERATION ${iteration}: Multiple result bundles found:\n${result_bundles}" - exit 1 + end_iteration_with_exit_value 1 fi echo "ITERATION ${iteration}: Report bundle found: ${result_bundles}" @@ -144,7 +155,7 @@ do if [[ upload_exit_value -ne 0 ]] then echo "ITERATION ${iteration}: Terminating due to failed upload." - exit $upload_exit_value + end_iteration_with_exit_value $upload_exit_value fi echo "END ITERATION ${iteration}" 2>&1 From 4374542396ba67007b06e17fa28e93eca1a63b0c Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Tue, 12 Apr 2022 10:58:21 -0300 Subject: [PATCH 13/14] Tar and zip the xcresult bundles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’m hoping this will reduce the upload time (more by reducing the number of files than the size). --- .github/workflows/integration-test-iOS16_2.yaml | 4 ++-- Scripts/continuously-run-tests-and-upload-results.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test-iOS16_2.yaml b/.github/workflows/integration-test-iOS16_2.yaml index 937995a43..febfcec50 100644 --- a/.github/workflows/integration-test-iOS16_2.yaml +++ b/.github/workflows/integration-test-iOS16_2.yaml @@ -66,5 +66,5 @@ jobs: uses: actions/upload-artifact@v3 if: always() with: - name: xcresult-bundles - path: xcresult-bundles + name: xcresult-bundles.tar.gz + path: xcresult-bundles.tar.gz diff --git a/Scripts/continuously-run-tests-and-upload-results.sh b/Scripts/continuously-run-tests-and-upload-results.sh index c64f1a6a3..2b9346109 100755 --- a/Scripts/continuously-run-tests-and-upload-results.sh +++ b/Scripts/continuously-run-tests-and-upload-results.sh @@ -44,6 +44,8 @@ end_iteration_with_exit_value() { if [[ -e xcresult-bundles ]] then echo "There are `du -d0 -h xcresult-bundles | awk -F '\t' '{print $1}'` of xcresult bundles to be uploaded." + tar --create --gzip xcresult-bundles > xcresult-bundles.tar.gz + echo "The file xcresult-bundles.tar.gz that will be uploaded as an artifact is `du -d0 -h xcresult-bundles.tar.gz | awk -F '\t' '{print $1}'`." else echo "There are no xcresult bundles to be uploaded." fi From c9c00d2357533cdb0782ace33fce96c2231d1cb6 Mon Sep 17 00:00:00 2001 From: Lawrence Forooghian Date: Thu, 12 May 2022 17:18:00 -0300 Subject: [PATCH 14/14] Add script for generating multiple jobs and workflows This lets us choose the length and parallelism of our test runs. --- Scripts/set-ci-length-and-parallelism.sh | 104 +++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100755 Scripts/set-ci-length-and-parallelism.sh diff --git a/Scripts/set-ci-length-and-parallelism.sh b/Scripts/set-ci-length-and-parallelism.sh new file mode 100755 index 000000000..efd705441 --- /dev/null +++ b/Scripts/set-ci-length-and-parallelism.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -e + +# Usage: +# ./set-ci-length-and-parallelism.sh --workflows --jobs-per-workflow + +# Check dependencies. +if ! which yq > /dev/null; then + echo "You need to install yq." 2>&1 + exit 1 +fi + +# Grab and validate command-line options. + +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +while [[ "$#" -gt 0 ]]; do + case $1 in + --workflows) + if [[ -z "$2" ]]; then + echo "You must specify the number of workflows." 2>&1 + exit 1 + fi + num_workflows="$2" + shift + ;; + --jobs-per-workflow) + if [[ -z "$2" ]]; then + echo "You must specify the number of jobs per workflow." 2>&1 + exit 1 + fi + jobs_per_workflow="$2" + shift + ;; + *) + echo "Unknown parameter passed: $1" 2>&1 + exit 1 + ;; + esac + shift +done + +if [[ -z $num_workflows ]]; then + echo "You need to specify the number of workflows (--workflows)." 2>&1 + exit 1 +fi + +if [[ ! $num_workflows =~ ^-?[0-9]+$ ]]; then + echo "The number of workflows must be a number." 2>&1 + exit 1 +fi + +if [[ $num_workflows -lt 1 ]]; then + echo "The number of workflows must be 1 or more." 2>&1 + exit 1 +fi + +if [[ -z $jobs_per_workflow ]]; then + echo "You need to specify the number of jobs per workflow (--jobs-per-workflow)." 2>&1 + exit 1 +fi + +if [[ ! $jobs_per_workflow =~ ^-?[0-9]+$ ]]; then + echo "The number of jobs per workflow must be a number." 2>&1 + exit 1 +fi + +if [[ $jobs_per_workflow -lt 1 ]]; then + echo "The number of jobs per workflow must be 1 or more." 2>&1 + exit 1 +fi + +workflow_file_without_extension=".github/workflows/integration-test-iOS16_2" +workflow_file_extension=".yaml" + +workflow_file="${workflow_file_without_extension}${workflow_file_extension}" +workflow_name=$(yq .name $workflow_file) + +# First, we apply the number of jobs per workflow. + +yq -i '(.jobs.check | key) = "check-1"' $workflow_file +yq -i "(.jobs.check-1.steps[] | select(.with.path == \"xcresult-bundles.tar.gz\")).with.name = \"xcresult-bundles-1.tar.gz\"" $workflow_file + +for ((i=2; i <= $jobs_per_workflow; i += 1)) +do + yq -i ".jobs.check-${i} = .jobs.check-$(($i-1))" $workflow_file + yq -i ".jobs.check-${i}.needs = [\"check-$(($i-1))\"]" $workflow_file + yq -i "(.jobs.check-${i}.steps[] | select(.with.path == \"xcresult-bundles.tar.gz\")).with.name = \"xcresult-bundles-${i}.tar.gz\"" $workflow_file +done + +# Now, we duplicate the workflow file the requested number of times. + +mv $workflow_file "${workflow_file_without_extension}-1${workflow_file_extension}" + +for ((i=1; i <= $num_workflows; i += 1)) +do + new_workflow_file="${workflow_file_without_extension}-${i}${workflow_file_extension}" + + if [[ $i -gt 1 ]]; then + cp "${workflow_file_without_extension}-$((i-1))${workflow_file_extension}" $new_workflow_file + fi + + yq -i ".name = \"${workflow_name} (workflow ${i})\"" $new_workflow_file +done