From a565f6dc8fb367d206a213efaeb2176cd3c2a281 Mon Sep 17 00:00:00 2001 From: Cameron Pepin Date: Mon, 9 Dec 2024 08:15:08 -0800 Subject: [PATCH] Updating run command to dynamically create bundles based on file path (#871) Summary: Pull Request resolved: https://github.com/facebook/idb/pull/871 ## Context The installation step for XCTest bundles creates symlinks to the test bundle source on the host machine in a static directory (`idb-test-bundles`), or copies the files from the source location to that directory. This is not necessary for XCTest if the bundle lives in the host machine, as the test bundle is injected as a dylib. By removing these symlinks, we get rid of some additional complexity, and remove the risk of leaving broken symlinks around on the host device. These old/broken symlinks create noise in test logs, which make them hard to read. ## This diff - Enables a test request API which takes a `testPath`, which points directly to the original test bundle location. - Creates a bundle & artifcat objects in memory on run, as opposed to finding them on disk in `idb-test-bundles`. ## Future Considerations - Possible enable code signing in the test run creation request. The install step, by default, code signs the test bundles. - We could create a `CommandExecutor` that is not dependent on an instance of the `StorageManager` class. The initialization of the `StorageManager` creates a static directory that we no longer need to exist if we skip the installation step. This is a pretty big undertaking, and this diff removes the symlinks which have been creating Differential Revision: D66895643 fbshipit-source-id: 1d5dd7499f44d5f5d721a6479d872765707b97ad --- CompanionLib/Request/FBXCTestRunRequest.h | 21 +++++++ CompanionLib/Request/FBXCTestRunRequest.m | 69 +++++++++++++++++++++- CompanionLib/Utility/FBIDBStorageManager.h | 8 +++ CompanionLib/Utility/FBIDBStorageManager.m | 34 +++++------ 4 files changed, 114 insertions(+), 18 deletions(-) diff --git a/CompanionLib/Request/FBXCTestRunRequest.h b/CompanionLib/Request/FBXCTestRunRequest.h index f387c2da5..12c39f2e1 100644 --- a/CompanionLib/Request/FBXCTestRunRequest.h +++ b/CompanionLib/Request/FBXCTestRunRequest.h @@ -39,6 +39,22 @@ NS_ASSUME_NONNULL_BEGIN */ + (instancetype)logicTestWithTestBundleID:(NSString *)testBundleID environment:(NSDictionary *)environment arguments:(NSArray *)arguments testsToRun:(nullable NSSet *)testsToRun testsToSkip:(NSSet *)testsToSkip testTimeout:(NSNumber *)testTimeout reportActivities:(BOOL)reportActivities reportAttachments:(BOOL)reportAttachments coverageRequest:(FBCodeCoverageRequest *)coverageRequest collectLogs:(BOOL)collectLogs waitForDebugger:(BOOL)waitForDebugger collectResultBundle:(BOOL)collectResultBundle; +/** + Initializer for Logic Tests from a test path. + + @param testPath the path of the .xctest or .xctestrun file. + @param environment environment for the logic test process. + @param arguments arguments for the logic test process. + @param testsToRun the tests to run. + @param testsToSkip the tests to skip + @param testTimeout the timeout for the entire execution, nil if no timeout should be applied. + @param reportActivities if set activities and their data will be reported + @param coverageRequest information about llvm code coverage collection + @param waitForDebugger a boolean describing whether the tests should stop after Run and wait for a debugger to be attached. + @return an FBXCTestRunRequest instance. + */ ++ (instancetype)logicTestWithTestPath:(NSURL *)testPath environment:(NSDictionary *)environment arguments:(NSArray *)arguments testsToRun:(nullable NSSet *)testsToRun testsToSkip:(NSSet *)testsToSkip testTimeout:(NSNumber *)testTimeout reportActivities:(BOOL)reportActivities reportAttachments:(BOOL)reportAttachments coverageRequest:(FBCodeCoverageRequest *)coverageRequest collectLogs:(BOOL)collectLogs waitForDebugger:(BOOL)waitForDebugger collectResultBundle:(BOOL)collectResultBundle; + /** The Initializer for App Tests. @@ -88,6 +104,11 @@ The Initializer for UI Tests. */ @property (nonatomic, copy, readonly) NSString *testBundleID; +/** + The path of the .xctest or .xctestrun file. + */ + @property (nonatomic, copy, readonly) NSURL *testPath; + /** The Bundle ID of the Test Host, if relevant. */ diff --git a/CompanionLib/Request/FBXCTestRunRequest.m b/CompanionLib/Request/FBXCTestRunRequest.m index ab0218e77..bf5dfcb02 100644 --- a/CompanionLib/Request/FBXCTestRunRequest.m +++ b/CompanionLib/Request/FBXCTestRunRequest.m @@ -209,6 +209,7 @@ - (BOOL)isUITest @implementation FBXCTestRunRequest @synthesize testBundleID = _testBundleID; +@synthesize testPath = _testPath; @synthesize testHostAppBundleID = _testHostAppBundleID; @synthesize environment = _environment; @synthesize arguments = _arguments; @@ -225,6 +226,11 @@ + (instancetype)logicTestWithTestBundleID:(NSString *)testBundleID environment:( return [[FBXCTestRunRequest_LogicTest alloc] initWithTestBundleID:testBundleID testHostAppBundleID:nil testTargetAppBundleID:nil environment:environment arguments:arguments testsToRun:testsToRun testsToSkip:testsToSkip testTimeout:testTimeout reportActivities:reportActivities reportAttachments:reportAttachments coverageRequest:coverageRequest collectLogs:collectLogs waitForDebugger:waitForDebugger collectResultBundle:collectResultBundle]; } ++ (instancetype)logicTestWithTestPath:(NSURL *)testPath environment:(NSDictionary *)environment arguments:(NSArray *)arguments testsToRun:(NSSet *)testsToRun testsToSkip:(NSSet *)testsToSkip testTimeout:(NSNumber *)testTimeout reportActivities:(BOOL)reportActivities reportAttachments:(BOOL)reportAttachments coverageRequest:(FBCodeCoverageRequest *)coverageRequest collectLogs:(BOOL)collectLogs waitForDebugger:(BOOL)waitForDebugger collectResultBundle:(BOOL)collectResultBundle +{ + return [[FBXCTestRunRequest_LogicTest alloc] initWithTestPath:testPath testHostAppBundleID:nil testTargetAppBundleID:nil environment:environment arguments:arguments testsToRun:testsToRun testsToSkip:testsToSkip testTimeout:testTimeout reportActivities:reportActivities reportAttachments:reportAttachments coverageRequest:coverageRequest collectLogs:collectLogs waitForDebugger:waitForDebugger collectResultBundle:collectResultBundle]; +} + + (instancetype)applicationTestWithTestBundleID:(NSString *)testBundleID testHostAppBundleID:(NSString *)testHostAppBundleID environment:(NSDictionary *)environment arguments:(NSArray *)arguments testsToRun:(NSSet *)testsToRun testsToSkip:(NSSet *)testsToSkip testTimeout:(NSNumber *)testTimeout reportActivities:(BOOL)reportActivities reportAttachments:(BOOL)reportAttachments coverageRequest:(FBCodeCoverageRequest *)coverageRequest collectLogs:(BOOL)collectLogs waitForDebugger:(BOOL)waitForDebugger collectResultBundle:(BOOL)collectResultBundle { return [[FBXCTestRunRequest_AppTest alloc] initWithTestBundleID:testBundleID testHostAppBundleID:testHostAppBundleID testTargetAppBundleID:nil environment:environment arguments:arguments testsToRun:testsToRun testsToSkip:testsToSkip testTimeout:testTimeout reportActivities:reportActivities reportAttachments:reportAttachments coverageRequest:coverageRequest collectLogs:collectLogs waitForDebugger:waitForDebugger collectResultBundle:collectResultBundle]; @@ -260,6 +266,31 @@ - (instancetype)initWithTestBundleID:(NSString *)testBundleID testHostAppBundleI return self; } +- (instancetype)initWithTestPath:(NSURL *)testPath testHostAppBundleID:(NSString *)testHostAppBundleID testTargetAppBundleID:(NSString *)testTargetAppBundleID environment:(NSDictionary *)environment arguments:(NSArray *)arguments testsToRun:(NSSet *)testsToRun testsToSkip:(NSSet *)testsToSkip testTimeout:(NSNumber *)testTimeout reportActivities:(BOOL)reportActivities reportAttachments:(BOOL)reportAttachments coverageRequest:(FBCodeCoverageRequest *)coverageRequest collectLogs:(BOOL)collectLogs waitForDebugger:(BOOL)waitForDebugger collectResultBundle:(BOOL)collectResultBundle +{ + self = [super init]; + if (!self) { + return nil; + } + + _testPath = testPath; + _testHostAppBundleID = testHostAppBundleID; + _testTargetAppBundleID = testTargetAppBundleID; + _environment = environment; + _arguments = arguments; + _testsToRun = testsToRun; + _testsToSkip = testsToSkip; + _testTimeout = testTimeout; + _reportActivities = reportActivities; + _reportAttachments = reportAttachments; + _coverageRequest = coverageRequest; + _collectLogs = collectLogs; + _waitForDebugger = waitForDebugger; + _collectResultBundle = collectResultBundle; + + return self; +} + - (BOOL)isLogicTest { return NO; @@ -298,10 +329,46 @@ - (BOOL)isUITest - (FBFuture> *)fetchAndSetupDescriptorWithBundleStorage:(FBXCTestBundleStorage *)bundleStorage target:(id)target { NSError *error = nil; - id testDescriptor = [bundleStorage testDescriptorWithID:self.testBundleID error:&error]; + id testDescriptor = nil; + + /* + * If a test path is provided, we will create a Descriptor object from it (i.e. the original file location). + * Otherwise, we'll look up the test bundle by ID on disk in the idb-test-bundles. + */ + if (self.testPath) { + NSURL *filePath = self.testPath; + + if ([filePath.pathExtension isEqualToString:@"xctest"]) { + FBBundleDescriptor *bundle = [FBBundleDescriptor bundleWithFallbackIdentifierFromPath:filePath.path error:&error]; + + if (!bundle) { + return [FBFuture futureWithError:error]; + } + + testDescriptor = [[FBXCTestBootstrapDescriptor alloc] initWithURL:filePath name:bundle.name testBundle:bundle]; + } + if ([filePath.pathExtension isEqualToString:@"xctestrun"]) { + NSArray> *descriptors = [bundleStorage getXCTestRunDescriptorsFromURL:filePath error:&error]; + + if (!descriptors) { + return [FBFuture futureWithError:error]; + } + if (descriptors.count != 1) { + return [[FBIDBError + describeFormat:@"Expected exactly one test in the xctestrun file, got: %lu", descriptors.count] + failFuture]; + } + + testDescriptor = descriptors[0]; + } + } else { + testDescriptor = [bundleStorage testDescriptorWithID:self.testBundleID error:&error]; + } + if (!testDescriptor) { return [FBFuture futureWithError:error]; } + return [[testDescriptor setupWithRequest:self target:target] mapReplace:testDescriptor]; } diff --git a/CompanionLib/Utility/FBIDBStorageManager.h b/CompanionLib/Utility/FBIDBStorageManager.h index 314e8b296..eeb4314b0 100644 --- a/CompanionLib/Utility/FBIDBStorageManager.h +++ b/CompanionLib/Utility/FBIDBStorageManager.h @@ -214,6 +214,14 @@ extern NSString *const IdbFrameworksFolder; */ - (nullable id)testDescriptorWithID:(NSString *)bundleId error:(NSError **)error; +/** + Get test run descriptors from xctestrun file. + + @param xctestrunURL URL of xctestrun file + @return test descriptor of the test + */ +- (nullable NSArray> *)getXCTestRunDescriptorsFromURL:(NSURL *)xctestrunURL error:(NSError **)error; + @end /** diff --git a/CompanionLib/Utility/FBIDBStorageManager.m b/CompanionLib/Utility/FBIDBStorageManager.m index f825bebb9..7d9e5ac1f 100644 --- a/CompanionLib/Utility/FBIDBStorageManager.m +++ b/CompanionLib/Utility/FBIDBStorageManager.m @@ -382,6 +382,23 @@ @implementation FBXCTestBundleStorage return [[FBIDBError describeFormat:@"Couldn't find test with id: %@", bundleId] fail:error]; } +- (nullable NSArray> *)getXCTestRunDescriptorsFromURL:(NSURL *)xctestrunURL error:(NSError **)error +{ + NSDictionary *contentDict = [FBXCTestRunFileReader readContentsOf:xctestrunURL expandPlaceholderWithPath:self.target.auxillaryDirectory error:error]; + if (!contentDict) { + return nil; + } + NSDictionary *xctestrunMetadata = contentDict[@"__xctestrun_metadata__"]; + // The legacy format of xctestrun file does not contain __xctestrun_metadata__ + if (xctestrunMetadata) { + [self.logger.info logFormat:@"Using xctestrun format version: %@", xctestrunMetadata[@"FormatVersion"]]; + return [self getDescriptorsFrom:contentDict with:xctestrunURL]; + } else { + [self.logger.info log:@"Using the legacy xctestrun file format"]; + return [self legacyGetDescriptorsFrom:contentDict with:xctestrunURL]; + } +} + #pragma mark Private - (NSSet *)listTestBundlesWithError:(NSError **)error @@ -434,23 +451,6 @@ - (NSURL *)xctestBundleWithID:(NSString *)bundleID error:(NSError **)error return [[FBIDBError describeFormat:@"Couldn't find test with url: %@", url] fail:error]; } -- (nullable NSArray> *)getXCTestRunDescriptorsFromURL:(NSURL *)xctestrunURL error:(NSError **)error -{ - NSDictionary *contentDict = [FBXCTestRunFileReader readContentsOf:xctestrunURL expandPlaceholderWithPath:self.target.auxillaryDirectory error:error]; - if (!contentDict) { - return nil; - } - NSDictionary *xctestrunMetadata = contentDict[@"__xctestrun_metadata__"]; - // The legacy format of xctestrun file does not contain __xctestrun_metadata__ - if (xctestrunMetadata) { - [self.logger.info logFormat:@"Using xctestrun format version: %@", xctestrunMetadata[@"FormatVersion"]]; - return [self getDescriptorsFrom:contentDict with:xctestrunURL]; - } else { - [self.logger.info log:@"Using the legacy xctestrun file format"]; - return [self legacyGetDescriptorsFrom:contentDict with:xctestrunURL]; - } -} - // xctestrun for Xcode 11+ - (NSArray> *)getDescriptorsFrom:(NSDictionary *)xctestrunContents with:(NSURL *)xctestrunURL {