From 3be6d03733a5d66072c608f5be6d1939f21e8e77 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Mon, 14 Aug 2023 12:43:32 -0700 Subject: [PATCH] Add support for direct CASAPI access from libToolchainCASPlugin.dylib Add initial CAS support through libToolchainCASPlugin.dylib --- Package.swift | 5 + Sources/CMakeLists.txt | 1 + Sources/CSwiftScan/include/swiftscan_header.h | 5 + Sources/CToolchainCASPlugin/CMakeLists.txt | 10 + .../ToolchainCASPluginImpl.c | 3 + .../include/ToolchainCASPlugin_CAPI.h | 368 ++++++++++++++++++ .../include/module.modulemap | 4 + Sources/SwiftDriver/CMakeLists.txt | 4 +- Sources/SwiftDriver/Driver/Driver.swift | 40 +- .../InterModuleDependencyOracle.swift | 34 +- .../ModuleDependencyScanning.swift | 13 +- .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 9 +- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 17 +- .../ToolchainCASPlugin/CASExtensions.swift | 61 +++ .../ToolchainCASPlugin.swift | 133 +++++++ .../SwiftDriver/Toolchains/Toolchain.swift | 26 ++ .../{SwiftScan => Utilities}/Loader.swift | 0 17 files changed, 716 insertions(+), 17 deletions(-) create mode 100644 Sources/CToolchainCASPlugin/CMakeLists.txt create mode 100644 Sources/CToolchainCASPlugin/ToolchainCASPluginImpl.c create mode 100644 Sources/CToolchainCASPlugin/include/ToolchainCASPlugin_CAPI.h create mode 100644 Sources/CToolchainCASPlugin/include/module.modulemap create mode 100644 Sources/SwiftDriver/ToolchainCASPlugin/CASExtensions.swift create mode 100644 Sources/SwiftDriver/ToolchainCASPlugin/ToolchainCASPlugin.swift rename Sources/SwiftDriver/{SwiftScan => Utilities}/Loader.swift (100%) diff --git a/Package.swift b/Package.swift index e84ac6223..41ec897f2 100644 --- a/Package.swift +++ b/Package.swift @@ -46,6 +46,10 @@ let package = Package( .target(name: "CSwiftScan", exclude: [ "CMakeLists.txt" ]), + /// C modules wrapper for ToolchainCASPlugin. + .target(name: "CToolchainCASPlugin", + exclude: [ "CMakeLists.txt" ]), + /// The driver library. .target( name: "SwiftDriver", @@ -53,6 +57,7 @@ let package = Package( "SwiftOptions", .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), "CSwiftScan", + "CToolchainCASPlugin", .product(name: "Yams", package: "yams"), ], exclude: ["CMakeLists.txt"]), diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index 2be6a1bdd..a2e2a0537 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -7,6 +7,7 @@ # See http://swift.org/CONTRIBUTORS.txt for Swift project authors add_subdirectory(CSwiftScan) +add_subdirectory(CToolchainCASPlugin) add_subdirectory(SwiftOptions) add_subdirectory(SwiftDriver) add_subdirectory(SwiftDriverExecution) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 1409fba5f..264f347d5 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -275,6 +275,11 @@ typedef struct { //=== Scanner CAS Operations ----------------------------------------------===// swiftscan_cas_t (*swiftscan_cas_create)(const char *path); + swiftscan_cas_t (*swiftscan_cas_create_from_plugin)(const char *plugin_path, + const char *ondisk_path, + int argc, + const char **keys, + const char **values); void (*swiftscan_cas_dispose)(swiftscan_cas_t cas); swiftscan_string_ref_t (*swiftscan_cas_store)(swiftscan_cas_t cas, uint8_t *data, unsigned size); diff --git a/Sources/CToolchainCASPlugin/CMakeLists.txt b/Sources/CToolchainCASPlugin/CMakeLists.txt new file mode 100644 index 000000000..4529e21c6 --- /dev/null +++ b/Sources/CToolchainCASPlugin/CMakeLists.txt @@ -0,0 +1,10 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_library(CToolchainCASPlugin STATIC + ToolchainCASPluginImpl.c) diff --git a/Sources/CToolchainCASPlugin/ToolchainCASPluginImpl.c b/Sources/CToolchainCASPlugin/ToolchainCASPluginImpl.c new file mode 100644 index 000000000..860cff8de --- /dev/null +++ b/Sources/CToolchainCASPlugin/ToolchainCASPluginImpl.c @@ -0,0 +1,3 @@ +// This file is here to prevent the package manager from warning about a +// target with no sources. +#include "include/ToolchainCASPlugin_CAPI.h" diff --git a/Sources/CToolchainCASPlugin/include/ToolchainCASPlugin_CAPI.h b/Sources/CToolchainCASPlugin/include/ToolchainCASPlugin_CAPI.h new file mode 100644 index 000000000..7324f0e52 --- /dev/null +++ b/Sources/CToolchainCASPlugin/include/ToolchainCASPlugin_CAPI.h @@ -0,0 +1,368 @@ +// +// Copyright © 2023 Apple, Inc. All rights reserved. +// + +#ifndef TOOLCHAINCASPLUGIN_CAPI_H +#define TOOLCHAINCASPLUGIN_CAPI_H + +#include +#include +#include + +#define LLCAS_VERSION_MAJOR 0 +#define LLCAS_VERSION_MINOR 1 + +typedef struct llcas_cas_options_s *llcas_cas_options_t; +typedef struct llcas_cas_s *llcas_cas_t; + +/** + * Digest hash bytes. + */ +typedef struct { + const uint8_t *data; + size_t size; +} llcas_digest_t; + +/** + * Data buffer for stored CAS objects. + */ +typedef struct { + const void *data; + size_t size; +} llcas_data_t; + +/** + * Identifier for a CAS object. + */ +typedef struct { + uint64_t opaque; +} llcas_objectid_t; + +/** + * A loaded CAS object. + */ +typedef struct { + uint64_t opaque; +} llcas_loaded_object_t; + +/** + * Object references for a CAS object. + */ +typedef struct { + uint64_t opaque_b; + uint64_t opaque_e; +} llcas_object_refs_t; + +/** + * Return values for a load operation. + */ +typedef enum { + /** + * The object was found. + */ + LLCAS_LOOKUP_RESULT_SUCCESS = 0, + + /** + * The object was not found. + */ + LLCAS_LOOKUP_RESULT_NOTFOUND = 1, + + /** + * An error occurred. + */ + LLCAS_LOOKUP_RESULT_ERROR = 2, +} llcas_lookup_result_t; + +/** + * Callback for \c llcas_cas_load_object_async. + * + * \param ctx pointer passed through from the \c llcas_cas_load_object_async + * call. + * \param error message if an error occurred. If set, the memory it points to + * needs to be released via \c llcas_string_dispose. + */ +typedef void (*llcas_cas_load_object_cb)(void *ctx, llcas_lookup_result_t, + llcas_loaded_object_t, char *error); + +/** + * Callback for \c llcas_actioncache_get_for_digest_async. + * + * \param ctx pointer passed through from the + * \c llcas_actioncache_get_for_digest_async call. + * \param error message if an error occurred. If set, the memory it points to + * needs to be released via \c llcas_string_dispose. + */ +typedef void (*llcas_actioncache_get_cb)(void *ctx, llcas_lookup_result_t, + llcas_objectid_t, char *error); + +/** + * Callback for \c llcas_actioncache_put_for_digest_async. + * + * \param ctx pointer passed through from the + * \c llcas_actioncache_put_for_digest_async call. + * \param error message if an error occurred. If set, the memory it points to + * needs to be released via \c llcas_string_dispose. + */ +typedef void (*llcas_actioncache_put_cb)(void *ctx, bool failed, char *error); + +typedef struct { + /** + * Returns the \c LLCAS_VERSION_MAJOR and \c LLCAS_VERSION_MINOR values that + * the plugin was compiled with. Intended for assisting compatibility with + * different versions. + */ + void (*llcas_get_plugin_version)(unsigned *major, unsigned *minor); + + /** + * Releases memory of C string pointers provided by other functions. + */ + void (*llcas_string_dispose)(char *); + + /** + * Options object to configure creation of \c llcas_cas_t. After passing to + * \c llcas_cas_create, its memory can be released via + * \c llcas_cas_options_dispose. + */ + llcas_cas_options_t (*llcas_cas_options_create)(void); + void (*llcas_cas_options_dispose)(llcas_cas_options_t); + + /** + * Receives the \c LLCAS_VERSION_MAJOR and \c LLCAS_VERSION_MINOR values that + * the client was compiled with. + * Intended for assisting compatibility with different versions. + */ + void (*llcas_cas_options_set_client_version)(llcas_cas_options_t, + unsigned major, unsigned minor); + + /** + * Receives a local file-system path that the plugin should use for any + * on-disk resources/caches. + */ + void (*llcas_cas_options_set_ondisk_path)(llcas_cas_options_t, const char *path); + + /** + * Receives a name/value strings pair, for the plugin to set as a custom + * option it supports. These are usually passed through as invocation options + * and are opaque to the client. + * + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns true if there was an error, false otherwise. + */ + bool (*llcas_cas_options_set_option)(llcas_cas_options_t, const char *name, + const char *value, char **error); + + /** + * Creates a new \c llcas_cas_t object. The objects returned from the other + * functions are only valid to use while the \c llcas_cas_t object that they + * came from is still valid. + * + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns \c NULL if there was an error. + */ + llcas_cas_t (*llcas_cas_create)(llcas_cas_options_t, char **error); + + /** + * Releases memory of \c llcas_cas_t. After calling this it is invalid to keep + * using objects that originated from this \c llcas_cas_t instance. + */ + void (*llcas_cas_dispose)(llcas_cas_t); + + /** + * \returns the hash schema name that the plugin is using. The string memory + * it points to needs to be released via \c llcas_string_dispose. + */ + char *(*llcas_cas_get_hash_schema_name)(llcas_cas_t); + + /** + * Parses the printed digest and returns the digest hash bytes. + * + * \param printed_digest a C string that was previously provided by + * \c llcas_digest_print. + * \param bytes pointer to a buffer for writing the digest bytes. Can be \c + * NULL if \p bytes_size is 0. \param bytes_size the size of the buffer. + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns 0 if there was an error. If \p bytes_size is smaller than the + * required size to fit the digest bytes, returns the required buffer size + * without writing to \c bytes. Otherwise writes the digest bytes to \p bytes + * and returns the number of written bytes. + */ + unsigned (*llcas_digest_parse)(llcas_cas_t, const char *printed_digest, + uint8_t *bytes, size_t bytes_size, + char **error); + + /** + * Returns a string for the given digest bytes that can be passed to + * \c llcas_digest_parse. + * + * \param printed_id pointer to receive the printed digest string. The memory + * it points to needs to be released via \c llcas_string_dispose. \param error + * optional pointer to receive an error message if an error occurred. If set, + * the memory it points to needs to be released via \c llcas_string_dispose. + * \returns true if there was an error, false otherwise. + */ + bool (*llcas_digest_print)(llcas_cas_t, llcas_digest_t, char **printed_id, + char **error); + + /** + * Provides the \c llcas_objectid_t value for the given \c llcas_digest_t. + * + * \param digest the digest bytes that the returned \c llcas_objectid_t + * represents. + * \param p_id pointer to store the returned \c llcas_objectid_t object. + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns true if there was an error, false otherwise. + */ + bool (*llcas_cas_get_objectid)(llcas_cas_t, llcas_digest_t digest, + llcas_objectid_t *p_id, char **error); + + /** + * \returns the \c llcas_digest_t value for the given \c llcas_objectid_t. + * The memory that the buffer points to is valid for the lifetime of the + * \c llcas_cas_t object. + */ + llcas_digest_t (*llcas_objectid_get_digest)(llcas_cas_t, llcas_objectid_t); + + /** + * Checks whether a \c llcas_objectid_t points to an existing object. + * + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns one of \c llcas_lookup_result_t. + */ + llcas_lookup_result_t (*llcas_cas_contains_object)(llcas_cas_t, + llcas_objectid_t, + char **error); + + /** + * Loads the object that \c llcas_objectid_t points to. + * + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns one of \c llcas_lookup_result_t. + */ + llcas_lookup_result_t (*llcas_cas_load_object)(llcas_cas_t, llcas_objectid_t, + llcas_loaded_object_t *, + char **error); + + /** + * Like \c llcas_cas_load_object but loading happens via a callback function. + * Whether the call is asynchronous or not depends on the implementation. + * + * \param ctx_cb pointer to pass to the callback function. + */ + void (*llcas_cas_load_object_async)(llcas_cas_t, llcas_objectid_t, + void *ctx_cb, llcas_cas_load_object_cb); + + /** + * Stores the object with the provided data buffer and \c llcas_objectid_t + * references, and provides its associated \c llcas_objectid_t. + * + * \param refs pointer to array of \c llcas_objectid_t. Can be \c NULL if + * \p refs_count is 0. + * \param refs_count number of \c llcas_objectid_t objects in the array. + * \param p_id pointer to store the returned \c llcas_objectid_t object. + * \param error optional pointer to receive an error message if an error + * occurred. If set, the memory it points to needs to be released via + * \c llcas_string_dispose. + * \returns true if there was an error, false otherwise. + */ + bool (*llcas_cas_store_object)(llcas_cas_t, llcas_data_t, + const llcas_objectid_t *refs, + size_t refs_count, llcas_objectid_t *p_id, + char **error); + + /** + * \returns the data buffer of the provided \c llcas_loaded_object_t. The + * buffer pointer must be 8-byte aligned and \c NULL terminated. The memory + * that the buffer points to is valid for the lifetime of the \c llcas_cas_t + * object. + */ + llcas_data_t (*llcas_loaded_object_get_data)(llcas_cas_t, + llcas_loaded_object_t); + + /** + * \returns the references of the provided \c llcas_loaded_object_t. + */ + llcas_object_refs_t (*llcas_loaded_object_get_refs)(llcas_cas_t, + llcas_loaded_object_t); + + /** + * \returns the number of references in the provided \c llcas_object_refs_t. + */ + size_t (*llcas_object_refs_get_count)(llcas_cas_t, llcas_object_refs_t); + + /** + * \returns the \c llcas_objectid_t of the reference at \p index. It is + * invalid to pass an index that is out of the range of references. + */ + llcas_objectid_t (*llcas_object_refs_get_id)(llcas_cas_t, llcas_object_refs_t, + size_t index); + + /** + * Retrieves the \c llcas_objectid_t value associated with a \p key. + * + * \param p_value pointer to store the returned \c llcas_objectid_t object. + * \param globally if true it is a hint to the underlying implementation that + * the lookup is profitable to be done on a distributed caching level, not + * just locally. The implementation is free to ignore this flag. \param error + * optional pointer to receive an error message if an error occurred. If set, + * the memory it points to needs to be released via \c llcas_string_dispose. + * \returns one of \c llcas_lookup_result_t. + */ + llcas_lookup_result_t (*llcas_actioncache_get_for_digest)( + llcas_cas_t, llcas_digest_t key, llcas_objectid_t *p_value, bool globally, + char **error); + + /** + * Like \c llcas_actioncache_get_for_digest but result is provided to a + * callback function. Whether the call is asynchronous or not depends on the + * implementation. + * + * \param ctx_cb pointer to pass to the callback function. + */ + void (*llcas_actioncache_get_for_digest_async)(llcas_cas_t, + llcas_digest_t key, + bool globally, void *ctx_cb, + llcas_actioncache_get_cb); + + /** + * Associates a \c llcas_objectid_t \p value with a \p key. It is invalid to + * set a different \p value to the same \p key. + * + * \param globally if true it is a hint to the underlying implementation that + * the association is profitable to be done on a distributed caching level, + * not just locally. The implementation is free to ignore this flag. \param + * error optional pointer to receive an error message if an error occurred. If + * set, the memory it points to needs to be released via \c + * llcas_string_dispose. \returns true if there was an error, false otherwise. + */ + bool (*llcas_actioncache_put_for_digest)(llcas_cas_t, llcas_digest_t key, + llcas_objectid_t value, + bool globally, char **error); + + /** + * Like \c llcas_actioncache_put_for_digest but result is provided to a + * callback function. Whether the call is asynchronous or not depends on the + * implementation. + * + * \param ctx_cb pointer to pass to the callback function. + */ + void (*llcas_actioncache_put_for_digest_async)(llcas_cas_t, + llcas_digest_t key, + llcas_objectid_t value, + bool globally, void *ctx_cb, + llcas_actioncache_put_cb); + +} llcas_functions_t; + +#endif /* TOOLCHAINCASPLUGIN_CAPI_H */ diff --git a/Sources/CToolchainCASPlugin/include/module.modulemap b/Sources/CToolchainCASPlugin/include/module.modulemap new file mode 100644 index 000000000..55e709df7 --- /dev/null +++ b/Sources/CToolchainCASPlugin/include/module.modulemap @@ -0,0 +1,4 @@ +module CToolchainCASPlugin { + header "ToolchainCASPlugin_CAPI.h" + export * +} diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index d7785e768..473d4fed3 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -16,7 +16,6 @@ add_library(SwiftDriver "ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift" SwiftScan/DependencyGraphBuilder.swift - SwiftScan/Loader.swift SwiftScan/SwiftScan.swift Driver/CompilerMode.swift @@ -100,6 +99,8 @@ add_library(SwiftDriver Jobs/WindowsToolchain+LinkerSupport.swift Jobs/PrebuiltModulesJob.swift + ToolchainCASPlugin/ToolchainCASPlugin.swift + Toolchains/DarwinToolchain.swift Toolchains/GenericUnixToolchain.swift Toolchains/Toolchain.swift @@ -115,6 +116,7 @@ add_library(SwiftDriver Utilities/Diagnostics.swift Utilities/FileList.swift Utilities/FileType.swift + Utilities/Loader.swift Utilities/PredictableRandomNumberGenerator.swift Utilities/RelativePathAdditions.swift Utilities/Sanitizer.swift diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 21a0dd410..302af077d 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -270,7 +270,6 @@ public struct Driver { /// CAS/Caching related options. let enableCaching: Bool let useClangIncludeTree: Bool - let casPath: String /// Code & data for incremental compilation. Nil if not running in incremental mode. /// Set during planning because needs the jobs to look at outputs. @@ -595,13 +594,6 @@ public struct Driver { let cachingEnableOverride = parsedOptions.hasArgument(.driverExplicitModuleBuild) && env.keys.contains("SWIFT_ENABLE_CACHING") self.enableCaching = parsedOptions.hasArgument(.cacheCompileJob) || cachingEnableOverride self.useClangIncludeTree = enableCaching && env.keys.contains("SWIFT_CACHING_USE_INCLUDE_TREE") - if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { - self.casPath = casPathOpt.description - } else if let cacheEnv = env["CCHROOT"] { - self.casPath = cacheEnv - } else { - self.casPath = "" - } // Compute the working directory. workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in @@ -3470,3 +3462,35 @@ extension Driver { return try VirtualPath.intern(path: moduleName.appendingFileTypeExtension(type)) } } + +// CAS and Caching. +extension Driver { + mutating func getCASPluginPath() throws -> AbsolutePath? { + if let pluginOpt = parsedOptions.getLastArgument(.casPluginOption)?.asSingle { + return try AbsolutePath(validating: pluginOpt.description) + } + return try toolchain.lookupToolchainCASPluginLib() + } + + mutating func getOnDiskCASPath() throws -> AbsolutePath? { + if let casPathOpt = parsedOptions.getLastArgument(.casPath)?.asSingle { + return try AbsolutePath(validating: casPathOpt.description) + } + if let cacheEnv = env["CCHROOT"] { + return try AbsolutePath(validating: cacheEnv) + } + return nil; + } + + mutating func getCASPluginOptions() throws -> [String: String] { + var options : [String: String] = [:] + for opt in parsedOptions.arguments(for: .casPluginOption) { + let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1) + if pluginArg.count != 2 { + throw Error.invalidArgumentValue(Option.casPluginOption.spelling, opt.argument.asSingle) + } + options[String(pluginArg[0])] = String(pluginArg[1]) + } + return options + } +} diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index 29ffd97ea..cc6c12eab 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -96,6 +96,25 @@ public class InterModuleDependencyOracle { } } + @_spi(Testing) public func verifyOrCreateCASInstance(fileSystem: FileSystem, + toolchainCASLibPath: AbsolutePath, + onDiskPath: AbsolutePath?, + pluginOptions: [String: String]) throws { + return try queue.sync { + if toolchainCASPlugin == nil { + guard fileSystem.exists(toolchainCASLibPath) else { + throw CASError.failedToCreate("/(toolchainCASLibPath) does not exist") + } + toolchainCASPlugin = try CASPlugin(dylib: toolchainCASLibPath, path: onDiskPath, options: pluginOptions) + } else { + guard toolchainCASPlugin!.path == toolchainCASLibPath else { + throw DependencyScanningError + .scanningLibraryInvocationMismatch(toolchainCASPlugin!.path, toolchainCASLibPath) + } + } + } + } + @_spi(Testing) public func serializeScannerCache(to path: AbsolutePath) { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to serialize scanner cache with no scanner instance.") @@ -171,14 +190,21 @@ public class InterModuleDependencyOracle { return diags.isEmpty ? nil : diags } - public func createCAS(path: String) throws { + public func createCAS(toolchainCASLibPath: AbsolutePath?, + onDiskPath: AbsolutePath?, + pluginOptions: [String: String]) throws { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") } - try swiftScan.createCAS(casPath: path) + let pluginPath = toolchainCASLibPath?.pathString + let casPath = onDiskPath?.pathString + try swiftScan.createCAS(pluginPath: pluginPath, casPath: casPath, pluginOptions: pluginOptions) } public func store(data: Data) throws -> String { + if let cas = toolchainCASPlugin { + return try cas.store(data: data) + } guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") } @@ -194,6 +220,7 @@ public class InterModuleDependencyOracle { } private var hasScannerInstance: Bool { self.swiftScanLibInstance != nil } + private var hasCASInstance: Bool { self.toolchainCASPlugin != nil } /// Queue to sunchronize accesses to the scanner internal let queue = DispatchQueue(label: "org.swift.swift-driver.swift-scan") @@ -201,6 +228,9 @@ public class InterModuleDependencyOracle { /// A reference to an instance of the compiler's libSwiftScan shared library private var swiftScanLibInstance: SwiftScan? = nil + /// A reference to an instance of the compiler's libToolchainCASPlugin shared library + private var toolchainCASPlugin: CASPlugin? = nil + internal let scannerRequiresPlaceholderModules: Bool } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index b0971c81d..9e1487958 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -167,7 +167,18 @@ public extension Driver { } } if !fallbackToFrontend && enableCaching { - try interModuleDependencyOracle.createCAS(path: casPath) + let pluginPath = try getCASPluginPath() + let onDiskPath = try getOnDiskCASPath() + let pluginOptions = try getCASPluginOptions() + if let plugin = pluginPath { + try interModuleDependencyOracle.verifyOrCreateCASInstance(fileSystem: fileSystem, + toolchainCASLibPath: plugin, + onDiskPath: onDiskPath, + pluginOptions: pluginOptions) + } + try interModuleDependencyOracle.createCAS(toolchainCASLibPath: pluginPath, + onDiskPath: onDiskPath, + pluginOptions: pluginOptions) } return fallbackToFrontend } diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 4fe14b919..3d7f91a33 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -345,12 +345,15 @@ extension Driver { // CAS related options. if enableCaching { commandLine.appendFlag(.cacheCompileJob) - if !casPath.isEmpty { + if let casPath = try getOnDiskCASPath() { commandLine.appendFlag(.casPath) - commandLine.appendFlag(casPath) + commandLine.appendFlag(casPath.pathString) } try commandLine.appendLast(.cacheRemarks, from: &parsedOptions) - try commandLine.appendLast(.casPluginPath, from: &parsedOptions) + if let pluginPath = try getCASPluginPath() { + commandLine.appendFlag(.casPluginPath) + commandLine.appendFlag(pluginPath.pathString) + } try commandLine.appendAll(.casPluginOption, from: &parsedOptions) } if useClangIncludeTree { diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index a43ab6cd2..1876fd97f 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -282,6 +282,7 @@ internal extension swiftscan_diagnostic_severity_t { @_spi(Testing) public var supportsCaching : Bool { return api.swiftscan_cas_create != nil && + api.swiftscan_cas_create_from_plugin != nil && api.swiftscan_cas_dispose != nil && api.swiftscan_compute_cache_key != nil && api.swiftscan_cas_store != nil && @@ -382,8 +383,19 @@ internal extension swiftscan_diagnostic_severity_t { } } - func createCAS(casPath: String) throws { - self.cas = api.swiftscan_cas_create(casPath.cString(using: String.Encoding.utf8)) + func createCAS(pluginPath: String?, casPath: String?, pluginOptions: [String: String]) throws { + if let plugin = pluginPath?.cString(using: String.Encoding.utf8) { + let onDiskPath = casPath?.cString(using: String.Encoding.utf8) + let keys = Array(pluginOptions.keys) + let values = Array(pluginOptions.values) + withArrayOfCStrings(keys) { c_key_array in + withArrayOfCStrings(values) { c_value_array in + self.cas = api.swiftscan_cas_create_from_plugin(plugin, onDiskPath, Int32(pluginOptions.count), c_key_array, c_value_array) + } + } + } else if let onDiskPath = casPath { + self.cas = api.swiftscan_cas_create(onDiskPath.cString(using: String.Encoding.utf8)) + } guard self.cas != nil else { throw DependencyScanningError.failedToInstantiateCAS } @@ -514,6 +526,7 @@ private extension swiftscan_functions_t { try loadOptional("swiftscan_clang_detail_get_module_cache_key") self.swiftscan_cas_create = try loadOptional("swiftscan_cas_create") + self.swiftscan_cas_create_from_plugin = try loadOptional("swiftscan_cas_create_from_plugin") self.swiftscan_cas_dispose = try loadOptional("swiftscan_cas_dispose") self.swiftscan_compute_cache_key = try loadOptional("swiftscan_compute_cache_key") diff --git a/Sources/SwiftDriver/ToolchainCASPlugin/CASExtensions.swift b/Sources/SwiftDriver/ToolchainCASPlugin/CASExtensions.swift new file mode 100644 index 000000000..417204125 --- /dev/null +++ b/Sources/SwiftDriver/ToolchainCASPlugin/CASExtensions.swift @@ -0,0 +1,61 @@ +//===------------------------ CASExtensions.swift -------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +@_implementationOnly import CToolchainCASPlugin + +import struct Foundation.Data + +extension llcas_data_t { + init(data: UnsafeRawPointer?, size: Int) { + self.init() + self.data = data + self.size = size + } +} + +extension llcas_functions_t { + init(_ swiftscan: Loader.Handle) throws { + self.init() + // MARK: load APIs from the dylib. + func load(_ symbol: String) throws -> T { + guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else { + throw DependencyScanningError.missingRequiredSymbol(symbol) + } + return sym + } + self.llcas_get_plugin_version = try load("llcas_get_plugin_version") + self.llcas_string_dispose = try load("llcas_string_dispose") + self.llcas_cas_options_create = try load("llcas_cas_options_create") + self.llcas_cas_options_dispose = try load("llcas_cas_options_dispose") + self.llcas_cas_options_set_client_version = try load("llcas_cas_options_set_client_version") + self.llcas_cas_options_set_ondisk_path = try load("llcas_cas_options_set_ondisk_path") + self.llcas_cas_options_set_option = try load("llcas_cas_options_set_option") + self.llcas_cas_create = try load("llcas_cas_create") + self.llcas_cas_dispose = try load("llcas_cas_dispose") + self.llcas_cas_get_hash_schema_name = try load("llcas_cas_get_hash_schema_name") + self.llcas_digest_parse = try load("llcas_digest_parse") + self.llcas_digest_print = try load("llcas_digest_print") + self.llcas_cas_get_objectid = try load("llcas_cas_get_objectid") + self.llcas_objectid_get_digest = try load("llcas_objectid_get_digest") + self.llcas_cas_contains_object = try load("llcas_cas_contains_object") + self.llcas_cas_load_object = try load("llcas_cas_load_object") + self.llcas_cas_load_object_async = try load("llcas_cas_load_object_async") + self.llcas_cas_store_object = try load("llcas_cas_store_object") + self.llcas_loaded_object_get_data = try load("llcas_loaded_object_get_data") + self.llcas_loaded_object_get_refs = try load("llcas_loaded_object_get_refs") + self.llcas_object_refs_get_count = try load("llcas_object_refs_get_count") + self.llcas_object_refs_get_id = try load("llcas_object_refs_get_id") + self.llcas_actioncache_get_for_digest = try load("llcas_actioncache_get_for_digest") + self.llcas_actioncache_get_for_digest_async = try load("llcas_actioncache_get_for_digest_async") + self.llcas_actioncache_put_for_digest = try load("llcas_actioncache_put_for_digest") + self.llcas_actioncache_put_for_digest_async = try load("llcas_actioncache_put_for_digest_async") + } +} diff --git a/Sources/SwiftDriver/ToolchainCASPlugin/ToolchainCASPlugin.swift b/Sources/SwiftDriver/ToolchainCASPlugin/ToolchainCASPlugin.swift new file mode 100644 index 000000000..7ec890f8e --- /dev/null +++ b/Sources/SwiftDriver/ToolchainCASPlugin/ToolchainCASPlugin.swift @@ -0,0 +1,133 @@ +//===------------------------ ToolchainCASPlugin.swift --------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import CToolchainCASPlugin + +import struct Foundation.Data +import protocol TSCBasic.DiagnosticData +import struct TSCBasic.AbsolutePath +import struct TSCBasic.Diagnostic + +public enum CASError: Error, DiagnosticData { + case failedToCreateOption + case failedToCreate(String) + case failedToStore(String) + case cannotPrintDigest(String) + + public var description: String { + switch self { + case .failedToCreateOption: + return "failed to create CAS options" + case .failedToCreate(let reason): + return "failed to create CAS: '\(reason)'" + case .failedToStore(let reason): + return "failed to store into CAS: '\(reason)'" + case .cannotPrintDigest(let reason): + return "cannot convert CAS digest into printable ID: '\(reason)'" + } + } +} + +/// Wrapper for libToolchainCASPlugin.dylib, handling a CAS instance and its functions. +@_spi(Testing) public final class CASPlugin { + /// The path to the libToolchainCASPlugin dylib. + let path: AbsolutePath + + /// The handle to the dylib. + let dylib: Loader.Handle + + /// libToolchainCASPlugin.dylib APIs. + let api: llcas_functions_t; + + /// CASOptions. + let options: llcas_cas_options_t + + /// CAS instance. + let db: llcas_cas_t + + @_spi(Testing) public init(dylib path: AbsolutePath, + path ondisk: AbsolutePath?, + options args: [String: String]) throws { + self.path = path + #if os(Windows) + self.dylib = try Loader.load(path.pathString, mode: []) + #else + self.dylib = try Loader.load(path.pathString, mode: [.lazy, .local, .first]) + #endif + self.api = try llcas_functions_t(self.dylib) + guard let opts = api.llcas_cas_options_create() else { + throw CASError.failedToCreateOption + } + api.llcas_cas_options_set_client_version(opts, UInt32(LLCAS_VERSION_MAJOR), UInt32(LLCAS_VERSION_MINOR)) + if let onDiskPath = ondisk { + api.llcas_cas_options_set_ondisk_path(opts, onDiskPath.pathString) + } + var c_err_msg: UnsafeMutablePointer? + for (name, value) in args { + guard !api.llcas_cas_options_set_option(opts, name.cString(using: String.Encoding.utf8), + value.cString(using:String.Encoding.utf8), &c_err_msg) else { + let err_msg = String(cString: c_err_msg!) + api.llcas_string_dispose(c_err_msg!) + throw CASError.failedToCreate(err_msg) + } + } + self.options = opts + let c_cas = api.llcas_cas_create(options, &c_err_msg) + guard let cas = c_cas else { + let err_msg = String(cString: c_err_msg!) + api.llcas_string_dispose(c_err_msg!) + throw CASError.failedToCreate(err_msg) + } + self.db = cas + } + + deinit { + api.llcas_cas_options_dispose(options) + api.llcas_cas_dispose(db) + // Is it safe to close? + dylib.leak() + } + + // Store a data blob inside CAS and return the digest of CAS object. + public func store(data: Data) throws -> String { + var c_err_msg: UnsafeMutablePointer? + var cas_object_id = llcas_objectid_t() + try data.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + let cas_data = llcas_data_t(data: bytes.baseAddress, size: data.count) + if api.llcas_cas_store_object(db, cas_data, nil, 0, &cas_object_id, &c_err_msg) { + let msg = convertString(c_err_msg!) + throw CASError.failedToStore(msg) + } + } + return try getObjectIDString(id: cas_object_id) + } +} + +// Helper functions. +extension CASPlugin { + private func convertString(_ c_err_msg: UnsafeMutablePointer) -> String { + let err_msg = String(cString: c_err_msg) + api.llcas_string_dispose(c_err_msg) + return err_msg + } + + private func getObjectIDString(id: llcas_objectid_t) throws -> String { + let cas_digest = api.llcas_objectid_get_digest(db, id) + var cas_printed_id: UnsafeMutablePointer? + var c_err_msg: UnsafeMutablePointer? + if api.llcas_digest_print(db, cas_digest, &cas_printed_id, &c_err_msg) { + let msg = convertString(c_err_msg!) + throw CASError.cannotPrintDigest(msg) + } + return convertString(cas_printed_id!) + } +} diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index a629063f5..023d04e49 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -284,6 +284,32 @@ extension Toolchain { #endif } + /// Looks for the executable in the `SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB` environment variable, if found nothing, + /// looks in the `lib` relative to the compiler executable. + @_spi(Testing) public func lookupToolchainCASPluginLib() throws -> AbsolutePath? { + if let overrideString = env["SWIFT_DRIVER_TOOLCHAIN_CASPLUGIN_LIB"], + let path = try? AbsolutePath(validating: overrideString) { + return path + } +#if os(Windows) + return nil +#else + let libraryName = sharedLibraryName("libToolchainCASPlugin") + let compilerPath = try getToolPath(.swiftCompiler) + let developerPath = compilerPath.parentDirectory // bin + .parentDirectory // toolchain root + .parentDirectory // toolchains + .parentDirectory // developer + let libraryPath = developerPath.appending(component: "usr") + .appending(component: "lib") + .appending(component: libraryName) + if fileSystem.isFile(libraryPath) { + return libraryPath + } + return nil +#endif + } + private func xcrunFind(executable: String) throws -> AbsolutePath { let xcrun = "xcrun" guard lookupExecutablePath(filename: xcrun, searchPaths: searchPaths) != nil else { diff --git a/Sources/SwiftDriver/SwiftScan/Loader.swift b/Sources/SwiftDriver/Utilities/Loader.swift similarity index 100% rename from Sources/SwiftDriver/SwiftScan/Loader.swift rename to Sources/SwiftDriver/Utilities/Loader.swift