Merge branch 'main' into enable_MemberImportVisibility_check

.github/workflows/swift_test_matrix.yml

@@ -19,8 +19,8 @@ concurrency:
 
 jobs:
   execute-matrix:
-    name: ${{ matrix.swift.platform }} (${{ matrix.swift.name }})
-    runs-on: ${{ matrix.swift.runner }}
+    name: ${{ matrix.config.platform }} (${{ matrix.config.name }})
+    runs-on: ${{ matrix.config.runner }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(inputs.matrix_string) }}
@@ -31,37 +31,37 @@ jobs:
           persist-credentials: false
           submodules: true
       - name: Pull Docker image
-        run: docker pull ${{ matrix.swift.image }}
+        run: docker pull ${{ matrix.config.image }}
       - name: Run matrix job
-        if: ${{ matrix.swift.platform != 'Windows' }}
+        if: ${{ matrix.config.platform != 'Windows' }}
         run: |
-          if [[ -n "${{ matrix.swift.setup_command }}" ]]; then
-            setup_command_expression="${{ matrix.swift.setup_command }} &&"
+          if [[ -n "${{ matrix.config.setup_command }}" ]]; then
+            setup_command_expression="${{ matrix.config.setup_command }} &&"
           else
             setup_command_expression=""
           fi
           workspace="/$(basename ${{ github.workspace }})"
           docker run -v ${{ github.workspace }}:"$workspace" \
             -w "$workspace" \
-            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" \
+            -e SWIFT_VERSION="${{ matrix.config.swift_version }}" \
             -e setup_command_expression="$setup_command_expression" \
             -e workspace="$workspace" \
-            ${{ matrix.swift.image }} \
-            bash -c "swift --version && git config --global --add safe.directory \"$workspace\" && $setup_command_expression ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"
+            ${{ matrix.config.image }} \
+            bash -c "swift --version && git config --global --add safe.directory \"$workspace\" && $setup_command_expression ${{ matrix.config.command }} ${{ matrix.config.command_arguments }}"
       - name: Run matrix job (Windows)
-        if: ${{ matrix.swift.platform == 'Windows' }}
+        if: ${{ matrix.config.platform == 'Windows' }}
         run: |
-          if (-not [string]::IsNullOrEmpty("${{ matrix.swift.setup_command }}")) {
-              $setup_command_expression = "${{ matrix.swift.setup_command }} &"
+          if (-not [string]::IsNullOrEmpty("${{ matrix.config.setup_command }}")) {
+              $setup_command_expression = "${{ matrix.config.setup_command }} &"
           } else {
               $setup_command_expression = ""
           }
           $workspace = "C:\" + (Split-Path ${{ github.workspace }} -Leaf)
           docker run -v ${{ github.workspace }}:$($workspace) `
             -w $($workspace) `
-            -e SWIFT_VERSION="${{ matrix.swift.swift_version }}" `
+            -e SWIFT_VERSION="${{ matrix.config.swift_version }}" `
             -e setup_command_expression=%setup_command_expression% `
-            ${{ matrix.swift.image }} `
-            cmd /s /c  "swift --version & powershell Invoke-Expression ""$($setup_command_expression) ${{ matrix.swift.command }} ${{ matrix.swift.command_arguments }}"""
+            ${{ matrix.config.image }} `
+            cmd /s /c  "swift --version & powershell Invoke-Expression ""$($setup_command_expression) ${{ matrix.config.command }} ${{ matrix.config.command_arguments }}"""
     env:
-      SWIFT_VERSION: ${{ matrix.swift.swift_version }}
+      SWIFT_VERSION: ${{ matrix.config.swift_version }}

NOTICE.txt

@@ -95,3 +95,12 @@ This product contains a derivation of the mocking infrastructure from Swift Syst
     * https://www.apache.org/licenses/LICENSE-2.0
   * HOMEPAGE:
     * https://github.com/apple/swift-system
+
+---
+
+This product contains a derivation of "TokenBucket.swift" from Swift Package Manager.
+
+  * LICENSE (Apache License 2.0):
+    * https://www.apache.org/licenses/LICENSE-2.0
+  * HOMEPAGE:
+    * https://github.com/swiftlang/swift-package-manager

Sources/NIOAsyncAwaitDemo/AsyncChannelIO.swift

@@ -24,8 +24,11 @@ struct AsyncChannelIO<Request, Response> {
     }
 
     func start() async throws -> AsyncChannelIO<Request, Response> {
-        try await channel.pipeline.addHandler(RequestResponseHandler<HTTPRequestHead, NIOHTTPClientResponseFull>())
-            .get()
+        try await channel.eventLoop.submit {
+            try channel.pipeline.syncOperations.addHandler(
+                RequestResponseHandler<HTTPRequestHead, NIOHTTPClientResponseFull>()
+            )
+        }.get()
         return self
     }
 

Sources/NIOCore/ByteBuffer-aux.swift

@@ -733,13 +733,13 @@ extension ByteBuffer: Codable {
     public init(from decoder: Decoder) throws {
         let container = try decoder.singleValueContainer()
         let base64String = try container.decode(String.self)
-        self = try ByteBuffer(bytes: base64String.base64Decoded())
+        self = try ByteBuffer(bytes: base64String._base64Decoded())
     }
 
     /// Encodes this buffer as a base64 string in a single value container.
     public func encode(to encoder: Encoder) throws {
         var container = encoder.singleValueContainer()
-        let base64String = String(base64Encoding: self.readableBytesView)
+        let base64String = String(_base64Encoding: self.readableBytesView)
         try container.encode(base64String)
     }
 }

Sources/NIOCore/EventLoopFuture.swift

@@ -163,7 +163,12 @@ public struct EventLoopPromise<Value> {
     internal static func makeUnleakablePromise(eventLoop: EventLoop, line: UInt = #line) -> EventLoopPromise<Value> {
         EventLoopPromise<Value>(
             eventLoop: eventLoop,
-            file: "BUG in SwiftNIO (please report), unleakable promise leaked.",
+            file: """
+                EventLoopGroup shut down with unfulfilled promises remaining. \
+                This suggests that the EventLoopGroup was shut down with unfinished work outstanding which is \
+                illegal. Either switch to using the singleton EventLoopGroups or fix the issue by only shutting down \
+                the EventLoopGroups when all the work associated with them has finished.
+                """,
             line: line
         )
     }

Sources/NIOFileSystem/FileSystem.swift

@@ -366,6 +366,8 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         }
     }
 
+    /// See ``FileSystemProtocol/removeItem(at:strategy:recursively:)``
+    ///
     /// Deletes the file or directory (and its contents) at `path`.
     ///
     /// Only regular files, symbolic links and directories may be removed. If the file at `path` is
@@ -387,11 +389,14 @@ public struct FileSystem: Sendable, FileSystemProtocol {
     ///
     /// - Parameters:
     ///   - path: The path to delete.
+    ///   - removalStrategy: Whether to delete files sequentially (one-by-one), or perform a
+    ///       concurrent scan of the tree at `path` and delete files when they are found.
     ///   - removeItemRecursively: Whether or not to remove items recursively.
     /// - Returns: The number of deleted items which may be zero if `path` did not exist.
     @discardableResult
     public func removeItem(
         at path: FilePath,
+        strategy removalStrategy: RemovalStrategy,
         recursively removeItemRecursively: Bool
     ) async throws -> Int {
         // Try to remove the item: we might just get lucky.
@@ -421,39 +426,60 @@ public struct FileSystem: Sendable, FileSystemProtocol {
                 )
             }
 
-            var (subdirectories, filesRemoved) = try await self.withDirectoryHandle(
-                atPath: path
-            ) { directory in
-                var subdirectories = [FilePath]()
-                var filesRemoved = 0
+            switch removalStrategy.wrapped {
+            case .sequential:
+                return try await self.removeItemSequentially(at: path)
+            case let .parallel(maxDescriptors):
+                return try await self.removeConcurrently(at: path, maxDescriptors)
+            }
 
-                for try await batch in directory.listContents().batched() {
-                    for entry in batch {
-                        switch entry.type {
-                        case .directory:
-                            subdirectories.append(entry.path)
+        case let .failure(errno):
+            throw FileSystemError.remove(errno: errno, path: path, location: .here())
+        }
+    }
 
-                        default:
-                            filesRemoved += try await self.removeOneItem(at: entry.path)
-                        }
+    @discardableResult
+    private func removeItemSequentially(
+        at path: FilePath
+    ) async throws -> Int {
+        var (subdirectories, filesRemoved) = try await self.withDirectoryHandle(
+            atPath: path
+        ) { directory in
+            var subdirectories = [FilePath]()
+            var filesRemoved = 0
+
+            for try await batch in directory.listContents().batched() {
+                for entry in batch {
+                    switch entry.type {
+                    case .directory:
+                        subdirectories.append(entry.path)
+
+                    default:
+                        filesRemoved += try await self.removeOneItem(at: entry.path)
                     }
                 }
-
-                return (subdirectories, filesRemoved)
             }
 
-            for subdirectory in subdirectories {
-                filesRemoved += try await self.removeItem(at: subdirectory)
-            }
+            return (subdirectories, filesRemoved)
+        }
 
-            // The directory should be empty now. Remove ourself.
-            filesRemoved += try await self.removeOneItem(at: path)
+        for subdirectory in subdirectories {
+            filesRemoved += try await self.removeItemSequentially(at: subdirectory)
+        }
 
-            return filesRemoved
+        // The directory should be empty now. Remove ourself.
+        filesRemoved += try await self.removeOneItem(at: path)
 
-        case let .failure(errno):
-            throw FileSystemError.remove(errno: errno, path: path, location: .here())
-        }
+        return filesRemoved
+
+    }
+
+    private func removeConcurrently(
+        at path: FilePath,
+        _ maxDescriptors: Int
+    ) async throws -> Int {
+        let bucket: TokenBucket = .init(tokens: maxDescriptors)
+        return try await self.discoverAndRemoveItemsInTree(at: path, bucket)
     }
 
     /// Moves the named file or directory to a new location.
@@ -490,7 +516,7 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         case .differentLogicalDevices:
             // Fall back to copy and remove.
             try await self.copyItem(at: sourcePath, to: destinationPath)
-            try await self.removeItem(at: sourcePath)
+            try await self.removeItem(at: sourcePath, strategy: .platformDefault)
         }
     }
 
@@ -518,9 +544,9 @@ public struct FileSystem: Sendable, FileSystemProtocol {
         withItemAt existingPath: FilePath
     ) async throws {
         do {
-            try await self.removeItem(at: destinationPath)
+            try await self.removeItem(at: destinationPath, strategy: .platformDefault)
             try await self.moveItem(at: existingPath, to: destinationPath)
-            try await self.removeItem(at: existingPath)
+            try await self.removeItem(at: existingPath, strategy: .platformDefault)
         } catch let error as FileSystemError {
             throw FileSystemError(
                 message: "Can't replace '\(destinationPath)' with '\(existingPath)'.",