From 336ab8fa4ba40096570b49d75e16a7e9000bd185 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 22 Sep 2026 02:39:16 +0000 Subject: [PATCH 1/2] fix(storage): rename ByteBuffer to ByteChunk and hide NIOCore from public API Rename `GoogleCloudStorage.ByteBuffer` to `ByteChunk` (and conform it to `ContiguousBytes`) so that it does not shadow or get confused with `NIOCore.ByteBuffer` in compiler diagnostics and consumer code. Make all `NIOCore.ByteBuffer` and `NIOHTTP1` initializers, properties, and helpers (`ByteChunk.init(_: NIOCore.ByteBuffer)`, `ByteChunk.byteBuffer`, `StreamSource.init(sequence:) where S.Element == NIOCore.ByteBuffer`, and `StorageClient.parseReadObjectMetadata`) `internal` so that `swift-nio` types are completely encapsulated within `GoogleCloudStorage`. Fixes #973 --- Package.swift | 1 - Tests/StorageW1R3/BenchmarkRunner.swift | 17 ++--- Tests/StorageW1R3/StorageOperations.swift | 9 ++- .../{ByteBuffer.swift => ByteChunk.swift} | 64 +++++++------------ .../GoogleCloudStorage/BytesSource.swift | 13 ++-- .../ChecksumCalculator.swift | 4 +- .../ChecksummedSource.swift | 8 +-- .../GoogleCloudStorage/DownloadOptions.swift | 18 +++--- .../GoogleCloudStorage/FileSource.swift | 4 +- .../MultipartUploadStream.swift | 4 +- .../StorageClient+Download.swift | 4 +- .../StorageClient+Upload.swift | 14 ++-- .../GoogleCloudStorage/StreamSource.swift | 12 ++-- .../GoogleCloudStorage/UploadSource.swift | 3 +- ...BufferTests.swift => ByteChunkTests.swift} | 43 +++++++------ .../Tests/BytesSourceTests.swift | 2 +- .../Tests/ChecksumTests.swift | 38 +++++------ .../Tests/FileSourceTests.swift | 4 +- .../StorageClientIntegrationTests.swift | 4 +- .../Tests/ResumableUploadTests.swift | 8 +-- .../Tests/SimpleUploadTests.swift | 4 +- .../Tests/StreamSourceTests.swift | 10 +-- .../Tests/UrlMocks.swift | 8 +-- 23 files changed, 139 insertions(+), 157 deletions(-) rename pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/{ByteBuffer.swift => ByteChunk.swift} (77%) rename pkgs/swift-google-cloud-storage/Tests/{ByteBufferTests.swift => ByteChunkTests.swift} (79%) diff --git a/Package.swift b/Package.swift index 338131846b..3fec669aa2 100644 --- a/Package.swift +++ b/Package.swift @@ -183,7 +183,6 @@ let package = Package( .product(name: "GoogleAuth", package: "swift-google-auth"), .product(name: "GoogleGax", package: "swift-google-gax"), .product(name: "Logging", package: "swift-log"), - .product(name: "NIOCore", package: "swift-nio"), ], path: "Tests/StorageW1R3", exclude: ["README.md"] diff --git a/Tests/StorageW1R3/BenchmarkRunner.swift b/Tests/StorageW1R3/BenchmarkRunner.swift index eb5619febf..4752259442 100644 --- a/Tests/StorageW1R3/BenchmarkRunner.swift +++ b/Tests/StorageW1R3/BenchmarkRunner.swift @@ -13,7 +13,6 @@ // limitations under the License. import Foundation -import NIOCore import GoogleAuth import GoogleGax import GoogleCloudStorage @@ -23,7 +22,7 @@ extension StorageW1R3 { func runWorker( taskIndex: Int, counters: BenchmarkCounters, - buffer: NIOCore.ByteBuffer, + buffer: ByteChunk, storageClient: StorageClient, controlClient: StorageControlClient, ) async { @@ -55,7 +54,7 @@ extension StorageW1R3 { let objectName = Self.randomObjectName() let isResumable = Bool.random() let uploadCrc32c = self.pickCrc32c() - let uploadSlice = buffer.getSlice(at: 0, length: size) ?? buffer.slice() + let uploadSlice = buffer.subdata(in: 0.. GoogleCloudStorage.Object? { @@ -133,7 +132,7 @@ extension StorageW1R3 { let uploadBuilder = SampleBuilder( iterationId: iterationId, op: uploadOp, - targetSize: buffer.readableBytes, + targetSize: buffer.count, object: objectName, crc32cEnabled: crc32cEnabled ) @@ -246,10 +245,9 @@ extension StorageW1R3 { print(sample.toRow()) } - func generateRandomBuffer() -> NIOCore.ByteBuffer { + func generateRandomBuffer() -> ByteChunk { let size = self.maxObjectSize - var buffer = ByteBufferAllocator().buffer(capacity: size) - guard size > 0 else { return buffer } + guard size > 0 else { return ByteChunk() } // There is a lot going on here. Sometimes the benchmark is used with really large buffers, // 256MiB and 2GiB are not uncommon. To efficiently initialized the buffer with random data // we create an array of the desired size. @@ -268,8 +266,7 @@ extension StorageW1R3 { } initializedCount = size } - buffer.writeBytes(bytes) - return buffer + return ByteChunk(bytes) } private static func randomObjectName() -> String { diff --git a/Tests/StorageW1R3/StorageOperations.swift b/Tests/StorageW1R3/StorageOperations.swift index b6dd5a6110..22bd59b9c8 100644 --- a/Tests/StorageW1R3/StorageOperations.swift +++ b/Tests/StorageW1R3/StorageOperations.swift @@ -13,7 +13,6 @@ // limitations under the License. import Foundation -import NIOCore import GoogleAuth import GoogleGax import GoogleCloudStorage @@ -25,7 +24,7 @@ enum StorageOperations { controlClient: StorageControlClient, bucketName: String, objectName: String, - buffer: NIOCore.ByteBuffer, + buffer: ByteChunk, isResumable: Bool, crc32cEnabled: Bool ) async throws -> GoogleCloudStorage.Object { @@ -37,15 +36,15 @@ enum StorageOperations { // If resumable, chunk size is set to 32MiB; if simple, threshold handles it if isResumable { $0.chunkSize = 32 * 1024 * 1024 - $0.resumableUploadThreshold = buffer.readableBytes + $0.resumableUploadThreshold = buffer.count } else { - $0.resumableUploadThreshold = buffer.readableBytes + 256 * 1024 + $0.resumableUploadThreshold = buffer.count + 256 * 1024 } } do { return try await client.upload( - BytesSource(buffer: .init(buffer)), to: bucketName, as: objectName, options: options) + BytesSource(buffer: buffer), to: bucketName, as: objectName, options: options) } catch let reqError as RequestError where reqError.isFailedPrecondition { logToStderr("Precondition failed for \(objectName), fetching object details") let getReq = GetObjectRequest().with { diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteBuffer.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift similarity index 77% rename from pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteBuffer.swift rename to pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift index f487ca5d67..c4c4574ec1 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteBuffer.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift @@ -17,45 +17,38 @@ import NIOCore import NIOFoundationCompat /// A container representing a sequence of bytes backed either by `Foundation.Data` -/// or `NIOCore.ByteBuffer` without unnecessary memory copying. -public struct ByteBuffer: Sendable { - @usableFromInline +/// or an internal network buffer without unnecessary memory copying. +public struct ByteChunk: Sendable, ContiguousBytes { internal enum Storage: Sendable { case data(Data) case byteBuffer(NIOCore.ByteBuffer) } - @usableFromInline internal let storage: Storage // MARK: - Initializers - /// Creates a byte buffer wrapping a `Foundation.Data` instance (zero-copy). - @inlinable + /// Creates a byte chunk wrapping a `Foundation.Data` instance (zero-copy). public init(_ data: Data) { self.storage = .data(data) } - /// Creates a byte buffer wrapping a `NIOCore.ByteBuffer` instance (zero-copy). - @inlinable - public init(_ buffer: NIOCore.ByteBuffer) { + /// Creates a byte chunk wrapping a `NIOCore.ByteBuffer` instance (zero-copy). + internal init(_ buffer: NIOCore.ByteBuffer) { self.storage = .byteBuffer(buffer) } - /// Creates an empty byte buffer instance. - @inlinable + /// Creates an empty byte chunk instance. public init() { self.storage = .data(Data()) } - /// Creates a byte buffer from an array of bytes. - @inlinable + /// Creates a byte chunk from an array of bytes. public init(_ bytes: [UInt8]) { self.storage = .data(Data(bytes)) } - /// Creates a byte buffer from a contiguous raw buffer pointer. - @inlinable + /// Creates a byte chunk from a contiguous raw buffer pointer. public init(_ bufferPointer: UnsafeRawBufferPointer) { self.storage = .data(Data(bufferPointer)) } @@ -63,9 +56,8 @@ public struct ByteBuffer: Sendable { // MARK: - Core Properties & Accessors -extension ByteBuffer { +extension ByteChunk { /// The total number of readable bytes stored. - @inlinable public var count: Int { switch storage { case .data(let data): @@ -75,14 +67,12 @@ extension ByteBuffer { } } - /// Indicates whether the buffer contains zero bytes. - @inlinable + /// Indicates whether the chunk contains zero bytes. public var isEmpty: Bool { count == 0 } /// Calls a closure with a pointer to the contiguous bytes without copying. - @inlinable public func withUnsafeBytes(_ body: (UnsafeRawBufferPointer) throws -> R) rethrows -> R { switch storage { case .data(let data): @@ -95,8 +85,7 @@ extension ByteBuffer { /// The underlying contents as a `Foundation.Data` instance. /// /// - Returns: The original `Data` with zero copies if backed by `Data`, - /// or copies the bytes into a new `Data` instance if backed by `NIOCore.ByteBuffer`. - @inlinable + /// or copies the bytes into a new `Data` instance if backed by an internal network buffer. public var data: Data { switch storage { case .data(let data): @@ -110,8 +99,7 @@ extension ByteBuffer { /// /// - Returns: The original `NIOCore.ByteBuffer` with zero copies if backed by `NIOCore.ByteBuffer`, /// or copies the bytes into a new `NIOCore.ByteBuffer` instance if backed by `Data`. - @inlinable - public var byteBuffer: NIOCore.ByteBuffer { + internal var byteBuffer: NIOCore.ByteBuffer { switch storage { case .byteBuffer(let buffer): return buffer @@ -125,42 +113,38 @@ extension ByteBuffer { } /// Returns the bytes as a newly allocated `[UInt8]` array. - @inlinable public var byteArray: [UInt8] { withUnsafeBytes { Array($0) } } - /// Returns a zero-copy sub-buffer within the specified byte range. - public func subdata(in range: Range) -> ByteBuffer { + /// Returns a zero-copy sub-chunk within the specified byte range. + public func subdata(in range: Range) -> ByteChunk { switch storage { case .data(let data): let start = data.startIndex.advanced(by: range.lowerBound) let end = data.startIndex.advanced(by: range.upperBound) - return ByteBuffer(data[start.. UInt8 { precondition(position >= 0 && position < count, "Index \(position) out of bounds 0..<\(count)") switch storage { @@ -174,8 +158,8 @@ extension ByteBuffer: RandomAccessCollection { // MARK: - Equatable & Hashable -extension ByteBuffer: Equatable { - public static func == (lhs: ByteBuffer, rhs: ByteBuffer) -> Bool { +extension ByteChunk: Equatable { + public static func == (lhs: ByteChunk, rhs: ByteChunk) -> Bool { guard lhs.count == rhs.count else { return false } if lhs.isEmpty { return true } return lhs.withUnsafeBytes { lhsBytes in @@ -189,7 +173,7 @@ extension ByteBuffer: Equatable { } } -extension ByteBuffer: Hashable { +extension ByteChunk: Hashable { public func hash(into hasher: inout Hasher) { withUnsafeBytes { hasher.combine(bytes: $0) } } @@ -197,13 +181,13 @@ extension ByteBuffer: Hashable { // MARK: - Literal & Description Conformances -extension ByteBuffer: ExpressibleByArrayLiteral { +extension ByteChunk: ExpressibleByArrayLiteral { public init(arrayLiteral elements: UInt8...) { self.init(Data(elements)) } } -extension ByteBuffer: CustomStringConvertible, CustomDebugStringConvertible { +extension ByteChunk: CustomStringConvertible, CustomDebugStringConvertible { public var description: String { "\(count) bytes" } @@ -214,6 +198,6 @@ extension ByteBuffer: CustomStringConvertible, CustomDebugStringConvertible { case .data: backing = "Data" case .byteBuffer: backing = "NIOCore.ByteBuffer" } - return "ByteBuffer(\(count) bytes, backing: \(backing))" + return "ByteChunk(\(count) bytes, backing: \(backing))" } } diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/BytesSource.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/BytesSource.swift index 25d454de55..72aeb3d745 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/BytesSource.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/BytesSource.swift @@ -13,25 +13,28 @@ // limitations under the License. import Foundation -import NIOCore /// An upload source that wraps in-memory bytes or buffers. public struct BytesSource: SeekableUploadSource { - public let buffer: ByteBuffer + public let buffer: ByteChunk public var totalSize: UInt64? { return UInt64(buffer.count) } private var offset: UInt64 = 0 - public init(buffer: ByteBuffer) { + public init(buffer: ByteChunk) { self.buffer = buffer } + public init(_ chunk: ByteChunk) { + self.buffer = chunk + } + public init(data: Data) { - self.buffer = ByteBuffer(data) + self.buffer = ByteChunk(data) } - public mutating func read(maxBytes: Int) async throws -> ByteBuffer? { + public mutating func read(maxBytes: Int) async throws -> ByteChunk? { guard maxBytes > 0, offset < UInt64(buffer.count) else { return nil } let end = min(offset + UInt64(maxBytes), UInt64(buffer.count)) let chunk = buffer.subdata(in: Int(offset).. { var source: S let options: ChecksumOptions private var calculators: [any ChecksumCalculator] = [] - private var nextChunk: ByteBuffer? = nil + private var nextChunk: ByteChunk? = nil private var isInitialized = false private var isFinished = false /// The high-water mark of sequentially processed bytes in `calculators`. @@ -78,13 +78,13 @@ struct ChecksummedSource { /// To support seeking backward and retrying chunk uploads without corrupting checksums, /// this method skips any prefix of `data` that falls below `bytesHashed` (the high-water mark /// of bytes already fed into `calculators`). Only bytes beyond `bytesHashed` are accumulated. - private mutating func updateChecksums(data: ByteBuffer, startOffset: UInt64) { + private mutating func updateChecksums(data: ByteChunk, startOffset: UInt64) { guard !calculators.isEmpty else { return } let endOffset = startOffset + UInt64(data.count) guard endOffset > bytesHashed else { return } - let unhashedData: ByteBuffer + let unhashedData: ByteChunk if startOffset >= bytesHashed { unhashedData = data } else { diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/DownloadOptions.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/DownloadOptions.swift index 97a97f4cdb..f4a42e4436 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/DownloadOptions.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/DownloadOptions.swift @@ -387,9 +387,9 @@ public struct ReadObjectMetadata: Sendable, Hashable, Equatable { } } -/// An asynchronous sequence of `ByteBuffer` chunks representing an object payload being downloaded. +/// An asynchronous sequence of `ByteChunk` chunks representing an object payload being downloaded. public struct ReadObjectSequence: AsyncSequence, Sendable { - public typealias Element = ByteBuffer + public typealias Element = ByteChunk private let coordinator: ReadObjectCoordinator @@ -399,7 +399,7 @@ public struct ReadObjectSequence: AsyncSequence, Sendable { /// An asynchronous iterator for iterating over chunks of downloaded object payload data. public struct AsyncIterator: AsyncIteratorProtocol { - public typealias Element = ByteBuffer + public typealias Element = ByteChunk private let coordinator: ReadObjectCoordinator @@ -407,8 +407,8 @@ public struct ReadObjectSequence: AsyncSequence, Sendable { self.coordinator = coordinator } - /// Advances to the next `ByteBuffer` chunk in the downloaded object payload stream. - public mutating func next() async throws -> ByteBuffer? { + /// Advances to the next `ByteChunk` chunk in the downloaded object payload stream. + public mutating func next() async throws -> ByteChunk? { try await coordinator.nextChunk() } } @@ -496,7 +496,7 @@ package final class ReadObjectCoordinator: @unchecked Sendable { return try await ensureInitialFetch() } - package func nextChunk() async throws -> ByteBuffer? { + package func nextChunk() async throws -> ByteChunk? { guard !isFinished && !isCancelled else { return nil } if case .prefix(0) = options.range { @@ -516,7 +516,7 @@ package final class ReadObjectCoordinator: @unchecked Sendable { let chunk = try await it.next() self.streamIterator = it if let chunk { - let storage = ByteBuffer(chunk) + let storage = ByteChunk(chunk) bytesReceived += UInt64(storage.count) resumeState.details.bytesDownloaded = bytesReceived resumeLoop.onProgress(state: &resumeState) @@ -531,7 +531,7 @@ package final class ReadObjectCoordinator: @unchecked Sendable { let chunk = try await it.next() self.bodyIterator = it if let chunk { - let storage = ByteBuffer(chunk) + let storage = ByteChunk(chunk) bytesReceived += UInt64(storage.count) resumeState.details.bytesDownloaded = bytesReceived resumeLoop.onProgress(state: &resumeState) @@ -578,7 +578,7 @@ package final class ReadObjectCoordinator: @unchecked Sendable { return nil } - private func updateChecksums(with chunk: ByteBuffer) { + private func updateChecksums(with chunk: ByteChunk) { guard crc32cCalculator != nil || md5Calculator != nil else { return } chunk.withUnsafeBytes { buffer in crc32cCalculator?.update(buffer) diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/FileSource.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/FileSource.swift index 7cfae00ce7..3dc9eafc20 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/FileSource.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/FileSource.swift @@ -73,7 +73,7 @@ public struct FileSource: SeekableUploadSource { self.fileURL = fileURL } - public mutating func read(maxBytes: Int) async throws -> ByteBuffer? { + public mutating func read(maxBytes: Int) async throws -> ByteChunk? { guard maxBytes > 0 else { return nil } if let size = totalSize, offset >= size { if let box = handleBox { @@ -99,7 +99,7 @@ public struct FileSource: SeekableUploadSource { } offset += UInt64(nioBuffer.readableBytes) - return ByteBuffer(nioBuffer) + return ByteChunk(nioBuffer) } public mutating func seek(to offset: UInt64) async throws { diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/MultipartUploadStream.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/MultipartUploadStream.swift index ee44c1ad60..102fce0a30 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/MultipartUploadStream.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/MultipartUploadStream.swift @@ -106,7 +106,7 @@ struct MultipartUploadStream: AsyncSequence, Sendable { var nio = chunk.byteBuffer buffer.writeBuffer(&nio) } - preparedSource = BytesSource(buffer: ByteBuffer(buffer)) + preparedSource = BytesSource(buffer: ByteChunk(buffer)) } let checksum = @@ -172,7 +172,7 @@ struct MultipartUploadStream: AsyncSequence, Sendable { return buffer case .body: - let chunk: ByteBuffer? + let chunk: ByteChunk? chunk = try await source.read(maxBytes: chunkSize) if let chunk = chunk, !chunk.isEmpty { bytesYielded += UInt64(chunk.count) diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Download.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Download.swift index b7771dc718..8534fadaf6 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Download.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Download.swift @@ -17,7 +17,7 @@ import Foundation import NIOHTTP1 extension StorageClient { - /// Reads (downloads) an object from Cloud Storage as an async sequence of Data chunks. + /// Reads (downloads) an object from Cloud Storage as an async sequence of `ByteChunk` chunks. /// /// - Parameters: /// - bucket: The GCS bucket name. @@ -46,7 +46,7 @@ extension StorageClient { return ReadObjectTask(coordinator: coordinator) } - package static func parseReadObjectMetadata( + internal static func parseReadObjectMetadata( from headers: NIOHTTP1.HTTPHeaders, bucket: String, object: String diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Upload.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Upload.swift index 690b7160c0..d6aa2a2d21 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Upload.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StorageClient+Upload.swift @@ -303,7 +303,7 @@ extension StorageClient { fileprivate static func sendChunk( httpClient: GoogleGax._HTTPClient, uploadId: String, - data: ByteBuffer, + data: ByteChunk, offset: UInt64, totalSize: UInt64?, options: UploadOptions, @@ -379,7 +379,7 @@ extension StorageClient { maxBytesSent: inout UInt64 ) async throws -> (status: ResumableUploadStatus, crc32cSeed: UInt32?) { let chunkInfo = try await checksummedSource.readChunk(maxBytes: chunkSize) - let chunk: ByteBuffer + let chunk: ByteChunk let effectiveTotalSize: UInt64? let checksum: String? @@ -390,7 +390,7 @@ extension StorageClient { effectiveTotalSize = (isLast && totalSize == nil) ? (committedBytes + UInt64(chunk.count)) : totalSize } else { - chunk = ByteBuffer() + chunk = ByteChunk() effectiveTotalSize = totalSize ?? committedBytes checksum = checksummedSource.finalizeChecksum() } @@ -409,7 +409,7 @@ extension StorageClient { } private struct PendingChunk { - var data: ByteBuffer + var data: ByteChunk let isLast: Bool let checksum: String? var chunkStartOffset: UInt64 @@ -558,7 +558,7 @@ extension StorageClient { if pendingChunk == nil { let chunkInfo = try await checksummedSource!.readChunk(maxBytes: chunkSize) - let chunk: ByteBuffer + let chunk: ByteChunk let isLast: Bool let checksum: String? let effectiveTotalSize: UInt64? @@ -570,7 +570,7 @@ extension StorageClient { effectiveTotalSize = (isLast && totalSize == nil) ? (sourceBytesRead + UInt64(chunk.count)) : totalSize } else { - chunk = ByteBuffer() + chunk = ByteChunk() isLast = true effectiveTotalSize = totalSize ?? sourceBytesRead checksum = checksummedSource!.finalizeChecksum() @@ -931,7 +931,7 @@ extension StorageClient { fileprivate static func buildUploadChunkRequest( httpClient: GoogleGax._HTTPClient, uploadId: String, - data: ByteBuffer, + data: ByteChunk, offset: UInt64, totalSize: UInt64?, options: UploadOptions, diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StreamSource.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StreamSource.swift index c4c9b5491b..4b6ad30038 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StreamSource.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/StreamSource.swift @@ -15,7 +15,7 @@ import Foundation import NIOCore -/// An upload source that wraps an arbitrary AsyncSequence of ByteBuffer, Data, or NIOCore.ByteBuffer chunks. +/// An upload source that wraps an arbitrary AsyncSequence of ByteChunk or Data chunks. public struct StreamSource: UploadSource { private final class StateBox: @unchecked Sendable { var nextChunk: () async throws -> NIOCore.ByteBuffer? @@ -33,7 +33,7 @@ public struct StreamSource: UploadSource { public init( sequence: S, totalSize: UInt64? = nil - ) where S.Element == ByteBuffer { + ) where S.Element == ByteChunk { self.totalSizeValue = totalSize var iterator = sequence.makeAsyncIterator() self.stateBox = StateBox { @@ -50,11 +50,11 @@ public struct StreamSource: UploadSource { var iterator = sequence.makeAsyncIterator() self.stateBox = StateBox { guard let next = try await iterator.next() else { return nil } - return ByteBuffer(next).byteBuffer + return ByteChunk(next).byteBuffer } } - public init( + internal init( sequence: S, totalSize: UInt64? = nil ) where S.Element == NIOCore.ByteBuffer { @@ -65,7 +65,7 @@ public struct StreamSource: UploadSource { } } - public mutating func read(maxBytes: Int) async throws -> ByteBuffer? { + public mutating func read(maxBytes: Int) async throws -> ByteChunk? { guard maxBytes > 0 else { return nil } while buffer.readableBytes < maxBytes { guard let nextChunk = try await stateBox.nextChunk() else { @@ -83,6 +83,6 @@ public struct StreamSource: UploadSource { guard let slice = buffer.readSlice(length: chunkSize) else { return nil } - return ByteBuffer(slice) + return ByteChunk(slice) } } diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/UploadSource.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/UploadSource.swift index c7acbbd550..42dbb77c3b 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/UploadSource.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/UploadSource.swift @@ -13,13 +13,12 @@ // limitations under the License. import Foundation -import NIOCore /// Represents a data source that can be read from sequentially. public protocol UploadSource: Sendable { /// Reads the next chunk of data, up to `maxBytes`. /// Returns `nil` when the source is exhausted. - mutating func read(maxBytes: Int) async throws -> ByteBuffer? + mutating func read(maxBytes: Int) async throws -> ByteChunk? /// The total size of the source, if known. var totalSize: UInt64? { get } diff --git a/pkgs/swift-google-cloud-storage/Tests/ByteBufferTests.swift b/pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift similarity index 79% rename from pkgs/swift-google-cloud-storage/Tests/ByteBufferTests.swift rename to pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift index 710d790cac..e858c9e57d 100644 --- a/pkgs/swift-google-cloud-storage/Tests/ByteBufferTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift @@ -13,13 +13,14 @@ // limitations under the License. import Foundation -import GoogleCloudStorage import NIOCore import Testing -@Suite struct ByteBufferTests { +@testable import GoogleCloudStorage + +@Suite struct ByteChunkTests { @Test func initEmpty() { - let empty = GoogleCloudStorage.ByteBuffer() + let empty = ByteChunk() #expect(empty.isEmpty) #expect(empty.count == 0) #expect(empty.byteArray.isEmpty) @@ -29,7 +30,7 @@ import Testing @Test func initWithData() { let original = Data([0x01, 0x02, 0x03, 0x04]) - let storage = GoogleCloudStorage.ByteBuffer(original) + let storage = ByteChunk(original) #expect(!storage.isEmpty) #expect(storage.count == 4) #expect(storage.data == original) @@ -43,7 +44,7 @@ import Testing @Test func initWithByteBuffer() { var buffer = ByteBufferAllocator().buffer(capacity: 8) buffer.writeBytes([0x0A, 0x0B, 0x0C, 0x0D]) - let storage = GoogleCloudStorage.ByteBuffer(buffer) + let storage = ByteChunk(buffer) #expect(!storage.isEmpty) #expect(storage.count == 4) #expect(storage.data == Data([0x0A, 0x0B, 0x0C, 0x0D])) @@ -53,7 +54,7 @@ import Testing @Test func initWithByteArray() { let bytes: [UInt8] = [0xDE, 0xAD, 0xBE, 0xEF] - let storage = GoogleCloudStorage.ByteBuffer(bytes) + let storage = ByteChunk(bytes) #expect(storage.count == 4) #expect(storage.byteArray == bytes) #expect(storage.data == Data(bytes)) @@ -62,21 +63,21 @@ import Testing @Test func initWithRawBufferPointer() { let bytes: [UInt8] = [10, 20, 30] bytes.withUnsafeBytes { rawBuffer in - let storage = GoogleCloudStorage.ByteBuffer(rawBuffer) + let storage = ByteChunk(rawBuffer) #expect(storage.count == 3) #expect(storage.byteArray == bytes) } } @Test func arrayLiteral() { - let storage: GoogleCloudStorage.ByteBuffer = [1, 2, 3] + let storage: ByteChunk = [1, 2, 3] #expect(storage.count == 3) #expect(storage.byteArray == [1, 2, 3]) } @Test func withUnsafeBytes() throws { let expected: [UInt8] = [100, 101, 102] - let dataStorage = GoogleCloudStorage.ByteBuffer(Data(expected)) + let dataStorage = ByteChunk(Data(expected)) let dataResult = dataStorage.withUnsafeBytes { ptr in Array(ptr) } @@ -84,7 +85,7 @@ import Testing var buffer = ByteBufferAllocator().buffer(capacity: 3) buffer.writeBytes(expected) - let bufferStorage = GoogleCloudStorage.ByteBuffer(buffer) + let bufferStorage = ByteChunk(buffer) let bufferResult = bufferStorage.withUnsafeBytes { ptr in Array(ptr) } @@ -94,7 +95,7 @@ import Testing @Test func collectionAccessWithDataOffset() { let baseData = Data([0, 1, 2, 3, 4, 5, 6, 7]) let subData = baseData.subdata(in: 2..<6) // contains [2, 3, 4, 5], startIndex may not be 0 - let storage = GoogleCloudStorage.ByteBuffer(subData) + let storage = ByteChunk(subData) #expect(storage.count == 4) #expect(storage[0] == 2) @@ -114,7 +115,7 @@ import Testing buffer.writeBytes([99, 99, 10, 20, 30, 40]) buffer.moveReaderIndex(forwardBy: 2) // skip first 2 bytes - let storage = GoogleCloudStorage.ByteBuffer(buffer) + let storage = ByteChunk(buffer) #expect(storage.count == 4) #expect(storage[0] == 10) #expect(storage[1] == 20) @@ -130,16 +131,16 @@ import Testing @Test func equalityAndHashing() { let bytes: [UInt8] = [1, 2, 3, 4, 5] - let dataStorage = GoogleCloudStorage.ByteBuffer(Data(bytes)) + let dataStorage = ByteChunk(Data(bytes)) var buffer = ByteBufferAllocator().buffer(capacity: 5) buffer.writeBytes(bytes) - let bufferStorage = GoogleCloudStorage.ByteBuffer(buffer) + let bufferStorage = ByteChunk(buffer) - let arrayStorage = GoogleCloudStorage.ByteBuffer(bytes) - let emptyStorage1 = GoogleCloudStorage.ByteBuffer() - let emptyStorage2 = GoogleCloudStorage.ByteBuffer(Data()) - let differentStorage = GoogleCloudStorage.ByteBuffer([1, 2, 3, 4, 6]) + let arrayStorage = ByteChunk(bytes) + let emptyStorage1 = ByteChunk() + let emptyStorage2 = ByteChunk(Data()) + let differentStorage = ByteChunk([1, 2, 3, 4, 6]) #expect(dataStorage == bufferStorage) #expect(dataStorage == arrayStorage) @@ -148,7 +149,7 @@ import Testing #expect(dataStorage != differentStorage) #expect(dataStorage != emptyStorage1) - var set = Set() + var set = Set() set.insert(dataStorage) #expect(set.contains(bufferStorage)) #expect(set.contains(arrayStorage)) @@ -157,13 +158,13 @@ import Testing } @Test func description() { - let dataStorage = GoogleCloudStorage.ByteBuffer(Data([1, 2, 3])) + let dataStorage = ByteChunk(Data([1, 2, 3])) #expect(dataStorage.description == "3 bytes") #expect(dataStorage.debugDescription.contains("Data")) var buffer = ByteBufferAllocator().buffer(capacity: 2) buffer.writeBytes([1, 2]) - let bufferStorage = GoogleCloudStorage.ByteBuffer(buffer) + let bufferStorage = ByteChunk(buffer) #expect(bufferStorage.description == "2 bytes") #expect(bufferStorage.debugDescription.contains("NIOCore.ByteBuffer")) } diff --git a/pkgs/swift-google-cloud-storage/Tests/BytesSourceTests.swift b/pkgs/swift-google-cloud-storage/Tests/BytesSourceTests.swift index ed1d4d8362..6c52a21d24 100644 --- a/pkgs/swift-google-cloud-storage/Tests/BytesSourceTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/BytesSourceTests.swift @@ -62,7 +62,7 @@ import Testing // With NIOCore buffer storage var nioBuf = NIOCore.ByteBuffer() nioBuf.writeBytes([10, 11, 12, 13]) - var nioSrc = BytesSource(buffer: GoogleCloudStorage.ByteBuffer(nioBuf)) + var nioSrc = BytesSource(buffer: ByteChunk(nioBuf)) #expect(nioSrc.totalSize == 4) let nioChunk1 = try await nioSrc.read(maxBytes: 2) diff --git a/pkgs/swift-google-cloud-storage/Tests/ChecksumTests.swift b/pkgs/swift-google-cloud-storage/Tests/ChecksumTests.swift index 7e3b2465d7..ac838d06a5 100644 --- a/pkgs/swift-google-cloud-storage/Tests/ChecksumTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/ChecksumTests.swift @@ -31,14 +31,14 @@ import Testing // Read first chunk let chunk1 = try await checksummedSource.readChunk(maxBytes: 7) #expect(chunk1 != nil) - #expect(chunk1!.data == ByteBuffer(data1)) + #expect(chunk1!.data == ByteChunk(data1)) #expect(chunk1!.isLast == false) #expect(chunk1!.checksum == nil) // Read second chunk let chunk2 = try await checksummedSource.readChunk(maxBytes: 7) #expect(chunk2 != nil) - #expect(chunk2!.data == ByteBuffer(data2)) + #expect(chunk2!.data == ByteChunk(data2)) #expect(chunk2!.isLast == true) #expect(chunk2!.checksum != nil) @@ -57,14 +57,14 @@ import Testing // Read first chunk let chunk1 = try await checksummedSource.readChunk(maxBytes: 7) #expect(chunk1 != nil) - #expect(chunk1!.data == ByteBuffer(data1)) + #expect(chunk1!.data == ByteChunk(data1)) #expect(chunk1!.isLast == false) #expect(chunk1!.checksum == nil) // Read second chunk let chunk2 = try await checksummedSource.readChunk(maxBytes: 7) #expect(chunk2 != nil) - #expect(chunk2!.data == ByteBuffer(data2)) + #expect(chunk2!.data == ByteChunk(data2)) #expect(chunk2!.isLast == true) #expect(chunk2!.checksum != nil) @@ -219,10 +219,10 @@ import Testing private var readCompleted = false init(data: Data) { self.data = data } var totalSize: UInt64? { UInt64(data.count) } - mutating func read(maxBytes: Int) async throws -> ByteBuffer? { + mutating func read(maxBytes: Int) async throws -> ByteChunk? { if readCompleted { return nil } readCompleted = true - return ByteBuffer(data) + return ByteChunk(data) } } @@ -232,7 +232,7 @@ import Testing let chunk = try await checksummedSource.readChunk(maxBytes: 100) #expect(chunk != nil) - #expect(chunk?.data == ByteBuffer(data)) + #expect(chunk?.data == ByteChunk(data)) #expect(chunk?.isLast == true) #expect(chunk?.checksum != nil) } @@ -357,12 +357,12 @@ import Testing // 1. Read chunk 1: 5 bytes ("Hello") let chunk1 = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk1?.data == ByteBuffer(Data("Hello".utf8))) + #expect(chunk1?.data == ByteChunk(Data("Hello".utf8))) #expect(chunk1?.isLast == false) // 2. Read chunk 2: 5 bytes (", Wor") -> bytesHashed becomes 10 let chunk2 = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk2?.data == ByteBuffer(Data(", Wor".utf8))) + #expect(chunk2?.data == ByteChunk(Data(", Wor".utf8))) #expect(chunk2?.isLast == false) // 3. Simulate upload failure of chunk 2: rewind to offset 5 @@ -370,12 +370,12 @@ import Testing // 4. Re-read chunk 2 from offset 5: 5 bytes (", Wor") -> should be skipped by updateChecksums let chunk2Retry = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk2Retry?.data == ByteBuffer(Data(", Wor".utf8))) + #expect(chunk2Retry?.data == ByteChunk(Data(", Wor".utf8))) #expect(chunk2Retry?.isLast == false) // 5. Read final chunk 3: 3 bytes ("ld!") -> bytesHashed becomes 13 let chunk3 = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk3?.data == ByteBuffer(Data("ld!".utf8))) + #expect(chunk3?.data == ByteChunk(Data("ld!".utf8))) #expect(chunk3?.isLast == true) #expect(chunk3?.checksum == "crc32c=TVUQaA==") } @@ -388,12 +388,12 @@ import Testing // 1. Read chunk 1: 5 bytes ("Hello") let chunk1 = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk1?.data == ByteBuffer(Data("Hello".utf8))) + #expect(chunk1?.data == ByteChunk(Data("Hello".utf8))) #expect(chunk1?.isLast == false) // 2. Read chunk 2: 5 bytes (", Wor") -> bytesHashed becomes 10 let chunk2 = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk2?.data == ByteBuffer(Data(", Wor".utf8))) + #expect(chunk2?.data == ByteChunk(Data(", Wor".utf8))) #expect(chunk2?.isLast == false) // 3. Simulate upload failure of chunk 2: rewind to offset 5 @@ -401,12 +401,12 @@ import Testing // 4. Re-read chunk 2 from offset 5: 5 bytes (", Wor") let chunk2Retry = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk2Retry?.data == ByteBuffer(Data(", Wor".utf8))) + #expect(chunk2Retry?.data == ByteChunk(Data(", Wor".utf8))) #expect(chunk2Retry?.isLast == false) // 5. Read final chunk 3: 3 bytes ("ld!") -> bytesHashed becomes 13 let chunk3 = try await checksummedSource.readChunk(maxBytes: 5) - #expect(chunk3?.data == ByteBuffer(Data("ld!".utf8))) + #expect(chunk3?.data == ByteChunk(Data("ld!".utf8))) #expect(chunk3?.isLast == true) #expect(chunk3?.checksum == "md5=ZajifYh5KDgxtmS9i38K1A==") } @@ -456,7 +456,7 @@ import Testing // Read first chunk (7 bytes: "Hello, ") -> both CRC32C and MD5 hash 0..<7 let chunk1 = try await checksummedSource.readChunk(maxBytes: 7) - #expect(chunk1?.data == ByteBuffer(part1)) + #expect(chunk1?.data == ByteChunk(part1)) // Server acknowledges commit at byte 7 with running CRC32C seed let seed = _CRC32C.compute(part1) @@ -464,7 +464,7 @@ import Testing // Read second chunk (6 bytes: "World!") let chunk2 = try await checksummedSource.readChunk(maxBytes: 7) - #expect(chunk2?.data == ByteBuffer(part2)) + #expect(chunk2?.data == ByteChunk(part2)) #expect(chunk2?.isLast == true) // MD5 was discarded because it cannot be safely reseeded; only CRC32C remains @@ -482,14 +482,14 @@ import Testing // Read first chunk (7 bytes: "Hello, ") -> MD5 hashes 0..<7 let chunk1 = try await checksummedSource.readChunk(maxBytes: 7) - #expect(chunk1?.data == ByteBuffer(part1)) + #expect(chunk1?.data == ByteChunk(part1)) // Calling seedCRC32C discards the MD5 calculator even though there is no CRC32C calculator checksummedSource.seedCRC32C(seed: 12345, bytesHashed: 7) // Read second chunk (6 bytes: "World!") let chunk2 = try await checksummedSource.readChunk(maxBytes: 7) - #expect(chunk2?.data == ByteBuffer(part2)) + #expect(chunk2?.data == ByteChunk(part2)) #expect(chunk2?.isLast == true) // All dynamic calculators were discarded, and no CRC32C was added; checksum is nil diff --git a/pkgs/swift-google-cloud-storage/Tests/FileSourceTests.swift b/pkgs/swift-google-cloud-storage/Tests/FileSourceTests.swift index 998fed3ecd..5037de06cc 100644 --- a/pkgs/swift-google-cloud-storage/Tests/FileSourceTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/FileSourceTests.swift @@ -59,10 +59,10 @@ import Testing var source = FileSource(fileURL: fileURL) let first = try await source.read(maxBytes: 5) - #expect(first == GoogleCloudStorage.ByteBuffer(Data("Hello".utf8))) + #expect(first == ByteChunk(Data("Hello".utf8))) let second = try await source.read(maxBytes: 100) - #expect(second == GoogleCloudStorage.ByteBuffer(Data(", World!".utf8))) + #expect(second == ByteChunk(Data(", World!".utf8))) let third = try await source.read(maxBytes: 10) #expect(third == nil) diff --git a/pkgs/swift-google-cloud-storage/Tests/IntegrationTests/StorageClientIntegrationTests.swift b/pkgs/swift-google-cloud-storage/Tests/IntegrationTests/StorageClientIntegrationTests.swift index 2ddd534a66..98e1c53649 100644 --- a/pkgs/swift-google-cloud-storage/Tests/IntegrationTests/StorageClientIntegrationTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/IntegrationTests/StorageClientIntegrationTests.swift @@ -794,11 +794,11 @@ private struct IntegrationDynamicSource: UploadSource { self.totalSize = totalSize } - mutating func read(maxBytes: Int) async throws -> GoogleCloudStorage.ByteBuffer? { + mutating func read(maxBytes: Int) async throws -> ByteChunk? { guard currentChunk < totalChunks else { return nil } let count = min(maxBytes, chunkSize) let byteVal = UInt8((currentChunk + 1) % 256) currentChunk += 1 - return GoogleCloudStorage.ByteBuffer(Data(repeating: byteVal, count: count)) + return ByteChunk(Data(repeating: byteVal, count: count)) } } diff --git a/pkgs/swift-google-cloud-storage/Tests/ResumableUploadTests.swift b/pkgs/swift-google-cloud-storage/Tests/ResumableUploadTests.swift index 8bf8a42eb1..d0fafcc216 100644 --- a/pkgs/swift-google-cloud-storage/Tests/ResumableUploadTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/ResumableUploadTests.swift @@ -2742,12 +2742,12 @@ private struct DynamicComputationSource: UploadSource { self.totalSize = totalSize } - mutating func read(maxBytes: Int) async throws -> ByteBuffer? { + mutating func read(maxBytes: Int) async throws -> ByteChunk? { guard currentChunk < totalChunks else { return nil } let count = min(maxBytes, chunkSize) let byteVal = UInt8((currentChunk + 1) % 256) currentChunk += 1 - return ByteBuffer(Data(repeating: byteVal, count: count)) + return ByteChunk(Data(repeating: byteVal, count: count)) } } @@ -2764,14 +2764,14 @@ private struct SeekableComputationSource: SeekableUploadSource { self.totalSize = UInt64(chunkSize * totalChunks) } - mutating func read(maxBytes: Int) async throws -> ByteBuffer? { + mutating func read(maxBytes: Int) async throws -> ByteChunk? { guard let totalSize = totalSize, currentOffset < totalSize else { return nil } let bytesToRead = min(UInt64(maxBytes), totalSize - currentOffset) guard bytesToRead > 0 else { return nil } let chunkIndex = Int(currentOffset / UInt64(chunkSize)) let byteVal = UInt8((chunkIndex + 1) % 256) currentOffset += bytesToRead - return ByteBuffer(Data(repeating: byteVal, count: Int(bytesToRead))) + return ByteChunk(Data(repeating: byteVal, count: Int(bytesToRead))) } mutating func seek(to offset: UInt64) async throws { diff --git a/pkgs/swift-google-cloud-storage/Tests/SimpleUploadTests.swift b/pkgs/swift-google-cloud-storage/Tests/SimpleUploadTests.swift index 5d11da324c..9ee3ab5bcb 100644 --- a/pkgs/swift-google-cloud-storage/Tests/SimpleUploadTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/SimpleUploadTests.swift @@ -346,12 +346,12 @@ import Testing UInt64(data.count) } - func read(maxBytes: Int) async throws -> GoogleCloudStorage.ByteBuffer? { + func read(maxBytes: Int) async throws -> ByteChunk? { guard offset < data.count else { return nil } let end = min(offset + maxBytes, data.count) let chunk = data.subdata(in: offset.. { continuation in - continuation.yield(GoogleCloudStorage.ByteBuffer([1, 2, 3])) - continuation.yield(GoogleCloudStorage.ByteBuffer([4, 5, 6, 7])) + /// Tests reading from an AsyncStream of ByteChunk. + @Test func readByteChunkSequence() async throws { + let stream = AsyncStream { continuation in + continuation.yield(ByteChunk([1, 2, 3])) + continuation.yield(ByteChunk([4, 5, 6, 7])) continuation.finish() } diff --git a/pkgs/swift-google-cloud-storage/Tests/UrlMocks.swift b/pkgs/swift-google-cloud-storage/Tests/UrlMocks.swift index d7af05e7c1..da90388898 100644 --- a/pkgs/swift-google-cloud-storage/Tests/UrlMocks.swift +++ b/pkgs/swift-google-cloud-storage/Tests/UrlMocks.swift @@ -59,14 +59,14 @@ struct RecordedRequest: Sendable { /// Mock UploadSource that can throw errors struct MockUploadSource: SeekableUploadSource { - var data: GoogleCloudStorage.ByteBuffer + var data: ByteChunk var totalSize: UInt64? var readError: (any Error)? var seekError: (any Error)? private var offset: UInt64 = 0 init( - data: GoogleCloudStorage.ByteBuffer, totalSize: UInt64? = nil, readError: (any Error)? = nil, + data: ByteChunk, totalSize: UInt64? = nil, readError: (any Error)? = nil, seekError: (any Error)? = nil ) { self.data = data @@ -80,11 +80,11 @@ struct MockUploadSource: SeekableUploadSource { seekError: (any Error)? = nil ) { self.init( - data: GoogleCloudStorage.ByteBuffer(data), totalSize: totalSize, readError: readError, + data: ByteChunk(data), totalSize: totalSize, readError: readError, seekError: seekError) } - mutating func read(maxBytes: Int) async throws -> GoogleCloudStorage.ByteBuffer? { + mutating func read(maxBytes: Int) async throws -> ByteChunk? { if let error = readError { throw error } From b3537da7988eb5c832e4e9d5d77bd2e362fb1651 Mon Sep 17 00:00:00 2001 From: chingor13 Date: Tue, 22 Sep 2026 03:37:39 +0000 Subject: [PATCH 2/2] fix: more efficient byte handling (prefer ByteBuffer storage) --- .../GoogleCloudStorage/ByteChunk.swift | 25 ++++++++++++++++--- .../Tests/ByteChunkTests.swift | 18 +++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift index c4c4574ec1..87535e1cbe 100644 --- a/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift +++ b/pkgs/swift-google-cloud-storage/Sources/GoogleCloudStorage/ByteChunk.swift @@ -40,17 +40,17 @@ public struct ByteChunk: Sendable, ContiguousBytes { /// Creates an empty byte chunk instance. public init() { - self.storage = .data(Data()) + self.storage = .byteBuffer(NIOCore.ByteBuffer()) } /// Creates a byte chunk from an array of bytes. public init(_ bytes: [UInt8]) { - self.storage = .data(Data(bytes)) + self.storage = .byteBuffer(NIOCore.ByteBuffer(bytes: bytes)) } /// Creates a byte chunk from a contiguous raw buffer pointer. public init(_ bufferPointer: UnsafeRawBufferPointer) { - self.storage = .data(Data(bufferPointer)) + self.storage = .byteBuffer(NIOCore.ByteBuffer(bytes: bufferPointer)) } } @@ -68,6 +68,7 @@ extension ByteChunk { } /// Indicates whether the chunk contains zero bytes. + @inlinable public var isEmpty: Bool { count == 0 } @@ -82,6 +83,18 @@ extension ByteChunk { } } + /// Executes a closure on the sequence's contiguous storage. + @inlinable + public func withContiguousStorageIfAvailable( + _ body: (UnsafeBufferPointer) throws -> R + ) rethrows -> R? { + try withUnsafeBytes { rawBuffer in + try rawBuffer.withMemoryRebound(to: UInt8.self) { buffer in + try body(buffer) + } + } + } + /// The underlying contents as a `Foundation.Data` instance. /// /// - Returns: The original `Data` with zero copies if backed by `Data`, @@ -113,6 +126,7 @@ extension ByteChunk { } /// Returns the bytes as a newly allocated `[UInt8]` array. + @inlinable public var byteArray: [UInt8] { withUnsafeBytes { Array($0) } } @@ -141,8 +155,10 @@ extension ByteChunk: RandomAccessCollection { public typealias Element = UInt8 public typealias Index = Int + @inlinable public var startIndex: Int { 0 } + @inlinable public var endIndex: Int { count } public subscript(position: Int) -> UInt8 { @@ -174,6 +190,7 @@ extension ByteChunk: Equatable { } extension ByteChunk: Hashable { + @inlinable public func hash(into hasher: inout Hasher) { withUnsafeBytes { hasher.combine(bytes: $0) } } @@ -183,7 +200,7 @@ extension ByteChunk: Hashable { extension ByteChunk: ExpressibleByArrayLiteral { public init(arrayLiteral elements: UInt8...) { - self.init(Data(elements)) + self.init(elements) } } diff --git a/pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift b/pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift index e858c9e57d..1421664665 100644 --- a/pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift +++ b/pkgs/swift-google-cloud-storage/Tests/ByteChunkTests.swift @@ -167,5 +167,23 @@ import Testing let bufferStorage = ByteChunk(buffer) #expect(bufferStorage.description == "2 bytes") #expect(bufferStorage.debugDescription.contains("NIOCore.ByteBuffer")) + + let arrayStorage = ByteChunk([4, 5, 6]) + #expect(arrayStorage.debugDescription.contains("NIOCore.ByteBuffer")) + } + + @Test func withContiguousStorageIfAvailable() { + let expected: [UInt8] = [10, 20, 30, 40] + let dataChunk = ByteChunk(Data(expected)) + let dataContiguous = dataChunk.withContiguousStorageIfAvailable { Array($0) } + #expect(dataContiguous == expected) + #expect(Array(dataChunk) == expected) + #expect(Data(dataChunk) == Data(expected)) + + let bufferChunk = ByteChunk(expected) + let bufferContiguous = bufferChunk.withContiguousStorageIfAvailable { Array($0) } + #expect(bufferContiguous == expected) + #expect(Array(bufferChunk) == expected) + #expect(Data(bufferChunk) == Data(expected)) } }