diff --git a/Sources/FoundationEssentials/Data/Data+Reading.swift b/Sources/FoundationEssentials/Data/Data+Reading.swift index 5132bfaf2..a7421ead4 100644 --- a/Sources/FoundationEssentials/Data/Data+Reading.swift +++ b/Sources/FoundationEssentials/Data/Data+Reading.swift @@ -193,33 +193,33 @@ struct ReadBytesResult { } #if os(Windows) +@lifetime(pBuffer: copy pBuffer) private func read(from hFile: HANDLE, at path: PathOrURL, - into pBuffer: UnsafeMutableRawPointer, length dwLength: Int, + into pBuffer: inout OutputRawSpan, chunkSize dwChunk: Int = 4096, progress bProgress: Bool) - throws -> Int { - var pBuffer = pBuffer - let progress = bProgress && Progress.current() != nil ? Progress(totalUnitCount: Int64(dwLength)) : nil + throws { + let progress = bProgress && Progress.current() != nil ? Progress(totalUnitCount: Int64(pBuffer.freeCapacity)) : nil - var dwBytesRemaining: DWORD = DWORD(dwLength) - while dwBytesRemaining > 0 { + while !pBuffer.isFull { if let progress, progress.isCancelled { throw CocoaError(.userCancelled) } let dwBytesToRead: DWORD = - DWORD(clamping: DWORD(min(DWORD(dwChunk), dwBytesRemaining))) + DWORD(clamping: min(dwChunk, pBuffer.freeCapacity)) + var dwBytesRead: DWORD = 0 - if !ReadFile(hFile, pBuffer, dwBytesToRead, &dwBytesRead, nil) { - throw CocoaError.errorWithFilePath(path, win32: GetLastError(), reading: true) + try pBuffer.withUnsafeMutableBytes { bytes, initializedCount in + if !ReadFile(hFile, bytes.baseAddress!.advanced(by: initializedCount), dwBytesToRead, &dwBytesRead, nil) { + throw CocoaError.errorWithFilePath(path, win32: GetLastError(), reading: true) + } + initializedCount += Int(dwBytesRead) } - dwBytesRemaining -= DWORD(clamping: dwBytesRead) - progress?.completedUnitCount = Int64(dwLength - Int(dwBytesRemaining)) + progress?.completedUnitCount += Int64(dwBytesRead) if dwBytesRead < dwBytesToRead { break } - pBuffer = pBuffer.advanced(by: Int(dwBytesRead)) } - return dwLength - Int(dwBytesRemaining) } #endif @@ -283,18 +283,20 @@ internal func readBytesFromFile(path inPath: PathOrURL, reportProgress: Bool, ma } })) } else { - guard let pBuffer: UnsafeMutableRawPointer = malloc(Int(szFileSize)) else { + guard let ptr: UnsafeMutableRawPointer = malloc(Int(szFileSize)) else { throw CocoaError.errorWithFilePath(inPath, errno: ENOMEM, reading: true) } + let buffer = UnsafeMutableRawBufferPointer(start: ptr, count: Int(szFileSize)) + var outputSpan = OutputRawSpan(buffer: buffer, initializedCount: 0) - localProgress?.becomeCurrent(withPendingUnitCount: Int64(szFileSize)) + localProgress?.becomeCurrent(withPendingUnitCount: Int64(outputSpan.freeCapacity)) do { - let dwLength = try read(from: hFile, at: inPath, into: pBuffer, length: Int(szFileSize), progress: reportProgress) + try read(from: hFile, at: inPath, into: &outputSpan, progress: reportProgress) localProgress?.resignCurrent() - return ReadBytesResult(bytes: pBuffer, length: dwLength, deallocator: .free) + return ReadBytesResult(bytes: ptr, length: outputSpan.finalize(for: buffer), deallocator: .free) } catch { localProgress?.resignCurrent() - free(pBuffer) + free(ptr) throw error } } @@ -367,18 +369,21 @@ internal func readBytesFromFile(path inPath: PathOrURL, reportProgress: Bool, ma #if os(Linux) || os(Android) // Linux has some files that may report a size of 0 but actually have contents let chunkSize = 1024 * 4 - var buffer = malloc(chunkSize)! + var ptr = malloc(chunkSize)! var totalRead = 0 while true { - let length = try readBytesFromFileDescriptor(fd, path: inPath, buffer: buffer.advanced(by: totalRead), length: chunkSize, readUntilLength: false, reportProgress: false) + let buffer = UnsafeMutableRawBufferPointer(start: ptr, count: totalRead + chunkSize) + var outputSpan = OutputRawSpan(buffer: buffer, initializedCount: totalRead) + try readBytesFromFileDescriptor(fd, path: inPath, buffer: &outputSpan, readUntilLength: false, reportProgress: false) + let length = outputSpan.finalize(for: buffer) totalRead += length if length != chunkSize { break } - buffer = realloc(buffer, totalRead + chunkSize) + ptr = realloc(ptr, totalRead + chunkSize) } - result = ReadBytesResult(bytes: buffer, length: totalRead, deallocator: .free) + result = ReadBytesResult(bytes: ptr, length: totalRead, deallocator: .free) #else result = ReadBytesResult(bytes: nil, length: 0, deallocator: nil) #endif @@ -415,13 +420,15 @@ internal func readBytesFromFile(path inPath: PathOrURL, reportProgress: Bool, ma guard let bytes = malloc(Int(fileSize)) else { throw CocoaError.errorWithFilePath(inPath, errno: ENOMEM, reading: true) } + let buffer = UnsafeMutableRawBufferPointer(start: bytes, count: Int(fileSize)) + var outputSpan = OutputRawSpan(buffer: buffer, initializedCount: 0) localProgress?.becomeCurrent(withPendingUnitCount: Int64(fileSize)) do { - let length = try readBytesFromFileDescriptor(fd, path: inPath, buffer: bytes, length: fileSize, reportProgress: reportProgress) + try readBytesFromFileDescriptor(fd, path: inPath, buffer: &outputSpan, reportProgress: reportProgress) localProgress?.resignCurrent() - result = ReadBytesResult(bytes: bytes, length: length, deallocator: .free) + result = ReadBytesResult(bytes: bytes, length: outputSpan.finalize(for: buffer), deallocator: .free) } catch { localProgress?.resignCurrent() free(bytes) @@ -440,25 +447,24 @@ internal func readBytesFromFile(path inPath: PathOrURL, reportProgress: Bool, ma /// Read data from a file descriptor. /// Takes an `Int` size and returns an `Int` to match `Data`'s count. If we are going to read more than Int.max, throws - because we won't be able to store it in `Data`. /// If `readUntilLength` is `false`, then we will end the read if we receive less than `length` bytes. This can be used to read from something like a socket, where the `length` simply represents the maximum size you can read at once. -private func readBytesFromFileDescriptor(_ fd: Int32, path: PathOrURL, buffer inBuffer: UnsafeMutableRawPointer, length: Int, readUntilLength: Bool = true, reportProgress: Bool) throws -> Int { - var buffer = inBuffer +private func readBytesFromFileDescriptor(_ fd: Int32, path: PathOrURL, buffer inBuffer: inout OutputRawSpan, readUntilLength: Bool = true, reportProgress: Bool) throws { // If chunkSize (8-byte value) is more than blksize_t.max (4 byte value), then use the 4 byte max and chunk let preferredChunkSize: size_t let localProgress: Progress? + let length = inBuffer.freeCapacity if Progress.current() != nil && reportProgress { - localProgress = Progress(totalUnitCount: Int64(length)) + localProgress = Progress(totalUnitCount: Int64(inBuffer.freeCapacity)) // To report progress, we have to try reading in smaller chunks than the whole file. Aim for about 1% increments. - preferredChunkSize = max(length / 100, 1024 * 4) + preferredChunkSize = max(inBuffer.freeCapacity / 100, 1024 * 4) } else { localProgress = nil // Get it all in one go, if possible - preferredChunkSize = length + preferredChunkSize = inBuffer.freeCapacity } - var numBytesRemaining = length - while numBytesRemaining > 0 { + while !inBuffer.isFull { if let localProgress, localProgress.isCancelled { throw CocoaError(.userCancelled) } @@ -467,22 +473,27 @@ private func readBytesFromFileDescriptor(_ fd: Int32, path: PathOrURL, buffer in var numBytesRequested = CUnsignedInt(clamping: min(preferredChunkSize, Int(CInt.max))) // Furthermore, don't request more than the number of bytes remaining - if numBytesRequested > numBytesRemaining { - numBytesRequested = CUnsignedInt(clamping: min(numBytesRemaining, Int(CInt.max))) + if numBytesRequested > inBuffer.freeCapacity { + numBytesRequested = CUnsignedInt(clamping: min(inBuffer.freeCapacity, Int(CInt.max))) } - var numBytesRead: CInt + var numBytesRead: CInt = 0 repeat { if let localProgress, localProgress.isCancelled { throw CocoaError(.userCancelled) } // read takes an Int-sized argument, which will always be at least the size of Int32. + inBuffer.withUnsafeMutableBytes { buffer, initializedCount in #if os(Windows) - numBytesRead = _read(fd, buffer, numBytesRequested) + numBytesRead = _read(fd, buffer.baseAddress!.advanced(by: initializedCount), numBytesRequested) #else - numBytesRead = CInt(read(fd, buffer, Int(numBytesRequested))) + numBytesRead = CInt(read(fd, buffer.baseAddress!.advanced(by: initializedCount), Int(numBytesRequested))) #endif + if numBytesRead >= 0 { + initializedCount += Int(clamping: numBytesRead) + } + } } while numBytesRead < 0 && errno == EINTR if numBytesRead < 0 { @@ -495,23 +506,14 @@ private func readBytesFromFileDescriptor(_ fd: Int32, path: PathOrURL, buffer in break } else { // Partial read - numBytesRemaining -= Int(clamping: numBytesRead) - if numBytesRemaining < 0 { - // Just in case; we do not want to have a negative amount of bytes remaining. We will just assume that is the end. - numBytesRemaining = 0 - } - localProgress?.completedUnitCount = Int64(length - numBytesRemaining) + localProgress?.completedUnitCount = Int64(length - inBuffer.freeCapacity) // The `readUntilLength` argument controls if we should end early when `read` returns less than the amount requested, or if we should continue to loop until we have reached `length` bytes. if !readUntilLength && numBytesRead < numBytesRequested { break } - - buffer = buffer.advanced(by: numericCast(numBytesRead)) } } - - return length - numBytesRemaining } @available(macOS 10.10, iOS 8.0, watchOS 2.0, tvOS 9.0, *) diff --git a/Sources/FoundationEssentials/Data/Data+Writing.swift b/Sources/FoundationEssentials/Data/Data+Writing.swift index 03d3211d5..354c5a254 100644 --- a/Sources/FoundationEssentials/Data/Data+Writing.swift +++ b/Sources/FoundationEssentials/Data/Data+Writing.swift @@ -56,9 +56,9 @@ private func openFileDescriptorProtected(path: UnsafePointer, flags: Int3 } #endif -private func writeToFileDescriptorWithProgress(_ fd: Int32, buffer: UnsafeRawBufferPointer, reportProgress: Bool) throws -> Int { +private func writeToFileDescriptorWithProgress(_ fd: Int32, buffer: RawSpan, reportProgress: Bool) throws -> Int { // Fetch this once - let length = buffer.count + let length = buffer.byteCount let preferredChunkSize: Int let localProgress: Progress? @@ -72,24 +72,22 @@ private func writeToFileDescriptorWithProgress(_ fd: Int32, buffer: UnsafeRawBuf localProgress = nil } - var nextRange = buffer.startIndex.. 0 { + var remaining = buffer + while !remaining.isEmpty { if let localProgress, localProgress.isCancelled { throw CocoaError(.userCancelled) } // Don't ever attempt to write more than (2GB - 1 byte). Some platforms will return an error over that amount. let numBytesRequested = CInt(clamping: min(preferredChunkSize, Int(CInt.max))) - let smallestAmountToRead = min(Int(numBytesRequested), numBytesRemaining) - let upperBound = nextRange.startIndex + smallestAmountToRead - nextRange = nextRange.startIndex..= 0 { _close(fd) } } - let callback = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.count)) : nil + let callback = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.byteCount)) : nil do { try write(buffer: buffer, toFileDescriptor: fd, path: inPath, parentProgress: callback) @@ -517,7 +503,7 @@ private func writeToFileAux(path inPath: PathOrURL, buffer: UnsafeRawBufferPoint defer { close(fd) } - let parentProgress = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.count)) : nil + let parentProgress = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.byteCount)) : nil do { try write(buffer: buffer, toFileDescriptor: fd, path: inPath, parentProgress: parentProgress) @@ -629,7 +615,7 @@ private func writeToFileAux(path inPath: PathOrURL, buffer: UnsafeRawBufferPoint } /// Create a new file out of `Data` at a path, not using atomic writing. -private func writeToFileNoAux(path inPath: PathOrURL, buffer: UnsafeRawBufferPointer, options: Data.WritingOptions, attributes: [String : Data], reportProgress: Bool) throws { +private func writeToFileNoAux(path inPath: PathOrURL, buffer: RawSpan, options: Data.WritingOptions, attributes: [String : Data], reportProgress: Bool) throws { #if !os(WASI) // `.atomic` is unavailable on WASI assert(!options.contains(.atomic)) #endif @@ -646,7 +632,7 @@ private func writeToFileNoAux(path inPath: PathOrURL, buffer: UnsafeRawBufferPoi } defer { _close(fd) } - let callback: Progress? = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.count)) : nil + let callback: Progress? = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.byteCount)) : nil do { try write(buffer: buffer, toFileDescriptor: fd, path: inPath, parentProgress: callback) @@ -681,7 +667,7 @@ private func writeToFileNoAux(path inPath: PathOrURL, buffer: UnsafeRawBufferPoi defer { close(fd) } - let parentProgress = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.count)) : nil + let parentProgress = (reportProgress && Progress.current() != nil) ? Progress(totalUnitCount: Int64(buffer.byteCount)) : nil do { try write(buffer: buffer, toFileDescriptor: fd, path: inPath, parentProgress: parentProgress) @@ -776,7 +762,7 @@ extension Data { } #if !NO_FILESYSTEM - try writeToFile(path: .url(url), data: self, options: options, reportProgress: true) + try writeToFile(path: .url(url), buffer: self.bytes, options: options, reportProgress: true) #else throw CocoaError(.featureUnsupported) #endif diff --git a/Sources/FoundationEssentials/String/String+IO.swift b/Sources/FoundationEssentials/String/String+IO.swift index 640efe422..39cf21b7e 100644 --- a/Sources/FoundationEssentials/String/String+IO.swift +++ b/Sources/FoundationEssentials/String/String+IO.swift @@ -456,7 +456,7 @@ extension StringProtocol { let options : Data.WritingOptions = useAuxiliaryFile ? [.atomic] : [] #endif - try writeToFile(path: .path(String(path)), data: data, options: options, attributes: attributes, reportProgress: false) + try writeToFile(path: .path(String(path)), buffer: data.bytes, options: options, attributes: attributes, reportProgress: false) } /// Writes the contents of the `String` to the URL specified by url using the specified encoding. @@ -479,7 +479,7 @@ extension StringProtocol { let options : Data.WritingOptions = useAuxiliaryFile ? [.atomic] : [] #endif - try writeToFile(path: .url(url), data: data, options: options, attributes: attributes, reportProgress: false) + try writeToFile(path: .url(url), buffer: data.bytes, options: options, attributes: attributes, reportProgress: false) } #endif } diff --git a/Tests/FoundationEssentialsTests/DataIOTests.swift b/Tests/FoundationEssentialsTests/DataIOTests.swift index 4ebca2557..c2ba8c6d1 100644 --- a/Tests/FoundationEssentialsTests/DataIOTests.swift +++ b/Tests/FoundationEssentialsTests/DataIOTests.swift @@ -133,7 +133,7 @@ private final class DataIOTests { // Data doesn't have a direct API to write with attributes, but our I/O code has it. Use it via @testable interface here. let writeAttrs: [String : Data] = [FileAttributeKey.hfsCreatorCode.rawValue : "abcd".data(using: .ascii)!] - try writeToFile(path: .url(url), data: writeData, options: [], attributes: writeAttrs) + try writeToFile(path: .url(url), buffer: writeData.bytes, options: [], attributes: writeAttrs) // Verify attributes var readAttrs: [String : Data] = [:]