From 85b59b40c5eb14e3ab04442e62d3150b1080e4a8 Mon Sep 17 00:00:00 2001 From: Jonathan Grynspan Date: Thu, 30 Oct 2025 10:52:37 -0400 Subject: [PATCH] [WIP, DNM] Prototype `#expect()` overloads outside the testing library. This PR introduces an overload of `#expect()`/`#require()` that's implemented outside the core Swift Testing library. It implements exit tests based on Foundation's `Process` class (AKA `NSTask`.) I do **not** plan to merge this PR. Rather, it's serving as a breadboard to let me prototype overloading `#expect()` outside Swift Testing, which we can then apply to other libraries like `swift-subprocess` and `swift-argument-parser`. --- Package.swift | 1 + .../Expectations/Expectation+Process.swift | 159 ++++++++++++++++++ .../ExitTests/ExitTest.Condition.swift | 2 +- Sources/Testing/Issues/Issue+Recording.swift | 2 +- Sources/Testing/Issues/Issue.swift | 2 +- .../SourceAttribution/Expression.swift | 4 +- Sources/TestingMacros/ConditionMacro.swift | 62 ++++++- Sources/TestingMacros/TestingMacrosMain.swift | 2 + Tests/TestingTests/ExitTestTests.swift | 40 +++++ 9 files changed, 267 insertions(+), 7 deletions(-) create mode 100644 Sources/Overlays/_Testing_Foundation/Expectations/Expectation+Process.swift diff --git a/Package.swift b/Package.swift index a82241527..65162bb8a 100644 --- a/Package.swift +++ b/Package.swift @@ -265,6 +265,7 @@ let package = Package( name: "_Testing_Foundation", dependencies: [ "Testing", + "_TestingInternals", ], path: "Sources/Overlays/_Testing_Foundation", exclude: ["CMakeLists.txt"], diff --git a/Sources/Overlays/_Testing_Foundation/Expectations/Expectation+Process.swift b/Sources/Overlays/_Testing_Foundation/Expectations/Expectation+Process.swift new file mode 100644 index 000000000..a41f71a68 --- /dev/null +++ b/Sources/Overlays/_Testing_Foundation/Expectations/Expectation+Process.swift @@ -0,0 +1,159 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if canImport(Foundation) +@_spi(ForToolsIntegrationOnly) public import Testing +public import Foundation +private import _TestingInternals + +@_spi(Experimental) +@freestanding(expression) +@discardableResult +#if !SWT_NO_EXIT_TESTS +@available(macOS 10.15.4, *) +#else +@_unavailableInEmbedded +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +public macro expect( + _ process: Process, + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation +) -> ExitTest.Result? = #externalMacro(module: "TestingMacros", type: "ExpectNSTaskExitsWithMacro") + +@_spi(Experimental) +@freestanding(expression) +@discardableResult +#if !SWT_NO_EXIT_TESTS +@available(macOS 10.15.4, *) +#else +@_unavailableInEmbedded +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +public macro require( + _ process: Process, + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], + _ comment: @autoclosure () -> Comment? = nil, + sourceLocation: SourceLocation = #_sourceLocation +) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "RequireNSTaskExitsWithMacro") + +// MARK: - + +@_spi(Experimental) +@discardableResult +#if !SWT_NO_EXIT_TESTS +@available(macOS 10.15.4, *) +#else +@_unavailableInEmbedded +@available(*, unavailable, message: "Exit tests are not available on this platform.") +#endif +public func __check( + _ process: Process, + exitsWith expectedExitCondition: ExitTest.Condition, + observing observedValues: [any PartialKeyPath & Sendable] = [], + comments: @autoclosure () -> [Comment], + isRequired: Bool, + isolation: isolated (any Actor)? = #isolation, + sourceLocation: SourceLocation +) async -> Result { +#if !SWT_NO_EXIT_TESTS + // The process may have already started and may already have a termination + // handler set, so it's not possible for us to asynchronously wait for it. + // As such, we'll have to block _some_ thread. + var result: ExitTest.Result + do { + try await withCheckedThrowingContinuation { continuation in + Thread.detachNewThread { + do { + // There's an obvious race condition here, but that's a limitation of + // the Process/NSTask API and we'll just have to accept it. + if !process.isRunning { + try process.run() + } + process.waitUntilExit() + continuation.resume() + } catch { + continuation.resume(throwing: error) + } + } + } + + let reason = process.terminationReason + let exitStatus: ExitStatus = switch reason { + case .exit: + .exitCode(process.terminationStatus) + case .uncaughtSignal: + #if os(Windows) + // On Windows, Foundation tries to map exit codes that look like HRESULT + // values to signals, which is not the model Swift Testing uses. The + // conversion is lossy, so there's not much we can do here other than treat + // it as an exit code too. + .exitCode(process.terminationStatus) + #else + .signal(process.terminationStatus) + #endif + @unknown default: + fatalError("Unexpected termination reason '\(reason)' from process \(process). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + } + + result = ExitTest.Result(exitStatus: exitStatus) + func makeContent(from streamObject: Any?) -> [UInt8] { + if let fileHandle = streamObject as? FileHandle { + if let content = try? fileHandle.readToEnd() { + return Array(content) + } + } else if let pipe = streamObject as? Pipe { + return makeContent(from: pipe.fileHandleForReading) + } + + return [] + } + if observedValues.contains(\.standardOutputContent) { + result.standardOutputContent = makeContent(from: process.standardOutput) + } + if observedValues.contains(\.standardErrorContent) { + result.standardErrorContent = makeContent(from: process.standardError) + } + } catch { + // As with the main exit test implementation, if an error occurs while + // trying to run the exit test, treat it as a system error and treat the + // condition as a mismatch. + let issue = Issue( + kind: .system, + comments: comments() + CollectionOfOne(Comment(rawValue: String(describingForTest: error))), + sourceContext: SourceContext(backtrace: nil, sourceLocation: sourceLocation) + ) + issue.record() + + let exitStatus: ExitStatus = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) { + .exitCode(EXIT_SUCCESS) + } else { + .exitCode(EXIT_FAILURE) + } + result = ExitTest.Result(exitStatus: exitStatus) + } + + let expression = Expression("expectedExitCondition") + return __checkValue( + expectedExitCondition.isApproximatelyEqual(to: result.exitStatus), + expression: expression, + expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(result.exitStatus), + comments: comments(), + isRequired: isRequired, + sourceLocation: sourceLocation + ).map { _ in result } +#else + swt_unreachable() +#endif +} +#endif diff --git a/Sources/Testing/ExitTests/ExitTest.Condition.swift b/Sources/Testing/ExitTests/ExitTest.Condition.swift index edd94193b..ea0cae58c 100644 --- a/Sources/Testing/ExitTests/ExitTest.Condition.swift +++ b/Sources/Testing/ExitTests/ExitTest.Condition.swift @@ -216,7 +216,7 @@ extension ExitTest.Condition { /// /// Two exit test conditions can be compared; if either instance is equal to /// ``failure``, it will compare equal to any instance except ``success``. - func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool { + package func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool { // Strictly speaking, the C standard treats 0 as a successful exit code and // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern // operating system defines EXIT_SUCCESS to any value other than 0, so the diff --git a/Sources/Testing/Issues/Issue+Recording.swift b/Sources/Testing/Issues/Issue+Recording.swift index 72f4c65a4..a39a638fc 100644 --- a/Sources/Testing/Issues/Issue+Recording.swift +++ b/Sources/Testing/Issues/Issue+Recording.swift @@ -18,7 +18,7 @@ extension Issue { /// /// - Returns: The issue that was recorded (`self` or a modified copy of it.) @discardableResult - func record(configuration: Configuration? = nil) -> Self { + package func record(configuration: Configuration? = nil) -> Self { // If this issue is a caught error that has a custom issue representation, // perform that customization now. if case let .errorCaught(error) = kind { diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index beeca101e..73b567f70 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -172,7 +172,7 @@ public struct Issue: Sendable { /// empty. /// - sourceContext: A ``SourceContext`` indicating where and how this issue /// occurred. - init( + package init( kind: Kind, severity: Severity = .error, comments: [Comment], diff --git a/Sources/Testing/SourceAttribution/Expression.swift b/Sources/Testing/SourceAttribution/Expression.swift index a294a81e0..c8c6a7ea8 100644 --- a/Sources/Testing/SourceAttribution/Expression.swift +++ b/Sources/Testing/SourceAttribution/Expression.swift @@ -326,7 +326,7 @@ public struct __Expression: Sendable { /// /// - Returns: A copy of `self` with information about the specified runtime /// value captured for future use. - func capturingRuntimeValue(_ value: (some Any)?) -> Self { + package func capturingRuntimeValue(_ value: (some Any)?) -> Self { var result = self result.runtimeValue = value.flatMap(Value.init(reflecting:)) if case let .negation(subexpression, isParenthetical) = kind, let value = value as? Bool { @@ -348,7 +348,7 @@ public struct __Expression: Sendable { /// /// If the ``kind`` of `self` is ``Kind/generic`` or ``Kind/stringLiteral``, /// this function is equivalent to ``capturingRuntimeValue(_:)``. - func capturingRuntimeValues(_ firstValue: (some Any)?, _ additionalValues: repeat (each T)?) -> Self { + package func capturingRuntimeValues(_ firstValue: (some Any)?, _ additionalValues: repeat (each T)?) -> Self { var result = self // Convert the variadic generic argument list to an array. diff --git a/Sources/TestingMacros/ConditionMacro.swift b/Sources/TestingMacros/ConditionMacro.swift index 6ba8ff124..01699eb58 100644 --- a/Sources/TestingMacros/ConditionMacro.swift +++ b/Sources/TestingMacros/ConditionMacro.swift @@ -43,8 +43,15 @@ public import SwiftSyntaxMacros /// argument (if present) and `sourceLocation` argument are placed at the end of /// the generated function call's argument list. public protocol ConditionMacro: ExpressionMacro, Sendable { + /// Whether or not the macro tries to parse/expand its condition argument. + static var parsesConditionArgument: Bool { get } + /// Whether or not the macro's expansion may throw an error. static var isThrowing: Bool { get } + + /// The name of the module containing the `__check()` function this macro + /// expands to call. + static var checkFunctionModuleName: DeclReferenceExprSyntax { get } } // MARK: - @@ -73,6 +80,14 @@ extension ConditionMacro { .disabled } + public static var parsesConditionArgument: Bool { + true + } + + public static var checkFunctionModuleName: DeclReferenceExprSyntax { + DeclReferenceExprSyntax(baseName: .identifier("Testing")) + } + /// Perform the expansion of this condition macro. /// /// - Parameters: @@ -115,7 +130,7 @@ extension ConditionMacro { // Construct the argument list to __check(). let expandedFunctionName: TokenSyntax var checkArguments = [Argument]() - do { + if parsesConditionArgument { if let trailingClosureIndex { // Include all arguments other than the "comment" and "sourceLocation" // arguments here. @@ -156,7 +171,19 @@ extension ConditionMacro { expandedFunctionName = conditionArgument.expandedFunctionName } + } else { + // Include all arguments other than the "comment" and "sourceLocation" + // arguments here. + checkArguments += macroArguments.indices.lazy + .filter { $0 != commentIndex } + .filter { $0 != isolationArgumentIndex } + .filter { $0 != sourceLocationArgumentIndex } + .map { macroArguments[$0] } + + expandedFunctionName = .identifier("__check") + } + do { // Capture any comments as well -- either in source, preceding the // expression macro or one of its lexical context nodes, or as an argument // to the macro. @@ -201,7 +228,7 @@ extension ConditionMacro { } // Construct and return the call to __check(). - let call: ExprSyntax = "Testing.\(expandedFunctionName)(\(LabeledExprListSyntax(checkArguments)))" + let call: ExprSyntax = "\(checkFunctionModuleName).\(expandedFunctionName)(\(LabeledExprListSyntax(checkArguments)))" if isThrowing { return "\(call).__required()" } @@ -279,6 +306,10 @@ extension RefinedConditionMacro { public static var isThrowing: Bool { Base.isThrowing } + + public static var parsesConditionArgument: Bool { + Base.parsesConditionArgument + } } // MARK: - Diagnostics-emitting condition macros @@ -671,3 +702,30 @@ public struct ExitTestExpectMacro: ExitTestConditionMacro { public struct ExitTestRequireMacro: ExitTestConditionMacro { public typealias Base = RequireMacro } + +// MARK: - Exit tests using Process/NSTask/Subprocess + +public struct ExpectNSTaskExitsWithMacro: RefinedConditionMacro { + public typealias Base = ExpectMacro + + public static var parsesConditionArgument: Bool { + false + } + + public static var checkFunctionModuleName: DeclReferenceExprSyntax { + DeclReferenceExprSyntax(baseName: .identifier("_Testing_Foundation")) + } +} + +public struct RequireNSTaskExitsWithMacro: RefinedConditionMacro { + public typealias Base = RequireMacro + + public static var parsesConditionArgument: Bool { + false + } + + public static var checkFunctionModuleName: DeclReferenceExprSyntax { + DeclReferenceExprSyntax(baseName: .identifier("_Testing_Foundation")) + } +} + diff --git a/Sources/TestingMacros/TestingMacrosMain.swift b/Sources/TestingMacros/TestingMacrosMain.swift index 074aeb86b..2c4032525 100644 --- a/Sources/TestingMacros/TestingMacrosMain.swift +++ b/Sources/TestingMacros/TestingMacrosMain.swift @@ -26,6 +26,8 @@ struct TestingMacrosMain: CompilerPlugin { NonOptionalRequireMacro.self, RequireThrowsMacro.self, RequireThrowsNeverMacro.self, + ExpectNSTaskExitsWithMacro.self, + RequireNSTaskExitsWithMacro.self, ExitTestExpectMacro.self, ExitTestRequireMacro.self, ExitTestCapturedValueMacro.self, diff --git a/Tests/TestingTests/ExitTestTests.swift b/Tests/TestingTests/ExitTestTests.swift index 5be229266..78d9347ad 100644 --- a/Tests/TestingTests/ExitTestTests.swift +++ b/Tests/TestingTests/ExitTestTests.swift @@ -11,6 +11,11 @@ @testable @_spi(ForToolsIntegrationOnly) import Testing private import _TestingInternals +#if canImport(Foundation) && canImport(_Testing_Foundation) +import Foundation +@_spi(Experimental) import _Testing_Foundation +#endif + #if !SWT_NO_EXIT_TESTS @Suite("Exit test tests") struct ExitTestTests { @Test("Signal names are reported (where supported)") func signalName() { @@ -627,6 +632,41 @@ private import _TestingInternals #endif } +#if canImport(Foundation) && canImport(_Testing_Foundation) +struct `Exit tests using Foundation.Process` { +#if !os(Windows) + @Test func `can consume stdout`() async throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/echo", isDirectory: false) + process.arguments = ["Hello world!"] + process.standardOutput = Pipe() + let result = try await #require(process, exitsWith: .success, observing: [\.standardOutputContent]) + #expect(result.standardOutputContent.contains("Hello world!".utf8)) + } + + @Test func `detects exit status`() async throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/cat", isDirectory: false) + process.arguments = ["ceci n'est pas un fichier"] + await #expect(process, exitsWith: .failure) + } + + @Test func `reports errors back to caller`() async throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/bin/this executable does not exist", isDirectory: false) + await withKnownIssue { + await #expect(process, exitsWith: .failure) + } matching: { issue in + if case .system = issue.kind { + return true + } + return false + } + } +#endif +} +#endif + // MARK: - Fixtures @Suite(.hidden) struct FailingExitTests {