[WIP, DNM] Prototype #expect() overloads outside the testing library.

Package.swift

@@ -265,6 +265,7 @@ let package = Package(
       name: "_Testing_Foundation",
       dependencies: [
         "Testing",
+        "_TestingInternals",
       ],
       path: "Sources/Overlays/_Testing_Foundation",
       exclude: ["CMakeLists.txt"],

Sources/Overlays/_Testing_Foundation/Expectations/Expectation+Process.swift

@@ -0,0 +1,159 @@
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2025 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+//
+
+#if canImport(Foundation)
+@_spi(ForToolsIntegrationOnly) public import Testing
+public import Foundation
+private import _TestingInternals
+
+@_spi(Experimental)
+@freestanding(expression)
+@discardableResult
+#if !SWT_NO_EXIT_TESTS
+@available(macOS 10.15.4, *)
+#else
+@_unavailableInEmbedded
+@available(*, unavailable, message: "Exit tests are not available on this platform.")
+#endif
+public macro expect(
+  _ process: Process,
+  exitsWith expectedExitCondition: ExitTest.Condition,
+  observing observedValues: [any PartialKeyPath<ExitTest.Result> & Sendable] = [],
+  _ comment: @autoclosure () -> Comment? = nil,
+  sourceLocation: SourceLocation = #_sourceLocation
+) -> ExitTest.Result? = #externalMacro(module: "TestingMacros", type: "ExpectNSTaskExitsWithMacro")
+
+@_spi(Experimental)
+@freestanding(expression)
+@discardableResult
+#if !SWT_NO_EXIT_TESTS
+@available(macOS 10.15.4, *)
+#else
+@_unavailableInEmbedded
+@available(*, unavailable, message: "Exit tests are not available on this platform.")
+#endif
+public macro require(
+  _ process: Process,
+  exitsWith expectedExitCondition: ExitTest.Condition,
+  observing observedValues: [any PartialKeyPath<ExitTest.Result> & Sendable] = [],
+  _ comment: @autoclosure () -> Comment? = nil,
+  sourceLocation: SourceLocation = #_sourceLocation
+) -> ExitTest.Result = #externalMacro(module: "TestingMacros", type: "RequireNSTaskExitsWithMacro")
+
+// MARK: -
+
+@_spi(Experimental)
+@discardableResult
+#if !SWT_NO_EXIT_TESTS
+@available(macOS 10.15.4, *)
+#else
+@_unavailableInEmbedded
+@available(*, unavailable, message: "Exit tests are not available on this platform.")
+#endif
+public func __check(
+  _ process: Process,
+  exitsWith expectedExitCondition: ExitTest.Condition,
+  observing observedValues: [any PartialKeyPath<ExitTest.Result> & Sendable] = [],
+  comments: @autoclosure () -> [Comment],
+  isRequired: Bool,
+  isolation: isolated (any Actor)? = #isolation,
+  sourceLocation: SourceLocation
+) async -> Result<ExitTest.Result?, any Error> {
+#if !SWT_NO_EXIT_TESTS
+  // The process may have already started and may already have a termination
+  // handler set, so it's not possible for us to asynchronously wait for it.
+  // As such, we'll have to block _some_ thread.
+  var result: ExitTest.Result
+  do {
+    try await withCheckedThrowingContinuation { continuation in
+      Thread.detachNewThread {
+        do {
+          // There's an obvious race condition here, but that's a limitation of
+          // the Process/NSTask API and we'll just have to accept it.
+          if !process.isRunning {
+            try process.run()
+          }
+          process.waitUntilExit()
+          continuation.resume()
+        } catch {
+          continuation.resume(throwing: error)
+        }
+      }
+    }
+
+    let reason = process.terminationReason
+    let exitStatus: ExitStatus = switch reason {
+    case .exit:
+      .exitCode(process.terminationStatus)
+    case .uncaughtSignal:
+  #if os(Windows)
+      // On Windows, Foundation tries to map exit codes that look like HRESULT
+      // values to signals, which is not the model Swift Testing uses. The
+      // conversion is lossy, so there's not much we can do here other than treat
+      // it as an exit code too.
+      .exitCode(process.terminationStatus)
+  #else
+      .signal(process.terminationStatus)
+  #endif
+    @unknown default:
+      fatalError("Unexpected termination reason '\(reason)' from process \(process). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new")
+    }
+
+    result = ExitTest.Result(exitStatus: exitStatus)
+    func makeContent(from streamObject: Any?) -> [UInt8] {
+      if let fileHandle = streamObject as? FileHandle {
+        if let content = try? fileHandle.readToEnd() {
+          return Array(content)
+        }
+      } else if let pipe = streamObject as? Pipe {
+        return makeContent(from: pipe.fileHandleForReading)
+      }
+
+      return []
+    }
+    if observedValues.contains(\.standardOutputContent) {
+      result.standardOutputContent = makeContent(from: process.standardOutput)
+    }
+    if observedValues.contains(\.standardErrorContent) {
+      result.standardErrorContent = makeContent(from: process.standardError)
+    }
+  } catch {
+    // As with the main exit test implementation, if an error occurs while
+    // trying to run the exit test, treat it as a system error and treat the
+    // condition as a mismatch.
+    let issue = Issue(
+      kind: .system,
+      comments: comments() + CollectionOfOne(Comment(rawValue: String(describingForTest: error))),
+      sourceContext: SourceContext(backtrace: nil, sourceLocation: sourceLocation)
+    )
+    issue.record()
+
+    let exitStatus: ExitStatus = if expectedExitCondition.isApproximatelyEqual(to: .exitCode(EXIT_FAILURE)) {
+      .exitCode(EXIT_SUCCESS)
+    } else {
+      .exitCode(EXIT_FAILURE)
+    }
+    result = ExitTest.Result(exitStatus: exitStatus)
+  }
+
+  let expression = Expression("expectedExitCondition")
+  return __checkValue(
+    expectedExitCondition.isApproximatelyEqual(to: result.exitStatus),
+    expression: expression,
+    expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(result.exitStatus),
+    comments: comments(),
+    isRequired: isRequired,
+    sourceLocation: sourceLocation
+  ).map { _ in result }
+#else
+  swt_unreachable()
+#endif
+}
+#endif

Sources/Testing/ExitTests/ExitTest.Condition.swift

@@ -216,7 +216,7 @@ extension ExitTest.Condition {
   ///
   /// Two exit test conditions can be compared; if either instance is equal to
   /// ``failure``, it will compare equal to any instance except ``success``.
-  func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool {
+  package func isApproximatelyEqual(to exitStatus: ExitStatus) -> Bool {
     // Strictly speaking, the C standard treats 0 as a successful exit code and
     // potentially distinct from EXIT_SUCCESS. To my knowledge, no modern
     // operating system defines EXIT_SUCCESS to any value other than 0, so the

Sources/Testing/Issues/Issue+Recording.swift

@@ -18,7 +18,7 @@ extension Issue {
   ///
   /// - Returns: The issue that was recorded (`self` or a modified copy of it.)
   @discardableResult
-  func record(configuration: Configuration? = nil) -> Self {
+  package func record(configuration: Configuration? = nil) -> Self {
     // If this issue is a caught error that has a custom issue representation,
     // perform that customization now.
     if case let .errorCaught(error) = kind {

Sources/Testing/Issues/Issue.swift

@@ -172,7 +172,7 @@ public struct Issue: Sendable {
   ///     empty.
   ///   - sourceContext: A ``SourceContext`` indicating where and how this issue
   ///     occurred.
-  init(
+  package init(
     kind: Kind,
     severity: Severity = .error,
     comments: [Comment],

Sources/Testing/SourceAttribution/Expression.swift

@@ -326,7 +326,7 @@ public struct __Expression: Sendable {
   ///
   /// - Returns: A copy of `self` with information about the specified runtime
   ///   value captured for future use.
-  func capturingRuntimeValue(_ value: (some Any)?) -> Self {
+  package func capturingRuntimeValue(_ value: (some Any)?) -> Self {
     var result = self
     result.runtimeValue = value.flatMap(Value.init(reflecting:))
     if case let .negation(subexpression, isParenthetical) = kind, let value = value as? Bool {
@@ -348,7 +348,7 @@ public struct __Expression: Sendable {
   ///
   /// If the ``kind`` of `self` is ``Kind/generic`` or ``Kind/stringLiteral``,
   /// this function is equivalent to ``capturingRuntimeValue(_:)``.
-  func capturingRuntimeValues<each T>(_ firstValue: (some Any)?, _ additionalValues: repeat (each T)?) -> Self {
+  package func capturingRuntimeValues<each T>(_ firstValue: (some Any)?, _ additionalValues: repeat (each T)?) -> Self {
     var result = self
 
     // Convert the variadic generic argument list to an array.

Sources/TestingMacros/ConditionMacro.swift

@@ -43,8 +43,15 @@ public import SwiftSyntaxMacros
 /// argument (if present) and `sourceLocation` argument are placed at the end of
 /// the generated function call's argument list.
 public protocol ConditionMacro: ExpressionMacro, Sendable {
+  /// Whether or not the macro tries to parse/expand its condition argument.
+  static var parsesConditionArgument: Bool { get }
+
   /// Whether or not the macro's expansion may throw an error.
   static var isThrowing: Bool { get }
+
+  /// The name of the module containing the `__check()` function this macro
+  /// expands to call.
+  static var checkFunctionModuleName: DeclReferenceExprSyntax { get }
 }
 
 // MARK: -
@@ -73,6 +80,14 @@ extension ConditionMacro {
     .disabled
   }
 
+  public static var parsesConditionArgument: Bool {
+    true
+  }
+
+  public static var checkFunctionModuleName: DeclReferenceExprSyntax {
+    DeclReferenceExprSyntax(baseName: .identifier("Testing"))
+  }
+
   /// Perform the expansion of this condition macro.
   ///
   /// - Parameters:
@@ -115,7 +130,7 @@ extension ConditionMacro {
     // Construct the argument list to __check().
     let expandedFunctionName: TokenSyntax
     var checkArguments = [Argument]()
-    do {
+    if parsesConditionArgument {
       if let trailingClosureIndex {
         // Include all arguments other than the "comment" and "sourceLocation"
         // arguments here.
@@ -156,7 +171,19 @@ extension ConditionMacro {
 
         expandedFunctionName = conditionArgument.expandedFunctionName
       }
+    } else {
+      // Include all arguments other than the "comment" and "sourceLocation"
+      // arguments here.
+      checkArguments += macroArguments.indices.lazy
+        .filter { $0 != commentIndex }
+        .filter { $0 != isolationArgumentIndex }
+        .filter { $0 != sourceLocationArgumentIndex }
+        .map { macroArguments[$0] }
+
+      expandedFunctionName = .identifier("__check")
+    }
 
+    do {
       // Capture any comments as well -- either in source, preceding the
       // expression macro or one of its lexical context nodes, or as an argument
       // to the macro.
@@ -201,7 +228,7 @@ extension ConditionMacro {
     }
 
     // Construct and return the call to __check().
-    let call: ExprSyntax = "Testing.\(expandedFunctionName)(\(LabeledExprListSyntax(checkArguments)))"
+    let call: ExprSyntax = "\(checkFunctionModuleName).\(expandedFunctionName)(\(LabeledExprListSyntax(checkArguments)))"
     if isThrowing {
       return "\(call).__required()"
     }
@@ -279,6 +306,10 @@ extension RefinedConditionMacro {
   public static var isThrowing: Bool {
     Base.isThrowing
   }
+
+  public static var parsesConditionArgument: Bool {
+    Base.parsesConditionArgument
+  }
 }
 
 // MARK: - Diagnostics-emitting condition macros
@@ -671,3 +702,30 @@ public struct ExitTestExpectMacro: ExitTestConditionMacro {
 public struct ExitTestRequireMacro: ExitTestConditionMacro {
   public typealias Base = RequireMacro
 }
+
+// MARK: - Exit tests using Process/NSTask/Subprocess
+
+public struct ExpectNSTaskExitsWithMacro: RefinedConditionMacro {
+  public typealias Base = ExpectMacro
+
+  public static var parsesConditionArgument: Bool {
+    false
+  }
+
+  public static var checkFunctionModuleName: DeclReferenceExprSyntax {
+    DeclReferenceExprSyntax(baseName: .identifier("_Testing_Foundation"))
+  }
+}
+
+public struct RequireNSTaskExitsWithMacro: RefinedConditionMacro {
+  public typealias Base = RequireMacro
+
+  public static var parsesConditionArgument: Bool {
+    false
+  }
+
+  public static var checkFunctionModuleName: DeclReferenceExprSyntax {
+    DeclReferenceExprSyntax(baseName: .identifier("_Testing_Foundation"))
+  }
+}
+

Sources/TestingMacros/TestingMacrosMain.swift

@@ -26,6 +26,8 @@ struct TestingMacrosMain: CompilerPlugin {
       NonOptionalRequireMacro.self,
       RequireThrowsMacro.self,
       RequireThrowsNeverMacro.self,
+      ExpectNSTaskExitsWithMacro.self,
+      RequireNSTaskExitsWithMacro.self,
       ExitTestExpectMacro.self,
       ExitTestRequireMacro.self,
       ExitTestCapturedValueMacro.self,

Tests/TestingTests/ExitTestTests.swift

@@ -11,6 +11,11 @@
 @testable @_spi(ForToolsIntegrationOnly) import Testing
 private import _TestingInternals
 
+#if canImport(Foundation) && canImport(_Testing_Foundation)
+import Foundation
+@_spi(Experimental) import _Testing_Foundation
+#endif
+
 #if !SWT_NO_EXIT_TESTS
 @Suite("Exit test tests") struct ExitTestTests {
   @Test("Signal names are reported (where supported)") func signalName() {
@@ -627,6 +632,41 @@ private import _TestingInternals
 #endif
 }
 
+#if canImport(Foundation) && canImport(_Testing_Foundation)
+struct `Exit tests using Foundation.Process` {
+#if !os(Windows)
+  @Test func `can consume stdout`() async throws {
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/bin/echo", isDirectory: false)
+    process.arguments = ["Hello world!"]
+    process.standardOutput = Pipe()
+    let result = try await #require(process, exitsWith: .success, observing: [\.standardOutputContent])
+    #expect(result.standardOutputContent.contains("Hello world!".utf8))
+  }
+
+  @Test func `detects exit status`() async throws {
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/bin/cat", isDirectory: false)
+    process.arguments = ["ceci n'est pas un fichier"]
+    await #expect(process, exitsWith: .failure)
+  }
+
+  @Test func `reports errors back to caller`() async throws {
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/bin/this executable does not exist", isDirectory: false)
+    await withKnownIssue {
+      await #expect(process, exitsWith: .failure)
+    } matching: { issue in
+      if case .system = issue.kind {
+        return true
+      }
+      return false
+    }
+  }
+#endif
+}
+#endif
+
 // MARK: - Fixtures
 
 @Suite(.hidden) struct FailingExitTests {