Implement test discovery on platforms without an Objective-C runtime (Linux, etc.)

This is a WIP PR for implementing test discovery on linux using indexing
data. The basic idea is to leverage the IndexStore library to get the
test methods and then generate a LinuxMain.swift file during the build
process.

This PR "works" but there are a number of serious hacks that require
resolving before we can merge it.

Author: aciidgh

Sources/Basic/Path.swift

@@ -121,6 +121,14 @@ public struct AbsolutePath: Hashable {
         return _impl.basename
     }
 
+    /// Returns the basename without the extension.
+    public var basenameWithoutExt: String {
+        if let ext = self.extension {
+            return String(basename.dropLast(ext.count + 1))
+        }
+        return basename
+    }
+
     /// Suffix (including leading `.` character) if any.  Note that a basename
     /// that starts with a `.` character is not considered a suffix, nor is a
     /// trailing `.` character.

Sources/Build/BuildDelegate.swift

@@ -186,6 +186,183 @@ extension SPMLLBuild.Diagnostic: DiagnosticDataConvertible {
     }
 }
 
+class CustomLLBuildCommand: ExternalCommand {
+    let ctx: BuildExecutionContext
+
+    required init(_ ctx: BuildExecutionContext) {
+        self.ctx = ctx
+    }
+
+    func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] {
+        return []
+    }
+
+    func execute(_ command: SPMLLBuild.Command) -> Bool {
+        fatalError("subclass responsibility")
+    }
+}
+
+final class TestDiscoveryCommand: CustomLLBuildCommand {
+
+    private func write(
+        tests: [IndexStore.TestCaseClass],
+        forModule module: String,
+        to path: AbsolutePath
+    ) throws {
+        let stream = try LocalFileOutputByteStream(path)
+
+        stream <<< "import XCTest" <<< "\n"
+        stream <<< "@testable import " <<< module <<< "\n"
+
+        for klass in tests {
+            stream <<< "\n"
+            stream <<< "fileprivate extension " <<< klass.name <<< " {" <<< "\n"
+            stream <<< indent(4) <<< "static let __allTests__\(klass.name) = [" <<< "\n"
+            for method in klass.methods {
+                let method = method.hasSuffix("()") ? String(method.dropLast(2)) : method
+                stream <<< indent(8) <<< "(\"\(method)\", \(method))," <<< "\n"
+            }
+            stream <<< indent(4) <<< "]" <<< "\n"
+            stream <<< "}" <<< "\n"
+        }
+
+        stream <<< """
+        func __allTests_\(module)() -> [XCTestCaseEntry] {
+            return [\n
+        """
+
+        for klass in tests {
+            stream <<< indent(8) <<< "testCase(\(klass.name).__allTests__\(klass.name)),\n"
+        }
+
+        stream <<< """
+            ]
+        }
+        """
+
+        stream.flush()
+    }
+
+    private func execute(with tool: ToolProtocol) throws {
+        assert(tool is TestDiscoveryTool, "Unexpected tool \(tool)")
+
+        let index = ctx.buildParameters.indexStore
+        let api = try ctx.indexStoreAPI.dematerialize()
+        let store = try IndexStore.open(store: index, api: api)
+
+        // FIXME: We can speed this up by having one llbuild command per object file.
+        let tests = try tool.inputs.flatMap {
+            try store.listTests(inObjectFile: AbsolutePath($0))
+        }
+
+        let outputs = tool.outputs.compactMap{ try? AbsolutePath(validating: $0) }
+        let testsByModule = Dictionary(grouping: tests, by: { $0.module })
+
+        func isMainFile(_ path: AbsolutePath) -> Bool {
+            return path.basename == "main.swift"
+        }
+
+        // Write one file for each test module.
+        //
+        // We could write everything in one file but that can easily run into type conflicts due
+        // in complex packages with large number of test targets.
+        for file in outputs {
+            if isMainFile(file) { continue }
+
+            // FIXME: This is relying on implementation detail of the output but passing the
+            // the context all the way through is not worth it right now.
+            let module = file.basenameWithoutExt
+
+            guard let tests = testsByModule[module] else {
+                // This module has no tests so just write an empty file for it.
+                try localFileSystem.writeFileContents(file, bytes: "")
+                continue
+            }
+            try write(tests: tests, forModule: module, to: file)
+        }
+
+        // Write the main file.
+        let mainFile = outputs.first(where: isMainFile)!
+        let stream = try LocalFileOutputByteStream(mainFile)
+
+        stream <<< "import XCTest" <<< "\n\n"
+        stream <<< "var tests = [XCTestCaseEntry]()" <<< "\n"
+        for module in testsByModule.keys {
+            stream <<< "tests += __allTests_\(module)()" <<< "\n"
+        }
+        stream <<< "\n"
+        stream <<< "XCTMain(tests)" <<< "\n"
+
+        stream.flush()
+    }
+
+    private func indent(_ spaces: Int) -> ByteStreamable {
+        return Format.asRepeating(string: " ", count: spaces)
+    }
+
+    override func execute(_ command: SPMLLBuild.Command) -> Bool {
+        guard let tool = ctx.buildTimeCmdToolMap[command.name] else {
+            print("command \(command.name) not registered")
+            return false
+        }
+        do {
+            try execute(with: tool)
+        } catch {
+            // FIXME: Shouldn't use "print" here.
+            print("error:", error)
+            return false
+        }
+        return true
+    }
+}
+
+private final class InProcessTool: Tool {
+    let ctx: BuildExecutionContext
+
+    init(_ ctx: BuildExecutionContext) {
+        self.ctx = ctx
+    }
+
+    func createCommand(_ name: String) -> ExternalCommand {
+        // FIXME: This should be able to dynamically look up the right command.
+        switch ctx.buildTimeCmdToolMap[name] {
+        case is TestDiscoveryTool:
+            return TestDiscoveryCommand(ctx)
+        default:
+            fatalError("Unhandled command \(name)")
+        }
+    }
+}
+
+/// The context available during build execution.
+public final class BuildExecutionContext {
+
+    /// Mapping of command-name to its tool.
+    let buildTimeCmdToolMap: [String: ToolProtocol]
+
+    var indexStoreAPI: Result<IndexStoreAPI, AnyError> {
+        indexStoreAPICache.getValue(self)
+    }
+
+    let buildParameters: BuildParameters
+
+    public init(_ plan: BuildPlan, buildTimeCmdToolMap: [String: ToolProtocol]) {
+        self.buildParameters = plan.buildParameters
+        self.buildTimeCmdToolMap = buildTimeCmdToolMap
+    }
+
+    // MARK:- Private
+
+    private var indexStoreAPICache = LazyCache(createIndexStoreAPI)
+    private func createIndexStoreAPI() -> Result<IndexStoreAPI, AnyError> {
+        Result {
+            let ext = buildParameters.triple.dynamicLibraryExtension
+            let indexStoreLib = buildParameters.toolchain.toolchainLibDir.appending(component: "libIndexStore" + ext)
+            return try IndexStoreAPI(dylib: indexStoreLib)
+        }
+    }
+}
+
 private let newLineByte: UInt8 = 10
 public final class BuildDelegate: BuildSystemDelegate, SwiftCompilerOutputParserDelegate {
     private let diagnostics: DiagnosticsEngine
@@ -201,7 +378,10 @@ public final class BuildDelegate: BuildSystemDelegate, SwiftCompilerOutputParser
     /// Target name keyed by llbuild command name.
     private let targetNames: [String: String]
 
+    let buildExecutionContext: BuildExecutionContext
+
     public init(
+        bctx: BuildExecutionContext,
         plan: BuildPlan,
         diagnostics: DiagnosticsEngine,
         outputStream: OutputByteStream,
@@ -212,6 +392,7 @@ public final class BuildDelegate: BuildSystemDelegate, SwiftCompilerOutputParser
         // https://forums.swift.org/t/allow-self-x-in-class-convenience-initializers/15924
         self.outputStream = outputStream as? ThreadSafeOutputByteStream ?? ThreadSafeOutputByteStream(outputStream)
         self.progressAnimation = progressAnimation
+        self.buildExecutionContext = bctx
 
         let buildConfig = plan.buildParameters.configuration.dirname
 
@@ -231,7 +412,12 @@ public final class BuildDelegate: BuildSystemDelegate, SwiftCompilerOutputParser
     }
 
     public func lookupTool(_ name: String) -> Tool? {
-        return nil
+        switch name {
+        case TestDiscoveryTool.name:
+            return InProcessTool(buildExecutionContext)
+        default:
+            return nil
+        }
     }
 
     public func hadCommandFailure() {

Sources/Build/BuildPlan.swift

@@ -128,6 +128,9 @@ public struct BuildParameters {
     /// Whether to enable code coverage.
     public let enableCodeCoverage: Bool
 
+    /// Whether to enable test discovery on platforms without Objective-C runtime.
+    public let enableTestDiscovery: Bool
+
     /// Whether to enable generation of `.swiftinterface` files alongside
     /// `.swiftmodule`s.
     public let enableParseableModuleInterfaces: Bool
@@ -156,7 +159,8 @@ public struct BuildParameters {
         sanitizers: EnabledSanitizers = EnabledSanitizers(),
         enableCodeCoverage: Bool = false,
         indexStoreMode: IndexStoreMode = .auto,
-        enableParseableModuleInterfaces: Bool = false
+        enableParseableModuleInterfaces: Bool = false,
+        enableTestDiscovery: Bool = false
     ) {
         self.dataPath = dataPath
         self.configuration = configuration
@@ -170,6 +174,7 @@ public struct BuildParameters {
         self.enableCodeCoverage = enableCodeCoverage
         self.indexStoreMode = indexStoreMode
         self.enableParseableModuleInterfaces = enableParseableModuleInterfaces
+        self.enableTestDiscovery = enableTestDiscovery
     }
 
     /// Returns the compiler arguments for the index store, if enabled.
@@ -445,6 +450,7 @@ public final class SwiftTargetBuildDescription {
 
     /// The objects in this target.
     var objects: [AbsolutePath] {
+        // Otherwise, use sources from the target.
         return target.sources.relativePaths.map({ tempsPath.appending(RelativePath("\($0.pathString).o")) })
     }
 
@@ -469,13 +475,22 @@ public final class SwiftTargetBuildDescription {
     /// If this target is a test target.
     public let isTestTarget: Bool
 
+    /// True if this is the test discovery target.
+    public let testDiscoveryTarget: Bool
+
     /// Create a new target description with target and build parameters.
-    init(target: ResolvedTarget, buildParameters: BuildParameters, isTestTarget: Bool? = nil) {
+    init(
+        target: ResolvedTarget,
+        buildParameters: BuildParameters,
+        isTestTarget: Bool? = nil,
+        testDiscoveryTarget: Bool = false
+    ) {
         assert(target.underlyingTarget is SwiftTarget, "underlying target type mismatch \(target)")
         self.target = target
         self.buildParameters = buildParameters
         // Unless mentioned explicitly, use the target type to determine if this is a test target.
         self.isTestTarget = isTestTarget ?? (target.type == .test)
+        self.testDiscoveryTarget = testDiscoveryTarget
     }
 
     /// The arguments needed to compile this target.
@@ -868,6 +883,65 @@ public class BuildPlan {
     /// Diagnostics Engine for emitting diagnostics.
     let diagnostics: DiagnosticsEngine
 
+    private static func planLinuxMain(
+        _ buildParameters: BuildParameters,
+        _ graph: PackageGraph
+    ) throws -> (ResolvedTarget, SwiftTargetBuildDescription)? {
+        guard buildParameters.triple.isLinux() else {
+            return nil
+        }
+
+        // Currently, there can be only one test product in a package graph.
+        guard let testProduct = graph.allProducts.first(where: { $0.type == .test }) else {
+            return nil
+        }
+
+        if !buildParameters.enableTestDiscovery {
+            guard let linuxMainTarget = testProduct.linuxMainTarget else {
+                throw Error.missingLinuxMain
+            }
+
+            let desc = SwiftTargetBuildDescription(
+                target: linuxMainTarget,
+                buildParameters: buildParameters,
+                isTestTarget: true
+            )
+            return (linuxMainTarget, desc)
+        }
+
+        // We'll generate sources containing the test names as part of the build process.
+        let derivedTestListDir = buildParameters.buildPath.appending(components: "testlist.derived")
+        let mainFile = derivedTestListDir.appending(component: "main.swift")
+
+        var paths: [AbsolutePath] = []
+        paths.append(mainFile)
+        let testTargets = graph.rootPackages.flatMap{ $0.targets }.filter{ $0.type == .test }
+        for testTarget in testTargets {
+            let path = derivedTestListDir.appending(components: testTarget.name + ".swift")
+            paths.append(path)
+        }
+
+        let src = Sources(paths: paths, root: derivedTestListDir)
+
+        let swiftTarget = SwiftTarget(
+            testDiscoverySrc: src,
+            name: testProduct.name,
+            dependencies: testProduct.underlyingProduct.targets)
+        let linuxMainTarget = ResolvedTarget(
+            target: swiftTarget,
+            dependencies: testProduct.targets.map(ResolvedTarget.Dependency.target)
+        )
+
+        let target = SwiftTargetBuildDescription(
+            target: linuxMainTarget,
+            buildParameters: buildParameters,
+            isTestTarget: true,
+            testDiscoveryTarget: true
+        )
+
+        return (linuxMainTarget, target)
+    }
+
     /// Create a build plan with build parameters and a package graph.
     public init(
         buildParameters: BuildParameters,
@@ -921,19 +995,10 @@ public class BuildPlan {
             throw Diagnostics.fatalError
         }
 
-        if buildParameters.triple.isLinux() {
-            // FIXME: Create a target for LinuxMain file on linux.
-            // This will go away once it is possible to auto detect tests.
-            let testProducts = graph.allProducts.filter({ $0.type == .test })
-
-            for product in testProducts {
-                guard let linuxMainTarget = product.linuxMainTarget else {
-                    throw Error.missingLinuxMain
-                }
-                let target = SwiftTargetBuildDescription(
-                        target: linuxMainTarget, buildParameters: buildParameters, isTestTarget: true)
-                targetMap[linuxMainTarget] = .swift(target)
-            }
+        // Plan the linux main target.
+        if let result = try Self.planLinuxMain(buildParameters, graph) {
+            targetMap[result.0] = .swift(result.1)
+            self.linuxMainTarget = result.0
         }
 
         var productMap: [ResolvedProduct: ProductBuildDescription] = [:]
@@ -953,6 +1018,8 @@ public class BuildPlan {
         try plan()
     }
 
+    private var linuxMainTarget: ResolvedTarget?
+
     static func validateDeploymentVersionOfProductDependency(
         _ product: ResolvedProduct,
         forTarget target: ResolvedTarget,
@@ -1094,7 +1161,7 @@ public class BuildPlan {
 
         if buildParameters.triple.isLinux() {
             if product.type == .test {
-                product.linuxMainTarget.map({ staticTargets.append($0) })
+                linuxMainTarget.map({ staticTargets.append($0) })
             }
         }