Merge pull request #2334 from ahoppen/relative-compile-commands

Interpret the working directories in `compile_commands.json` relative to the directory that contains `compile_commands.json`

Author: ahoppen

Sources/BuildServerIntegration/CompilationDatabase.swift

@@ -78,19 +78,35 @@ package struct CompilationDatabaseCompileCommand: Equatable, Codable {
     try container.encodeIfPresent(output, forKey: .output)
   }
 
-  /// The `DocumentURI` for this file. If `filename` is relative and `directory` is
-  /// absolute, returns the concatenation. However, if both paths are relative,
-  /// it falls back to `filename`, which is more likely to be the identifier
-  /// that a caller will be looking for.
-  package var uri: DocumentURI {
-    if filename.isAbsolutePath || !directory.isAbsolutePath {
-      return DocumentURI(filePath: filename, isDirectory: false)
-    } else {
-      return DocumentURI(URL(fileURLWithPath: directory).appending(component: filename, directoryHint: .notDirectory))
+  /// The `DocumentURI` for this file. If this a relative path, it will be interpreted relative to the compile command's
+  /// working directory, which in turn is relative to `compileCommandsDirectory`, the directory that contains the
+  /// `compile_commands.json` file.
+  package func uri(compileCommandsDirectory: URL) -> DocumentURI {
+    if filename.isAbsolutePath {
+      return DocumentURI(URL(fileURLWithPath: self.filename))
     }
+    return DocumentURI(
+      URL(
+        fileURLWithPath: self.filename,
+        relativeTo: self.directoryURL(compileCommandsDirectory: compileCommandsDirectory)
+      )
+    )
+  }
+
+  /// A file URL representing `directory`. If `directory` is relative, it's interpreted relative to
+  /// `compileCommandsDirectory`, the directory that contains the `compile_commands.json` file.
+  func directoryURL(compileCommandsDirectory: URL) -> URL {
+    return URL(fileURLWithPath: directory, isDirectory: true, relativeTo: compileCommandsDirectory)
   }
 }
 
+extension CodingUserInfoKey {
+  /// When decoding `JSONCompilationDatabase` a `URL` representing the directory that contains the
+  /// `compile_commands.json`.
+  package static let compileCommandsDirectoryKey: CodingUserInfoKey =
+    CodingUserInfoKey(rawValue: "lsp.compile-commands-dir")!
+}
+
 /// The JSON clang-compatible compilation database.
 ///
 /// Example:
@@ -110,13 +126,26 @@ package struct JSONCompilationDatabase: Equatable, Codable {
   private var pathToCommands: [DocumentURI: [Int]] = [:]
   var commands: [CompilationDatabaseCompileCommand] = []
 
-  package init(_ commands: [CompilationDatabaseCompileCommand] = []) {
+  /// The directory that contains the `compile_commands.json` file.
+  private let compileCommandsDirectory: URL
+
+  package init(_ commands: [CompilationDatabaseCompileCommand] = [], compileCommandsDirectory: URL) {
+    self.compileCommandsDirectory = compileCommandsDirectory
     for command in commands {
       add(command)
     }
   }
 
+  /// Decode the `JSONCompilationDatabase` from a decoder.
+  ///
+  /// A `URL` representing the directory that contains the `compile_commands.json` must be passed in the decoder's
+  /// `userInfo` via the `compileCommandsDirectoryKey`.
   package init(from decoder: Decoder) throws {
+    guard let compileCommandsDirectory = decoder.userInfo[.compileCommandsDirectoryKey] as? URL else {
+      struct MissingCompileCommandsDirectoryKeyError: Error {}
+      throw MissingCompileCommandsDirectoryKeyError()
+    }
+    self.compileCommandsDirectory = compileCommandsDirectory
     var container = try decoder.unkeyedContainer()
     while !container.isAtEnd {
       self.add(try container.decode(CompilationDatabaseCompileCommand.self))
@@ -135,7 +164,9 @@ package struct JSONCompilationDatabase: Equatable, Codable {
   /// - Returns: `nil` if the file does not exist
   package init(file: URL) throws {
     let data = try Data(contentsOf: file)
-    self = try JSONDecoder().decode(JSONCompilationDatabase.self, from: data)
+    let decoder = JSONDecoder()
+    decoder.userInfo[.compileCommandsDirectoryKey] = file.deletingLastPathComponent()
+    self = try decoder.decode(JSONCompilationDatabase.self, from: data)
   }
 
   package func encode(to encoder: Encoder) throws {
@@ -156,7 +187,7 @@ package struct JSONCompilationDatabase: Equatable, Codable {
   }
 
   private mutating func add(_ command: CompilationDatabaseCompileCommand) {
-    let uri = command.uri
+    let uri = command.uri(compileCommandsDirectory: compileCommandsDirectory)
     pathToCommands[uri, default: []].append(commands.count)
 
     if let symlinkTarget = uri.symlinkTarget {

Sources/BuildServerIntegration/JSONCompilationDatabaseBuildServer.swift

@@ -28,14 +28,14 @@ fileprivate extension CompilationDatabaseCompileCommand {
   ///
   /// If the compiler is a symlink to `swiftly`, it uses `swiftlyResolver` to find the corresponding executable in a
   /// real toolchain and returns that executable.
-  func compiler(swiftlyResolver: SwiftlyResolver) async -> String? {
+  func compiler(swiftlyResolver: SwiftlyResolver, compileCommandsDirectory: URL) async -> String? {
     guard let compiler = commandLine.first else {
       return nil
     }
     let swiftlyResolved = await orLog("Resolving swiftly") {
       try await swiftlyResolver.resolve(
         compiler: URL(fileURLWithPath: compiler),
-        workingDirectory: URL(fileURLWithPath: directory)
+        workingDirectory: directoryURL(compileCommandsDirectory: compileCommandsDirectory)
       )?.filePath
     }
     if let swiftlyResolved {
@@ -62,7 +62,17 @@ package actor JSONCompilationDatabaseBuildServer: BuiltInBuildServer {
 
   private let connectionToSourceKitLSP: any Connection
 
-  package let configPath: URL
+  /// The path of the `compile_commands.json` file
+  private let configPath: URL
+
+  /// The directory containing the `compile_commands.json` file and relative to which the working directories in the
+  /// compilation database will be interpreted.
+  ///
+  /// Note that while `configPath` might be a symlink, this is always the directory of the compilation database's
+  /// `realpath` since the user most likely wants to reference relative working directories relative to that path
+  /// instead of relative to eg. a symlink in the project's root directory, which was just placed there so SourceKit-LSP
+  /// finds the compilation database in a build directory.
+  private var configDirectory: URL
 
   private let swiftlyResolver = SwiftlyResolver()
 
@@ -107,12 +117,14 @@ package actor JSONCompilationDatabaseBuildServer: BuiltInBuildServer {
     self.toolchainRegistry = toolchainRegistry
     self.connectionToSourceKitLSP = connectionToSourceKitLSP
     self.configPath = configPath
+    // See comment on configDirectory why we do `realpath` here
+    self.configDirectory = try configPath.realpath.deletingLastPathComponent()
   }
 
   package func buildTargets(request: WorkspaceBuildTargetsRequest) async throws -> WorkspaceBuildTargetsResponse {
     let compilers = Set(
       await compdb.commands.asyncCompactMap { (command) -> String? in
-        await command.compiler(swiftlyResolver: swiftlyResolver)
+        await command.compiler(swiftlyResolver: swiftlyResolver, compileCommandsDirectory: configDirectory)
       }
     ).sorted { $0 < $1 }
     let targets = try await compilers.asyncMap { compiler in
@@ -142,10 +154,11 @@ package actor JSONCompilationDatabaseBuildServer: BuiltInBuildServer {
         return nil
       }
       let commandsWithRequestedCompilers = await compdb.commands.lazy.asyncFilter { command in
-        return await targetCompiler == command.compiler(swiftlyResolver: swiftlyResolver)
+        return await targetCompiler
+          == command.compiler(swiftlyResolver: swiftlyResolver, compileCommandsDirectory: configDirectory)
       }
       let sources = commandsWithRequestedCompilers.map {
-        SourceItem(uri: $0.uri, kind: .file, generated: false)
+        SourceItem(uri: $0.uri(compileCommandsDirectory: configDirectory), kind: .file, generated: false)
       }
       return SourcesItem(target: target, sources: Array(sources))
     }
@@ -172,14 +185,15 @@ package actor JSONCompilationDatabaseBuildServer: BuiltInBuildServer {
   ) async throws -> TextDocumentSourceKitOptionsResponse? {
     let targetCompiler = try request.target.compileCommandsCompiler
     let command = await compdb[request.textDocument.uri].asyncFilter {
-      return await $0.compiler(swiftlyResolver: swiftlyResolver) == targetCompiler
+      return await $0.compiler(swiftlyResolver: swiftlyResolver, compileCommandsDirectory: configDirectory)
+        == targetCompiler
     }.first
     guard let command else {
       return nil
     }
     return TextDocumentSourceKitOptionsResponse(
       compilerArguments: Array(command.commandLine.dropFirst()),
-      workingDirectory: command.directory
+      workingDirectory: try command.directoryURL(compileCommandsDirectory: configDirectory).filePath
     )
   }
 

Sources/SKTestSupport/CheckCoding.swift

@@ -20,10 +20,12 @@ import XCTest
 package func checkCoding<T: Codable & Equatable>(
   _ value: T,
   json: String,
+  userInfo: [CodingUserInfoKey: any Sendable] = [:],
   file: StaticString = #filePath,
   line: UInt = #line
 ) {
   let encoder = JSONEncoder()
+  encoder.userInfo = userInfo
   encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]
 
   let data = try! encoder.encode(WrapFragment(value: value))
@@ -42,6 +44,7 @@ package func checkCoding<T: Codable & Equatable>(
   XCTAssertEqual(json, str, file: file, line: line)
 
   let decoder = JSONDecoder()
+  decoder.userInfo = userInfo
   let decodedValue = try! decoder.decode(WrapFragment<T>.self, from: data).value
 
   XCTAssertEqual(value, decodedValue, file: file, line: line)

Sources/SKTestSupport/IndexedSingleSwiftFileTestProject.swift

@@ -153,7 +153,8 @@ package struct IndexedSingleSwiftFileTestProject {
           filename: try testFileURL.filePath,
           commandLine: [try swiftc.filePath] + compilerArguments
         )
-      ]
+      ],
+      compileCommandsDirectory: testWorkspaceDirectory
     )
     let encoder = JSONEncoder()
     encoder.outputFormatting = [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes]

Tests/BuildServerIntegrationTests/CompilationDatabaseTests.swift

@@ -121,18 +121,28 @@ final class CompilationDatabaseTests: XCTestCase {
   }
 
   func testJSONCompilationDatabaseCoding() {
+    #if os(Windows)
+    let fileSystemRoot = URL(filePath: #"C:\"#)
+    #else
+    let fileSystemRoot = URL(filePath: "/")
+    #endif
+    let userInfo = [CodingUserInfoKey.compileCommandsDirectoryKey: fileSystemRoot]
     checkCoding(
-      JSONCompilationDatabase([]),
+      JSONCompilationDatabase([], compileCommandsDirectory: fileSystemRoot),
       json: """
         [
 
         ]
-        """
+        """,
+      userInfo: userInfo
+    )
+    let db = JSONCompilationDatabase(
+      [
+        .init(directory: "a", filename: "b", commandLine: [], output: nil),
+        .init(directory: "c", filename: "b", commandLine: [], output: nil),
+      ],
+      compileCommandsDirectory: fileSystemRoot
     )
-    let db = JSONCompilationDatabase([
-      .init(directory: "a", filename: "b", commandLine: [], output: nil),
-      .init(directory: "c", filename: "b", commandLine: [], output: nil),
-    ])
     checkCoding(
       db,
       json: """
@@ -152,7 +162,8 @@ final class CompilationDatabaseTests: XCTestCase {
             "file" : "b"
           }
         ]
-        """
+        """,
+      userInfo: userInfo
     )
   }
 
@@ -177,9 +188,9 @@ final class CompilationDatabaseTests: XCTestCase {
       output: nil
     )
 
-    let db = JSONCompilationDatabase([cmd1, cmd2, cmd3])
+    let db = JSONCompilationDatabase([cmd1, cmd2, cmd3], compileCommandsDirectory: URL(filePath: fileSystemRoot))
 
-    XCTAssertEqual(db[DocumentURI(filePath: "b", isDirectory: false)], [cmd1])
+    XCTAssertEqual(db[DocumentURI(filePath: "\(fileSystemRoot)a/b", isDirectory: false)], [cmd1])
     XCTAssertEqual(db[DocumentURI(filePath: "\(fileSystemRoot)c/b", isDirectory: false)], [cmd2])
     XCTAssertEqual(db[DocumentURI(filePath: "\(fileSystemRoot)b", isDirectory: false)], [cmd3])
   }
@@ -256,28 +267,37 @@ final class CompilationDatabaseTests: XCTestCase {
   }
 
   func testCompilationDatabaseBuildServer() async throws {
-    try await checkCompilationDatabaseBuildServer(
-      """
+    #if os(Windows)
+    let fileSystemRoot = URL(filePath: #"C:\"#)
+    #else
+    let fileSystemRoot = URL(filePath: "/")
+    #endif
+    let workingDirectory = try fileSystemRoot.appending(components: "a").filePath
+    let filePath = try fileSystemRoot.appending(components: "a", "a.swift").filePath
+    let compileCommands = JSONCompilationDatabase(
       [
-        {
-          "file": "/a/a.swift",
-          "directory": "/a",
-          "arguments": ["swiftc", "-swift-version", "4", "/a/a.swift"]
-        }
-      ]
-      """
+        CompilationDatabaseCompileCommand(
+          directory: workingDirectory,
+          filename: filePath,
+          commandLine: ["swiftc", "-swift-version", "4", filePath]
+        )
+      ],
+      compileCommandsDirectory: fileSystemRoot
+    )
+    try await checkCompilationDatabaseBuildServer(
+      XCTUnwrap(String(data: JSONEncoder().encode(compileCommands), encoding: .utf8))
     ) { buildServer in
       let settings = try await buildServer.sourceKitOptions(
         request: TextDocumentSourceKitOptionsRequest(
-          textDocument: TextDocumentIdentifier(DocumentURI(URL(fileURLWithPath: "/a/a.swift"))),
+          textDocument: TextDocumentIdentifier(DocumentURI(URL(fileURLWithPath: "\(filePath)"))),
           target: BuildTargetIdentifier.createCompileCommands(compiler: "swiftc"),
           language: .swift
         )
       )
 
       XCTAssertNotNil(settings)
-      XCTAssertEqual(settings?.workingDirectory, "/a")
-      XCTAssertEqual(settings?.compilerArguments, ["-swift-version", "4", "/a/a.swift"])
+      XCTAssertEqual(settings?.workingDirectory, workingDirectory)
+      XCTAssertEqual(settings?.compilerArguments, ["-swift-version", "4", filePath])
       assertNil(await buildServer.indexStorePath)
       assertNil(await buildServer.indexDatabasePath)
     }

Tests/SourceKitLSPTests/CompilationDatabaseTests.swift

@@ -354,6 +354,40 @@ final class CompilationDatabaseTests: XCTestCase {
       }
     }
   }
+
+  func testCompilationDatabaseWithRelativeDirectory() async throws {
+    let project = try await MultiFileTestProject(files: [
+      "projectA/headers/header.h": """
+      int 1️⃣foo2️⃣() {}
+      """,
+      "projectA/main.cpp": """
+      #include "header.h"
+
+      int main() {
+        3️⃣foo();
+      }
+      """,
+      "compile_commands.json": """
+      [
+        {
+          "directory": "projectA",
+          "arguments": [
+            "clang",
+            "-I", "headers"
+          ],
+          "file": "main.cpp"
+        }
+      ]
+      """,
+    ])
+
+    let (mainUri, positions) = try project.openDocument("main.cpp")
+
+    let definition = try await project.testClient.send(
+      DefinitionRequest(textDocument: TextDocumentIdentifier(mainUri), position: positions["3️⃣"])
+    )
+    XCTAssertEqual(definition?.locations, [try project.location(from: "1️⃣", to: "2️⃣", in: "header.h")])
+  }
 }
 
 private let defaultSDKArgs: String = {

Tests/SourceKitLSPTests/WorkspaceTestDiscoveryTests.swift

@@ -787,13 +787,16 @@ final class WorkspaceTestDiscoveryTests: XCTestCase {
     let swiftc = try await unwrap(ToolchainRegistry.forTesting.default?.swiftc)
     let uri = try project.uri(for: "MyTests.swift")
 
-    let compilationDatabase = JSONCompilationDatabase([
-      CompilationDatabaseCompileCommand(
-        directory: try project.scratchDirectory.filePath,
-        filename: uri.pseudoPath,
-        commandLine: [try swiftc.filePath, uri.pseudoPath]
-      )
-    ])
+    let compilationDatabase = JSONCompilationDatabase(
+      [
+        CompilationDatabaseCompileCommand(
+          directory: try project.scratchDirectory.filePath,
+          filename: uri.pseudoPath,
+          commandLine: [try swiftc.filePath, uri.pseudoPath]
+        )
+      ],
+      compileCommandsDirectory: project.scratchDirectory
+    )
 
     try await project.changeFileOnDisk(
       JSONCompilationDatabaseBuildServer.dbName,