From d0a3e68f1a8aedf04b5d71e5bcb127a44609af69 Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 23 Sep 2025 15:15:52 -0700 Subject: [PATCH 1/4] [clang][gmodules] Use CASID to encode splitDwarfFilename Teach clang to encode CASID as splitDwarfFilename for gmodule when clang caching is enabled. This allows the outputs from compiler do not contain paths to the clang module files, thus allows distributed caching without the need of a unified clang module cache directory path. --- .../include/clang/Basic/ASTSourceDescriptor.h | 6 +- .../include/clang/Basic/DiagnosticCASKinds.td | 6 +- clang/include/clang/Basic/Module.h | 12 ++ .../include/clang/Frontend/CompilerInstance.h | 4 +- clang/include/clang/Serialization/ASTReader.h | 10 + .../clang/Serialization/InMemoryModuleCache.h | 11 +- .../include/clang/Serialization/ModuleFile.h | 3 + clang/lib/Basic/ASTSourceDescriptor.cpp | 2 + clang/lib/CodeGen/CGDebugInfo.cpp | 5 + .../CodeGen/ObjectFilePCHContainerWriter.cpp | 4 +- clang/lib/Frontend/CompilerInstance.cpp | 102 +++++++--- clang/lib/Frontend/FrontendAction.cpp | 18 +- clang/lib/Serialization/ASTReader.cpp | 56 ++++- clang/lib/Serialization/ASTWriter.cpp | 14 +- clang/lib/Serialization/GlobalModuleIndex.cpp | 3 + .../lib/Serialization/InMemoryModuleCache.cpp | 13 +- .../DependencyScannerImpl.cpp | 10 +- .../IncludeTreeActionController.cpp | 7 +- .../DependencyScanning/ModuleDepCollector.cpp | 18 +- .../DependencyScanning/ScanAndUpdateArgs.cpp | 12 +- clang/test/CAS/depscan-include-tree.c | 4 +- .../ClangScanDeps/cas-fs-multiple-commands.c | 12 +- clang/test/ClangScanDeps/gmodules.c | 187 +++++++++++++++++ .../include-tree-multiple-commands.c | 12 +- .../include-tree-preserve-pch-path.c | 21 +- .../ClangScanDeps/modules-cas-context-hash.c | 4 +- .../modules-cas-fs-prefix-mapping.c | 8 +- .../modules-cas-trees-with-pch.c | 10 +- clang/test/ClangScanDeps/modules-cas-trees.c | 8 +- .../modules-include-tree-dependency-file.c | 2 +- ...include-tree-pch-common-stale-prefix-map.c | 192 ++++++++++++++++++ .../modules-include-tree-prefix-map.c | 16 +- .../modules-include-tree-with-pch.c | 8 +- .../test/ClangScanDeps/modules-include-tree.c | 16 +- .../modules-pch-cas-fs-prefix-mapping.c | 16 +- 35 files changed, 685 insertions(+), 147 deletions(-) create mode 100644 clang/test/ClangScanDeps/gmodules.c create mode 100644 clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c diff --git a/clang/include/clang/Basic/ASTSourceDescriptor.h b/clang/include/clang/Basic/ASTSourceDescriptor.h index 175e0551db765..c0ce5486d4d4a 100644 --- a/clang/include/clang/Basic/ASTSourceDescriptor.h +++ b/clang/include/clang/Basic/ASTSourceDescriptor.h @@ -30,14 +30,15 @@ class ASTSourceDescriptor { StringRef Path; StringRef ASTFile; ASTFileSignature Signature; + StringRef CASID; Module *ClangModule = nullptr; public: ASTSourceDescriptor() = default; ASTSourceDescriptor(StringRef Name, StringRef Path, StringRef ASTFile, - ASTFileSignature Signature) + ASTFileSignature Signature, StringRef CASID) : PCHModuleName(std::move(Name)), Path(std::move(Path)), - ASTFile(std::move(ASTFile)), Signature(Signature) {} + ASTFile(std::move(ASTFile)), Signature(Signature), CASID(CASID) {} ASTSourceDescriptor(Module &M); std::string getModuleName() const; @@ -45,6 +46,7 @@ class ASTSourceDescriptor { StringRef getASTFile() const { return ASTFile; } ASTFileSignature getSignature() const { return Signature; } Module *getModuleOrNull() const { return ClangModule; } + StringRef getCASID() const { return CASID; } }; } // namespace clang diff --git a/clang/include/clang/Basic/DiagnosticCASKinds.td b/clang/include/clang/Basic/DiagnosticCASKinds.td index 7dd533c17d53d..f37b134859ff7 100644 --- a/clang/include/clang/Basic/DiagnosticCASKinds.td +++ b/clang/include/clang/Basic/DiagnosticCASKinds.td @@ -23,8 +23,10 @@ def err_cas_depscan_daemon_connection: Error< def err_cas_depscan_failed: Error< "CAS-based dependency scan failed: %0">, DefaultFatal; def err_cas_store: Error<"failed to store to CAS: %0">, DefaultFatal; -def err_cas_unloadable_module : Error< - "module file '%0' not found: unloadable module cache key %1: %2">, DefaultFatal; +def err_cas_unloadable_module + : Error<"module file '%0' not found: unloadable %select{casid|module cache " + "key}1 %2: %3">, + DefaultFatal; def err_cas_missing_module : Error< "module file '%0' not found: missing module cache key %1: %2">, DefaultFatal; def err_cas_missing_root_id : Error< diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index cddf9d2b81ce8..8082822c254ad 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -274,6 +274,9 @@ class alignas(8) Module { /// The \c ActionCache key for this module, if any. std::optional ModuleCacheKey; + /// The \c CASID for the loaded module, if any. + std::optional CASID; + /// The top-level headers associated with this module. llvm::SmallSetVector TopHeaders; @@ -773,6 +776,15 @@ class alignas(8) Module { getTopLevelModule()->ModuleCacheKey = std::move(Key); } + std::optional getCASID() const { + return getTopLevelModule()->CASID; + } + + void setCASID(std::string ID) { + assert(!getCASID() || *getCASID() == ID); + getTopLevelModule()->CASID = std::move(ID); + } + /// Retrieve the umbrella directory as written. std::optional getUmbrellaDirAsWritten() const { if (const auto *Dir = std::get_if(&Umbrella)) diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 0ed1455335e17..d5a2a4eba7093 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -1025,8 +1025,8 @@ class CompilerInstance : public ModuleLoader { /// "-fmodule-file-cache-key", or an imported pcm file. Used in diagnostics. /// /// \returns true on failure. - bool addCachedModuleFile(StringRef Path, StringRef CacheKey, - StringRef Provider); + bool addCachedModuleFile(StringRef Path, StringRef CASID, StringRef Provider, + bool IsKey); ModuleCache &getModuleCache() const { return *ModCache; } diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 53ce87faf3d78..d1036a183b3a8 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -276,6 +276,14 @@ class ASTReaderListener { return false; } + /// Called for each module CASID. + /// + /// \returns true to indicate the key cannot be loaded. + virtual bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) { + return false; + } + /// Indicates that a particular module file extension has been read. virtual void readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) {} @@ -329,6 +337,8 @@ class ChainedASTReaderListener : public ASTReaderListener { bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override; void readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) override; }; diff --git a/clang/include/clang/Serialization/InMemoryModuleCache.h b/clang/include/clang/Serialization/InMemoryModuleCache.h index fc3ba334fc64d..8aa3a48db6542 100644 --- a/clang/include/clang/Serialization/InMemoryModuleCache.h +++ b/clang/include/clang/Serialization/InMemoryModuleCache.h @@ -30,6 +30,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase { struct PCM { std::unique_ptr Buffer; + std::string CASID; + /// Track whether this PCM is known to be good (either built or /// successfully imported by a CompilerInstance/ASTReader using this /// cache). @@ -38,6 +40,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase { PCM() = default; PCM(std::unique_ptr Buffer) : Buffer(std::move(Buffer)) {} + + PCM(std::unique_ptr Buffer, llvm::StringRef CASID) + : Buffer(std::move(Buffer)), CASID(CASID.str()) {} }; /// Cache of buffers. @@ -64,7 +69,8 @@ class InMemoryModuleCache : public llvm::RefCountedBase { /// \post state is Tentative /// \return a reference to the buffer as a convenience. llvm::MemoryBuffer &addPCM(llvm::StringRef Filename, - std::unique_ptr Buffer); + std::unique_ptr Buffer, + llvm::StringRef CASID = ""); /// Store a just-built PCM under the Filename. /// @@ -90,6 +96,9 @@ class InMemoryModuleCache : public llvm::RefCountedBase { /// Get a pointer to the pCM if it exists; else nullptr. llvm::MemoryBuffer *lookupPCM(llvm::StringRef Filename) const; + /// Get the PCM if it exits; else nullptr. + const PCM *lookup(llvm::StringRef Filename) const; + /// Check whether the PCM is final and has been shown to work. /// /// \return true iff state is Final. diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 4eeed6a65dad1..c9dd1c8bb1985 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -147,6 +147,9 @@ class ModuleFile { /// The \c ActionCache key for this module, or empty. std::string ModuleCacheKey; + /// The \c CASID for the module, or empty. + std::string CASID; + /// The CAS filesystem root ID for implicit modules built with the dependency /// scanner, or empty. std::string CASFileSystemRootID; diff --git a/clang/lib/Basic/ASTSourceDescriptor.cpp b/clang/lib/Basic/ASTSourceDescriptor.cpp index 8072c08a51d3a..cdc03a4bbea6b 100644 --- a/clang/lib/Basic/ASTSourceDescriptor.cpp +++ b/clang/lib/Basic/ASTSourceDescriptor.cpp @@ -21,6 +21,8 @@ ASTSourceDescriptor::ASTSourceDescriptor(Module &M) Path = M.Directory->getName(); if (auto File = M.getASTFile()) ASTFile = File->getName(); + if (M.getCASID()) + CASID = *M.getCASID(); } std::string ASTSourceDescriptor::getModuleName() const { diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp index 7c547a8c466af..003fdf90b87a5 100644 --- a/clang/lib/CodeGen/CGDebugInfo.cpp +++ b/clang/lib/CodeGen/CGDebugInfo.cpp @@ -3402,6 +3402,11 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod, PCM = Mod.getPath(); } llvm::sys::path::append(PCM, Mod.getASTFile()); + + // If module file has CASID, use CASID as PCM path. + if (!Mod.getCASID().empty()) + PCM = Mod.getCASID(); + DIB.createCompileUnit( TheCU->getSourceLanguage(), // TODO: Support "Source" from external AST providers? diff --git a/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp b/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp index 8b54552623872..0b20fc00df716 100644 --- a/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp +++ b/clang/lib/CodeGen/ObjectFilePCHContainerWriter.cpp @@ -191,8 +191,8 @@ class PCHContainerGenerator : public ASTConsumer { // Prepare CGDebugInfo to emit debug info for a clang module. auto *DI = Builder->getModuleDebugInfo(); StringRef ModuleName = llvm::sys::path::filename(MainFileName); - DI->setPCHDescriptor( - {ModuleName, "", OutputFileName, ASTFileSignature::createDISentinel()}); + DI->setPCHDescriptor({ModuleName, "", OutputFileName, + ASTFileSignature::createDISentinel(), /*CASID=*/""}); DI->setModuleMap(MMap); } diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 9ace84ff4f0c9..4ddedc159d0b0 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -644,6 +644,8 @@ class CompileCacheASTReaderHelper : public ASTReaderListener { bool readIncludeTreeID(StringRef ID, bool Complain) override; bool readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) override; + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override; private: bool checkCASID(bool Complain, StringRef RootID, unsigned ParseDiagID, @@ -2522,25 +2524,51 @@ void CompilerInstance::setExternalSemaSource( ExternalSemaSrc = std::move(ESS); } -static bool addCachedModuleFileToInMemoryCache( - StringRef Path, StringRef CacheKey, StringRef Provider, - cas::ObjectStore &CAS, cas::ActionCache &Cache, - ModuleCache &ModCache, DiagnosticsEngine &Diags) { +static bool addCachedModuleFileToInMemoryCache(StringRef Path, + cas::ObjectStore &CAS, + cas::ObjectRef Object, + ModuleCache &ModCache, + DiagnosticsEngine &Diags) { + // FIXME: We wait to materialize each module file before proceeding, which + // introduces latency for a network CAS. Instead we should collect all the + // module keys and materialize them concurrently using \c getProxyFuture, for + // better network utilization. + auto OutputProxy = CAS.getProxy(Object); + if (!OutputProxy) { + Diags.Report(diag::err_cas_unloadable_module) + << Path << 0 << CAS.getID(Object).toString() << OutputProxy.takeError(); + return true; + } + + auto PCMID = OutputProxy->getID().toString(); + if (const auto *PCM = ModCache.getInMemoryModuleCache().lookup(Path)) { + // If the CASID in the module cache differs, return error. + if (!PCM->CASID.empty() && PCM->CASID != PCMID) + return true; - if (ModCache.getInMemoryModuleCache().lookupPCM(Path)) return false; + } + + ModCache.getInMemoryModuleCache().addPCM(Path, OutputProxy->getMemoryBuffer(), + OutputProxy->getID().toString()); + return false; +} +static bool addCachedModuleFileToInMemoryCacheFromKey( + StringRef Path, StringRef CacheKey, StringRef Provider, + cas::ObjectStore &CAS, cas::ActionCache &Cache, ModuleCache &ModCache, + DiagnosticsEngine &Diags) { auto ID = CAS.parseID(CacheKey); if (!ID) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << ID.takeError(); + << Path << 1 << CacheKey << ID.takeError(); return true; } auto Value = Cache.get(*ID); if (!Value) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << Value.takeError(); + << Path << 1 << CacheKey << Value.takeError(); return true; } if (!*Value) { @@ -2561,7 +2589,7 @@ static bool addCachedModuleFileToInMemoryCache( auto ValueRef = CAS.getReference(**Value); if (!ValueRef) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << "result module cannot be loaded from CAS"; + << Path << 1 << CacheKey << "result module cannot be loaded from CAS"; return true; } @@ -2570,44 +2598,68 @@ static bool addCachedModuleFileToInMemoryCache( cas::CompileJobResultSchema Schema(CAS); if (llvm::Error E = Schema.load(*ValueRef).moveInto(Result)) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << std::move(E); + << Path << 1 << CacheKey << std::move(E); return true; } auto Output = Result->getOutput(cas::CompileJobCacheResult::OutputKind::MainOutput); if (!Output) llvm::report_fatal_error("missing main output"); - // FIXME: We wait to materialize each module file before proceeding, which - // introduces latency for a network CAS. Instead we should collect all the - // module keys and materialize them concurrently using \c getProxyFuture, for - // better network utilization. - auto OutputProxy = CAS.getProxy(Output->Object); - if (!OutputProxy) { + + return addCachedModuleFileToInMemoryCache(Path, CAS, Output->Object, ModCache, + Diags); +} + +static bool addCachedModuleFileToInMemoryCacheFromID(StringRef Filename, + cas::ObjectStore &CAS, + StringRef CASID, + ModuleCache &ModCache, + DiagnosticsEngine &Diags) { + auto ID = CAS.parseID(CASID); + if (!ID) { Diags.Report(diag::err_cas_unloadable_module) - << Path << CacheKey << OutputProxy.takeError(); + << Filename << 0 << CASID << ID.takeError(); return true; } + auto ModuleRef = CAS.getReference(*ID); + if (!ModuleRef) { + Diags.Report(diag::err_cas_unloadable_module) + << Filename << 1 << CASID << "does not exist in CAS"; - ModCache.getInMemoryModuleCache().addPCM(Path, - OutputProxy->getMemoryBuffer()); - return false; + return true; + } + + return addCachedModuleFileToInMemoryCache(Filename, CAS, *ModuleRef, ModCache, + Diags); } -bool CompilerInstance::addCachedModuleFile(StringRef Path, StringRef CacheKey, - StringRef Provider) { - return addCachedModuleFileToInMemoryCache( - Path, CacheKey, Provider, getOrCreateObjectStore(), - getOrCreateActionCache(), getModuleCache(), getDiagnostics()); +bool CompilerInstance::addCachedModuleFile(StringRef Path, StringRef CASID, + StringRef Provider, bool IsKey) { + if (IsKey) + return addCachedModuleFileToInMemoryCacheFromKey( + Path, CASID, Provider, getOrCreateObjectStore(), + getOrCreateActionCache(), getModuleCache(), getDiagnostics()); + + return addCachedModuleFileToInMemoryCacheFromID( + Path, getOrCreateObjectStore(), CASID, getModuleCache(), + getDiagnostics()); } bool CompileCacheASTReaderHelper::readModuleCacheKey(StringRef ModuleName, StringRef Filename, StringRef CacheKey) { // FIXME: add name/path of the importing module? - return addCachedModuleFileToInMemoryCache( + return addCachedModuleFileToInMemoryCacheFromKey( Filename, CacheKey, "imported module", CAS, Cache, ModCache, Diags); } +bool CompileCacheASTReaderHelper::readModuleCASID(StringRef ModuleName, + StringRef Filename, + StringRef CASID) { + return addCachedModuleFileToInMemoryCacheFromID(Filename, CAS, CASID, + ModCache, Diags); +} + /// Verify that ID is in the CAS. Otherwise the module cache probably was /// created with a different CAS. bool CompileCacheASTReaderHelper::checkCASID(bool Complain, StringRef RootID, diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp index f52bbe880e685..861b08c28b227 100644 --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -1259,7 +1259,8 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, // Provide any modules from the action cache. for (const auto &KeyPair : CI.getFrontendOpts().ModuleCacheKeys) if (CI.addCachedModuleFile(KeyPair.first, KeyPair.second, - "-fmodule-file-cache-key")) + "-fmodule-file-cache-key", + /*IsKey=*/true)) return false; @@ -1324,9 +1325,10 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, DeserialListener, DeleteDeserialListener); DeleteDeserialListener = true; } + std::string PCHCASID; if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty() || IncludeTreePCH) { - StringRef PCHPath; + std::string PCHPath; DisableValidationForModuleKind DisableValidation; std::unique_ptr PCHBuffer; if (IncludeTreePCH) { @@ -1335,7 +1337,11 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, if (llvm::Error E = IncludeTreePCH->getMemoryBuffer().moveInto(PCHBuffer)) return reportError(std::move(E)); - PCHPath = PCHBuffer->getBufferIdentifier(); + auto PCHFile = IncludeTreePCH->getContents(); + if (!PCHFile) + return reportError(PCHFile.takeError()); + PCHCASID = PCHFile->getID().toString(); + PCHPath = PCHCASID; } else { PCHPath = CI.getPreprocessorOpts().ImplicitPCHInclude; DisableValidation = @@ -1347,6 +1353,12 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI, DeserialListener, DeleteDeserialListener, std::move(PCHBuffer)); if (!CI.getASTContext().getExternalSource()) return false; + + // After loading PCH, set its CASID for content. + if (!PCHCASID.empty()) { + auto &MF = CI.getASTReader()->getModuleManager().getPrimaryModule(); + MF.CASID = PCHCASID; + } } // If modules are enabled, create the AST reader before creating // any builtins, so that all declarations know that they might be diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index ed1fe9d6c3227..086976d72a784 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -267,6 +267,12 @@ bool ChainedASTReaderListener::readModuleCacheKey(StringRef ModuleName, return First->readModuleCacheKey(ModuleName, Filename, CacheKey) || Second->readModuleCacheKey(ModuleName, Filename, CacheKey); } +bool ChainedASTReaderListener::readModuleCASID(StringRef ModuleName, + StringRef Filename, + StringRef CASID) { + return First->readModuleCASID(ModuleName, Filename, CASID) || + Second->readModuleCASID(ModuleName, Filename, CASID); +} void ChainedASTReaderListener::readModuleFileExtension( const ModuleFileExtensionMetadata &Metadata) { @@ -3291,7 +3297,8 @@ ASTReader::ReadControlBlock(ModuleFile &F, std::string ImportedFile; std::string StoredFile; bool IgnoreImportedByNote = false; - StringRef ImportedCacheKey; + bool CASIDIsKey = false; + StringRef ImportedCASKey; // For prebuilt and explicit modules first consult the file map for // an override. Note that here we don't search prebuilt module @@ -3311,6 +3318,7 @@ ASTReader::ReadControlBlock(ModuleFile &F, if (!IsImportingStdCXXModule) { StoredSize = (off_t)Record[Idx++]; StoredModTime = (time_t)Record[Idx++]; + CASIDIsKey = (bool)Record[Idx++]; StringRef SignatureBytes = Blob.substr(0, ASTFileSignature::size); StoredSignature = ASTFileSignature::create(SignatureBytes.begin(), @@ -3339,18 +3347,32 @@ ASTReader::ReadControlBlock(ModuleFile &F, } } - ImportedCacheKey = ReadStringBlob(Record, Idx, Blob); + ImportedCASKey = ReadStringBlob(Record, Idx, Blob); } - if (!ImportedCacheKey.empty()) { - if (!Listener || Listener->readModuleCacheKey( - ImportedName, ImportedFile, ImportedCacheKey)) { + if (!ImportedCASKey.empty()) { + if (!Listener) { Diag(diag::err_ast_file_not_found) << moduleKindForDiagnostic(ImportedKind) << ImportedFile << true - << ("missing or unloadable module cache key" + ImportedCacheKey) + << ("missing listener to read CAS reference" + ImportedCASKey) .str(); return Failure; } + if (CASIDIsKey) { + assert(ImportedKind == MK_ImplicitModule && + "implicit module encodes cache key, not CASID"); + if (Listener->readModuleCacheKey(ImportedName, ImportedFile, + ImportedCASKey)) { + // If the module cache key failed to read, treat the module as + // out-of-date so it can be rebuilt. + return OutOfDate; + } + } else { + if (Listener->readModuleCASID(ImportedName, ImportedFile, + ImportedCASKey)) { + return OutOfDate; + } + } } // If our client can't cope with us being out of date, we can't cope with @@ -6046,11 +6068,17 @@ bool ASTReader::readASTFileControlBlock( // Skip signature. Blob = Blob.substr(ASTFileSignature::size); + bool CASIDIsKey = Record[Idx++]; + StringRef FilenameStr = ReadStringBlob(Record, Idx, Blob); auto Filename = ResolveImportedPath(PathBuf, FilenameStr, ModuleDir); - StringRef CacheKey = ReadStringBlob(Record, Idx, Blob); - if (!CacheKey.empty()) - Listener.readModuleCacheKey(ModuleName, *Filename, CacheKey); + StringRef CASID = ReadStringBlob(Record, Idx, Blob); + if (!CASID.empty()) { + if (CASIDIsKey) + Listener.readModuleCacheKey(ModuleName, *Filename, CASID); + else + Listener.readModuleCASID(ModuleName, *Filename, CASID); + } Listener.visitImport(ModuleName, *Filename); break; } @@ -6272,6 +6300,14 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, CurrentModule->PresumedModuleMapFile = F.ModuleMapPath; if (!F.ModuleCacheKey.empty()) CurrentModule->setModuleCacheKey(F.ModuleCacheKey); + + // Set CASID for current module. + if (const auto *PCM = + ModuleMgr.getModuleCache().getInMemoryModuleCache().lookup( + F.FileName)) { + CurrentModule->setCASID(PCM->CASID); + F.CASID = PCM->CASID; + } } CurrentModule->Kind = Kind; @@ -9857,7 +9893,7 @@ std::optional ASTReader::getSourceDescriptor(unsigned ID) { StringRef FileName = llvm::sys::path::filename(MF.FileName); return ASTSourceDescriptor(ModuleName, llvm::sys::path::parent_path(MF.FileName), - FileName, MF.Signature); + FileName, MF.Signature, MF.CASID); } return std::nullopt; } diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index c4108dc5c8e68..640a5037cd714 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1624,6 +1624,7 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, StringRef isysroot) { Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Standard C++ mod Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // File size Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // File timestamp + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // CASIDIsKey Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // File name len Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Cache key len Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Strings @@ -1653,16 +1654,27 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, StringRef isysroot) { Record.push_back(0); // FIXME: cache support for standard C++ modules. Record.push_back(0); + Record.push_back(0); } else { // If we have calculated signature, there is no need to store // the size or timestamp. Record.push_back(M.Signature ? 0 : M.File.getSize()); Record.push_back(M.Signature ? 0 : getTimestampForOutput(M.File)); + // Encode CacheKey if it is implicit module, otherwise encode CASID of + // module file. + bool CASIDIsKey = M.Kind == serialization::MK_ImplicitModule; + Record.push_back(CASIDIsKey); llvm::append_range(Blob, M.Signature); AddPathBlob(M.FileName, Record, Blob); - AddStringBlob(M.ModuleCacheKey, Record, Blob); + if (CASIDIsKey) + AddStringBlob(M.ModuleCacheKey, Record, Blob); + else { + assert(!(M.CASID.empty() && !M.ModuleCacheKey.empty()) && + "should not have module cache key without CASID"); + AddStringBlob(M.CASID, Record, Blob); + } } Stream.EmitRecordWithBlob(AbbrevCode, Record, Blob); diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp index 1e2272c48bd04..69a43683b6334 100644 --- a/clang/lib/Serialization/GlobalModuleIndex.cpp +++ b/clang/lib/Serialization/GlobalModuleIndex.cpp @@ -635,6 +635,9 @@ llvm::Error GlobalModuleIndexBuilder::loadModuleFile(FileEntryRef File) { off_t StoredSize = (off_t)Record[Idx++]; time_t StoredModTime = (time_t)Record[Idx++]; + // Skip CASIDIsKey + ++Idx; + // Skip the stored signature. // FIXME: we could read the signature out of the import and validate it. StringRef SignatureBytes = Blob.substr(0, ASTFileSignature::size); diff --git a/clang/lib/Serialization/InMemoryModuleCache.cpp b/clang/lib/Serialization/InMemoryModuleCache.cpp index d35fa2a807f4d..556f9453a1351 100644 --- a/clang/lib/Serialization/InMemoryModuleCache.cpp +++ b/clang/lib/Serialization/InMemoryModuleCache.cpp @@ -23,8 +23,9 @@ InMemoryModuleCache::getPCMState(llvm::StringRef Filename) const { llvm::MemoryBuffer & InMemoryModuleCache::addPCM(llvm::StringRef Filename, - std::unique_ptr Buffer) { - auto Insertion = PCMs.insert(std::make_pair(Filename, std::move(Buffer))); + std::unique_ptr Buffer, + llvm::StringRef CASID) { + auto Insertion = PCMs.try_emplace(Filename, PCM{std::move(Buffer), CASID}); assert(Insertion.second && "Already has a PCM"); return *Insertion.first->second.Buffer; } @@ -48,6 +49,14 @@ InMemoryModuleCache::lookupPCM(llvm::StringRef Filename) const { return I->second.Buffer.get(); } +const InMemoryModuleCache::PCM * +InMemoryModuleCache::lookup(llvm::StringRef Filename) const { + auto I = PCMs.find(Filename); + if (I == PCMs.end()) + return nullptr; + return &I->second; +} + bool InMemoryModuleCache::isPCMFinal(llvm::StringRef Filename) const { return getPCMState(Filename) == Final; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp index 5bd206626fac3..8516942afed85 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp @@ -186,7 +186,15 @@ class PrebuiltModuleListener : public ASTReaderListener { CI.getFrontendOpts().ModuleCacheKeys.emplace_back(std::string(Filename), std::string(CacheKey)); // FIXME: add name/path of the importing module? - return CI.addCachedModuleFile(Filename, CacheKey, "imported module"); + return CI.addCachedModuleFile(Filename, CacheKey, "imported module", + /*IsKey=*/true); + } + + bool readModuleCASID(StringRef ModuleName, StringRef Filename, + StringRef CASID) override { + // FIXME: add name/path of the importing module? + return CI.addCachedModuleFile(Filename, CASID, "imported module", + /*IsKey=*/false); } private: diff --git a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp index da1855ed4dd87..06ddb3dec8492 100644 --- a/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp +++ b/clang/lib/Tooling/DependencyScanning/IncludeTreeActionController.cpp @@ -706,12 +706,7 @@ IncludeTreeBuilder::finishIncludeTree(CompilerInstance &ScanInstance, if (!CASContents) return llvm::errorCodeToError(CASContents.getError()); - StringRef PCHFilename = ""; - if (NewInvocation.getFrontendOpts().IncludeTreePreservePCHPath) - PCHFilename = PPOpts.ImplicitPCHInclude; - - auto PCHFile = - cas::IncludeTree::File::create(DB, PCHFilename, *CASContents); + auto PCHFile = cas::IncludeTree::File::create(DB, "", *CASContents); if (!PCHFile) return PCHFile.takeError(); PCHRef = PCHFile->getRef(); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index e0301b865c470..3e798000c874d 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -363,10 +363,14 @@ ModuleDepCollector::getInvocationAdjustedForModuleBuildWithoutOutputs( // Report the prebuilt modules this module uses. for (const auto &PrebuiltModule : Deps.PrebuiltModuleDeps) { - CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile); - if (PrebuiltModule.ModuleCacheKey) + if (PrebuiltModule.ModuleCacheKey) { + // canonicalize the PCM path if using CAS. + auto PCMFile = llvm::sys::path::filename(PrebuiltModule.PCMFile); + CI.getMutFrontendOpts().ModuleFiles.push_back(PCMFile.str()); CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back( - PrebuiltModule.PCMFile, *PrebuiltModule.ModuleCacheKey); + PCMFile, *PrebuiltModule.ModuleCacheKey); + } else + CI.getMutFrontendOpts().ModuleFiles.push_back(PrebuiltModule.PCMFile); } // Add module file inputs from dependencies. @@ -421,9 +425,11 @@ void ModuleDepCollector::addModuleFiles( Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile); assert(MD && "Inconsistent dependency info"); - if (MD->ModuleCacheKey) + if (MD->ModuleCacheKey) { + PCMPath = llvm::sys::path::filename(PCMPath); CI.getFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); + } if (Service.shouldEagerLoadModules()) CI.getFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else @@ -440,9 +446,11 @@ void ModuleDepCollector::addModuleFiles( Controller.lookupModuleOutput(*MD, ModuleOutputKind::ModuleFile); assert(MD && "Inconsistent dependency info"); - if (MD->ModuleCacheKey) + if (MD->ModuleCacheKey) { + PCMPath = llvm::sys::path::filename(PCMPath); CI.getMutFrontendOpts().ModuleCacheKeys.emplace_back(PCMPath, *MD->ModuleCacheKey); + } if (Service.shouldEagerLoadModules()) CI.getMutFrontendOpts().ModuleFiles.push_back(std::move(PCMPath)); else diff --git a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp index 9e3377381a489..639b8310efedb 100644 --- a/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp +++ b/clang/lib/Tooling/DependencyScanning/ScanAndUpdateArgs.cpp @@ -93,13 +93,6 @@ void tooling::dependencies::configureInvocationForCaching( PPOpts.MacroIncludes.clear(); PPOpts.Includes.clear(); } - if (!FrontendOpts.IncludeTreePreservePCHPath) { - // Disable `-gmodules` to avoid debug info referencing a non-existent PCH - // filename. - // FIXME: we should also allow -gmodules if there is no PCH involved. - CodeGenOpts.DebugTypeExtRefs = false; - HSOpts.ModuleFormat = "raw"; - } // Clear APINotes options. CI.getAPINotesOpts().ModuleSearchPaths = {}; @@ -110,6 +103,9 @@ void tooling::dependencies::configureInvocationForCaching( updateRelativePath(CI.getDiagnosticOpts().DiagnosticLogFile, CWD); updateRelativePath(CI.getDependencyOutputOpts().OutputFile, CWD); FileSystemOpts.WorkingDir.clear(); + + // IncludeTree always use absolute path. Do not set debug comp dir. + CodeGenOpts.DebugCompilationDir.clear(); break; } case CachingInputKind::FileSystemRoot: { @@ -210,8 +206,6 @@ void DepscanPrefixMapping::remapInvocationPaths(CompilerInvocation &Invocation, mapInPlaceAll(FrontendOpts.ASTMergeFiles); Mapper.mapInPlace(FrontendOpts.OverrideRecordLayoutsFile); Mapper.mapInPlace(FrontendOpts.StatsFile); - for (auto &[Path, _] : FrontendOpts.ModuleCacheKeys) - Mapper.mapInPlace(Path); // Filesystem options. Mapper.mapInPlace(FileSystemOpts.WorkingDir); diff --git a/clang/test/CAS/depscan-include-tree.c b/clang/test/CAS/depscan-include-tree.c index a75fb2e0f8b72..628993b6ca0a2 100644 --- a/clang/test/CAS/depscan-include-tree.c +++ b/clang/test/CAS/depscan-include-tree.c @@ -14,6 +14,8 @@ // CHECK: "-fcas-path" "[[PREFIX]]/cas" // CHECK: "-fcas-include-tree" // CHECK: "-isysroot" +// CHECK: "-fmodule-format=obj" +// CHECK: "-dwarf-ext-refs" // CHECK: "-coverage-data-file=[[PREFIX]]/t.gcda" // CHECK: "-coverage-notes-file=[[PREFIX]]/t.gcno" // SHOULD-NOT: "-fcas-fs" @@ -21,8 +23,6 @@ // SHOULD-NOT: "-I" // SHOULD-NOT: "[[PREFIX]]/t.c" // SHOULD-NOT: "-D" -// SHOULD-NOT: "-dwarf-ext-refs" -// SHOULD-NOT: "-fmodule-format=obj" // RUN: FileCheck %s -input-file %t/inline.d -check-prefix=DEPS -DPREFIX=%t diff --git a/clang/test/ClangScanDeps/cas-fs-multiple-commands.c b/clang/test/ClangScanDeps/cas-fs-multiple-commands.c index c03ba69384e6d..40b7d8cf40bbb 100644 --- a/clang/test/ClangScanDeps/cas-fs-multiple-commands.c +++ b/clang/test/ClangScanDeps/cas-fs-multiple-commands.c @@ -109,12 +109,12 @@ // CHECK-NOT: "-fcas-input-file-cache-key" // CHECK: "-E" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c" // CHECK: "[[PREFIX]]/tu.c" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "file-deps": [ // CHECK-NEXT: "[[PREFIX]]/tu.c" @@ -139,12 +139,12 @@ // CHECK-NEXT: "[[CPP_CACHE_KEY]]" // CHECK: "-emit-llvm-bc" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c-cpp-output" // CHECK-NOT: "{{.*}}tu.i" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "input-file": "[[PREFIX]]{{.}}tu.c" // CHECK-NEXT: } @@ -217,7 +217,7 @@ // CHECK-LIBCLANG-NEXT: [[PREFIX]]/tu.c // CHECK-LIBCLANG-NOT: -fcas-input-file-cache-key // CHECK-LIBCLANG-NOT: {{.*}}tu.c -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 1: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. @@ -229,7 +229,7 @@ // CHECK-LIBCLANG-NEXT: [[PREFIX]]/tu.c // CHECK-LIBCLANG-NOT: -fcas-fs // CHECK-LIBCLANG-NOT: {{.*}}tu.i -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c-cpp-output {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[PREFIX]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] -x c-cpp-output {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 2: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c new file mode 100644 index 0000000000000..b483edad85982 --- /dev/null +++ b/clang/test/ClangScanDeps/gmodules.c @@ -0,0 +1,187 @@ +// REQUIRES: ondisk_cas +// REQUIRES: system-darwin + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: sed "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json +// RUN: sed "s|DIR|%/t|g" %t/cas-config.template > %t/.cas-config + +/// Scan PCH +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch.json + +/// Build PCH +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Top > %t/Top.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --module-name Left > %t/Left.rsp +// RUN: %deps-to-rsp %t/deps_pch.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/Top.rsp +// RUN: %clang @%t/Left.rsp +// RUN: %clang @%t/pch.rsp + +/// Scan TU with PCH +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps.json + +// RUN: cat %t/deps.json | %PathSanitizingFileCheck --sanitize PREFIX=%/t %s + +/// Build TU +// RUN: %deps-to-rsp %t/deps.json --module-name Right > %t/Right.rsp +// RUN: %deps-to-rsp %t/deps.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/Right.rsp +// RUN: %clang @%t/tu.rsp + +/// Check module in a different directory and compare output. +// RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_pch_2.json +// RUN: %deps-to-rsp %t/deps_pch_2.json --module-name Top > %t/Top_2.rsp +// RUN: %deps-to-rsp %t/deps_pch_2.json --module-name Left > %t/Left_2.rsp +// RUN: %deps-to-rsp %t/deps_pch_2.json --tu-index 0 > %t/pch_2.rsp +// RUN: %clang @%t/Top_2.rsp +// RUN: %clang @%t/Left_2.rsp +// RUN: %clang @%t/pch_2.rsp -o %t/prefix_2.pch +// RUN: clang-scan-deps -compilation-database %t/cdb.json \ +// RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ +// RUN: -format experimental-include-tree-full -mode preprocess-dependency-directives \ +// RUN: > %t/deps_2.json +// RUN: %deps-to-rsp %t/deps_2.json --module-name Right > %t/Right_2.rsp +// RUN: %deps-to-rsp %t/deps_2.json --tu-index 0 > %t/tu_2.rsp +// RUN: %clang @%t/Right_2.rsp +// RUN: %clang @%t/tu_2.rsp -o %t/tu_2.o + +/// Diff all outputs +// RUN: diff %t/prefix.h.pch %t/prefix_2.pch +// RUN: diff %t/tu.o %t/tu_2.o + + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [] +// CHECK: "clang-modulemap-file": "PREFIX{{/|\\\\}}module.modulemap" +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "PREFIX{{/|\\\\}}cas" +// CHECK: "-o" +// CHECK-NEXT: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK: "-disable-free" +// CHECK: "-fno-pch-timestamp" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" +// CHECK: "-fcache-compile-job" +// CHECK: "-emit-module" +// CHECK: "-fmodule-file=Top-{{.*}}.pcm" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "Top-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodules" +// CHECK: "-fmodule-name=Right" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "PREFIX{{/|\\\\}}module.modulemap" +// CHECK-NEXT: "PREFIX{{/|\\\\}}Right.h" +// CHECK-NEXT: ] +// CHECK: "name": "Right" +// CHECK: } +// CHECK-NOT: "clang-modulemap-file" +// CHECK: ] +// CHECK-NEXT: "translation-units": [ +// CHECK-NEXT: { +// CHECK-NEXT: "commands": [ +// CHECK-NEXT: { +// CHECK: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK: "module-name": "Right" +// CHECK: } +// CHECK-NEXT: ] +// CHECK: "command-line": [ +// CHECK-NEXT: "-cc1" +// CHECK: "-fcas-path" +// CHECK-NEXT: "PREFIX{{/|\\\\}}cas" +// CHECK-NOT: -fmodule-map-file= +// CHECK: "-disable-free" +// CHECK: "-fcas-include-tree" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-fcache-compile-job" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "Right-{{.*}}.pcm" +// CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" +// CHECK: "-x" +// CHECK-NEXT: "c" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" +// CHECK: "-fmodules" +// CHECK: "-fno-implicit-modules" +// CHECK: ] +// CHECK: "file-deps": [ +// CHECK-NEXT: "PREFIX{{/|\\\\}}tu.c" +// CHECK-NEXT: "PREFIX{{/|\\\\}}prefix.h.pch" +// CHECK-NEXT: ] +// CHECK: "input-file": "PREFIX{{/|\\\\}}tu.c" +// CHECK: } +// CHECK-NEXT: ] +// CHECK-NEXT: } +// CHECK-NEXT: ] +// CHECK-NEXT: } + +//--- cdb_pch.json.template +[{ + "file": "DIR/prefix.h", + "directory": "DIR", + "command": "clang -x c-header DIR/prefix.h -o DIR/prefix.h.pch -fmodules -fimplicit-modules -fimplicit-module-maps -gmodules -fmodules-cache-path=DIR/module-cache" +}] + +//--- cdb.json.template +[{ + "file": "DIR/tu.c", + "directory": "DIR", + "command": "clang -c -o DIR/tu.o DIR/tu.c -include DIR/prefix.h -fmodules -fimplicit-modules -fimplicit-module-maps -gmodules -fmodules-cache-path=DIR/module-cache" +}] + +//--- module.modulemap +module Top { header "Top.h" export *} +module Left { header "Left.h" export *} +module Right { header "Right.h" export *} + +//--- Top.h +#pragma once +struct Top { int x; }; + +//--- Left.h +#pragma once +#include "Top.h" +struct Left { struct Top top; }; + +//--- Right.h +#pragma once +#include "Top.h" +struct Right { struct Top top; }; + +//--- prefix.h +#include "Left.h" +struct Prefix { struct Top top; }; + +//--- tu.c +#include "Right.h" + +int main(void) { + struct Left _left; + struct Right _right; + struct Top _top; + struct Prefix _prefix; + return 0; +} + +//--- cas-config.template +{ + "CASPath": "DIR/cas" +} diff --git a/clang/test/ClangScanDeps/include-tree-multiple-commands.c b/clang/test/ClangScanDeps/include-tree-multiple-commands.c index e13a42f8d7727..a3868d948907b 100644 --- a/clang/test/ClangScanDeps/include-tree-multiple-commands.c +++ b/clang/test/ClangScanDeps/include-tree-multiple-commands.c @@ -104,12 +104,12 @@ // CHECK-NOT: "-fcas-input-file-cache-key" // CHECK: "-E" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c" // CHECK-NOT: "{{.*}}tu.c" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "file-deps": [ // CHECK-NEXT: "[[SRC]]/tu.c" @@ -137,12 +137,12 @@ // CHECK-NEXT: "[[CPP_CACHE_KEY]]" // CHECK: "-emit-llvm-bc" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK-NEXT: "Mod-{{.*}}.pcm" // CHECK-NEXT: "[[M_CACHE_KEY]]" // CHECK: "-x" // CHECK-NEXT: "c-cpp-output" // CHECK-NOT: "{{.*}}tu.i" -// CHECK: "-fmodule-file={{.*}}[[PREFIX]]/modules/{{.*}}/Mod-{{.*}}.pcm" +// CHECK: "-fmodule-file=Mod=Mod-{{.*}}.pcm" // CHECK: ] // CHECK: "input-file": "[[SRC]]/tu.c" // CHECK-NEXT: } @@ -218,7 +218,7 @@ // CHECK-LIBCLANG-NEXT: [[SRC]]/header.h // CHECK-LIBCLANG-NOT: -fcas-input-file-cache-key // CHECK-LIBCLANG-NOT: {{.*}}tu.c -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.i {{.*}} -E -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 1: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. @@ -231,7 +231,7 @@ // CHECK-LIBCLANG-NEXT: [[SRC]]/header.h // CHECK-LIBCLANG-NOT: -fcas-include-tree // CHECK-LIBCLANG-NOT: {{.*}}tu.i -// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c-cpp-output {{.*}} -fmodule-file={{.*}}[[PREFIX]]/modules/Mod_{{.*}}.pcm +// CHECK-LIBCLANG-NEXT: build-args: -cc1 {{.*}} -o [[DST]]/tu.bc {{.*}} -fcas-input-file-cache-key [[CPP_CACHE_KEY]] {{.*}} -emit-llvm-bc -fmodule-file-cache-key {{.*}} [[M_CACHE_KEY]] {{.*}} -x c-cpp-output {{.*}} -fmodule-file=Mod=Mod_{{.*}}.pcm // CHECK-LIBCLANG-NEXT: command 2: // CHECK-LIBCLANG-NEXT: context-hash: {{.*}} // FIXME: This should be empty. diff --git a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c index 3ef420105bbd5..c9bd12097ffde 100644 --- a/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c +++ b/clang/test/ClangScanDeps/include-tree-preserve-pch-path.c @@ -3,7 +3,6 @@ // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb_pch.json.template > %t/cdb_pch.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: sed -e "s|DIR|%/t|g" %t/cdb_no_preserve.json.template > %t/cdb_no_preserve.json // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_pch.json // RUN: FileCheck %s -input-file %t/deps_pch.json -DPREFIX=%/t @@ -21,37 +20,23 @@ // RUN: %clang @%t/tu.rsp // RUN: cat %t/tu.ll | %PathSanitizingFileCheck --sanitize PREFIX=%/t --enable-yaml-compatibility %s -check-prefix=LLVMIR -// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "prefix.pch" +// LLVMIR: !DICompileUnit({{.*}}, splitDebugFilename: "llvmcas://{{.*}}" // Extract include-tree casid // RUN: cat %t/tu.rsp | sed -E 's|.*"-fcas-include-tree" "(llvmcas://[[:xdigit:]]+)".*|\1|' > %t/tu.casid // RUN: clang-cas-test -cas %t/cas -print-include-tree @%t/tu.casid | FileCheck %s -check-prefix=INCLUDE_TREE -DPREFIX=%/t -// INCLUDE_TREE: (PCH) [[PREFIX]]/prefix.pch llvmcas:// - -// RUN: clang-scan-deps -compilation-database %t/cdb_no_preserve.json -format experimental-include-tree-full -cas-path %t/cas > %t/deps_no_preserve.json -// RUN: FileCheck %s -input-file %t/deps_no_preserve.json -DPREFIX=%/t -check-prefix=NO_PRESERVE - -// Note: "raw" is the default format, so it will not show up in the arguments. -// NO_PRESERVE-NOT: "-fmodule-format= -// NO_PRESERVE-NOT: "-dwarf-ext-refs" +// INCLUDE_TREE: (PCH) llvmcas:// //--- cdb_pch.json.template [{ "directory": "DIR", - "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g -Xclang -finclude-tree-preserve-pch-path", + "command": "clang -x c-header DIR/prefix.h -target x86_64-apple-macos12 -o DIR/prefix.pch -gmodules -g", "file": "DIR/prefix.h" }] //--- cdb.json.template -[{ - "directory": "DIR", - "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g -Xclang -finclude-tree-preserve-pch-path", - "file": "DIR/tu.c" -}] - -//--- cdb_no_preserve.json.template [{ "directory": "DIR", "command": "clang -S -emit-llvm DIR/tu.c -o DIR/tu.ll -include-pch DIR/prefix.pch -target x86_64-apple-macos12 -gmodules -g", diff --git a/clang/test/ClangScanDeps/modules-cas-context-hash.c b/clang/test/ClangScanDeps/modules-cas-context-hash.c index 3ff42bbe08743..1722a2339e607 100644 --- a/clang/test/ClangScanDeps/modules-cas-context-hash.c +++ b/clang/test/ClangScanDeps/modules-cas-context-hash.c @@ -33,7 +33,7 @@ // CHECK: ], // CHECK: "command-line": [ // CHECK: "-fmodule-file-cache-key" -// CHECK: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}[[HASH]]{{/|\\\\}}Mod-[[HASH]].pcm" +// CHECK: "Mod-[[HASH]].pcm" // CHECK: "llvmcas://[[KEY]]" // CHECK: ] @@ -57,7 +57,7 @@ // CHECK: ], // CHECK: "command-line": [ // CHECK: "-fmodule-file-cache-key" -// CHECK: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}[[HASH]]{{/|\\\\}}Mod-[[HASH]].pcm" +// CHECK: "Mod-[[HASH]].pcm" // CHECK: "llvmcas://[[KEY]]" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c b/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c index 63892d1a87595..dbf5c00a50646 100644 --- a/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c +++ b/clang/test/ClangScanDeps/modules-cas-fs-prefix-mapping.c @@ -66,7 +66,7 @@ // CHECK: "-o" // CHECK: "PREFIX{{/|\\\\}}modules{{/|\\\\}}{{.*}}{{/|\\\\}}A-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}B-[[B_CONTEXT_HASH:[^.]+]].pcm" +// CHECK: "B-[[B_CONTEXT_HASH:[^.]+]].pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK: "c" @@ -75,7 +75,7 @@ // CHECK: "ROOT^sdk" // CHECK: "-resource-dir" // CHECK: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}" -// CHECK: "-fmodule-file=B=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}B-[[B_CONTEXT_HASH]].pcm" +// CHECK: "-fmodule-file=B=B-[[B_CONTEXT_HASH]].pcm" // CHECK: "-isystem" // CHECK: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}{{/|\\\\}}include" // CHECK: "-internal-externc-isystem" @@ -142,7 +142,7 @@ // CHECK: "ROOT^src" // CHECK: "-fmodule-map-file=ROOT^src{{/|\\\\}}module.modulemap" // CHECK: "-fmodule-file-cache-key" -// CHECK: "ROOT^modules{{/|\\\\}}{{.*}}A-{{.*}}.pcm" +// CHECK: "{{.*}}A-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK: "c" @@ -151,7 +151,7 @@ // CHECK: "ROOT^sdk" // CHECK: "-resource-dir" // CHECK: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}" -// CHECK: "-fmodule-file=A=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}A-{{.*}}.pcm" +// CHECK: "-fmodule-file=A=A-{{.*}}.pcm" // CHECK: "-isystem" // CHECK: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}{{/|\\\\}}include" // CHECK: "-internal-externc-isystem" diff --git a/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c b/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c index 7c1dcaedec128..b9dddf5d738ae 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c +++ b/clang/test/ClangScanDeps/modules-cas-trees-with-pch.c @@ -64,11 +64,11 @@ // PCH: "-fcas-fs" // PCH-NEXT: "[[A_ROOT_ID]]" // PCH: "-o" -// PCH-NEXT: "[[A_PCM:.*outputs.*A-.*\.pcm]]" +// PCH-NEXT: "{{.*}}[[A_PCM:A-.*\.pcm]]" // PCH: "-fcache-compile-job" // PCH: "-emit-module" // PCH: "-fmodule-file-cache-key" -// PCH: "[[B_PCM:.*outputs.*B-.*\.pcm]]" +// PCH: "[[B_PCM:B-.*\.pcm]]" // PCH: "[[B_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // PCH: "-fmodule-file={{(B=)?}}[[B_PCM]]" // PCH: ] @@ -88,7 +88,7 @@ // PCH: "-fcas-fs" // PCH-NEXT: "[[B_ROOT_ID]]" // PCH: "-o" -// PCH-NEXT: "[[B_PCM]]" +// PCH-NEXT: "{{.*}}[[B_PCM]]" // PCH: "-fcache-compile-job" // PCH: "-emit-module" // PCH: ] @@ -140,10 +140,10 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[C_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[C_PCM:.*outputs.*C-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[C_PCM:C-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file={{(B=)?}}[[B_PCM:.*outputs.*B-.*\.pcm]]" +// CHECK: "-fmodule-file={{(B=)?}}[[B_PCM:B-.*\.pcm]]" // CHECK: "-fmodule-file-cache-key" // CHECK: "[[B_PCM]]" // CHECK: "[[B_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" diff --git a/clang/test/ClangScanDeps/modules-cas-trees.c b/clang/test/ClangScanDeps/modules-cas-trees.c index 48d6ef11bfee2..1d17865219b88 100644 --- a/clang/test/ClangScanDeps/modules-cas-trees.c +++ b/clang/test/ClangScanDeps/modules-cas-trees.c @@ -78,11 +78,11 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[LEFT_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[LEFT_PCM:.*outputs.*Left-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[LEFT_PCM:Left-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK: "[[TOP_PCM:.*outputs.*Top-.*\.pcm]]" +// CHECK: "{{.*}}[[TOP_PCM:Top-.*\.pcm]]" // CHECK: "[[TOP_CACHE_KEY:llvmcas://[[:xdigit:]]+]]" // CHECK: "-fmodule-file={{(Top=)?}}[[TOP_PCM]]" // CHECK: ] @@ -107,7 +107,7 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[RIGHT_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[RIGHT_PCM:.*outputs.*Right-.*\.pcm]]" +// CHECK-NEXT: "{{.*}}[[RIGHT_PCM:Right-.*\.pcm]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" @@ -132,7 +132,7 @@ // CHECK: "-fcas-fs" // CHECK-NEXT: "[[TOP_ROOT_ID]]" // CHECK: "-o" -// CHECK-NEXT: "[[TOP_PCM]]" +// CHECK-NEXT: "{{.*}}[[TOP_PCM]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c b/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c index fa5c123975401..084b0f5837fd0 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c +++ b/clang/test/ClangScanDeps/modules-include-tree-dependency-file.c @@ -25,7 +25,7 @@ // DOT: digraph "dependencies" // DOT-DAG: [[TU:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}tu.m"]; // DOT-DAG: [[HEADER:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}A.h"]; -// DOT-DAG: [[PCM:header_[0-9]+]] [ shape="box", label="{{.*}}{{/|\\}}Mod-{{.*}}.pcm"]; +// DOT-DAG: [[PCM:header_[0-9]+]] [ shape="box", label="Mod-{{.*}}.pcm"]; // DOT-DAG: [[TU]] -> [[HEADER]] // DOT-DAG: [[HEADER]] -> [[PCM]] diff --git a/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c new file mode 100644 index 0000000000000..f15c4244f2fd8 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-include-tree-pch-common-stale-prefix-map.c @@ -0,0 +1,192 @@ +// Test that modifications to a common header (imported from both a PCH and a TU) +// cause rebuilds of dependent modules imported from the TU on incremental build. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +//--- module.modulemap +module mod_common { header "mod_common.h" } +module mod_tu { header "mod_tu.h" } +module mod_tu_extra { header "mod_tu_extra.h" } + +//--- mod_common.h +#define MOD_COMMON_MACRO 0 +//--- mod_tu.h +#include "mod_common.h" +#if MOD_COMMON_MACRO +#include "mod_tu_extra.h" +#endif + +//--- mod_tu_extra.h + +//--- prefix.h +#include "mod_common.h" + +//--- tu.c +#include "mod_tu.h" + +// Clean: scan the PCH. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_pch_clean.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -x c-header %t/prefix.h -o %t/prefix.h.pch -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache + +// Clean: build the PCH. +// RUN: %deps-to-rsp %t/deps_pch_clean.json --module-name mod_common > %t/mod_common.rsp +// RUN: %deps-to-rsp %t/deps_pch_clean.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/mod_common.rsp +// RUN: %clang @%t/pch.rsp + +// Clean: scan the TU. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_clean.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// RUN: cat %t/deps_tu_clean.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-CLEAN -DPREFIX=%/t +// CHECK-TU-CLEAN: { +// CHECK-TU-CLEAN-NEXT: "modules": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-module-deps": [], +// CHECK-TU-CLEAN-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-CLEAN-NEXT: "command-line": [ +// CHECK-TU-CLEAN: ], +// CHECK-TU-CLEAN-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "file-deps": [ +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/mod_tu.h" +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "link-libraries": [], +// CHECK-TU-CLEAN-NEXT: "name": "mod_tu" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "translation-units": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "commands": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "clang-module-deps": [ +// CHECK-TU-CLEAN-NEXT: { +// CHECK-TU-CLEAN-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "module-name": "mod_tu" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "command-line": [ +// CHECK-TU-CLEAN: ], +// CHECK-TU-CLEAN-NEXT: "executable": "{{.*}}", +// CHECK-TU-CLEAN-NEXT: "file-deps": [ +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/tu.c", +// CHECK-TU-CLEAN-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-TU-CLEAN-NEXT: ], +// CHECK-TU-CLEAN-NEXT: "input-file": "[[PREFIX]]/tu.c" +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN-NEXT: ] +// CHECK-TU-CLEAN-NEXT: } +// CHECK-TU-CLEAN: ] +// CHECK-TU-CLEAN: } + +// Clean: build the TU. +// RUN: %deps-to-rsp %t/deps_tu_clean.json --module-name mod_tu > %t/mod_tu.rsp +// RUN: %deps-to-rsp %t/deps_tu_clean.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/mod_tu.rsp +// RUN: %clang @%t/tu.rsp + +// Incremental: modify the common module. +// RUN: sleep 1 +// RUN: echo "#define MOD_COMMON_MACRO 1" > %t/mod_common.h +// RUN: echo "#define MOD_COMMON_MACRO_NEW 1" >> %t/mod_common.h + +// Incremental: scan the PCH. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_pch_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -x c-header %t/prefix.h -o %t/prefix.h.pch -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache + +// Incremental: build the PCH. +// RUN: %deps-to-rsp %t/deps_pch_incremental.json --module-name mod_common > %t/mod_common.rsp +// RUN: %deps-to-rsp %t/deps_pch_incremental.json --tu-index 0 > %t/pch.rsp +// RUN: %clang @%t/mod_common.rsp +// RUN: %clang @%t/pch.rsp + +// Incremental: scan the TU. This needs to invalidate modules imported from the +// TU that depend on modules imported from the PCH and discover the +// new dependency on 'mod_tu_extra'. +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// RUN: cat %t/deps_tu_incremental.json | sed 's:\\\\\?:/:g' | FileCheck %s --check-prefix=CHECK-TU-INCREMENTAL -DPREFIX=%/t +// RUN: clang-scan-deps -format experimental-include-tree-full -cas-path %t/cas -o %t/deps_tu_incremental.json -prefix-map=%t=/^tmp -- \ +// RUN: %clang -c %t/tu.c -o %t/tu.o -include %t/prefix.h -F %t \ +// RUN: -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/cache +// CHECK-TU-INCREMENTAL: { +// CHECK-TU-INCREMENTAL-NEXT: "modules": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "module-name": "mod_tu_extra" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/mod_tu.h" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "link-libraries": [], +// CHECK-TU-INCREMENTAL-NEXT: "name": "mod_tu" +// CHECK-TU-INCREMENTAL-NEXT: }, +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [], +// CHECK-TU-INCREMENTAL-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/mod_tu_extra.h" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "link-libraries": [], +// CHECK-TU-INCREMENTAL-NEXT: "name": "mod_tu_extra" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "translation-units": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "commands": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "cache-key": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "cas-include-tree-id": "llvmcas://{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "clang-module-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: { +// CHECK-TU-INCREMENTAL-NEXT: "context-hash": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "module-name": "mod_tu" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "command-line": [ +// CHECK-TU-INCREMENTAL: ], +// CHECK-TU-INCREMENTAL-NEXT: "executable": "{{.*}}", +// CHECK-TU-INCREMENTAL-NEXT: "file-deps": [ +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/tu.c", +// CHECK-TU-INCREMENTAL-NEXT: "[[PREFIX]]/prefix.h.pch" +// CHECK-TU-INCREMENTAL-NEXT: ], +// CHECK-TU-INCREMENTAL-NEXT: "input-file": "[[PREFIX]]/tu.c" +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL-NEXT: ] +// CHECK-TU-INCREMENTAL-NEXT: } +// CHECK-TU-INCREMENTAL: ] +// CHECK-TU-INCREMENTAL: } + +// Incremental: build the TU. +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --module-name mod_tu_extra > %t/mod_tu_extra.rsp +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --module-name mod_tu > %t/mod_tu.rsp +// RUN: %deps-to-rsp %t/deps_tu_incremental.json --tu-index 0 > %t/tu.rsp +// RUN: %clang @%t/mod_tu_extra.rsp +// RUN: %clang @%t/mod_tu.rsp +// RUN: %clang @%t/tu.rsp diff --git a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c index cde604544d5fc..38c47a90cda13 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c +++ b/clang/test/ClangScanDeps/modules-include-tree-prefix-map.c @@ -133,9 +133,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" // CHECK: "-fno-implicit-modules" @@ -168,9 +168,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key -// CHECK-NEXT: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" // CHECK: "-fno-implicit-modules" @@ -262,13 +262,13 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Left-{{.*}}.pcm" +// CHECK-NEXT: "Left-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Left=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Left-{{.*}}.pcm" -// CHECK: "-fmodule-file=Right=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Left=Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c index c9300de986180..a08757e15a6d0 100644 --- a/clang/test/ClangScanDeps/modules-include-tree-with-pch.c +++ b/clang/test/ClangScanDeps/modules-include-tree-with-pch.c @@ -53,9 +53,9 @@ // CHECK-NEXT: "[[RIGHT_TREE:llvmcas://[[:xdigit:]]+]]" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file=PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "PREFIX{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -91,11 +91,11 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-x" // CHECK-NEXT: "c" -// CHECK: "-fmodule-file=Right=PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-include-tree.c b/clang/test/ClangScanDeps/modules-include-tree.c index 46485b1f99ff0..fe05c1640a5f6 100644 --- a/clang/test/ClangScanDeps/modules-include-tree.c +++ b/clang/test/ClangScanDeps/modules-include-tree.c @@ -116,9 +116,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Left" // CHECK: "-fno-implicit-modules" @@ -151,9 +151,9 @@ // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" // CHECK: "-fmodule-file-cache-key -// CHECK-NEXT: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK-NEXT: "Top-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Top=PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Top-{{.*}}.pcm" +// CHECK: "-fmodule-file=Top=Top-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fmodule-name=Right" // CHECK: "-fno-implicit-modules" @@ -273,13 +273,13 @@ // CHECK: "-fcache-compile-job" // CHECK: "-fsyntax-only" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Left-{{.*}}.pcm" +// CHECK-NEXT: "Left-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" // CHECK: "-fmodule-file-cache-key" -// CHECK-NEXT: "PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK-NEXT: "Right-{{.*}}.pcm" // CHECK-NEXT: "llvmcas://{{[[:xdigit:]]+}}" -// CHECK: "-fmodule-file=Left=PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Left-{{.*}}.pcm" -// CHECK: "-fmodule-file=Right=PREFIX{{/|\\\\}}outputs{{/|\\\\}}{{.*}}{{/|\\\\}}Right-{{.*}}.pcm" +// CHECK: "-fmodule-file=Left=Left-{{.*}}.pcm" +// CHECK: "-fmodule-file=Right=Right-{{.*}}.pcm" // CHECK: "-fmodules" // CHECK: "-fno-implicit-modules" // CHECK: ] diff --git a/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c b/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c index 8dad299143de9..bee4720c7a921 100644 --- a/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c +++ b/clang/test/ClangScanDeps/modules-pch-cas-fs-prefix-mapping.c @@ -96,7 +96,7 @@ // PCH: "-o" // PCH: "PREFIX{{/|\\\\}}modules{{/|\\\\}}{{.*}}{{/|\\\\}}A-{{.*}}.pcm" // PCH: "-fmodule-file-cache-key" -// PCH: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}B-[[B_CONTEXT_HASH:[^.]+]].pcm" +// PCH: "B-[[B_CONTEXT_HASH:[^.]+]].pcm" // PCH: "llvmcas://{{.*}}" // PCH: "-x" // PCH: "c" @@ -105,7 +105,7 @@ // PCH: "ROOT^sdk" // PCH: "-resource-dir" // PCH: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}" -// PCH: "-fmodule-file=B=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}B-[[B_CONTEXT_HASH]].pcm" +// PCH: "-fmodule-file=B=B-[[B_CONTEXT_HASH]].pcm" // PCH: "-isystem" // PCH: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}{{/|\\\\}}include" // PCH: "-internal-externc-isystem" @@ -172,7 +172,7 @@ // PCH: "ROOT^src" // PCH: "-fmodule-map-file=ROOT^src{{/|\\\\}}module.modulemap" // PCH: "-fmodule-file-cache-key" -// PCH: "ROOT^modules{{/|\\\\}}{{.*}}A-{{.*}}.pcm" +// PCH: "{{.*}}A-{{.*}}.pcm" // PCH: "llvmcas://{{.*}}" // PCH: "-x" // PCH: "c-header" @@ -181,7 +181,7 @@ // PCH: "ROOT^sdk" // PCH: "-resource-dir" // PCH: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}" -// PCH: "-fmodule-file=A=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}A-{{.*}}.pcm" +// PCH: "-fmodule-file=A=A-{{.*}}.pcm" // PCH: "-isystem" // PCH: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}{{/|\\\\}}include" // PCH: "-internal-externc-isystem" @@ -208,9 +208,9 @@ // CHECK-NEXT: "ROOT^src" // CHECK: "-fcache-compile-job" // CHECK: "-emit-module" -// CHECK: "-fmodule-file=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}B-{{.*}}.pcm" +// CHECK: "-fmodule-file=B-{{.*}}.pcm" // CHECK: "-fmodule-file-cache-key" -// CHECK: "ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}B-{{.*}}.pcm" +// CHECK: "B-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -244,7 +244,7 @@ // CHECK-NEXT: "ROOT^src" // CHECK: "-fmodule-map-file=ROOT^src{{/|\\\\}}module.modulemap" // CHECK: "-fmodule-file-cache-key" -// CHECK: "ROOT^modules{{/|\\\\}}{{.*}}C-{{.*}}.pcm" +// CHECK: "C-{{.*}}.pcm" // CHECK: "llvmcas://{{.*}}" // CHECK: "-x" // CHECK-NEXT: "c" @@ -253,7 +253,7 @@ // CHECK-NEXT: "ROOT^sdk" // CHECK: "-resource-dir" // CHECK-NEXT: "ROOT^tc{{/|\\\\}}lib{{/|\\\\}}clang{{/|\\\\}}{{.*}}" -// CHECK: "-fmodule-file=C=ROOT^modules{{/|\\\\}}{{.*}}{{/|\\\\}}C-{{.*}}.pcm" +// CHECK: "-fmodule-file=C=C-{{.*}}.pcm" // CHECK: "-isystem" // CHECK-NEXT: "ROOT^sdk{{/|\\\\}}usr{{/|\\\\}}local{{/|\\\\}}include" // CHECK: "-isystem" From 64ed8015d1de3e5d21a7e92aa5cda9d9577dc04c Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Tue, 23 Sep 2025 15:19:12 -0700 Subject: [PATCH 2/4] [dsymutil] Teach dsymutil about CAS and loading clang modules from CAS Teach dsymutil how to use CAS and how to load clang modules from CAS when building dSYM when gmodule is used. --- clang/test/CMakeLists.txt | 1 + clang/test/ClangScanDeps/gmodules.c | 41 ++++++++++++++++++ llvm/tools/dsymutil/BinaryHolder.cpp | 64 +++++++++++++++++++++++++--- llvm/tools/dsymutil/BinaryHolder.h | 9 +++- llvm/tools/dsymutil/Options.td | 13 ++++++ llvm/tools/dsymutil/dsymutil.cpp | 29 ++++++++++++- 6 files changed, 150 insertions(+), 7 deletions(-) diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt index 6230a51bc5db2..62eb51bff3d72 100644 --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -155,6 +155,7 @@ if( NOT CLANG_BUILT_STANDALONE ) llvm-config FileCheck count not CASPluginTest + dsymutil llc llvm-ar llvm-as diff --git a/clang/test/ClangScanDeps/gmodules.c b/clang/test/ClangScanDeps/gmodules.c index b483edad85982..762d2f6cf07d9 100644 --- a/clang/test/ClangScanDeps/gmodules.c +++ b/clang/test/ClangScanDeps/gmodules.c @@ -35,6 +35,14 @@ // RUN: %clang @%t/Right.rsp // RUN: %clang @%t/tu.rsp +/// Check debug info is correct. +// RUN: %clang %t/tu.o -o %t/a.out +// RUN: dsymutil -cas %t/cas %t/a.out -o %t/a.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty +// RUN: dsymutil %t/a.out -o %t/a2.dSYM 2>&1 | FileCheck %s --check-prefix=WARN --allow-empty +// WARN-NOT: warning: + +// RUN: llvm-dwarfdump --debug-info %t/a.dSYM | FileCheck %s --check-prefix=DWARF + /// Check module in a different directory and compare output. // RUN: clang-scan-deps -compilation-database %t/cdb_pch.json \ // RUN: -cas-path %t/cas -module-files-dir %t/outputs-2 \ @@ -59,6 +67,39 @@ // RUN: diff %t/prefix.h.pch %t/prefix_2.pch // RUN: diff %t/tu.o %t/tu_2.o +// Check all the types are available +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Top") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Top") +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Left") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Left") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF-NEXT: DW_AT_name ("Right") +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Right") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" +// DWARF: DW_TAG_compile_unit +// DWARF: DW_TAG_module +// DWARF: DW_TAG_structure_type +// DWARF-NEXT: DW_AT_name ("Prefix") +// DWARF: DW_TAG_member +// DWARF-NEXT: DW_AT_name ("top") +// DWARF-NEXT: DW_AT_type +// DWARF-SAME: "Top::Top" + // CHECK: { // CHECK-NEXT: "modules": [ diff --git a/llvm/tools/dsymutil/BinaryHolder.cpp b/llvm/tools/dsymutil/BinaryHolder.cpp index 58751bddd3ba7..af04248fac226 100644 --- a/llvm/tools/dsymutil/BinaryHolder.cpp +++ b/llvm/tools/dsymutil/BinaryHolder.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "BinaryHolder.h" +#include "llvm/CAS/CASConfiguration.h" #include "llvm/Object/MachO.h" #include "llvm/Support/WithColor.h" #include "llvm/Support/raw_ostream.h" @@ -42,8 +43,9 @@ getMachOFatMemoryBuffers(StringRef Filename, MemoryBuffer &Mem, } BinaryHolder::BinaryHolder(IntrusiveRefCntPtr VFS, - BinaryHolder::Options Opts) - : VFS(VFS), Opts(Opts) {} + BinaryHolder::Options Opts, + std::shared_ptr CAS) + : VFS(VFS), CAS(std::move(CAS)), Opts(Opts) {} Error BinaryHolder::ArchiveEntry::load(IntrusiveRefCntPtr VFS, StringRef Filename, @@ -144,6 +146,37 @@ Error BinaryHolder::ObjectEntry::load(IntrusiveRefCntPtr VFS, return Error::success(); } +bool BinaryHolder::ObjectEntry::load(cas::ObjectStore &CAS, + StringRef PossibleID, Options Opts) { + auto ID = CAS.parseID(PossibleID); + if (!ID) { + consumeError(ID.takeError()); + return false; + } + + if (Opts.Verbose) + WithColor::note() << "try load cas object " << PossibleID << "\n"; + + auto Ref = CAS.getProxy(*ID); + if (!Ref) { + WithColor::warning() << "failed to load CAS object " << PossibleID << "\n"; + return false; + } + + // Only module files should be loaded here. Always not fat. + auto ObjectBuf = Ref->getMemoryBuffer(); + auto ErrOrObjectFile = + object::ObjectFile::createObjectFile(ObjectBuf->getMemBufferRef()); + if (!ErrOrObjectFile) { + WithColor::warning() << "malformed object loaded via " << PossibleID + << "\n"; + return false; + } + + Objects.push_back(std::move(*ErrOrObjectFile)); + return true; +} + std::vector BinaryHolder::ObjectEntry::getObjects() const { std::vector Result; @@ -231,6 +264,22 @@ BinaryHolder::ArchiveEntry::getObjectEntry(StringRef Filename, return *(MemberCache[Key] = std::move(OE)); } +static std::shared_ptr searchAndCreateCAS(StringRef Path) { + auto Config = cas::CASConfiguration::createFromSearchConfigFile(Path); + if (!Config) + return nullptr; + + auto DB = Config->second.createDatabases(); + if (!DB) { + consumeError(DB.takeError()); + return nullptr; + } + + WithColor::note() << "create CAS using configuration: " << Config->first + << "'\n"; + return DB->first; +} + Expected BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { if (Opts.Verbose) @@ -265,9 +314,14 @@ BinaryHolder::getObjectEntry(StringRef Filename, TimestampTy Timestamp) { ObjectRefCounter[Filename]++; if (!ObjectCache.count(Filename)) { auto OE = std::make_unique(); - auto Err = OE->load(VFS, Filename, Timestamp, Opts); - if (Err) - return std::move(Err); + if (!CAS) + CAS = searchAndCreateCAS(Filename); + + if (!(CAS && OE->load(*CAS, Filename, Opts))) { + auto Err = OE->load(VFS, Filename, Timestamp, Opts); + if (Err) + return std::move(Err); + } ObjectCache[Filename] = std::move(OE); } diff --git a/llvm/tools/dsymutil/BinaryHolder.h b/llvm/tools/dsymutil/BinaryHolder.h index cb5bd95978144..6a438deaa4efe 100644 --- a/llvm/tools/dsymutil/BinaryHolder.h +++ b/llvm/tools/dsymutil/BinaryHolder.h @@ -15,6 +15,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringMap.h" +#include "llvm/CAS/ObjectStore.h" #include "llvm/Object/Archive.h" #include "llvm/Object/Error.h" #include "llvm/Object/MachOUniversal.h" @@ -46,7 +47,8 @@ class BinaryHolder { }; BinaryHolder(IntrusiveRefCntPtr VFS, - BinaryHolder::Options Opts = {}); + BinaryHolder::Options Opts = {}, + std::shared_ptr CAS = nullptr); // Forward declarations for friend declaration. class ObjectEntry; @@ -67,6 +69,9 @@ class BinaryHolder { Error load(IntrusiveRefCntPtr VFS, StringRef Filename, TimestampTy Timestamp, BinaryHolder::Options = {}); + bool load(cas::ObjectStore &CAS, StringRef Filename, + BinaryHolder::Options = {}); + /// Access all owned ObjectFiles. std::vector getObjects() const; @@ -150,6 +155,8 @@ class BinaryHolder { /// Virtual File System instance. IntrusiveRefCntPtr VFS; + std::shared_ptr CAS; + Options Opts; }; diff --git a/llvm/tools/dsymutil/Options.td b/llvm/tools/dsymutil/Options.td index ad35e55e33b12..8f7ab541c7cb1 100644 --- a/llvm/tools/dsymutil/Options.td +++ b/llvm/tools/dsymutil/Options.td @@ -218,3 +218,16 @@ def dsym_search_path: Separate<["-", "--"], "D">, MetaVarName<"">, HelpText<"Specify a directory that contain dSYM files to search for.">, Group; + +def cas : Separate<["-", "--"], "cas">, + MetaVarName<"">, + HelpText<"Specify CAS directory to load CAS references from.">, + Group; +def cas_plugin_path : Separate<["-", "--"], "cas-plugin-path">, + MetaVarName<"">, + HelpText<"Path to CAS plugin library if used">, + Group; +def cas_plugin_option : Separate<["-", "--"], "cas-plugin-option">, + MetaVarName<"