Vendor import of stripped clang trunk r375505, the last commit beforevendor/clang/clang-trunk-r375505vendor/clang

the upstream Subversion repository was made read-only, and the LLVM
project migrated to GitHub:

https://llvm.org/svn/llvm-project/cfe/trunk@375505

4 files changed, 334 insertions, 9 deletions

diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
new file mode 100644
index 000000000000..7d0881343478
--- /dev/null
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -0,0 +1,188 @@
+//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
+#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
+
+#include "clang/Basic/LLVM.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include <mutex>
+
+namespace clang {
+namespace tooling {
+namespace dependencies {
+
+/// An in-memory representation of a file system entity that is of interest to
+/// the dependency scanning filesystem.
+///
+/// It represents one of the following:
+/// - an opened source file with minimized contents and a stat value.
+/// - an opened source file with original contents and a stat value.
+/// - a directory entry with its stat value.
+/// - an error value to represent a file system error.
+/// - a placeholder with an invalid stat indicating a not yet initialized entry.
+class CachedFileSystemEntry {
+public:
+  /// Default constructor creates an entry with an invalid stat.
+  CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {}
+
+  CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {}
+
+  /// Create an entry that represents an opened source file with minimized or
+  /// original contents.
+  ///
+  /// The filesystem opens the file even for `stat` calls open to avoid the
+  /// issues with stat + open of minimized files that might lead to a
+  /// mismatching size of the file. If file is not minimized, the full file is
+  /// read and copied into memory to ensure that it's not memory mapped to avoid
+  /// running out of file descriptors.
+  static CachedFileSystemEntry createFileEntry(StringRef Filename,
+                                               llvm::vfs::FileSystem &FS,
+                                               bool Minimize = true);
+
+  /// Create an entry that represents a directory on the filesystem.
+  static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat);
+
+  /// \returns True if the entry is valid.
+  bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); }
+
+  /// \returns True if the current entry points to a directory.
+  bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); }
+
+  /// \returns The error or the file's contents.
+  llvm::ErrorOr<StringRef> getContents() const {
+    if (!MaybeStat)
+      return MaybeStat.getError();
+    assert(!MaybeStat->isDirectory() && "not a file");
+    assert(isValid() && "not initialized");
+    return StringRef(Contents);
+  }
+
+  /// \returns The error or the status of the entry.
+  llvm::ErrorOr<llvm::vfs::Status> getStatus() const {
+    assert(isValid() && "not initialized");
+    return MaybeStat;
+  }
+
+  /// \returns the name of the file.
+  StringRef getName() const {
+    assert(isValid() && "not initialized");
+    return MaybeStat->getName();
+  }
+
+  /// Return the mapping between location -> distance that is used to speed up
+  /// the block skipping in the preprocessor.
+  const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const {
+    return PPSkippedRangeMapping;
+  }
+
+  CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
+  CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
+
+  CachedFileSystemEntry(const CachedFileSystemEntry &) = delete;
+  CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete;
+
+private:
+  llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
+  // Store the contents in a small string to allow a
+  // move from the small string for the minimized contents.
+  // Note: small size of 1 allows us to store an empty string with an implicit
+  // null terminator without any allocations.
+  llvm::SmallString<1> Contents;
+  PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
+};
+
+/// This class is a shared cache, that caches the 'stat' and 'open' calls to the
+/// underlying real file system.
+///
+/// It is sharded based on the hash of the key to reduce the lock contention for
+/// the worker threads.
+class DependencyScanningFilesystemSharedCache {
+public:
+  struct SharedFileSystemEntry {
+    std::mutex ValueLock;
+    CachedFileSystemEntry Value;
+  };
+
+  DependencyScanningFilesystemSharedCache();
+
+  /// Returns a cache entry for the corresponding key.
+  ///
+  /// A new cache entry is created if the key is not in the cache. This is a
+  /// thread safe call.
+  SharedFileSystemEntry &get(StringRef Key);
+
+private:
+  struct CacheShard {
+    std::mutex CacheLock;
+    llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
+  };
+  std::unique_ptr<CacheShard[]> CacheShards;
+  unsigned NumShards;
+};
+
+/// A virtual file system optimized for the dependency discovery.
+///
+/// It is primarily designed to work with source files whose contents was was
+/// preprocessed to remove any tokens that are unlikely to affect the dependency
+/// computation.
+///
+/// This is not a thread safe VFS. A single instance is meant to be used only in
+/// one thread. Multiple instances are allowed to service multiple threads
+/// running in parallel.
+class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
+public:
+  DependencyScanningWorkerFilesystem(
+      DependencyScanningFilesystemSharedCache &SharedCache,
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
+      : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
+        PPSkipMappings(PPSkipMappings) {}
+
+  llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
+  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
+  openFileForRead(const Twine &Path) override;
+
+  /// The set of files that should not be minimized.
+  llvm::StringSet<> IgnoredFiles;
+
+private:
+  void setCachedEntry(StringRef Filename, const CachedFileSystemEntry *Entry) {
+    bool IsInserted = Cache.try_emplace(Filename, Entry).second;
+    (void)IsInserted;
+    assert(IsInserted && "local cache is updated more than once");
+  }
+
+  const CachedFileSystemEntry *getCachedEntry(StringRef Filename) {
+    auto It = Cache.find(Filename);
+    return It == Cache.end() ? nullptr : It->getValue();
+  }
+
+  llvm::ErrorOr<const CachedFileSystemEntry *>
+  getOrCreateFileSystemEntry(const StringRef Filename);
+
+  DependencyScanningFilesystemSharedCache &SharedCache;
+  /// The local cache is used by the worker thread to cache file system queries
+  /// locally instead of querying the global cache every time.
+  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+  /// The optional mapping structure which records information about the
+  /// excluded conditional directive skip mappings that are used by the
+  /// currently active preprocessor.
+  ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
+};
+
+} // end namespace dependencies
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
new file mode 100644
index 000000000000..fd8ed80b143c
--- /dev/null
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -0,0 +1,65 @@
+//===- DependencyScanningService.h - clang-scan-deps service ===-*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H
+#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H
+
+#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+
+namespace clang {
+namespace tooling {
+namespace dependencies {
+
+/// The mode in which the dependency scanner will operate to find the
+/// dependencies.
+enum class ScanningMode {
+  /// This mode is used to compute the dependencies by running the preprocessor
+  /// over
+  /// the unmodified source files.
+  CanonicalPreprocessing,
+
+  /// This mode is used to compute the dependencies by running the preprocessor
+  /// over
+  /// the source files that have been minimized to contents that might affect
+  /// the dependencies.
+  MinimizedSourcePreprocessing
+};
+
+/// The dependency scanning service contains the shared state that is used by
+/// the invidual dependency scanning workers.
+class DependencyScanningService {
+public:
+  DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true,
+                            bool SkipExcludedPPRanges = true);
+
+  ScanningMode getMode() const { return Mode; }
+
+  bool canReuseFileManager() const { return ReuseFileManager; }
+
+  bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
+
+  DependencyScanningFilesystemSharedCache &getSharedCache() {
+    return SharedCache;
+  }
+
+private:
+  const ScanningMode Mode;
+  const bool ReuseFileManager;
+  /// Set to true to use the preprocessor optimization that skips excluded PP
+  /// ranges by bumping the buffer pointer in the lexer instead of lexing the
+  /// tokens in the range until reaching the corresponding directive.
+  const bool SkipExcludedPPRanges;
+  /// The global file system cache.
+  DependencyScanningFilesystemSharedCache SharedCache;
+};
+
+} // end namespace dependencies
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_SERVICE_H
diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
new file mode 100644
index 000000000000..0c9efccb1d8b
--- /dev/null
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
@@ -0,0 +1,48 @@
+//===- DependencyScanningTool.h - clang-scan-deps service ------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_TOOL_H
+#define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_TOOL_H
+
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
+#include <string>
+
+namespace clang{
+namespace tooling{
+namespace dependencies{
+
+/// The high-level implementation of the dependency discovery tool that runs on
+/// an individual worker thread.
+class DependencyScanningTool {
+public:
+  /// Construct a dependency scanning tool.
+  ///
+  /// \param Compilations     The reference to the compilation database that's
+  /// used by the clang tool.
+  DependencyScanningTool(DependencyScanningService &Service, const clang::tooling::CompilationDatabase &Compilations);
+
+  /// Print out the dependency information into a string using the dependency
+  /// file format that is specified in the options (-MD is the default) and
+  /// return it.
+  ///
+  /// \returns A \c StringError with the diagnostic output if clang errors
+  /// occurred, dependency file contents otherwise.
+  llvm::Expected<std::string> getDependencyFile(const std::string &Input, StringRef CWD);
+
+private:
+  DependencyScanningWorker Worker;
+  const tooling::CompilationDatabase &Compilations;
+};
+
+} // end namespace dependencies
+} // end namespace tooling
+} // end namespace clang
+
+#endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_TOOL_H
diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
index 3ea261a30d0f..45c9fb4f029d 100644
--- a/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -10,17 +10,35 @@
 #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_WORKER_H
 
 #include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include <string>
 
 namespace clang {
+
+class DependencyOutputOptions;
+
 namespace tooling {
 namespace dependencies {
 
+class DependencyScanningService;
+class DependencyScanningWorkerFilesystem;
+
+class DependencyConsumer {
+public:
+  virtual ~DependencyConsumer() {}
+
+  virtual void handleFileDependency(const DependencyOutputOptions &Opts,
+                                    StringRef Filename) = 0;
+
+  // FIXME: Add support for reporting modular dependencies.
+};
+
 /// An individual dependency scanning worker that is able to run on its own
 /// thread.
 ///
@@ -29,26 +47,32 @@ namespace dependencies {
 /// using the regular processing run.
 class DependencyScanningWorker {
 public:
-  DependencyScanningWorker();
+  DependencyScanningWorker(DependencyScanningService &Service);
 
-  /// Print out the dependency information into a string using the dependency
-  /// file format that is specified in the options (-MD is the default) and
-  /// return it.
+  /// Run the dependency scanning tool for a given clang driver invocation (as
+  /// specified for the given Input in the CDB), and report the discovered
+  /// dependencies to the provided consumer.
   ///
   /// \returns A \c StringError with the diagnostic output if clang errors
-  /// occurred, dependency file contents otherwise.
-  llvm::Expected<std::string> getDependencyFile(const std::string &Input,
-                                                StringRef WorkingDirectory,
-                                                const CompilationDatabase &CDB);
+  /// occurred, success otherwise.
+  llvm::Error computeDependencies(const std::string &Input,
+                                  StringRef WorkingDirectory,
+                                  const CompilationDatabase &CDB,
+                                  DependencyConsumer &Consumer);
 
 private:
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+  std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
 
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The file system that is used by each worker when scanning for
   /// dependencies. This filesystem persists accross multiple compiler
   /// invocations.
-  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> WorkerFS;
+  llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
+  /// The file manager that is reused accross multiple invocations by this
+  /// worker. If null, the file manager will not be reused.
+  llvm::IntrusiveRefCntPtr<FileManager> Files;
 };
 
 } // end namespace dependencies