Merge from 'master' to 'sycl-web' (#1)

  CONFLICT (content): Merge conflict in clang/include/clang/AST/PrettyPrinter.h

Author: aelovikov-intel

clang-tools-extra/clangd/CodeComplete.cpp

@@ -1104,7 +1104,7 @@ bool semaCodeComplete(std::unique_ptr<CodeCompleteConsumer> Consumer,
   // overriding the preamble will break sema completion. Fortunately we can just
   // skip all includes in this case; these completions are really simple.
   PreambleBounds PreambleRegion =
-      ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
+      ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0);
   bool CompletingInPreamble = PreambleRegion.Size > Input.Offset;
   if (Input.Patch)
     Input.Patch->apply(*CI);

clang-tools-extra/clangd/Preamble.cpp

@@ -249,8 +249,7 @@ scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
   // This means we're scanning (though not preprocessing) the preamble section
   // twice. However, it's important to precisely follow the preamble bounds used
   // elsewhere.
-  auto Bounds =
-      ComputePreambleBounds(*CI->getLangOpts(), ContentsBuffer.get(), 0);
+  auto Bounds = ComputePreambleBounds(*CI->getLangOpts(), *ContentsBuffer, 0);
   auto PreambleContents =
       llvm::MemoryBuffer::getMemBufferCopy(Contents.substr(0, Bounds.Size));
   auto Clang = prepareCompilerInstance(
@@ -322,8 +321,7 @@ buildPreamble(PathRef FileName, CompilerInvocation CI,
   // without those.
   auto ContentsBuffer =
       llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
-  auto Bounds =
-      ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0);
+  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
 
   trace::Span Tracer("BuildPreamble");
   SPAN_ATTACH(Tracer, "File", FileName);
@@ -376,8 +374,7 @@ bool isPreambleCompatible(const PreambleData &Preamble,
                           const CompilerInvocation &CI) {
   auto ContentsBuffer =
       llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
-  auto Bounds =
-      ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0);
+  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), *ContentsBuffer, 0);
   auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
   return compileCommandsAreEqual(Inputs.CompileCommand,
                                  Preamble.CompileCommand) &&

clang-tools-extra/clangd/index/Serialization.cpp

@@ -16,6 +16,7 @@
 #include "support/Trace.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
@@ -104,6 +105,20 @@ class Reader {
     llvm::StringRef Raw = consume(SymbolID::RawSize); // short if truncated.
     return LLVM_UNLIKELY(err()) ? SymbolID() : SymbolID::fromRaw(Raw);
   }
+
+  // Read a varint (as consumeVar) and resize the container accordingly.
+  // If the size is invalid, return false and mark an error.
+  // (The caller should abort in this case).
+  template <typename T> LLVM_NODISCARD bool consumeSize(T &Container) {
+    auto Size = consumeVar();
+    // Conservatively assume each element is at least one byte.
+    if (Size > (End - Begin)) {
+      Err = true;
+      return false;
+    }
+    Container.resize(Size);
+    return true;
+  }
 };
 
 void write32(uint32_t I, llvm::raw_ostream &OS) {
@@ -257,7 +272,8 @@ IncludeGraphNode readIncludeGraphNode(Reader &Data,
   IGN.URI = Data.consumeString(Strings);
   llvm::StringRef Digest = Data.consume(IGN.Digest.size());
   std::copy(Digest.bytes_begin(), Digest.bytes_end(), IGN.Digest.begin());
-  IGN.DirectIncludes.resize(Data.consumeVar());
+  if (!Data.consumeSize(IGN.DirectIncludes))
+    return IGN;
   for (llvm::StringRef &Include : IGN.DirectIncludes)
     Include = Data.consumeString(Strings);
   return IGN;
@@ -323,7 +339,8 @@ Symbol readSymbol(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
   Sym.Documentation = Data.consumeString(Strings);
   Sym.ReturnType = Data.consumeString(Strings);
   Sym.Type = Data.consumeString(Strings);
-  Sym.IncludeHeaders.resize(Data.consumeVar());
+  if (!Data.consumeSize(Sym.IncludeHeaders))
+    return Sym;
   for (auto &I : Sym.IncludeHeaders) {
     I.IncludeHeader = Data.consumeString(Strings);
     I.References = Data.consumeVar();
@@ -353,7 +370,8 @@ std::pair<SymbolID, std::vector<Ref>>
 readRefs(Reader &Data, llvm::ArrayRef<llvm::StringRef> Strings) {
   std::pair<SymbolID, std::vector<Ref>> Result;
   Result.first = Data.consumeID();
-  Result.second.resize(Data.consumeVar());
+  if (!Data.consumeSize(Result.second))
+    return Result;
   for (auto &Ref : Result.second) {
     Ref.Kind = static_cast<RefKind>(Data.consume8());
     Ref.Location = readLocation(Data, Strings);
@@ -400,7 +418,8 @@ InternedCompileCommand
 readCompileCommand(Reader CmdReader, llvm::ArrayRef<llvm::StringRef> Strings) {
   InternedCompileCommand Cmd;
   Cmd.Directory = CmdReader.consumeString(Strings);
-  Cmd.CommandLine.resize(CmdReader.consumeVar());
+  if (!CmdReader.consumeSize(Cmd.CommandLine))
+    return Cmd;
   for (llvm::StringRef &C : Cmd.CommandLine)
     C = CmdReader.consumeString(Strings);
   return Cmd;
@@ -499,10 +518,10 @@ llvm::Expected<IndexFileIn> readRIFF(llvm::StringRef Data) {
   }
   if (Chunks.count("cmdl")) {
     Reader CmdReader(Chunks.lookup("cmdl"));
-    if (CmdReader.err())
-      return error("malformed or truncated commandline section");
     InternedCompileCommand Cmd =
         readCompileCommand(CmdReader, Strings->Strings);
+    if (CmdReader.err())
+      return error("malformed or truncated commandline section");
     Result.Cmd.emplace();
     Result.Cmd->Directory = std::string(Cmd.Directory);
     Result.Cmd->CommandLine.reserve(Cmd.CommandLine.size());

clang-tools-extra/clangd/index/remote/server/Server.cpp

@@ -94,10 +94,14 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
   }
 
 private:
+  using stopwatch = std::chrono::steady_clock;
+
   grpc::Status Lookup(grpc::ServerContext *Context,
                       const LookupRequest *Request,
                       grpc::ServerWriter<LookupReply> *Reply) override {
-    WithContextValue(CurrentRequest, Context);
+    auto StartTime = stopwatch::now();
+    WithContextValue WithRequestContext(CurrentRequest, Context);
+    logRequest(*Request);
     trace::Span Tracer("LookupRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
@@ -116,21 +120,26 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
       }
       LookupReply NextMessage;
       *NextMessage.mutable_stream_result() = *SerializedItem;
+      logResponse(NextMessage);
       Reply->Write(NextMessage);
       ++Sent;
     });
     LookupReply LastMessage;
     LastMessage.mutable_final_result()->set_has_more(true);
+    logResponse(LastMessage);
     Reply->Write(LastMessage);
     SPAN_ATTACH(Tracer, "Sent", Sent);
     SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
+    logRequestSummary("v1/Lookup", Sent, StartTime);
     return grpc::Status::OK;
   }
 
   grpc::Status FuzzyFind(grpc::ServerContext *Context,
                          const FuzzyFindRequest *Request,
                          grpc::ServerWriter<FuzzyFindReply> *Reply) override {
-    WithContextValue(CurrentRequest, Context);
+    auto StartTime = stopwatch::now();
+    WithContextValue WithRequestContext(CurrentRequest, Context);
+    logRequest(*Request);
     trace::Span Tracer("FuzzyFindRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
@@ -150,20 +159,25 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
       }
       FuzzyFindReply NextMessage;
       *NextMessage.mutable_stream_result() = *SerializedItem;
+      logResponse(NextMessage);
       Reply->Write(NextMessage);
       ++Sent;
     });
     FuzzyFindReply LastMessage;
     LastMessage.mutable_final_result()->set_has_more(HasMore);
+    logResponse(LastMessage);
     Reply->Write(LastMessage);
     SPAN_ATTACH(Tracer, "Sent", Sent);
     SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
+    logRequestSummary("v1/FuzzyFind", Sent, StartTime);
     return grpc::Status::OK;
   }
 
   grpc::Status Refs(grpc::ServerContext *Context, const RefsRequest *Request,
                     grpc::ServerWriter<RefsReply> *Reply) override {
-    WithContextValue(CurrentRequest, Context);
+    auto StartTime = stopwatch::now();
+    WithContextValue WithRequestContext(CurrentRequest, Context);
+    logRequest(*Request);
     trace::Span Tracer("RefsRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
@@ -182,21 +196,26 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
       }
       RefsReply NextMessage;
       *NextMessage.mutable_stream_result() = *SerializedItem;
+      logResponse(NextMessage);
       Reply->Write(NextMessage);
       ++Sent;
     });
     RefsReply LastMessage;
     LastMessage.mutable_final_result()->set_has_more(HasMore);
+    logResponse(LastMessage);
     Reply->Write(LastMessage);
     SPAN_ATTACH(Tracer, "Sent", Sent);
     SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
+    logRequestSummary("v1/Refs", Sent, StartTime);
     return grpc::Status::OK;
   }
 
   grpc::Status Relations(grpc::ServerContext *Context,
                          const RelationsRequest *Request,
                          grpc::ServerWriter<RelationsReply> *Reply) override {
-    WithContextValue(CurrentRequest, Context);
+    auto StartTime = stopwatch::now();
+    WithContextValue WithRequestContext(CurrentRequest, Context);
+    logRequest(*Request);
     trace::Span Tracer("RelationsRequest");
     auto Req = ProtobufMarshaller->fromProtobuf(Request);
     if (!Req) {
@@ -217,17 +236,44 @@ class RemoteIndexServer final : public v1::SymbolIndex::Service {
           }
           RelationsReply NextMessage;
           *NextMessage.mutable_stream_result() = *SerializedItem;
+          logResponse(NextMessage);
           Reply->Write(NextMessage);
           ++Sent;
         });
     RelationsReply LastMessage;
     LastMessage.mutable_final_result()->set_has_more(true);
+    logResponse(LastMessage);
     Reply->Write(LastMessage);
     SPAN_ATTACH(Tracer, "Sent", Sent);
     SPAN_ATTACH(Tracer, "Failed to send", FailedToSend);
+    logRequestSummary("v1/Relations", Sent, StartTime);
     return grpc::Status::OK;
   }
 
+  // Proxy object to allow proto messages to be lazily serialized as text.
+  struct TextProto {
+    const google::protobuf::Message &M;
+    friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+                                         const TextProto &P) {
+      return OS << P.M.DebugString();
+    }
+  };
+
+  void logRequest(const google::protobuf::Message &M) {
+    vlog("<<< {0}\n{1}", M.GetDescriptor()->name(), TextProto{M});
+  }
+  void logResponse(const google::protobuf::Message &M) {
+    vlog(">>> {0}\n{1}", M.GetDescriptor()->name(), TextProto{M});
+  }
+  void logRequestSummary(llvm::StringLiteral RequestName, unsigned Sent,
+                         stopwatch::time_point StartTime) {
+    auto Duration = stopwatch::now() - StartTime;
+    auto Millis =
+        std::chrono::duration_cast<std::chrono::milliseconds>(Duration).count();
+    log("[public] request {0} => OK: {1} results in {2}ms", RequestName, Sent,
+        Millis);
+  }
+
   std::unique_ptr<Marshaller> ProtobufMarshaller;
   clangd::SymbolIndex &Index;
 };

clang-tools-extra/clangd/test/remote-index/pipeline_helper.py

@@ -28,6 +28,8 @@ def main():
   parser.add_argument('--input-file-name', required=True)
   parser.add_argument('--project-root', required=True)
   parser.add_argument('--index-file', required=True)
+  parser.add_argument('--server-arg', action='append', default=[])
+  parser.add_argument('--server-log', nargs='?', type=argparse.FileType('wb'), default=os.devnull)
 
   args = parser.parse_args()
 
@@ -40,7 +42,7 @@ def main():
   index_server_process = subprocess.Popen([
       'clangd-index-server', '--server-address=' + server_address,
       args.index_file, args.project_root
-  ],
+  ] + args.server_arg,
                                           stderr=subprocess.PIPE)
 
   # This will kill index_server_process if it hangs without printing init
@@ -53,7 +55,10 @@ def main():
   # Wait for the server to warm-up.
   found_init_message = False
   while index_server_process.poll() is None:
-    if b'Server listening' in index_server_process.stderr.readline():
+    line = index_server_process.stderr.readline()
+    args.server_log.write(line)
+    args.server_log.flush()
+    if b'Server listening' in line:
       print('Server initialization complete.', file=sys.stderr)
       found_init_message = True
       break
@@ -70,12 +75,14 @@ def main():
       '--project-root=' + args.project_root, '--lit-test', '--sync'
   ],
                                     stdin=in_file)
-
   clangd_process.wait()
   print(
       'Clangd executed successfully, shutting down child processes.',
       file=sys.stderr)
   index_server_process.kill()
+  for line in index_server_process.stderr:
+    args.server_log.write(line)
+    args.server_log.flush()
 
 
 if __name__ == '__main__':

clang-tools-extra/clangd/test/remote-index/public-log.test

@@ -0,0 +1,26 @@
+# RUN: rm -rf %t
+# RUN: clangd-indexer %S/Inputs/Source.cpp > %t.idx
+# RUN: %python %S/pipeline_helper.py --input-file-name=%s --server-arg=--log=verbose --server-arg=-log-public --server-log=%t.public.log --project-root=%S --index-file=%t.idx > /dev/null
+# RUN: %python %S/pipeline_helper.py --input-file-name=%s --server-arg=--log=verbose --server-log=%t.log --project-root=%S --index-file=%t.idx > /dev/null
+# RUN: FileCheck --check-prefixes=LOG,LOG-PUBLIC %s < %t.public.log
+# RUN: FileCheck --check-prefixes=LOG,LOG-ALL %s < %t.log
+# REQUIRES: clangd-remote-index
+
+# LOG: Server listening on
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+---
+# Verify that request and response bodies are included in the verbose logs,
+# but not when --log-public is on.
+# The request summary should be included in either case.
+{"jsonrpc":"2.0","id":1,"method":"workspace/symbol","params":{"query":"gFoo"}}
+# LOG-ALL: <<< FuzzyFindRequest
+# LOG-ALL: query: "gFoo"
+# LOG-ALL: >>> FuzzyFindReply
+# LOG-ALL: name: "getFoo"
+# LOG-PUBLIC-NOT: gFoo
+# LOG-PUBLIC-NOT: getFoo
+# LOG: request v1/FuzzyFind => OK: 1 results in {{.*}}ms
+---
+{"jsonrpc":"2.0","id":4,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}