Cache calculated file state in LSP

toolchain/diagnostics/diagnostic_kind.def

@@ -28,11 +28,6 @@ CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError)
 CARBON_DIAGNOSTIC_KIND(CompileInputNotRegularFile)
 CARBON_DIAGNOSTIC_KIND(CompileOutputFileOpenError)
 CARBON_DIAGNOSTIC_KIND(FormatMultipleFilesToOneOutput)
-CARBON_DIAGNOSTIC_KIND(LanguageServerMissingInputStream)
-CARBON_DIAGNOSTIC_KIND(LanguageServerNotificationParseError)
-CARBON_DIAGNOSTIC_KIND(LanguageServerTransportError)
-CARBON_DIAGNOSTIC_KIND(LanguageServerUnexpectedReply)
-CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedNotification)
 
 // ============================================================================
 // SourceBuffer diagnostics
@@ -441,6 +436,23 @@ CARBON_DIAGNOSTIC_KIND(AssociatedConstantWithDifferentValues)
 CARBON_DIAGNOSTIC_KIND(ImplsOnNonFacetType)
 CARBON_DIAGNOSTIC_KIND(WhereOnNonFacetType)
 
+// ============================================================================
+// Language server diagnostics
+// ============================================================================
+
+CARBON_DIAGNOSTIC_KIND(LanguageServerFileUnknown)
+CARBON_DIAGNOSTIC_KIND(LanguageServerFileUnsupported)
+CARBON_DIAGNOSTIC_KIND(LanguageServerMissingInputStream)
+CARBON_DIAGNOSTIC_KIND(LanguageServerNotificationParseError)
+CARBON_DIAGNOSTIC_KIND(LanguageServerTransportError)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnexpectedReply)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedNotification)
+
+// Document handling.
+CARBON_DIAGNOSTIC_KIND(LanguageServerOpenDuplicateFile)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedChanges)
+CARBON_DIAGNOSTIC_KIND(LanguageServerCloseUnknownFile)
+
 // ============================================================================
 // Other diagnostics
 // ============================================================================

toolchain/language_server/BUILD

@@ -28,10 +28,20 @@ cc_library(
 
 cc_library(
     name = "context",
+    srcs = ["context.cpp"],
     hdrs = ["context.h"],
     deps = [
         "//common:map",
+        "//toolchain/base:shared_value_stores",
         "//toolchain/diagnostics:diagnostic_emitter",
+        "//toolchain/diagnostics:file_diagnostics",
+        "//toolchain/diagnostics:null_diagnostics",
+        "//toolchain/lex",
+        "//toolchain/lex:tokenized_buffer",
+        "//toolchain/parse",
+        "//toolchain/parse:tree",
+        "//toolchain/sem_ir:file",
+        "//toolchain/source:source_buffer",
     ],
 )
 

toolchain/language_server/context.cpp

@@ -0,0 +1,68 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/language_server/context.h"
+
+#include <memory>
+
+#include "toolchain/base/shared_value_stores.h"
+#include "toolchain/diagnostics/null_diagnostics.h"
+#include "toolchain/lex/lex.h"
+#include "toolchain/lex/tokenized_buffer.h"
+#include "toolchain/parse/parse.h"
+#include "toolchain/parse/tree_and_subtrees.h"
+
+namespace Carbon::LanguageServer {
+
+auto Context::File::SetText(Context& context, llvm::StringRef text) -> void {
+  // Clear state dependent on the source text.
+  tree_and_subtrees_.reset();
+  tree_.reset();
+  tokens_.reset();
+  value_stores_.reset();
+  source_.reset();
+
+  // TODO: Make the processing asynchronous, to better handle rapid text
+  // updates.
+  CARBON_CHECK(!source_ && !value_stores_ && !tokens_ && !tree_,
+               "We currently cache everything together");
+  // TODO: Diagnostics should be passed to the LSP instead of dropped.
+  auto& null_consumer = NullDiagnosticConsumer();
+  std::optional source =
+      SourceBuffer::MakeFromStringCopy(filename_, text, null_consumer);
+  if (!source) {
+    // Failing here should be rare, but provide stub data for recovery so that
+    // we can have a simple API.
+    source = SourceBuffer::MakeFromStringCopy(filename_, "", null_consumer);
+    CARBON_CHECK(source, "Making an empty buffer should always succeed");
+  }
+  source_ = std::make_unique<SourceBuffer>(std::move(*source));
+  value_stores_ = std::make_unique<SharedValueStores>();
+  tokens_ = std::make_unique<Lex::TokenizedBuffer>(
+      Lex::Lex(*value_stores_, *source_, null_consumer));
+  tree_ = std::make_unique<Parse::Tree>(
+      Parse::Parse(*tokens_, null_consumer, context.vlog_stream()));
+  tree_and_subtrees_ =
+      std::make_unique<Parse::TreeAndSubtrees>(*tokens_, *tree_);
+}
+
+auto Context::LookupFile(llvm::StringRef filename) -> File* {
+  if (!filename.ends_with(".carbon")) {
+    CARBON_DIAGNOSTIC(LanguageServerFileUnsupported, Warning,
+                      "non-Carbon file requested");
+    file_emitter_.Emit(filename, LanguageServerFileUnsupported);
+    return nullptr;
+  }
+
+  if (auto lookup_result = files().Lookup(filename)) {
+    return &lookup_result.value();
+  } else {
+    CARBON_DIAGNOSTIC(LanguageServerFileUnknown, Warning,
+                      "unknown file requested");
+    file_emitter_.Emit(filename, LanguageServerFileUnknown);
+    return nullptr;
+  }
+}
+
+}  // namespace Carbon::LanguageServer

toolchain/language_server/context.h

@@ -5,29 +5,72 @@
 #ifndef CARBON_TOOLCHAIN_LANGUAGE_SERVER_CONTEXT_H_
 #define CARBON_TOOLCHAIN_LANGUAGE_SERVER_CONTEXT_H_
 
+#include <memory>
 #include <string>
 
 #include "common/map.h"
+#include "toolchain/base/shared_value_stores.h"
 #include "toolchain/diagnostics/diagnostic_consumer.h"
 #include "toolchain/diagnostics/diagnostic_emitter.h"
+#include "toolchain/diagnostics/file_diagnostics.h"
+#include "toolchain/lex/tokenized_buffer.h"
+#include "toolchain/parse/tree_and_subtrees.h"
+#include "toolchain/sem_ir/file.h"
+#include "toolchain/source/source_buffer.h"
 
 namespace Carbon::LanguageServer {
 
 // Context for LSP call handling.
 class Context {
  public:
-  // `consumer` is required.
-  explicit Context(DiagnosticConsumer* consumer) : no_loc_emitter_(consumer) {}
+  // Cached information for an open file.
+  class File {
+   public:
+    explicit File(std::string filename) : filename_(std::move(filename)) {}
 
+    // Changes the file's text, updating dependent state.
+    auto SetText(Context& context, llvm::StringRef text) -> void;
+
+    auto tree_and_subtrees() const -> const Parse::TreeAndSubtrees& {
+      return *tree_and_subtrees_;
+    }
+
+   private:
+    // The filename, stable across instances.
+    std::string filename_;
+
+    // Current file content, and derived values.
+    std::unique_ptr<SourceBuffer> source_;
+    std::unique_ptr<SharedValueStores> value_stores_;
+    std::unique_ptr<Lex::TokenizedBuffer> tokens_;
+    std::unique_ptr<Parse::Tree> tree_;
+    std::unique_ptr<Parse::TreeAndSubtrees> tree_and_subtrees_;
+  };
+
+  // `consumer` and `emitter` are required. `vlog_stream` is optional.
+  explicit Context(llvm::raw_ostream* vlog_stream, DiagnosticConsumer* consumer)
+      : vlog_stream_(vlog_stream),
+        file_emitter_(consumer),
+        no_loc_emitter_(consumer) {}
+
+  // Returns a reference to the file if it's known, or diagnoses and returns
+  // null.
+  auto LookupFile(llvm::StringRef filename) -> File*;
+
+  auto file_emitter() -> FileDiagnosticEmitter& { return file_emitter_; }
   auto no_loc_emitter() -> NoLocDiagnosticEmitter& { return no_loc_emitter_; }
+  auto vlog_stream() -> llvm::raw_ostream* { return vlog_stream_; }
 
-  auto files() -> Map<std::string, std::string>& { return files_; }
+  auto files() -> Map<std::string, File>& { return files_; }
 
  private:
+  // Diagnostic and output streams.
+  llvm::raw_ostream* vlog_stream_;
+  FileDiagnosticEmitter file_emitter_;
   NoLocDiagnosticEmitter no_loc_emitter_;
 
   // Content of files managed by the language client.
-  Map<std::string, std::string> files_;
+  Map<std::string, File> files_;
 };
 
 }  // namespace Carbon::LanguageServer

toolchain/language_server/handle.h

@@ -15,6 +15,11 @@ auto HandleDidChangeTextDocument(
     Context& context, const clang::clangd::DidChangeTextDocumentParams& params)
     -> void;
 
+// Closes a document.
+auto HandleDidCloseTextDocument(
+    Context& context, const clang::clangd::DidCloseTextDocumentParams& params)
+    -> void;
+
 // Updates the content of already-open documents.
 auto HandleDidOpenTextDocument(
     Context& context, const clang::clangd::DidOpenTextDocumentParams& params)

toolchain/language_server/handle_document_symbol.cpp

@@ -15,18 +15,17 @@ namespace Carbon::LanguageServer {
 
 // Returns the text of first child of kind IdentifierNameBeforeParams or
 // IdentifierNameNotBeforeParams.
-static auto GetIdentifierName(const SharedValueStores& value_stores,
-                              const Lex::TokenizedBuffer& tokens,
-                              const Parse::TreeAndSubtrees& tree_and_subtrees,
+static auto GetIdentifierName(const Parse::TreeAndSubtrees& tree_and_subtrees,
                               Parse::NodeId node)
     -> std::optional<llvm::StringRef> {
+  const auto& tokens = tree_and_subtrees.tree().tokens();
   for (auto child : tree_and_subtrees.children(node)) {
     switch (tree_and_subtrees.tree().node_kind(child)) {
       case Parse::NodeKind::IdentifierNameBeforeParams:
       case Parse::NodeKind::IdentifierNameNotBeforeParams: {
         auto token = tree_and_subtrees.tree().node_token(child);
         if (tokens.GetKind(token) == Lex::TokenKind::Identifier) {
-          return value_stores.identifiers().Get(tokens.GetIdentifier(token));
+          return tokens.GetTokenText(token);
         }
         break;
       }
@@ -42,22 +41,19 @@ auto HandleDocumentSymbol(
     llvm::function_ref<
         void(llvm::Expected<std::vector<clang::clangd::DocumentSymbol>>)>
         on_done) -> void {
-  SharedValueStores value_stores;
-  llvm::vfs::InMemoryFileSystem vfs;
-  auto lookup = context.files().Lookup(params.textDocument.uri.file());
-  CARBON_CHECK(lookup);
-  vfs.addFile(lookup.key(), /*mtime=*/0,
-              llvm::MemoryBuffer::getMemBufferCopy(lookup.value()));
+  auto* file = context.LookupFile(params.textDocument.uri.file());
+  if (!file) {
+    return;
+  }
+
+  const auto& tree_and_subtrees = file->tree_and_subtrees();
+  const auto& tree = tree_and_subtrees.tree();
+  const auto& tokens = tree.tokens();
 
-  auto source =
-      SourceBuffer::MakeFromFile(vfs, lookup.key(), NullDiagnosticConsumer());
-  auto tokens = Lex::Lex(value_stores, *source, NullDiagnosticConsumer());
-  auto tree = Parse::Parse(tokens, NullDiagnosticConsumer(), nullptr);
-  Parse::TreeAndSubtrees tree_and_subtrees(tokens, tree);
   std::vector<clang::clangd::DocumentSymbol> result;
-  for (const auto& node : tree.postorder()) {
+  for (const auto& node_id : tree.postorder()) {
     clang::clangd::SymbolKind symbol_kind;
-    switch (tree.node_kind(node)) {
+    switch (tree.node_kind(node_id)) {
       case Parse::NodeKind::FunctionDecl:
       case Parse::NodeKind::FunctionDefinitionStart:
         symbol_kind = clang::clangd::SymbolKind::Function;
@@ -76,9 +72,8 @@ auto HandleDocumentSymbol(
         continue;
     }
 
-    if (auto name =
-            GetIdentifierName(value_stores, tokens, tree_and_subtrees, node)) {
-      auto token = tree.node_token(node);
+    if (auto name = GetIdentifierName(tree_and_subtrees, node_id)) {
+      auto token = tree.node_token(node_id);
       clang::clangd::Position pos{tokens.GetLineNumber(token) - 1,
                                   tokens.GetColumnNumber(token) - 1};
 

toolchain/language_server/handle_text_document.cpp

@@ -9,17 +9,58 @@ namespace Carbon::LanguageServer {
 auto HandleDidOpenTextDocument(
     Context& context, const clang::clangd::DidOpenTextDocumentParams& params)
     -> void {
-  context.files().Update(params.textDocument.uri.file(),
-                         params.textDocument.text);
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
+  auto insert_result = context.files().Insert(
+      filename, [&] { return Context::File(filename.str()); });
+  insert_result.value().SetText(context, params.textDocument.text);
+  if (!insert_result.is_inserted()) {
+    CARBON_DIAGNOSTIC(LanguageServerOpenDuplicateFile, Warning,
+                      "duplicate open file request; updating content");
+    context.file_emitter().Emit(filename, LanguageServerOpenDuplicateFile);
+  }
 }
 
 auto HandleDidChangeTextDocument(
     Context& context, const clang::clangd::DidChangeTextDocumentParams& params)
     -> void {
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
   // Full text is sent if full sync is specified in capabilities.
-  CARBON_CHECK(params.contentChanges.size() == 1);
-  context.files().Update(params.textDocument.uri.file(),
-                         params.contentChanges[0].text);
+  if (params.contentChanges.size() != 1) {
+    CARBON_DIAGNOSTIC(LanguageServerUnsupportedChanges, Warning,
+                      "received unsupported contentChanges count: {0}", int);
+    context.file_emitter().Emit(filename, LanguageServerUnsupportedChanges,
+                                params.contentChanges.size());
+    return;
+  }
+  if (auto* file = context.LookupFile(filename)) {
+    file->SetText(context, params.contentChanges[0].text);
+  }
+}
+
+auto HandleDidCloseTextDocument(
+    Context& context, const clang::clangd::DidCloseTextDocumentParams& params)
+    -> void {
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
+  if (!context.files().Erase(filename)) {
+    CARBON_DIAGNOSTIC(LanguageServerCloseUnknownFile, Warning,
+                      "tried closing unknown file; ignoring request");
+    context.file_emitter().Emit(filename, LanguageServerCloseUnknownFile);
+  }
 }
 
 }  // namespace Carbon::LanguageServer

toolchain/language_server/incoming_messages.cpp

@@ -76,6 +76,7 @@ IncomingMessages::IncomingMessages(clang::clangd::Transport* transport,
   AddCallHandler("initialize", &HandleInitialize);
   AddNotificationHandler("textDocument/didChange",
                          &HandleDidChangeTextDocument);
+  AddNotificationHandler("textDocument/didClose", &HandleDidCloseTextDocument);
   AddNotificationHandler("textDocument/didOpen", &HandleDidOpenTextDocument);
 }
 

toolchain/language_server/language_server.cpp

@@ -55,7 +55,7 @@ auto Run(FILE* input_stream, llvm::raw_ostream& output_stream,
       clang::clangd::newJSONTransport(input_stream, output_stream,
                                       /*InMirror=*/nullptr,
                                       /*Pretty=*/true));
-  Context context(&consumer);
+  Context context(vlog_stream, &consumer);
   // TODO: Use error_stream in IncomingMessages to report dropped errors.
   IncomingMessages incoming(transport.get(), &context);
   OutgoingMessages outgoing(transport.get());

toolchain/language_server/testdata/exit.carbon → toolchain/language_server/testdata/basics/exit.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/exit.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/exit.carbon
 
 // --- STDIN
 [[@LSP-NOTIFY:exit]]

toolchain/language_server/testdata/fail_empty_stdin.carbon → toolchain/language_server/testdata/basics/fail_empty_stdin.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/fail_empty_stdin.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/fail_empty_stdin.carbon
 
 // --- STDIN
 // --- AUTOUPDATE-SPLIT

toolchain/language_server/testdata/fail_no_stdin.carbon → toolchain/language_server/testdata/basics/fail_no_stdin.carbon

@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_no_stdin.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/fail_no_stdin.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_no_stdin.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/fail_no_stdin.carbon
 
 // --- AUTOUPDATE-SPLIT