From 3547038d5cc40ceb781c9f4fe8d69b0aa8fe18fa Mon Sep 17 00:00:00 2001
From: Vinicius Stock <vinicius.stock@shopify.com>
Date: Tue, 20 Aug 2024 17:17:39 -0400
Subject: [PATCH] Support semantic token deltas

---
 lib/ruby_lsp/document.rb                      |   9 +-
 lib/ruby_lsp/listeners/inlay_hints.rb         |  11 +
 .../listeners/semantic_highlighting.rb        |  50 +--
 .../requests/semantic_highlighting.rb         | 109 +++++-
 lib/ruby_lsp/requests/support/common.rb       |   9 -
 .../semantic_highlighting.rb                  |   8 +-
 lib/ruby_lsp/server.rb                        |  64 +++-
 ...semantic_highlighting_expectations_test.rb |   8 +-
 test/requests/semantic_tokens_delta_test.rb   | 319 ++++++++++++++++++
 .../support/semantic_token_encoder_test.rb    |   4 +-
 test/ruby_document_test.rb                    |  11 +
 11 files changed, 516 insertions(+), 86 deletions(-)
 create mode 100644 test/requests/semantic_tokens_delta_test.rb

diff --git a/lib/ruby_lsp/document.rb b/lib/ruby_lsp/document.rb
index f6c503eb6..61e6187b2 100644
--- a/lib/ruby_lsp/document.rb
+++ b/lib/ruby_lsp/document.rb
@@ -16,6 +16,7 @@ class LanguageId < T::Enum
     extend T::Generic
 
     ParseResultType = type_member
+    EMPTY_CACHE = T.let(Object.new.freeze, Object)
 
     abstract!
 
@@ -34,9 +35,13 @@ class LanguageId < T::Enum
     sig { returns(Encoding) }
     attr_reader :encoding
 
+    sig { returns(T.any(Interface::SemanticTokens, Object)) }
+    attr_accessor :semantic_tokens
+
     sig { params(source: String, version: Integer, uri: URI::Generic, encoding: Encoding).void }
     def initialize(source:, version:, uri:, encoding: Encoding::UTF_8)
-      @cache = T.let({}, T::Hash[String, T.untyped])
+      @cache = T.let(Hash.new(EMPTY_CACHE), T::Hash[String, T.untyped])
+      @semantic_tokens = T.let(EMPTY_CACHE, T.any(Interface::SemanticTokens, Object))
       @encoding = T.let(encoding, Encoding)
       @source = T.let(source, String)
       @version = T.let(version, Integer)
@@ -63,7 +68,7 @@ def language_id; end
     end
     def cache_fetch(request_name, &block)
       cached = @cache[request_name]
-      return cached if cached
+      return cached if cached != EMPTY_CACHE
 
       result = block.call(self)
       @cache[request_name] = result
diff --git a/lib/ruby_lsp/listeners/inlay_hints.rb b/lib/ruby_lsp/listeners/inlay_hints.rb
index 3ee0ba204..c08cfc141 100644
--- a/lib/ruby_lsp/listeners/inlay_hints.rb
+++ b/lib/ruby_lsp/listeners/inlay_hints.rb
@@ -69,6 +69,17 @@ def on_implicit_node_enter(node)
           tooltip: tooltip,
         )
       end
+
+      private
+
+      sig { params(node: T.nilable(Prism::Node), range: T.nilable(T::Range[Integer])).returns(T::Boolean) }
+      def visible?(node, range)
+        return true if range.nil?
+        return false if node.nil?
+
+        loc = node.location
+        range.cover?(loc.start_line - 1) && range.cover?(loc.end_line - 1)
+      end
     end
   end
 end
diff --git a/lib/ruby_lsp/listeners/semantic_highlighting.rb b/lib/ruby_lsp/listeners/semantic_highlighting.rb
index 42f317c1f..51e51bde5 100644
--- a/lib/ruby_lsp/listeners/semantic_highlighting.rb
+++ b/lib/ruby_lsp/listeners/semantic_highlighting.rb
@@ -22,12 +22,10 @@ class SemanticHighlighting
         params(
           dispatcher: Prism::Dispatcher,
           response_builder: ResponseBuilders::SemanticHighlighting,
-          range: T.nilable(T::Range[Integer]),
         ).void
       end
-      def initialize(dispatcher, response_builder, range: nil)
+      def initialize(dispatcher, response_builder)
         @response_builder = response_builder
-        @range = range
         @special_methods = T.let(nil, T.nilable(T::Array[String]))
         @current_scope = T.let(ParameterScope.new, ParameterScope)
         @inside_regex_capture = T.let(false, T::Boolean)
@@ -74,7 +72,6 @@ def initialize(dispatcher, response_builder, range: nil)
       sig { params(node: Prism::CallNode).void }
       def on_call_node_enter(node)
         return if @inside_implicit_node
-        return unless visible?(node, @range)
 
         message = node.message
         return unless message
@@ -107,51 +104,38 @@ def on_match_write_node_leave(node)
       sig { params(node: Prism::ConstantReadNode).void }
       def on_constant_read_node_enter(node)
         return if @inside_implicit_node
-        return unless visible?(node, @range)
 
         @response_builder.add_token(node.location, :namespace)
       end
 
       sig { params(node: Prism::ConstantWriteNode).void }
       def on_constant_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, :namespace)
       end
 
       sig { params(node: Prism::ConstantAndWriteNode).void }
       def on_constant_and_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, :namespace)
       end
 
       sig { params(node: Prism::ConstantOperatorWriteNode).void }
       def on_constant_operator_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, :namespace)
       end
 
       sig { params(node: Prism::ConstantOrWriteNode).void }
       def on_constant_or_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, :namespace)
       end
 
       sig { params(node: Prism::ConstantTargetNode).void }
       def on_constant_target_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.location, :namespace)
       end
 
       sig { params(node: Prism::DefNode).void }
       def on_def_node_enter(node)
         @current_scope = ParameterScope.new(@current_scope)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, :method, [:declaration])
       end
 
@@ -184,7 +168,6 @@ def on_block_parameter_node_enter(node)
       sig { params(node: Prism::RequiredKeywordParameterNode).void }
       def on_required_keyword_parameter_node_enter(node)
         @current_scope << node.name
-        return unless visible?(node, @range)
 
         location = node.name_loc
         @response_builder.add_token(location.copy(length: location.length - 1), :parameter)
@@ -193,7 +176,6 @@ def on_required_keyword_parameter_node_enter(node)
       sig { params(node: Prism::OptionalKeywordParameterNode).void }
       def on_optional_keyword_parameter_node_enter(node)
         @current_scope << node.name
-        return unless visible?(node, @range)
 
         location = node.name_loc
         @response_builder.add_token(location.copy(length: location.length - 1), :parameter)
@@ -205,24 +187,19 @@ def on_keyword_rest_parameter_node_enter(node)
 
         if name
           @current_scope << name.to_sym
-
-          @response_builder.add_token(T.must(node.name_loc), :parameter) if visible?(node, @range)
+          @response_builder.add_token(T.must(node.name_loc), :parameter)
         end
       end
 
       sig { params(node: Prism::OptionalParameterNode).void }
       def on_optional_parameter_node_enter(node)
         @current_scope << node.name
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, :parameter)
       end
 
       sig { params(node: Prism::RequiredParameterNode).void }
       def on_required_parameter_node_enter(node)
         @current_scope << node.name
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.location, :parameter)
       end
 
@@ -232,29 +209,23 @@ def on_rest_parameter_node_enter(node)
 
         if name
           @current_scope << name.to_sym
-
-          @response_builder.add_token(T.must(node.name_loc), :parameter) if visible?(node, @range)
+          @response_builder.add_token(T.must(node.name_loc), :parameter)
         end
       end
 
       sig { params(node: Prism::SelfNode).void }
       def on_self_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.location, :variable, [:default_library])
       end
 
       sig { params(node: Prism::LocalVariableWriteNode).void }
       def on_local_variable_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, @current_scope.type_for(node.name))
       end
 
       sig { params(node: Prism::LocalVariableReadNode).void }
       def on_local_variable_read_node_enter(node)
         return if @inside_implicit_node
-        return unless visible?(node, @range)
 
         # Numbered parameters
         if /_\d+/.match?(node.name)
@@ -267,22 +238,16 @@ def on_local_variable_read_node_enter(node)
 
       sig { params(node: Prism::LocalVariableAndWriteNode).void }
       def on_local_variable_and_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, @current_scope.type_for(node.name))
       end
 
       sig { params(node: Prism::LocalVariableOperatorWriteNode).void }
       def on_local_variable_operator_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, @current_scope.type_for(node.name))
       end
 
       sig { params(node: Prism::LocalVariableOrWriteNode).void }
       def on_local_variable_or_write_node_enter(node)
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.name_loc, @current_scope.type_for(node.name))
       end
 
@@ -294,15 +259,11 @@ def on_local_variable_target_node_enter(node)
         # prevent pushing local variable target tokens. See https://github.com/ruby/prism/issues/1912
         return if @inside_regex_capture
 
-        return unless visible?(node, @range)
-
         @response_builder.add_token(node.location, @current_scope.type_for(node.name))
       end
 
       sig { params(node: Prism::ClassNode).void }
       def on_class_node_enter(node)
-        return unless visible?(node, @range)
-
         constant_path = node.constant_path
 
         if constant_path.is_a?(Prism::ConstantReadNode)
@@ -342,8 +303,6 @@ def on_class_node_enter(node)
 
       sig { params(node: Prism::ModuleNode).void }
       def on_module_node_enter(node)
-        return unless visible?(node, @range)
-
         constant_path = node.constant_path
 
         if constant_path.is_a?(Prism::ConstantReadNode)
@@ -365,8 +324,6 @@ def on_module_node_enter(node)
 
       sig { params(node: Prism::ImplicitNode).void }
       def on_implicit_node_enter(node)
-        return unless visible?(node, @range)
-
         @inside_implicit_node = true
       end
 
@@ -378,7 +335,6 @@ def on_implicit_node_leave(node)
       sig { params(node: Prism::ConstantPathNode).void }
       def on_constant_path_node_enter(node)
         return if @inside_implicit_node
-        return unless visible?(node, @range)
 
         @response_builder.add_token(node.name_loc, :namespace)
       end
diff --git a/lib/ruby_lsp/requests/semantic_highlighting.rb b/lib/ruby_lsp/requests/semantic_highlighting.rb
index 546c988b2..9e1a639d7 100644
--- a/lib/ruby_lsp/requests/semantic_highlighting.rb
+++ b/lib/ruby_lsp/requests/semantic_highlighting.rb
@@ -35,28 +35,125 @@ def provider
               token_modifiers: ResponseBuilders::SemanticHighlighting::TOKEN_MODIFIERS.keys,
             ),
             range: true,
-            full: { delta: false },
+            full: { delta: true },
           )
         end
+
+        # The compute_delta method receives the current semantic tokens and the previous semantic tokens and then tries
+        # to compute the smallest possible semantic token edit that will turn previous into current
+        sig do
+          params(
+            current_tokens: T::Array[Integer],
+            previous_tokens: T::Array[Integer],
+            result_id: String,
+          ).returns(Interface::SemanticTokensDelta)
+        end
+        def compute_delta(current_tokens, previous_tokens, result_id)
+          # Find the index of the first token that is different between the two sets of tokens
+          first_different_position = current_tokens.zip(previous_tokens).find_index { |new, old| new != old }
+
+          # When deleting a token from the end, the first_different_position will be nil, but since we're removing at
+          # the end, then we have to initialize it to the length of the current tokens after the deletion
+          if !first_different_position && current_tokens.length < previous_tokens.length
+            first_different_position = current_tokens.length
+          end
+
+          unless first_different_position
+            return Interface::SemanticTokensDelta.new(result_id: result_id, edits: [])
+          end
+
+          # Filter the tokens based on the first different position. This must happen at this stage, before we try to
+          # find the next position from the end or else we risk confusing sets of token that may have different lengths,
+          # but end with the exact same token
+          old_tokens = T.must(previous_tokens[first_different_position...])
+          new_tokens = T.must(current_tokens[first_different_position...])
+
+          # Then search from the end to find the first token that doesn't match. Since the user is normally editing the
+          # middle of the file, this will minimize the number of edits since the end of the token array has not changed
+          first_different_token_from_end = new_tokens.reverse.zip(old_tokens.reverse).find_index do |new, old|
+            new != old
+          end || 0
+
+          # Filter the old and new tokens to only the section that will be replaced/inserted/deleted
+          old_tokens = T.must(old_tokens[...old_tokens.length - first_different_token_from_end])
+          new_tokens = T.must(new_tokens[...new_tokens.length - first_different_token_from_end])
+
+          # And we send back a single edit, replacing an entire section for the new tokens
+          Interface::SemanticTokensDelta.new(
+            result_id: result_id,
+            edits: [{ start: first_different_position, deleteCount: old_tokens.length, data: new_tokens }],
+          )
+        end
+
+        sig { returns(Integer) }
+        def next_result_id
+          @mutex.synchronize do
+            @result_id += 1
+          end
+        end
       end
 
-      sig { params(global_state: GlobalState, dispatcher: Prism::Dispatcher, range: T.nilable(T::Range[Integer])).void }
-      def initialize(global_state, dispatcher, range: nil)
+      @result_id = T.let(0, Integer)
+      @mutex = T.let(Mutex.new, Mutex)
+
+      sig do
+        params(
+          global_state: GlobalState,
+          dispatcher: Prism::Dispatcher,
+          document: T.any(RubyDocument, ERBDocument),
+          previous_result_id: T.nilable(String),
+          range: T.nilable(T::Range[Integer]),
+        ).void
+      end
+      def initialize(global_state, dispatcher, document, previous_result_id, range: nil)
         super()
+
+        @document = document
+        @previous_result_id = previous_result_id
+        @range = range
+        @result_id = T.let(SemanticHighlighting.next_result_id.to_s, String)
         @response_builder = T.let(
           ResponseBuilders::SemanticHighlighting.new(global_state.encoding),
           ResponseBuilders::SemanticHighlighting,
         )
-        Listeners::SemanticHighlighting.new(dispatcher, @response_builder, range: range)
+        Listeners::SemanticHighlighting.new(dispatcher, @response_builder)
 
         Addon.addons.each do |addon|
           addon.create_semantic_highlighting_listener(@response_builder, dispatcher)
         end
       end
 
-      sig { override.returns(Interface::SemanticTokens) }
+      sig { override.returns(T.any(Interface::SemanticTokens, Interface::SemanticTokensDelta)) }
       def perform
-        @response_builder.response
+        previous_tokens = @document.semantic_tokens
+        tokens = @response_builder.response
+        encoded_tokens = ResponseBuilders::SemanticHighlighting::SemanticTokenEncoder.new.encode(tokens)
+        full_response = Interface::SemanticTokens.new(result_id: @result_id, data: encoded_tokens)
+        @document.semantic_tokens = full_response
+
+        if @range
+          tokens_within_range = tokens.select { |token| @range.cover?(token.start_line - 1) }
+
+          return Interface::SemanticTokens.new(
+            result_id: @result_id,
+            data: ResponseBuilders::SemanticHighlighting::SemanticTokenEncoder.new.encode(tokens_within_range),
+          )
+        end
+
+        # Semantic tokens full delta
+        if @previous_result_id
+          response = if previous_tokens.is_a?(Interface::SemanticTokens) &&
+              previous_tokens.result_id == @previous_result_id
+            Requests::SemanticHighlighting.compute_delta(encoded_tokens, previous_tokens.data, @result_id)
+          else
+            full_response
+          end
+
+          return response
+        end
+
+        # Semantic tokens full
+        full_response
       end
     end
   end
diff --git a/lib/ruby_lsp/requests/support/common.rb b/lib/ruby_lsp/requests/support/common.rb
index 7d8c894ff..7a86b91fd 100644
--- a/lib/ruby_lsp/requests/support/common.rb
+++ b/lib/ruby_lsp/requests/support/common.rb
@@ -37,15 +37,6 @@ def range_from_location(location)
           )
         end
 
-        sig { params(node: T.nilable(Prism::Node), range: T.nilable(T::Range[Integer])).returns(T::Boolean) }
-        def visible?(node, range)
-          return true if range.nil?
-          return false if node.nil?
-
-          loc = node.location
-          range.cover?(loc.start_line - 1) && range.cover?(loc.end_line - 1)
-        end
-
         sig do
           params(
             node: Prism::Node,
diff --git a/lib/ruby_lsp/response_builders/semantic_highlighting.rb b/lib/ruby_lsp/response_builders/semantic_highlighting.rb
index 3d83720d2..c7354464b 100644
--- a/lib/ruby_lsp/response_builders/semantic_highlighting.rb
+++ b/lib/ruby_lsp/response_builders/semantic_highlighting.rb
@@ -91,9 +91,9 @@ def last
         @stack.last
       end
 
-      sig { override.returns(Interface::SemanticTokens) }
+      sig { override.returns(T::Array[SemanticToken]) }
       def response
-        SemanticTokenEncoder.new.encode(@stack)
+        @stack
       end
 
       class SemanticToken
@@ -162,7 +162,7 @@ def initialize
         sig do
           params(
             tokens: T::Array[SemanticToken],
-          ).returns(Interface::SemanticTokens)
+          ).returns(T::Array[Integer])
         end
         def encode(tokens)
           sorted_tokens = tokens.sort_by.with_index do |token, index|
@@ -176,7 +176,7 @@ def encode(tokens)
             compute_delta(token)
           end
 
-          Interface::SemanticTokens.new(data: delta)
+          delta
         end
 
         # The delta array is computed according to the LSP specification:
diff --git a/lib/ruby_lsp/server.rb b/lib/ruby_lsp/server.rb
index bcb891342..57f6e7565 100644
--- a/lib/ruby_lsp/server.rb
+++ b/lib/ruby_lsp/server.rb
@@ -23,6 +23,7 @@ def process_message(message)
         run_initialize(message)
       when "initialized"
         send_log_message("Finished initializing Ruby LSP!") unless @test_mode
+
         run_initialized
       when "textDocument/didOpen"
         text_document_did_open(message)
@@ -40,6 +41,8 @@ def process_message(message)
         text_document_code_lens(message)
       when "textDocument/semanticTokens/full"
         text_document_semantic_tokens_full(message)
+      when "textDocument/semanticTokens/full/delta"
+        text_document_semantic_tokens_delta(message)
       when "textDocument/foldingRange"
         text_document_folding_range(message)
       when "textDocument/semanticTokens/range"
@@ -378,7 +381,7 @@ def run_combined_requests(message)
 
       # If the response has already been cached by another request, return it
       cached_response = document.cache_get(message[:method])
-      if cached_response
+      if cached_response != Document::EMPTY_CACHE
         send_message(Result.new(id: message[:id], response: cached_response))
         return
       end
@@ -391,8 +394,6 @@ def run_combined_requests(message)
       document_symbol = Requests::DocumentSymbol.new(uri, dispatcher)
       document_link = Requests::DocumentLink.new(uri, parse_result.comments, dispatcher)
       code_lens = Requests::CodeLens.new(@global_state, uri, dispatcher)
-
-      semantic_highlighting = Requests::SemanticHighlighting.new(@global_state, dispatcher)
       dispatcher.dispatch(parse_result.value)
 
       # Store all responses retrieve in this round of visits in the cache and then return the response for the request
@@ -401,19 +402,51 @@ def run_combined_requests(message)
       document.cache_set("textDocument/documentSymbol", document_symbol.perform)
       document.cache_set("textDocument/documentLink", document_link.perform)
       document.cache_set("textDocument/codeLens", code_lens.perform)
-      document.cache_set(
-        "textDocument/semanticTokens/full",
-        semantic_highlighting.perform,
-      )
+
       send_message(Result.new(id: message[:id], response: document.cache_get(message[:method])))
     end
 
     alias_method :text_document_document_symbol, :run_combined_requests
     alias_method :text_document_document_link, :run_combined_requests
     alias_method :text_document_code_lens, :run_combined_requests
-    alias_method :text_document_semantic_tokens_full, :run_combined_requests
     alias_method :text_document_folding_range, :run_combined_requests
 
+    sig { params(message: T::Hash[Symbol, T.untyped]).void }
+    def text_document_semantic_tokens_full(message)
+      document = @store.get(message.dig(:params, :textDocument, :uri))
+
+      unless document.is_a?(RubyDocument) || document.is_a?(ERBDocument)
+        send_empty_response(message[:id])
+        return
+      end
+
+      dispatcher = Prism::Dispatcher.new
+      semantic_highlighting = Requests::SemanticHighlighting.new(@global_state, dispatcher, document, nil)
+      dispatcher.visit(document.parse_result.value)
+
+      send_message(Result.new(id: message[:id], response: semantic_highlighting.perform))
+    end
+
+    sig { params(message: T::Hash[Symbol, T.untyped]).void }
+    def text_document_semantic_tokens_delta(message)
+      document = @store.get(message.dig(:params, :textDocument, :uri))
+
+      unless document.is_a?(RubyDocument) || document.is_a?(ERBDocument)
+        send_empty_response(message[:id])
+        return
+      end
+
+      dispatcher = Prism::Dispatcher.new
+      request = Requests::SemanticHighlighting.new(
+        @global_state,
+        dispatcher,
+        document,
+        message.dig(:params, :previousResultId),
+      )
+      dispatcher.visit(document.parse_result.value)
+      send_message(Result.new(id: message[:id], response: request.perform))
+    end
+
     sig { params(message: T::Hash[Symbol, T.untyped]).void }
     def text_document_semantic_tokens_range(message)
       params = message[:params]
@@ -426,15 +459,16 @@ def text_document_semantic_tokens_range(message)
         return
       end
 
-      start_line = range.dig(:start, :line)
-      end_line = range.dig(:end, :line)
-
       dispatcher = Prism::Dispatcher.new
-      request = Requests::SemanticHighlighting.new(@global_state, dispatcher, range: start_line..end_line)
+      request = Requests::SemanticHighlighting.new(
+        @global_state,
+        dispatcher,
+        document,
+        nil,
+        range: range.dig(:start, :line)..range.dig(:end, :line),
+      )
       dispatcher.visit(document.parse_result.value)
-
-      response = request.perform
-      send_message(Result.new(id: message[:id], response: response))
+      send_message(Result.new(id: message[:id], response: request.perform))
     end
 
     sig { params(message: T::Hash[Symbol, T.untyped]).void }
diff --git a/test/requests/semantic_highlighting_expectations_test.rb b/test/requests/semantic_highlighting_expectations_test.rb
index 8b4a87622..6909c4bd0 100644
--- a/test/requests/semantic_highlighting_expectations_test.rb
+++ b/test/requests/semantic_highlighting_expectations_test.rb
@@ -20,7 +20,13 @@ def run_expectations(source)
     dispatcher = Prism::Dispatcher.new
     global_state = RubyLsp::GlobalState.new
     global_state.apply_options({})
-    listener = RubyLsp::Requests::SemanticHighlighting.new(global_state, dispatcher, range: processed_range)
+    listener = RubyLsp::Requests::SemanticHighlighting.new(
+      global_state,
+      dispatcher,
+      document,
+      nil,
+      range: processed_range,
+    )
 
     dispatcher.dispatch(document.parse_result.value)
     listener.perform
diff --git a/test/requests/semantic_tokens_delta_test.rb b/test/requests/semantic_tokens_delta_test.rb
new file mode 100644
index 000000000..2005b5524
--- /dev/null
+++ b/test/requests/semantic_tokens_delta_test.rb
@@ -0,0 +1,319 @@
+# typed: true
+# frozen_string_literal: true
+
+require "test_helper"
+
+class SemanticTokensDeltaTest < Minitest::Test
+  def test_inserting_something_at_the_end
+    assert_expected_token_result(
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100, 200, 300, 400, 500],
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+    )
+  end
+
+  def test_inserting_something_in_the_beginning
+    assert_expected_token_result(
+      [10, 100, 200, 300, 400, 500, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+    )
+  end
+
+  def test_inserting_something_in_the_middle
+    assert_expected_token_result(
+      [1, 2, 3, 4, 5, 10, 100, 200, 300, 400, 500, 6, 7, 8, 9, 10],
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+    )
+  end
+
+  def test_deleting_at_the_end
+    assert_expected_token_result(
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 100, 200, 300, 400, 500],
+    )
+  end
+
+  def test_deleting_at_the_beginning
+    assert_expected_token_result(
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+      [10, 100, 200, 300, 400, 500, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+    )
+  end
+
+  def test_deleting_in_the_middle
+    assert_expected_token_result(
+      [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+      [1, 2, 3, 4, 5, 10, 100, 200, 300, 400, 500, 6, 7, 8, 9, 10],
+    )
+  end
+
+  # The scenarios below were captured with the responses from the language server automatically, so that we can have a
+  # few more real case tests. There are confounding aspects to these examples, like having the exact same semantic token
+  # in arrays with different lengths
+
+  def test_computing_deltas_1
+    assert_expected_token_result(
+      [3, 4, 3, 13, 1, 1, 2, 1, 8, 0, 1, 0, 1, 13, 0, 1, 2, 9, 13, 0, 1, 2, 1, 8, 0],
+      [3, 4, 3, 13, 1, 1, 2, 1, 8, 0, 2, 2, 9, 13, 0, 1, 2, 1, 8, 0],
+    )
+  end
+
+  def test_computing_deltas_2
+    assert_expected_token_result(
+      [
+        3,
+        4,
+        21,
+        13,
+        1,
+        1,
+        2,
+        12,
+        13,
+        0,
+        0,
+        73,
+        5,
+        13,
+        0,
+        2,
+        2,
+        5,
+        8,
+        0,
+        0,
+        8,
+        7,
+        0,
+        0,
+        0,
+        9,
+        8,
+        0,
+        0,
+        0,
+        10,
+        20,
+        0,
+        0,
+        0,
+        21,
+        13,
+        13,
+        0,
+        4,
+        4,
+        5,
+        13,
+        0,
+        2,
+        0,
+        1,
+        8,
+        0,
+        1,
+        0,
+        1,
+        8,
+        0,
+        2,
+        2,
+        12,
+        13,
+        0,
+        0,
+        73,
+        5,
+        8,
+        0,
+      ],
+      [
+        3,
+        4,
+        21,
+        13,
+        1,
+        1,
+        2,
+        12,
+        13,
+        0,
+        0,
+        73,
+        5,
+        13,
+        0,
+        2,
+        2,
+        5,
+        8,
+        0,
+        0,
+        8,
+        7,
+        0,
+        0,
+        0,
+        9,
+        8,
+        0,
+        0,
+        0,
+        10,
+        20,
+        0,
+        0,
+        0,
+        21,
+        13,
+        13,
+        0,
+        4,
+        4,
+        5,
+        13,
+        0,
+        2,
+        0,
+        1,
+        8,
+        0,
+        3,
+        2,
+        12,
+        13,
+        0,
+        0,
+        73,
+        5,
+        8,
+        0,
+      ],
+    )
+  end
+
+  def test_computing_deltas_3
+    assert_expected_token_result(
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0],
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1],
+    )
+  end
+
+  def test_computing_delta_4
+    assert_expected_token_result(
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0, 1, 4, 1, 8, 0],
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0],
+    )
+  end
+
+  def test_computing_delta_5
+    assert_expected_token_result(
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0, 1, 4, 1, 8, 0, 0, 2, 4, 13, 0],
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0, 1, 4, 1, 8, 0, 0, 2, 4, 13, 0, 1, 4, 1, 8, 0],
+    )
+  end
+
+  def test_computing_delta_6
+    assert_expected_token_result(
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0, 1, 4, 1, 8, 0, 0, 2, 4, 13, 0],
+      [3, 6, 3, 2, 1, 0, 0, 3, 0, 0, 1, 6, 4, 13, 1, 1, 4, 1, 8, 0, 1, 4, 1, 8, 0, 1, 4, 1, 8, 0, 0, 2, 4, 13, 0],
+    )
+  end
+
+  def test_computing_delta_7
+    assert_expected_token_result(
+      [
+        3,
+        6,
+        3,
+        2,
+        1,
+        0,
+        0,
+        3,
+        0,
+        0,
+        1,
+        6,
+        4,
+        13,
+        1,
+        1,
+        4,
+        1,
+        8,
+        0,
+        1,
+        4,
+        1,
+        8,
+        0,
+        0,
+        2,
+        2,
+        13,
+        0,
+        1,
+        4,
+        1,
+        8,
+        0,
+        0,
+        2,
+        4,
+        13,
+        0,
+      ],
+      [
+        3,
+        6,
+        3,
+        2,
+        1,
+        0,
+        0,
+        3,
+        0,
+        0,
+        1,
+        6,
+        4,
+        13,
+        1,
+        1,
+        4,
+        1,
+        8,
+        0,
+        1,
+        4,
+        1,
+        8,
+        0,
+        0,
+        2,
+        1,
+        13,
+        0,
+        1,
+        4,
+        1,
+        8,
+        0,
+        0,
+        2,
+        4,
+        13,
+        0,
+      ],
+    )
+  end
+
+  private
+
+  def assert_expected_token_result(current_tokens, previous_tokens)
+    edit = RubyLsp::Requests::SemanticHighlighting.compute_delta(current_tokens, previous_tokens, "1").edits.first
+
+    previous_tokens[edit[:start]...(edit[:start] + edit[:deleteCount])] = edit[:data]
+    assert_equal(current_tokens, previous_tokens)
+  end
+end
diff --git a/test/requests/support/semantic_token_encoder_test.rb b/test/requests/support/semantic_token_encoder_test.rb
index 983497170..4a22a5860 100644
--- a/test/requests/support/semantic_token_encoder_test.rb
+++ b/test/requests/support/semantic_token_encoder_test.rb
@@ -38,7 +38,7 @@ def test_tokens_encoded_to_relative_positioning
 
     assert_equal(
       expected_encoding,
-      SemanticTokenEncoder.new.encode(tokens).data,
+      SemanticTokenEncoder.new.encode(tokens),
     )
   end
 
@@ -73,7 +73,7 @@ def test_tokens_sorted_before_encoded
       16,
     ]
 
-    assert_equal(expected_encoding, SemanticTokenEncoder.new.encode(tokens).data)
+    assert_equal(expected_encoding, SemanticTokenEncoder.new.encode(tokens))
   end
 
   def test_encoded_modifiers_with_no_modifiers
diff --git a/test/ruby_document_test.rb b/test/ruby_document_test.rb
index 862ae63e4..7e50b6a08 100644
--- a/test/ruby_document_test.rb
+++ b/test/ruby_document_test.rb
@@ -739,6 +739,17 @@ def test_locate_first_within_range
     assert_equal("each", T.cast(target, Prism::CallNode).message)
   end
 
+  def test_uncached_requests_return_empty_cache_object
+    document = RubyLsp::RubyDocument.new(source: +<<~RUBY, version: 1, uri: URI("file:///foo/bar.rb"))
+      class Foo
+      end
+    RUBY
+
+    assert_same(document.cache_get("textDocument/codeLens"), RubyLsp::Document::EMPTY_CACHE)
+    document.cache_set("textDocument/codeLens", nil)
+    assert_nil(document.cache_get("textDocument/codeLens"))
+  end
+
   private
 
   def assert_error_edit(actual, error_range)