Merge pull request #112 from Shopify/introduce-token-modifiers

Implement token modifiers in SemanticTokenEncoder
Shopify · Jun 7, 2022 · 9debb27 · 9debb27
2 parents b4fc8ef + e2c3eab
commit 9debb27
Show file tree

Hide file tree

Showing 4 changed files with 59 additions and 21 deletions.
diff --git a/lib/ruby_lsp/handler.rb b/lib/ruby_lsp/handler.rb
@@ -85,7 +85,7 @@ def respond_with_capabilities(enabled_features)
           document_selector: { scheme: "file", language: "ruby" },
           legend: Interface::SemanticTokensLegend.new(
             token_types: Requests::SemanticHighlighting::TOKEN_TYPES,
-            token_modifiers: Requests::SemanticHighlighting::TOKEN_MODIFIERS
+            token_modifiers: Requests::SemanticHighlighting::TOKEN_MODIFIERS.keys
           ),
           range: false,
           full: {

diff --git a/lib/ruby_lsp/requests/semantic_highlighting.rb b/lib/ruby_lsp/requests/semantic_highlighting.rb
@@ -21,7 +21,19 @@ class SemanticHighlighting < BaseRequest
         :variable,
         :method,
       ].freeze
-      TOKEN_MODIFIERS = [].freeze
+
+      TOKEN_MODIFIERS = {
+        declaration: 0,
+        definition: 1,
+        readonly: 2,
+        static: 3,
+        deprecated: 4,
+        abstract: 5,
+        async: 6,
+        modification: 7,
+        documentation: 8,
+        default_library: 9,
+      }.freeze
 
       SemanticToken = Struct.new(:location, :length, :type, :modifier)
 
@@ -90,9 +102,10 @@ def visit_vcall(node)
         add_token(node.value.location, :method)
       end
 
-      def add_token(location, type)
+      def add_token(location, type, modifiers = [])
         length = location.end_char - location.start_char
-        @tokens.push(SemanticToken.new(location, length, TOKEN_TYPES.index(type), 0))
+        modifiers_indices = modifiers.filter_map { |modifier| TOKEN_MODIFIERS[modifier] }
+        @tokens.push(SemanticToken.new(location, length, TOKEN_TYPES.index(type), modifiers_indices))
       end
     end
   end

diff --git a/lib/ruby_lsp/requests/support/semantic_token_encoder.rb b/lib/ruby_lsp/requests/support/semantic_token_encoder.rb
@@ -39,11 +39,21 @@ def compute_delta(token)
           delta_column = column
           delta_column -= @current_column if delta_line == 0
 
-          [delta_line, delta_column, token.length, token.type, token.modifier]
+          [delta_line, delta_column, token.length, token.type, encode_modifiers(token.modifier)]
         ensure
           @current_row = row
           @current_column = column
         end
+
+        # Encode an array of modifiers to positions onto a bit flag
+        # For example, [:default_library] will be encoded as
+        # 0b1000000000, as :default_library is the 10th bit according
+        # to the token modifiers index map.
+        def encode_modifiers(modifiers)
+          modifiers.inject(0) do |encoded_modifiers, modifier|
+            encoded_modifiers | (1 << modifier)
+          end
+        end
       end
     end
   end

diff --git a/test/requests/support/semantic_token_encoder_test.rb b/test/requests/support/semantic_token_encoder_test.rb
@@ -9,17 +9,17 @@ class SemanticTokenEncoderTest < Minitest::Test
 
   def test_tokens_encoded_to_relative_positioning
     tokens = [
-      TokenStub.new(LocationStub.new(1, 2), 1, 0, 0),
-      TokenStub.new(LocationStub.new(1, 4), 2, 9, 0),
-      TokenStub.new(LocationStub.new(2, 2), 3, 0, 6),
-      TokenStub.new(LocationStub.new(5, 6), 10, 4, 4),
+      TokenStub.new(LocationStub.new(1, 2), 1, 0, [0]),
+      TokenStub.new(LocationStub.new(1, 4), 2, 9, [0]),
+      TokenStub.new(LocationStub.new(2, 2), 3, 0, [6]),
+      TokenStub.new(LocationStub.new(5, 6), 10, 4, [4]),
     ]
 
     expected_encoding = [
-      0, 2, 1, 0, 0,
-      0, 2, 2, 9, 0,
-      1, 2, 3, 0, 6,
-      3, 6, 10, 4, 4,
+      0, 2, 1, 0, 1,
+      0, 2, 2, 9, 1,
+      1, 2, 3, 0, 64,
+      3, 6, 10, 4, 16,
     ]
 
     assert_equal(expected_encoding,
@@ -28,20 +28,35 @@ def test_tokens_encoded_to_relative_positioning
 
   def test_tokens_sorted_before_encoded
     tokens = [
-      TokenStub.new(LocationStub.new(1, 2), 1, 0, 0),
-      TokenStub.new(LocationStub.new(5, 6), 10, 4, 4),
-      TokenStub.new(LocationStub.new(2, 2), 3, 0, 6),
-      TokenStub.new(LocationStub.new(1, 4), 2, 9, 0),
+      TokenStub.new(LocationStub.new(1, 2), 1, 0, [0]),
+      TokenStub.new(LocationStub.new(5, 6), 10, 4, [4]),
+      TokenStub.new(LocationStub.new(2, 2), 3, 0, [6]),
+      TokenStub.new(LocationStub.new(1, 4), 2, 9, [0]),
     ]
 
     expected_encoding = [
-      0, 2, 1, 0, 0,
-      0, 2, 2, 9, 0,
-      1, 2, 3, 0, 6,
-      3, 6, 10, 4, 4,
+      0, 2, 1, 0, 1,
+      0, 2, 2, 9, 1,
+      1, 2, 3, 0, 64,
+      3, 6, 10, 4, 16,
     ]
 
     assert_equal(expected_encoding,
       RubyLsp::Requests::Support::SemanticTokenEncoder.new.encode(tokens).data)
   end
+
+  def test_encoded_modifiers_with_no_modifiers
+    bit_flag = RubyLsp::Requests::Support::SemanticTokenEncoder.new.encode_modifiers([])
+    assert_equal(0b0000000000, bit_flag)
+  end
+
+  def test_encoded_modifiers_with_one_modifier
+    bit_flag = RubyLsp::Requests::Support::SemanticTokenEncoder.new.encode_modifiers([9])
+    assert_equal(0b1000000000, bit_flag)
+  end
+
+  def test_encoded_modifiers_with_some_modifiers
+    bit_flag = RubyLsp::Requests::Support::SemanticTokenEncoder.new.encode_modifiers([1, 3, 9, 7, 5])
+    assert_equal(0b1010101010, bit_flag)
+  end
 end