Skip to content

Commit

Permalink
Ensure new, longer token data is always correctly applied
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattmassicotte
mattmassicotte committed Jun 5, 2023
1 parent 56febf6 commit dc438f7
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 9 deletions.
5 changes: 4 additions & 1 deletion Sources/LanguageServerProtocol/Additions/TokenRepresentation.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ public class TokenRepresentation {

/// Merge new token data with existing representation
///
/// - Warning: The returned range diffs are not currently calculated and will always be empty.
///
/// - Returns: Ranges affected by the new data (currently unimplemented).
public func applyData(_ newData: [UInt32]) -> [LSPRange] {
let minLength = min(data.count, newData.count)
Expand All @@ -41,7 +43,8 @@ public class TokenRepresentation {

let diffRange = diffStartIndex..<data.count

if diffRange.isEmpty && data.count > 0 {
// data is the same
if diffRange.isEmpty && data.count == newData.count {
return []
}

Expand Down
53 changes: 45 additions & 8 deletions Tests/LanguageServerProtocolTests/TokenRepresentationTests.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@ import XCTest
// }

final class TokenRepresentationTests: XCTestCase {
private static let tokenTypes = ["namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator"]
private static let tokenModifiers = ["declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary"]
private static let legend = SemanticTokensLegend(tokenTypes: tokenTypes, tokenModifiers: tokenModifiers)

func testDataDecode() {
let tokenTypes = ["namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", "event", "function", "method", "macro", "keyword", "modifier", "comment", "string", "number", "regexp", "operator"]
let tokenModifiers = ["declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary"]
let legend = SemanticTokensLegend(tokenTypes: tokenTypes, tokenModifiers: tokenModifiers)
let rep = TokenRepresentation(legend: legend)
let rep = TokenRepresentation(legend: Self.legend)

let codedData: [UInt32] = [0,0,7,15,0,0,8,4,0,0,2,0,6,15,0,0,8,3,0,0,1,0,6,15,0,0,8,3,0,0,2,0,4,15,0,0,5,4,12,2,1,4,3,0,0,0,4,7,12,0,0,8,13,18,0]
let codedData: [UInt32] = [0,0,7,15,0,0,8,4,0,0,2,0,6,15,0,0,8,3,0,0,1,0,6,15,0,0,8,3,0,0,2,0,4,15,0,0,5,4,12,2,1,1,3,0,0,0,4,7,12,0,0,8,13,18,0]

_ = rep.applyData(codedData)

Expand All @@ -35,9 +36,45 @@ final class TokenRepresentationTests: XCTestCase {
Token(range: LSPRange(startPair: (3, 8), endPair: (3, 11)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (5, 0), endPair: (5, 4)), tokenType: "keyword"),
Token(range: LSPRange(startPair: (5, 5), endPair: (5, 9)), tokenType: "function"),
Token(range: LSPRange(startPair: (6, 4), endPair: (6, 7)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (6, 8), endPair: (6, 15)), tokenType: "function"),
Token(range: LSPRange(startPair: (6, 16), endPair: (6, 29)), tokenType: "string"),
Token(range: LSPRange(startPair: (6, 1), endPair: (6, 4)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (6, 5), endPair: (6, 12)), tokenType: "function"),
Token(range: LSPRange(startPair: (6, 13), endPair: (6, 26)), tokenType: "string"),
]

XCTAssertEqual(tokens, expectedTokens)
}

func testApplyNewData() {
let rep = TokenRepresentation(legend: Self.legend)

let codedData: [UInt32] = [0,0,7,15,0,0,8,4,0,0,2,0,6,15,0,0,8,3,0,0,1,0,6,15,0,0,8,3,0,0,2,0,4,15,0,0,5,4,12,2,1,1,3,0,0,0,4,7,12,0,0,8,13,18,0]

// Represents duplicating the line " fmt.Println("hello world")" in the function body
let newCodedData: [UInt32] = [0,0,7,15,0,0,8,4,0,0,2,0,6,15,0,0,8,3,0,0,1,0,6,15,0,0,8,3,0,0,2,0,4,15,0,0,5,4,12,2,1,1,3,0,0,0,4,7,12,0,0,8,13,18,0,1,1,3,0,0,0,4,7,12,0,0,8,13,18,0]

_ = rep.applyData(codedData)
_ = rep.applyData(newCodedData)

let range = LSPRange(startPair: (0, 0), endPair: (8, 1))
let tokens = rep.decodeTokens(in: range)

let expectedTokens = [
Token(range: LSPRange(startPair: (0, 0), endPair: (0, 7)), tokenType: "keyword"),
Token(range: LSPRange(startPair: (0, 8), endPair: (0, 12)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (2, 0), endPair: (2, 6)), tokenType: "keyword"),
Token(range: LSPRange(startPair: (2, 8), endPair: (2, 11)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (3, 0), endPair: (3, 6)), tokenType: "keyword"),
Token(range: LSPRange(startPair: (3, 8), endPair: (3, 11)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (5, 0), endPair: (5, 4)), tokenType: "keyword"),
Token(range: LSPRange(startPair: (5, 5), endPair: (5, 9)), tokenType: "function"),
Token(range: LSPRange(startPair: (6, 1), endPair: (6, 4)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (6, 5), endPair: (6, 12)), tokenType: "function"),
Token(range: LSPRange(startPair: (6, 13), endPair: (6, 26)), tokenType: "string"),

// this should actually be added
Token(range: LSPRange(startPair: (7, 1), endPair: (7, 4)), tokenType: "namespace"),
Token(range: LSPRange(startPair: (7, 5), endPair: (7, 12)), tokenType: "function"),
Token(range: LSPRange(startPair: (7, 13), endPair: (7, 26)), tokenType: "string"),
]

XCTAssertEqual(tokens, expectedTokens)
Expand Down

0 comments on commit dc438f7

Please sign in to comment.