refactor: Use semanticdb and move everything to Indexed symbol class

metals-bench/src/main/scala/bench/ClasspathSymbolsBench.scala

@@ -5,7 +5,6 @@ import java.util.concurrent.TimeUnit
 import scala.meta.dialects
 import scala.meta.internal.metals.EmptyReportContext
 import scala.meta.internal.metals.MetalsEnrichments._
-import scala.meta.internal.mtags.OnDemandSymbolIndex
 import scala.meta.internal.tvp.IndexedSymbols
 import scala.meta.io.AbsolutePath
 
@@ -37,8 +36,7 @@ class ClasspathSymbolsBench {
   def run(): Unit = {
     implicit val reporting = EmptyReportContext
     val jars = new IndexedSymbols(
-      OnDemandSymbolIndex.empty(),
-      isStatisticsEnabled = false,
+      isStatisticsEnabled = false
     )
     classpath.foreach { jar =>
       jars.jarSymbols(jar, "cats/", dialects.Scala213)

metals/src/main/scala/scala/meta/internal/metals/MetalsLspService.scala

@@ -575,12 +575,17 @@ class MetalsLspService(
       trees,
     )
 
+  val classpathTreeIndex = new IndexedSymbols(
+    isStatisticsEnabled = clientConfig.initialConfig.statistics.isTreeView
+  )
+
   private val semanticDBIndexer: SemanticdbIndexer = new SemanticdbIndexer(
     List(
       referencesProvider,
       implementationProvider,
       syntheticsDecorator,
       testProvider,
+      classpathTreeIndex,
     ),
     buildTargets,
     folder,
@@ -806,9 +811,9 @@ class MetalsLspService(
       buildTargets,
       () => buildClient.ongoingCompilations(),
       definitionIndex,
-      clientConfig.initialConfig.statistics,
       optJavaHome,
       scalaVersionSelector,
+      classpathTreeIndex,
     )
 
   private val popupChoiceReset: PopupChoiceReset = new PopupChoiceReset(

metals/src/main/scala/scala/meta/internal/tvp/IndexedSymbols.scala

@@ -1,36 +1,70 @@
 package scala.meta.internal.tvp
 
+import java.io.UncheckedIOException
+
 import scala.collection.concurrent.TrieMap
 
 import scala.meta.Dialect
+import scala.meta.internal.io.FileIO
 import scala.meta.internal.metals.MetalsEnrichments._
+import scala.meta.internal.metals.ReportContext
+import scala.meta.internal.metals.SemanticdbFeatureProvider
 import scala.meta.internal.metals.Time
 import scala.meta.internal.metals.Timer
-import scala.meta.internal.mtags.GlobalSymbolIndex
+import scala.meta.internal.mtags.Mtags
+import scala.meta.internal.mtags.SemanticdbPath
 import scala.meta.internal.mtags.Symbol
 import scala.meta.internal.mtags.SymbolDefinition
 import scala.meta.internal.semanticdb.SymbolInformation
+import scala.meta.internal.semanticdb.TextDocuments
 import scala.meta.io.AbsolutePath
 
-class IndexedSymbols(index: GlobalSymbolIndex, isStatisticsEnabled: Boolean) {
+class IndexedSymbols(isStatisticsEnabled: Boolean)(implicit rc: ReportContext)
+    extends SemanticdbFeatureProvider {
 
+  private val mtags = new Mtags()
   // Used for workspace, is eager
   private val workspaceCache = TrieMap.empty[
     AbsolutePath,
-    Array[TreeViewSymbolInformation],
+    AllSymbols,
   ]
 
   type TopLevel = SymbolDefinition
+  type ToplevelSymbol = String
   type AllSymbols = Array[TreeViewSymbolInformation]
-  // Used for dependencies, is lazy, TopLevel is changed to AllSymbols when needed
+
+  /* Used for dependencies lazily calculates all symbols in a jar.
+   * At the start it only contains the definition of a toplevel Symbol, later
+   * resolves to information about all symbols contained in the top level.
+   */
   private val jarCache = TrieMap.empty[
     AbsolutePath,
-    TrieMap[String, Either[TopLevel, AllSymbols]],
+    TrieMap[ToplevelSymbol, Either[TopLevel, AllSymbols]],
   ]
 
-  def clearCache(path: AbsolutePath): Unit = {
-    jarCache.remove(path)
-    workspaceCache.remove(path)
+  // TODO still not refreshing on changes
+  override def onChange(docs: TextDocuments, path: AbsolutePath): Unit = {
+    // pprint.log(path)
+    val all = docs.documents.flatMap { doc =>
+      val existing = doc.occurrences.collect {
+        case occ if occ.role.isDefinition => occ.symbol
+      }.toSet
+      doc.symbols.distinct.collect {
+        case info if existing(info.symbol) && !info.kind.isConstructor =>
+          TreeViewSymbolInformation(
+            info.symbol,
+            info.kind,
+            info.properties,
+          )
+      }
+    }.toList
+    workspaceCache.put(path, all.toArray)
+  }
+
+  override def onDelete(path: SemanticdbPath): Unit = {
+    // pprint.log(path)
+    // TODO path is actually text document path, so nothing is removed
+    workspaceCache.remove(path.absolutePath)
   }
 
   def reset(): Unit = {
@@ -57,13 +91,9 @@ class IndexedSymbols(index: GlobalSymbolIndex, isStatisticsEnabled: Boolean) {
   def workspaceSymbols(
       in: AbsolutePath,
       symbol: String,
-      dialect: Dialect,
   ): Iterator[TreeViewSymbolInformation] = withTimer(s"$in/!$symbol") {
     val syms = workspaceCache
-      .getOrElseUpdate(
-        in,
-        members(in, dialect).map(toTreeView),
-      )
+      .getOrElse(in, Array.empty)
     if (Symbol(symbol).isRootPackage) syms.iterator
     else
       syms.collect {
@@ -92,8 +122,7 @@ class IndexedSymbols(index: GlobalSymbolIndex, isStatisticsEnabled: Boolean) {
       val realIn = if (!in.isSourcesJar) potentialSourceJar else in
       val jarSymbols = jarCache.getOrElseUpdate(
         realIn, {
-          val toplevels = index
-            .toplevelsAt(in, dialect)
+          val toplevels = toplevelsAt(in, dialect)
             .map(defn => defn.definitionSymbol.value -> Left(defn))
 
           TrieMap.empty[
@@ -128,7 +157,10 @@ class IndexedSymbols(index: GlobalSymbolIndex, isStatisticsEnabled: Boolean) {
         jarSymbols.get(toplevelOwner(Symbol(symbol)).value) match {
           case Some(Left(toplevelOnly)) =>
             val allSymbols = members(toplevelOnly.path, dialect).map(toTreeView)
-            jarSymbols.put(symbol, Right(allSymbols))
+            jarSymbols.put(
+              toplevelOnly.definitionSymbol.value,
+              Right(allSymbols),
+            )
             allSymbols.iterator
           case Some(Right(calculated)) =>
             calculated.iterator
@@ -167,8 +199,7 @@ class IndexedSymbols(index: GlobalSymbolIndex, isStatisticsEnabled: Boolean) {
       path: AbsolutePath,
       dialect: Dialect,
   ): Array[SymbolDefinition] = {
-    index
-      .symbolsAt(path, dialect)
+    symbolsAt(path, dialect)
       .filter(defn =>
         defn.kind.isEmpty || !defn.kind.exists(kind =>
           kind.isParameter || kind.isTypeParameter
@@ -196,4 +227,62 @@ class IndexedSymbols(index: GlobalSymbolIndex, isStatisticsEnabled: Boolean) {
     )
   }
 
+  private def toplevelsAt(
+      path: AbsolutePath,
+      dialect: Dialect,
+  ): List[SymbolDefinition] = {
+
+    def indexJar(jar: AbsolutePath) = {
+      FileIO.withJarFileSystem(jar, create = false) { root =>
+        try {
+          root.listRecursive.toList.collect {
+            case source if source.isFile =>
+              (source, mtags.toplevels(source.toInput, dialect).symbols)
+          }
+        } catch {
+          // this happens in broken jars since file from FileWalker should exists
+          case _: UncheckedIOException => Nil
+        }
+      }
+    }
+
+    val pathSymbolInfos = if (path.isSourcesJar) {
+      indexJar(path)
+    } else {
+      List((path, mtags.toplevels(path.toInput, dialect).symbols))
+    }
+    pathSymbolInfos.collect { case (path, infos) =>
+      infos.map { info =>
+        SymbolDefinition(
+          Symbol("_empty_"),
+          Symbol(info.symbol),
+          path,
+          dialect,
+          None,
+          Some(info.kind),
+          info.properties,
+        )
+      }
+    }.flatten
+  }
+
+  private def symbolsAt(
+      path: AbsolutePath,
+      dialect: Dialect,
+  ): List[SymbolDefinition] = {
+    val document = mtags.allSymbols(path, dialect)
+    document.symbols.map { info =>
+      SymbolDefinition(
+        Symbol("_empty_"),
+        Symbol(info.symbol),
+        path,
+        dialect,
+        None,
+        Some(info.kind),
+        info.properties,
+      )
+    }.toList
+
+  }
+
 }

metals/src/main/scala/scala/meta/internal/tvp/MetalsTreeViewProvider.scala

@@ -253,20 +253,16 @@ class FolderTreeViewProvider(
     buildTargets: BuildTargets,
     compilations: () => TreeViewCompilations,
     definitionIndex: GlobalSymbolIndex,
-    statistics: StatisticsConfig,
     userJavaHome: Option[AbsolutePath],
     scalaVersionSelector: ScalaVersionSelector,
+    classpath: IndexedSymbols,
 ) {
   def dialectOf(path: AbsolutePath): Option[Dialect] =
     scalaVersionSelector.dialectFromBuildTarget(path)
   private val maybeUsedJdkVersion =
     userJavaHome.flatMap { path =>
       JdkVersion.fromReleaseFileString(path)
     }
-  private val classpath = new IndexedSymbols(
-    definitionIndex,
-    isStatisticsEnabled = statistics.isTreeView,
-  )
   private val isVisible = TrieMap.empty[String, Boolean].withDefaultValue(false)
   private val isCollapsed = TrieMap.empty[BuildTargetIdentifier, Boolean]
   private val pendingProjectUpdates =
@@ -317,7 +313,7 @@ class FolderTreeViewProvider(
         scalaTarget <- buildTargets.scalaTarget(id).iterator
         source <- buildTargets.buildTargetSources(id)
         dialect = scalaTarget.dialect(source)
-      } yield classpath.workspaceSymbols(source, symbol, dialect)
+      } yield classpath.workspaceSymbols(source, symbol)
       tops.flatten
     },
   )
@@ -345,9 +341,6 @@ class FolderTreeViewProvider(
   def onBuildTargetDidCompile(
       id: BuildTargetIdentifier
   ): Option[Array[TreeViewNode]] = {
-    buildTargets
-      .targetClassDirectories(id)
-      .foreach(cd => classpath.clearCache(cd.toAbsolutePath))
     if (isCollapsed.contains(id)) {
       pendingProjectUpdates.add(id)
       flushPendingProjectUpdates()

mtags/src/main/scala/scala/meta/internal/mtags/CharArrayReader.scala

@@ -5,7 +5,7 @@ import scala.meta.inputs._
 import scala.meta.internal.tokenizers.Chars._
 import scala.meta.internal.tokenizers.Reporter
 
-private[mtags] case class CharArrayReader private (
+private[meta] case class CharArrayReader private (
     buf: Array[Char],
     dialect: Dialect,
     reporter: Reporter,
@@ -61,16 +61,18 @@ private[mtags] case class CharArrayReader private (
     } else {
       begCharOffset = endCharOffset
       val (hi, hiEnd) = readUnicodeChar(endCharOffset)
+
       if (!Character.isHighSurrogate(hi)) {
         ch = hi
         endCharOffset = hiEnd
       } else if (hiEnd >= buf.length)
         readerError("invalid unicode surrogate pair", at = begCharOffset)
       else {
         val (lo, loEnd) = readUnicodeChar(hiEnd)
-        if (!Character.isLowSurrogate(lo))
-          readerError("invalid unicode surrogate pair", at = begCharOffset)
-        else {
+        if (!Character.isLowSurrogate(lo)) {
+          ch = hi
+          endCharOffset = hiEnd
+        } else {
           ch = Character.toCodePoint(hi, lo)
           endCharOffset = loEnd
         }

mtags/src/main/scala/scala/meta/internal/mtags/GlobalSymbolIndex.scala

@@ -37,16 +37,6 @@ trait GlobalSymbolIndex {
 
   def definitions(symbol: mtags.Symbol): List[SymbolDefinition]
 
-  def toplevelsAt(
-      path: AbsolutePath,
-      dialect: Dialect
-  ): List[SymbolDefinition] = Nil
-
-  def symbolsAt(
-      path: AbsolutePath,
-      dialect: Dialect
-  ): List[SymbolDefinition] = Nil
-
   /**
    * Add an individual Java or Scala source file to the index.
    *

mtags/src/main/scala/scala/meta/internal/mtags/Mtags.scala

@@ -9,11 +9,23 @@ import scala.meta.internal.mtags.ScalametaCommonEnrichments._
 import scala.meta.internal.semanticdb.Language
 import scala.meta.internal.semanticdb.Scala._
 import scala.meta.internal.semanticdb.TextDocument
+import scala.meta.io.AbsolutePath
 
 final class Mtags(implicit rc: ReportContext) {
   def totalLinesOfCode: Long = javaLines + scalaLines
   def totalLinesOfScala: Long = scalaLines
   def totalLinesOfJava: Long = javaLines
+
+  def allSymbols(path: AbsolutePath, dialect: Dialect): TextDocument = {
+    val language = path.toLanguage
+    val input = path.toInput
+
+    Mtags.stdLibPatches.patchDocument(
+      path,
+      index(language, input, dialect)
+    )
+  }
+
   def toplevels(
       input: Input.VirtualFile,
       dialect: Dialect = dialects.Scala213
@@ -126,4 +138,37 @@ object Mtags {
     new Mtags().topLevelSymbols(input, dialect)
   }
 
+  /**
+   * Scala 3 has a specific package that adds / replaces some symbols in scala.Predef + scala.language
+   * https://github.com/lampepfl/dotty/blob/main/library/src/scala/runtime/stdLibPatches/
+   * We need to do the same to correctly provide location for symbols obtained from semanticdb.
+   */
+  object stdLibPatches {
+    val packageName = "scala/runtime/stdLibPatches"
+
+    def isScala3Library(jar: AbsolutePath): Boolean =
+      jar.filename.startsWith("scala3-library_3")
+
+    def isScala3LibraryPatchSource(file: AbsolutePath): Boolean = {
+      file.jarPath.exists(
+        isScala3Library(_)
+      ) && file.parent.filename == "stdLibPatches"
+    }
+
+    def patchSymbol(sym: String): String =
+      sym.replace(packageName, "scala")
+
+    def patchDocument(
+        file: AbsolutePath,
+        doc: TextDocument
+    ): TextDocument = {
+      if (isScala3LibraryPatchSource(file)) {
+        val occs =
+          doc.occurrences.map(occ => occ.copy(symbol = patchSymbol(occ.symbol)))
+
+        doc.copy(occurrences = occs)
+      } else doc
+    }
+
+  }
 }