record end markers in trees and semanticdb, exclude top level def wrappers

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -182,6 +182,7 @@ object desugar {
         tpt     = TypeTree(defn.UnitType),
         rhs     = setterRhs
       ).withMods((mods | Accessor) &~ (CaseAccessor | GivenOrImplicit | Lazy))
+       .dropEndIndex() // the end marker should only appear on the getter definition
       Thicket(vdef1, setter)
     }
     else vdef1
@@ -874,6 +875,7 @@ object desugar {
       val modul = ValDef(moduleName, clsRef, New(clsRef, Nil))
         .withMods(mods.toTermFlags & RetainedModuleValFlags | ModuleValCreationFlags)
         .withSpan(mdef.span.startPos)
+        .withEndIndex(copyFrom = mdef) // copy over the end market position to the module val
       val ValDef(selfName, selfTpt, _) = impl.self
       val selfMods = impl.self.mods
       if (!selfTpt.isEmpty) report.error(ObjectMayNotHaveSelfType(mdef), impl.self.srcPos)

compiler/src/dotty/tools/dotc/ast/Trees.scala

@@ -327,9 +327,52 @@ object Trees {
 
   extension (mdef: untpd.DefTree) def mods: untpd.Modifiers = mdef.rawMods
 
-  abstract class NamedDefTree[-T >: Untyped](implicit @constructorOnly src: SourceFile) extends NameTree[T] with DefTree[T] {
+  /** PackageDef | NamedDefTree */
+  sealed trait WithEndMarker:
+    self: Attachment.Container =>
+
+    import WithEndMarker.*
+
+    final def endSpan(using Context): Span =
+      self.getAttachment(EndIndex) match
+        case Some(end) =>
+          val realName = endName.stripModuleClassSuffix.lastPart
+          Span(end - realName.length, end)
+        case none => NoSpan
+
+    protected def endName(using Context): Name
+
+    final def withEndIndex(index: Int): self.type =
+      self.withAttachment(EndIndex, index)
+
+    final def withEndIndex(copyFrom: WithEndMarker): self.type =
+      copyFrom.endIndex.foreach(withEndIndex)
+      this
+
+    final def dropEndIndex(): self.type =
+      self.removeAttachment(EndIndex)
+      this
+
+    protected def endIndex: Option[Int] = self.getAttachment(EndIndex)
+
+  object WithEndMarker:
+    /** Property key for trees with an `end` marker */
+    private val EndIndex: Property.StickyKey[Int] = Property.StickyKey()
+
+  end WithEndMarker
+
+  abstract class NamedDefTree[-T >: Untyped](implicit @constructorOnly src: SourceFile)
+  extends NameTree[T] with DefTree[T] with WithEndMarker {
     type ThisTree[-T >: Untyped] <: NamedDefTree[T]
 
+    protected def endName(using Context) =
+      if name == nme.CONSTRUCTOR then nme.this_
+      else srcName
+
+    protected def srcName(using Context): Name =
+      if symbol.isPackageObject then symbol.owner.name
+      else name
+
     /** The position of the name defined by this definition.
      *  This is a point position if the definition is synthetic, or a range position
      *  if the definition comes from source.
@@ -342,7 +385,7 @@ object Trees {
         val point = span.point
         if (rawMods.is(Synthetic) || span.isSynthetic || name.toTermName == nme.ERROR) Span(point)
         else {
-          val realName = name.stripModuleClassSuffix.lastPart
+          val realName = srcName.stripModuleClassSuffix.lastPart
           Span(point, point + realName.length, point)
         }
       }
@@ -857,9 +900,10 @@ object Trees {
 
   /** package pid { stats } */
   case class PackageDef[-T >: Untyped] private[ast] (pid: RefTree[T], stats: List[Tree[T]])(implicit @constructorOnly src: SourceFile)
-    extends ProxyTree[T] {
+    extends ProxyTree[T] with WithEndMarker {
     type ThisTree[-T >: Untyped] = PackageDef[T]
     def forwardTo: RefTree[T] = pid
+    protected def endName(using Context): Name = pid.name
   }
 
   /** arg @annot */

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -1283,8 +1283,6 @@ object Parsers {
           in.token == IDENTIFIER && in.name == nme.extension
         case PackageDef(pid: RefTree, _) =>
           in.isIdent && in.name == pid.name
-        case PatDef(_, IdPattern(id, _) :: Nil, _, _) =>
-         in.isIdent && in.name == id.name
         case stat: MemberDef if stat.mods.is(Given) => in.token == GIVEN
         case _: PatDef => in.token == VAL
         case _: If => in.token == IF
@@ -1295,9 +1293,20 @@ object Parsers {
         case _: (ForYield | ForDo) => in.token == FOR
         case _ => false
 
+      def matchesAndSetEnd(stat: Tree): Boolean = {
+        val didMatch = matches(stat)
+        if didMatch then
+          stat match
+            case stat: WithEndMarker =>
+              stat.withEndIndex(index = in.lastCharOffset)
+            case _ =>
+              ()
+        didMatch
+      }
+
       if in.token == END then
         val start = in.skipToken()
-        if stats.isEmpty || !matches(stats.last) then
+        if stats.isEmpty || !matchesAndSetEnd(stats.last) then
           syntaxError("misaligned end marker", Span(start, in.lastCharOffset))
         in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion
         in.nextToken()

compiler/src/dotty/tools/dotc/semanticdb/ExtractSemanticDB.scala

@@ -6,7 +6,7 @@ import core._
 import Phases._
 import ast.tpd._
 import ast.untpd.given
-import ast.Trees.mods
+import ast.Trees.{mods, WithEndMarker}
 import Contexts._
 import Symbols._
 import Flags._
@@ -139,47 +139,46 @@ class ExtractSemanticDB extends Phase:
             case tree => registerDefinition(tree.symbol, tree.span, Set.empty, tree.source)
           tree.stats.foreach(traverse)
         case tree: NamedDefTree =>
-          if tree.symbol.isAllOf(ModuleValCreationFlags) then
-            return
-          if !excludeDef(tree.symbol)
-          && tree.span.hasLength then
-            registerDefinition(tree.symbol, tree.nameSpan, symbolKinds(tree), tree.source)
-            val privateWithin = tree.symbol.privateWithin
-            if privateWithin.exists then
-              registerUseGuarded(None, privateWithin, spanOfSymbol(privateWithin, tree.span, tree.source), tree.source)
-          else if !excludeSymbol(tree.symbol) then
-            registerSymbol(tree.symbol, symbolName(tree.symbol), symbolKinds(tree))
-          tree match
-          case tree: ValDef
-          if tree.symbol.isAllOf(EnumValue) =>
-            tree.rhs match
-            case Block(TypeDef(_, template: Template) :: _, _) => // simple case with specialised extends clause
-              template.parents.filter(!_.span.isZeroExtent).foreach(traverse)
-            case _ => // calls $new
-          case tree: ValDef
-          if tree.symbol.isSelfSym =>
-            if tree.tpt.span.hasLength then
-              traverse(tree.tpt)
-          case tree: DefDef
-          if tree.symbol.isConstructor => // ignore typeparams for secondary ctors
-            tree.trailingParamss.foreach(_.foreach(traverse))
-            traverse(tree.rhs)
-          case tree: (DefDef | ValDef)
-          if tree.symbol.isSyntheticWithIdent =>
+          if !tree.symbol.isAllOf(ModuleValCreationFlags) then
+            if !excludeDef(tree.symbol)
+            && tree.span.hasLength then
+              registerDefinition(tree.symbol, tree.nameSpan, symbolKinds(tree), tree.source)
+              val privateWithin = tree.symbol.privateWithin
+              if privateWithin.exists then
+                registerUseGuarded(None, privateWithin, spanOfSymbol(privateWithin, tree.span, tree.source), tree.source)
+            else if !excludeSymbol(tree.symbol) then
+              registerSymbol(tree.symbol, symbolName(tree.symbol), symbolKinds(tree))
             tree match
-              case tree: DefDef =>
-                tree.paramss.foreach(_.foreach(param => registerSymbolSimple(param.symbol)))
-              case tree: ValDef if tree.symbol.is(Given) => traverse(tree.tpt)
-              case _ =>
-            if !tree.symbol.isGlobal then
-              localBodies(tree.symbol) = tree.rhs
-            // ignore rhs
-          case PatternValDef(pat, rhs) =>
-            traverse(rhs)
-            PatternValDef.collectPats(pat).foreach(traverse)
-          case tree =>
-            if !excludeChildren(tree.symbol) then
-              traverseChildren(tree)
+            case tree: ValDef
+            if tree.symbol.isAllOf(EnumValue) =>
+              tree.rhs match
+              case Block(TypeDef(_, template: Template) :: _, _) => // simple case with specialised extends clause
+                template.parents.filter(!_.span.isZeroExtent).foreach(traverse)
+              case _ => // calls $new
+            case tree: ValDef
+            if tree.symbol.isSelfSym =>
+              if tree.tpt.span.hasLength then
+                traverse(tree.tpt)
+            case tree: DefDef
+            if tree.symbol.isConstructor => // ignore typeparams for secondary ctors
+              tree.trailingParamss.foreach(_.foreach(traverse))
+              traverse(tree.rhs)
+            case tree: (DefDef | ValDef)
+            if tree.symbol.isSyntheticWithIdent =>
+              tree match
+                case tree: DefDef =>
+                  tree.paramss.foreach(_.foreach(param => registerSymbolSimple(param.symbol)))
+                case tree: ValDef if tree.symbol.is(Given) => traverse(tree.tpt)
+                case _ =>
+              if !tree.symbol.isGlobal then
+                localBodies(tree.symbol) = tree.rhs
+              // ignore rhs
+            case PatternValDef(pat, rhs) =>
+              traverse(rhs)
+              PatternValDef.collectPats(pat).foreach(traverse)
+            case tree =>
+              if !excludeChildren(tree.symbol) then
+                traverseChildren(tree)
         case tree: Template =>
           val ctorSym = tree.constr.symbol
           if !excludeDef(ctorSym) then
@@ -240,6 +239,13 @@ class ExtractSemanticDB extends Phase:
         case _ =>
           traverseChildren(tree)
 
+      tree match
+        case tree: WithEndMarker =>
+          val endSpan = tree.endSpan
+          if endSpan.exists then
+            registerUseGuarded(None, tree.symbol, endSpan, tree.source)
+        case _ =>
+
     end traverse
 
     private def funParamSymbol(funSym: Symbol)(using Context): Name => String =

tests/semanticdb/expect/EndMarkers.expect.scala

@@ -0,0 +1,59 @@
+package endmarkers:
+
+  class MultiCtor/*<-endmarkers::MultiCtor#*/(val i/*<-endmarkers::MultiCtor#i.*/: Int/*->scala::Int#*/):
+    def this()/*<-endmarkers::MultiCtor#`<init>`(+1).*/ =
+      this(23)
+    end this/*->endmarkers::MultiCtor#`<init>`(+1).*/
+  end MultiCtor/*->endmarkers::MultiCtor#*/
+
+  /*<-endmarkers::EndMarkers$package.*/def topLevelMethod/*<-endmarkers::EndMarkers$package.topLevelMethod().*/: String/*->scala::Predef.String#*/ =
+    "hello"
+  end topLevelMethod/*->endmarkers::EndMarkers$package.topLevelMethod().*/
+
+  val topLevelVal/*<-endmarkers::EndMarkers$package.topLevelVal.*/: Int/*->scala::Int#*/ =
+    23
+  end topLevelVal/*->endmarkers::EndMarkers$package.topLevelVal.*/
+
+  var topLevelVar/*<-endmarkers::EndMarkers$package.topLevelVar().*/: String/*->scala::Predef.String#*/ =
+    ""
+  end topLevelVar/*->endmarkers::EndMarkers$package.topLevelVar().*/
+
+  class Container/*<-endmarkers::Container#*/:
+
+    def foo/*<-endmarkers::Container#foo().*/ =
+      (/*->scala::Tuple3.apply().*/1,2,3)
+    end foo/*->endmarkers::Container#foo().*/
+
+    val bar/*<-endmarkers::Container#bar.*/ =
+      (/*->scala::Tuple3.apply().*/4,5,6)
+    end bar/*->endmarkers::Container#bar.*/
+
+    var baz/*<-endmarkers::Container#baz().*/ =
+      15
+    end baz/*->endmarkers::Container#baz().*/
+
+  end Container/*->endmarkers::Container#*/
+
+  def topLevelWithLocals/*<-endmarkers::EndMarkers$package.topLevelWithLocals().*/: Unit/*->scala::Unit#*/ =
+
+    val localVal/*<-local0*/ =
+      37
+    end localVal/*->local0*/
+
+    var localVar/*<-local1*/ =
+      43
+    end localVar/*->local1*/
+
+    def localDef/*<-local2*/ =
+      97
+    end localDef/*->local2*/
+
+  end topLevelWithLocals/*->endmarkers::EndMarkers$package.topLevelWithLocals().*/
+
+  object TestObj/*<-endmarkers::TestObj.*/:
+
+    def foo/*<-endmarkers::TestObj.foo().*/ = 23
+
+  end TestObj/*->endmarkers::TestObj.*/
+
+end endmarkers

tests/semanticdb/expect/EndMarkers.scala

@@ -0,0 +1,59 @@
+package endmarkers:
+
+  class MultiCtor(val i: Int):
+    def this() =
+      this(23)
+    end this
+  end MultiCtor
+
+  def topLevelMethod: String =
+    "hello"
+  end topLevelMethod
+
+  val topLevelVal: Int =
+    23
+  end topLevelVal
+
+  var topLevelVar: String =
+    ""
+  end topLevelVar
+
+  class Container:
+
+    def foo =
+      (1,2,3)
+    end foo
+
+    val bar =
+      (4,5,6)
+    end bar
+
+    var baz =
+      15
+    end baz
+
+  end Container
+
+  def topLevelWithLocals: Unit =
+
+    val localVal =
+      37
+    end localVal
+
+    var localVar =
+      43
+    end localVar
+
+    def localDef =
+      97
+    end localDef
+
+  end topLevelWithLocals
+
+  object TestObj:
+
+    def foo = 23
+
+  end TestObj
+
+end endmarkers

tests/semanticdb/expect/EndMarkers2.expect.scala

@@ -0,0 +1,7 @@
+package object endmarkers2/*<-endmarkers2::package.*/:
+
+  type Foo/*<-endmarkers2::package.Foo#*/ =
+    Unit/*->scala::Unit#*/
+  end Foo/*->endmarkers2::package.Foo#*/
+
+end endmarkers2/*->endmarkers2::package.*/

tests/semanticdb/expect/EndMarkers2.scala

@@ -0,0 +1,7 @@
+package object endmarkers2:
+
+  type Foo =
+    Unit
+  end Foo
+
+end endmarkers2

tests/semanticdb/expect/semanticdb-Flags.expect.scala

@@ -2,8 +2,8 @@ package flags
 
 import scala.language/*->scala::language.*/.experimental/*->scala::language.experimental.*/.macros/*->scala::language.experimental.macros.*/
 
-package object p {
-  p/*<-flags::p::package.*/rivate lazy val x/*<-flags::p::package.x.*/ = 1
+package object p/*<-flags::p::package.*/ {
+  private lazy val x/*<-flags::p::package.x.*/ = 1
   protected implicit var y/*<-flags::p::package.y().*/: Int/*->scala::Int#*/ = 2
   def z/*<-flags::p::package.z().*/(pp/*<-flags::p::package.z().(pp)*/: Int/*->scala::Int#*/) = 3
   def m/*<-flags::p::package.m().*/[TT/*<-flags::p::package.m().[TT]*/]: Int/*->scala::Int#*/ = macro ???/*->scala::Predef.`???`().*/