Add source file attribute to TASTY files

This introduces the first non-boolean attribute to the TASTY format. We Split the tags into a set of boolean tags and string tags to handle This new kind of attribute. We keep some tags as unknown to allow for additional kinds of tags in the future.
scala · Nov 27, 2023 · 8f71739 · 8f71739
1 parent 438250f
commit 8f71739
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 28 deletions.
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala
@@ -12,12 +12,18 @@ object AttributePickler:
     pickler: TastyPickler,
     buf: TastyBuffer
   ): Unit =
-    if attributes.booleanTags.nonEmpty then
-      pickler.newSection(AttributesSection, buf)
+    pickler.newSection(AttributesSection, buf)
 
     for tag <- attributes.booleanTags do
+      assert(tag < TastyFormat.firstStringAttrTag, "Not a boolean attribute tag: " + tag)
       buf.writeByte(tag)
 
+    assert(attributes.stringTagValues.contains(TastyFormat.SOURCEFILEattr))
+    for (tag, value) <- attributes.stringTagValues do
+      assert(TastyFormat.firstStringAttrTag <= tag && tag < TastyFormat.firstUnassignedAttrTag, "Not a string attribute tag: " + tag)
+      buf.writeByte(tag)
+      buf.writeUtf8(value)
+
   end pickleAttributes
 
 end AttributePickler
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala
@@ -2,6 +2,7 @@ package dotty.tools.dotc
 package core.tasty
 
 import scala.language.unsafeNulls
+import scala.collection.immutable.TreeMap
 
 import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}
 
@@ -10,11 +11,18 @@ class AttributeUnpickler(reader: TastyReader):
 
   lazy val attributes: Attributes = {
     val booleanTags = List.newBuilder[Int]
+    val stringTagValue = TreeMap.newBuilder[Int, String]
 
     while !isAtEnd do
-      booleanTags += readByte()
+      val tag = readByte()
+      if tag < TastyFormat.firstStringAttrTag then
+        booleanTags += tag
+      else if tag < TastyFormat.firstUnassignedAttrTag then
+        stringTagValue += ((tag, readUtf8()))
+      else
+        assert(false, "unknown attribute tag: " + tag)
 
-    new Attributes(booleanTags.result())
+    new Attributes(booleanTags.result(), stringTagValue.result())
   }
 
 end AttributeUnpickler
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala b/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala
@@ -2,22 +2,34 @@ package dotty.tools.dotc.core.tasty
 
 import dotty.tools.tasty.TastyFormat
 
+import scala.collection.immutable.TreeMap
+
 class Attributes(
   val booleanTags: List[Int],
+  val stringTagValues: TreeMap[Int, String],
 ) {
   def scala2StandardLibrary: Boolean =
     booleanTags.contains(TastyFormat.SCALA2STANDARDLIBRARYattr)
   def explicitNulls: Boolean =
     booleanTags.contains(TastyFormat.EXPLICITNULLSattr)
+   def sourceFile: String =
+    stringTagValues(TastyFormat.SOURCEFILEattr)
 }
 
 object Attributes:
   def apply(
+    sourceFile: String,
     scala2StandardLibrary: Boolean,
     explicitNulls: Boolean,
   ): Attributes =
     val booleanTags = List.newBuilder[Int]
     if scala2StandardLibrary then booleanTags += TastyFormat.SCALA2STANDARDLIBRARYattr
     if explicitNulls then booleanTags += TastyFormat.EXPLICITNULLSattr
-    new Attributes(booleanTags.result())
+
+    val stringTagValues = TreeMap.newBuilder[Int, String]
+    stringTagValues += TastyFormat.SOURCEFILEattr -> sourceFile
+
+    new Attributes(booleanTags.result(), stringTagValues.result())
   end apply
+
+  def empty: Attributes = new Attributes(Nil, TreeMap.empty)
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala
@@ -57,7 +57,7 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
   private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get
 
   def tastyAttributes: Attributes =
-    attributeUnpicklerOpt.map(_.attributes).getOrElse(new Attributes(booleanTags = Nil))
+    attributeUnpicklerOpt.map(_.attributes).getOrElse(Attributes.empty)
 
   /** Enter all toplevel classes and objects into their scopes
    *  @param roots          a set of SymDenotations that should be overwritten by unpickling

diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala
@@ -237,7 +237,9 @@ class TastyPrinter(bytes: Array[Byte]) {
 
       for tag <- attributes.booleanTags do
         sb.append("   ").append(attributeTagToString(tag)).append("\n")
-
+      for (tag, value) <- attributes.stringTagValues do
+        sb.append("   ").append(attributeTagToString(tag))
+          .append(" ").append(value).append("\n")
       sb.result
     }
   }

diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala
@@ -108,7 +108,11 @@ class Pickler extends Phase {
                 pickler, treePkl.buf.addrOfTree, treePkl.docString, tree,
                 scratch.commentBuffer)
 
+          val sourceRelativePath =
+            val reference = ctx.settings.sourceroot.value
+            util.SourceFile.relativePath(unit.source, reference)
           val attributes = Attributes(
+            sourceFile = sourceRelativePath,
             scala2StandardLibrary = ctx.settings.YcompileScala2Library.value,
             explicitNulls = ctx.settings.YexplicitNulls.value,
           )

diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala
@@ -235,11 +235,11 @@ Note: The signature of a SELECTin or TERMREFin node is the signature of the sele
 
 Note: Tree tags are grouped into 5 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way.
 ```none
-  Category 1 (tags 1-59)   :  tag
-  Category 2 (tags 60-89)  :  tag Nat
-  Category 3 (tags 90-109) :  tag AST
-  Category 4 (tags 110-127):  tag Nat AST
-  Category 5 (tags 128-255):  tag Length <payload>
+  Tree Category 1 (tags 1-59)   :  tag
+  Tree Category 2 (tags 60-89)  :  tag Nat
+  Tree Category 3 (tags 90-109) :  tag AST
+  Tree Category 4 (tags 110-127):  tag Nat AST
+  Tree Category 5 (tags 128-255):  tag Length <payload>
 ```
 
 Standard-Section: "Positions" LinesSizes Assoc*
@@ -273,6 +273,13 @@ Standard Section: "Attributes" Attribute*
 ```none
   Attribute     = SCALA2STANDARDLIBRARYattr
                   EXPLICITNULLSattr
+                  SOURCEFILEattr Utf8
+```
+
+Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way.
+```none
+  Attribute Category 1 (tags 1-64)  :  tag
+  Attribute Category 2 (tags 65-128):  tag UTF8
 ```
 
 **************************************************************************************/
@@ -436,9 +443,9 @@ object TastyFormat {
 
   final val SOURCE = 4
 
- // AST tags
-  // Cat. 1:    tag
+  // AST tags
 
+  // Tree Cat. 1:    tag
   final val firstSimpleTreeTag = UNITconst
   // final val ??? = 1
   final val UNITconst = 2
@@ -487,8 +494,8 @@ object TastyFormat {
   final val EMPTYCLAUSE = 45
   final val SPLITCLAUSE = 46
 
-  // Cat. 2:    tag Nat
-
+  // Tree Cat. 2:    tag Nat
+  final val firstNatTreeTag = SHAREDterm
   final val SHAREDterm = 60
   final val SHAREDtype = 61
   final val TERMREFdirect = 62
@@ -507,8 +514,8 @@ object TastyFormat {
   final val IMPORTED = 75
   final val RENAMED = 76
 
-  // Cat. 3:    tag AST
-
+  // Tree Cat. 3:    tag AST
+  final val firstASTTreeTag = THIS
   final val THIS = 90
   final val QUALTHIS = 91
   final val CLASSconst = 92
@@ -525,8 +532,8 @@ object TastyFormat {
   final val EXPLICITtpt = 103
 
 
-  // Cat. 4:    tag Nat AST
-
+  // Tree Cat. 4:    tag Nat AST
+  final val firstNatASTTreeTag = IDENT
   final val IDENT = 110
   final val IDENTtpt = 111
   final val SELECT = 112
@@ -538,8 +545,8 @@ object TastyFormat {
   final val SELFDEF = 118
   final val NAMEDARG = 119
 
-  // Cat. 5:    tag Length ...
-
+  // Tree Cat. 5:    tag Length ...
+  final val firstLengthTreeTag = PACKAGE
   final val PACKAGE = 128
   final val VALDEF = 129
   final val DEFDEF = 130
@@ -601,17 +608,21 @@ object TastyFormat {
 
   final val HOLE = 255
 
-  final val firstNatTreeTag = SHAREDterm
-  final val firstASTTreeTag = THIS
-  final val firstNatASTTreeTag = IDENT
-  final val firstLengthTreeTag = PACKAGE
-
-
   // Attributes tags
 
+  // Attr Cat. 1: tag
+  final val firstBooleanAttrTag = SCALA2STANDARDLIBRARYattr
   final val SCALA2STANDARDLIBRARYattr = 1
   final val EXPLICITNULLSattr = 2
 
+  // Attr Cat. 2: tag UTF8
+  final val firstStringAttrTag = SOURCEFILEattr
+  final val SOURCEFILEattr = 128
+
+  // Attr Cat. 3: unassigned
+  final val firstUnassignedAttrTag = 129
+
+
   /** Useful for debugging */
   def isLegalTag(tag: Int): Boolean =
     firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE ||
@@ -830,6 +841,7 @@ object TastyFormat {
   def attributeTagToString(tag: Int): String = tag match {
     case SCALA2STANDARDLIBRARYattr => "SCALA2STANDARDLIBRARYattr"
     case EXPLICITNULLSattr => "EXPLICITNULLSattr"
+    case SOURCEFILEattr => "SOURCEFILEattr"
   }
 
   /** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry.