From 8f71739a7e52c7f54e843228421df87c56b6e1b1 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Mon, 27 Nov 2023 14:19:48 +0100 Subject: [PATCH] Add source file attribute to TASTY files This introduces the first non-boolean attribute to the TASTY format. We Split the tags into a set of boolean tags and string tags to handle This new kind of attribute. We keep some tags as unknown to allow for additional kinds of tags in the future. --- .../dotc/core/tasty/AttributePickler.scala | 10 +++- .../dotc/core/tasty/AttributeUnpickler.scala | 12 ++++- .../tools/dotc/core/tasty/Attributes.scala | 14 ++++- .../dotc/core/tasty/DottyUnpickler.scala | 2 +- .../tools/dotc/core/tasty/TastyPrinter.scala | 4 +- .../dotty/tools/dotc/transform/Pickler.scala | 4 ++ tasty/src/dotty/tools/tasty/TastyFormat.scala | 54 +++++++++++-------- 7 files changed, 72 insertions(+), 28 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala index d7c268a6b4da..d7dc25738eba 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala @@ -12,12 +12,18 @@ object AttributePickler: pickler: TastyPickler, buf: TastyBuffer ): Unit = - if attributes.booleanTags.nonEmpty then - pickler.newSection(AttributesSection, buf) + pickler.newSection(AttributesSection, buf) for tag <- attributes.booleanTags do + assert(tag < TastyFormat.firstStringAttrTag, "Not a boolean attribute tag: " + tag) buf.writeByte(tag) + assert(attributes.stringTagValues.contains(TastyFormat.SOURCEFILEattr)) + for (tag, value) <- attributes.stringTagValues do + assert(TastyFormat.firstStringAttrTag <= tag && tag < TastyFormat.firstUnassignedAttrTag, "Not a string attribute tag: " + tag) + buf.writeByte(tag) + buf.writeUtf8(value) + end pickleAttributes end AttributePickler diff --git a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala index 5fa61ef887df..101fbc26cce2 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala @@ -2,6 +2,7 @@ package dotty.tools.dotc package core.tasty import scala.language.unsafeNulls +import scala.collection.immutable.TreeMap import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer} @@ -10,11 +11,18 @@ class AttributeUnpickler(reader: TastyReader): lazy val attributes: Attributes = { val booleanTags = List.newBuilder[Int] + val stringTagValue = TreeMap.newBuilder[Int, String] while !isAtEnd do - booleanTags += readByte() + val tag = readByte() + if tag < TastyFormat.firstStringAttrTag then + booleanTags += tag + else if tag < TastyFormat.firstUnassignedAttrTag then + stringTagValue += ((tag, readUtf8())) + else + assert(false, "unknown attribute tag: " + tag) - new Attributes(booleanTags.result()) + new Attributes(booleanTags.result(), stringTagValue.result()) } end AttributeUnpickler diff --git a/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala b/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala index 874767166229..541f911beff5 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala @@ -2,22 +2,34 @@ package dotty.tools.dotc.core.tasty import dotty.tools.tasty.TastyFormat +import scala.collection.immutable.TreeMap + class Attributes( val booleanTags: List[Int], + val stringTagValues: TreeMap[Int, String], ) { def scala2StandardLibrary: Boolean = booleanTags.contains(TastyFormat.SCALA2STANDARDLIBRARYattr) def explicitNulls: Boolean = booleanTags.contains(TastyFormat.EXPLICITNULLSattr) + def sourceFile: String = + stringTagValues(TastyFormat.SOURCEFILEattr) } object Attributes: def apply( + sourceFile: String, scala2StandardLibrary: Boolean, explicitNulls: Boolean, ): Attributes = val booleanTags = List.newBuilder[Int] if scala2StandardLibrary then booleanTags += TastyFormat.SCALA2STANDARDLIBRARYattr if explicitNulls then booleanTags += TastyFormat.EXPLICITNULLSattr - new Attributes(booleanTags.result()) + + val stringTagValues = TreeMap.newBuilder[Int, String] + stringTagValues += TastyFormat.SOURCEFILEattr -> sourceFile + + new Attributes(booleanTags.result(), stringTagValues.result()) end apply + + def empty: Attributes = new Attributes(Nil, TreeMap.empty) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index b4fba925df4d..033d8cd40c1f 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -57,7 +57,7 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get def tastyAttributes: Attributes = - attributeUnpicklerOpt.map(_.attributes).getOrElse(new Attributes(booleanTags = Nil)) + attributeUnpicklerOpt.map(_.attributes).getOrElse(Attributes.empty) /** Enter all toplevel classes and objects into their scopes * @param roots a set of SymDenotations that should be overwritten by unpickling diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index 2b5360349305..30b4630ff696 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -237,7 +237,9 @@ class TastyPrinter(bytes: Array[Byte]) { for tag <- attributes.booleanTags do sb.append(" ").append(attributeTagToString(tag)).append("\n") - + for (tag, value) <- attributes.stringTagValues do + sb.append(" ").append(attributeTagToString(tag)) + .append(" ").append(value).append("\n") sb.result } } diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 57b8da058073..55162dd02f2e 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -108,7 +108,11 @@ class Pickler extends Phase { pickler, treePkl.buf.addrOfTree, treePkl.docString, tree, scratch.commentBuffer) + val sourceRelativePath = + val reference = ctx.settings.sourceroot.value + util.SourceFile.relativePath(unit.source, reference) val attributes = Attributes( + sourceFile = sourceRelativePath, scala2StandardLibrary = ctx.settings.YcompileScala2Library.value, explicitNulls = ctx.settings.YexplicitNulls.value, ) diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index a77ad5d400d5..d29e8e2b8956 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -235,11 +235,11 @@ Note: The signature of a SELECTin or TERMREFin node is the signature of the sele Note: Tree tags are grouped into 5 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. ```none - Category 1 (tags 1-59) : tag - Category 2 (tags 60-89) : tag Nat - Category 3 (tags 90-109) : tag AST - Category 4 (tags 110-127): tag Nat AST - Category 5 (tags 128-255): tag Length + Tree Category 1 (tags 1-59) : tag + Tree Category 2 (tags 60-89) : tag Nat + Tree Category 3 (tags 90-109) : tag AST + Tree Category 4 (tags 110-127): tag Nat AST + Tree Category 5 (tags 128-255): tag Length ``` Standard-Section: "Positions" LinesSizes Assoc* @@ -273,6 +273,13 @@ Standard Section: "Attributes" Attribute* ```none Attribute = SCALA2STANDARDLIBRARYattr EXPLICITNULLSattr + SOURCEFILEattr Utf8 +``` + +Note: Attribute tags are grouped into 2 categories that determine what follows, and thus allow to compute the size of the tagged tree in a generic way. +```none + Attribute Category 1 (tags 1-64) : tag + Attribute Category 2 (tags 65-128): tag UTF8 ``` **************************************************************************************/ @@ -436,9 +443,9 @@ object TastyFormat { final val SOURCE = 4 - // AST tags - // Cat. 1: tag + // AST tags + // Tree Cat. 1: tag final val firstSimpleTreeTag = UNITconst // final val ??? = 1 final val UNITconst = 2 @@ -487,8 +494,8 @@ object TastyFormat { final val EMPTYCLAUSE = 45 final val SPLITCLAUSE = 46 - // Cat. 2: tag Nat - + // Tree Cat. 2: tag Nat + final val firstNatTreeTag = SHAREDterm final val SHAREDterm = 60 final val SHAREDtype = 61 final val TERMREFdirect = 62 @@ -507,8 +514,8 @@ object TastyFormat { final val IMPORTED = 75 final val RENAMED = 76 - // Cat. 3: tag AST - + // Tree Cat. 3: tag AST + final val firstASTTreeTag = THIS final val THIS = 90 final val QUALTHIS = 91 final val CLASSconst = 92 @@ -525,8 +532,8 @@ object TastyFormat { final val EXPLICITtpt = 103 - // Cat. 4: tag Nat AST - + // Tree Cat. 4: tag Nat AST + final val firstNatASTTreeTag = IDENT final val IDENT = 110 final val IDENTtpt = 111 final val SELECT = 112 @@ -538,8 +545,8 @@ object TastyFormat { final val SELFDEF = 118 final val NAMEDARG = 119 - // Cat. 5: tag Length ... - + // Tree Cat. 5: tag Length ... + final val firstLengthTreeTag = PACKAGE final val PACKAGE = 128 final val VALDEF = 129 final val DEFDEF = 130 @@ -601,17 +608,21 @@ object TastyFormat { final val HOLE = 255 - final val firstNatTreeTag = SHAREDterm - final val firstASTTreeTag = THIS - final val firstNatASTTreeTag = IDENT - final val firstLengthTreeTag = PACKAGE - - // Attributes tags + // Attr Cat. 1: tag + final val firstBooleanAttrTag = SCALA2STANDARDLIBRARYattr final val SCALA2STANDARDLIBRARYattr = 1 final val EXPLICITNULLSattr = 2 + // Attr Cat. 2: tag UTF8 + final val firstStringAttrTag = SOURCEFILEattr + final val SOURCEFILEattr = 128 + + // Attr Cat. 3: unassigned + final val firstUnassignedAttrTag = 129 + + /** Useful for debugging */ def isLegalTag(tag: Int): Boolean = firstSimpleTreeTag <= tag && tag <= SPLITCLAUSE || @@ -830,6 +841,7 @@ object TastyFormat { def attributeTagToString(tag: Int): String = tag match { case SCALA2STANDARDLIBRARYattr => "SCALA2STANDARDLIBRARYattr" case EXPLICITNULLSattr => "EXPLICITNULLSattr" + case SOURCEFILEattr => "SOURCEFILEattr" } /** @return If non-negative, the number of leading references (represented as nats) of a length/trees entry.