fixes #12264 [backport] (#12302)

compiler/int128.nim

@@ -73,6 +73,15 @@ proc toInt64*(arg: Int128): int64 =
 
   cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
 
+proc toInt64Checked*(arg: Int128; onError: int64): int64 =
+  if isNegative(arg):
+    if arg.sdata(3) != -1 or arg.sdata(2) != -1:
+      return onError
+  else:
+    if arg.sdata(3) != 0 or arg.sdata(2) != 0:
+      return onError
+  return cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
 proc toInt32*(arg: Int128): int32 =
   if isNegative(arg):
     assert(arg.sdata(3) == -1, "out of range")

compiler/sizealignoffsetimpl.nim

@@ -21,6 +21,7 @@ const
   szUnknownSize* = -3
   szIllegalRecursion* = -2
   szUncomputedSize* = -1
+  szTooBigSize* = -4
 
 type IllegalTypeRecursionError = object of Exception
 
@@ -33,14 +34,14 @@ type
     offset: int
 
 proc inc(arg: var OffsetAccum; value: int) =
-  if unlikely(value == szIllegalRecursion):  raiseIllegalTypeRecursion()
+  if unlikely(value == szIllegalRecursion): raiseIllegalTypeRecursion()
   if value == szUnknownSize or arg.offset == szUnknownSize:
     arg.offset = szUnknownSize
   else:
     arg.offset += value
 
 proc alignmentMax(a,b: int): int =
-  if unlikely(a == szIllegalRecursion or b == szIllegalRecursion):  raiseIllegalTypeRecursion()
+  if unlikely(a == szIllegalRecursion or b == szIllegalRecursion): raiseIllegalTypeRecursion()
   if a == szUnknownSize or b == szUnknownSize:
     szUnknownSize
   else:
@@ -57,7 +58,7 @@ proc align(arg: var OffsetAccum; value: int) =
 
 proc mergeBranch(arg: var OffsetAccum; value: OffsetAccum) =
   if value.maxAlign == szUnknownSize or arg.maxAlign == szUnknownSize or
-     value.offset   == szUnknownSize or arg.offset == szUnknownSize:
+     value.offset == szUnknownSize or arg.offset == szUnknownSize:
     arg.maxAlign = szUnknownSize
     arg.offset = szUnknownSize
   else:
@@ -111,7 +112,7 @@ proc setOffsetsToUnknown(n: PNode) =
     for i in 0 ..< safeLen(n):
       setOffsetsToUnknown(n[i])
 
-proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, packed: bool, accum: var OffsetAccum): void =
+proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, packed: bool, accum: var OffsetAccum) =
   ## ``offset`` is the offset within the object, after the node has been written, no padding bytes added
   ## ``align`` maximum alignment from all sub nodes
   assert n != nil
@@ -256,7 +257,7 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) =
       typ.size = elemSize
       typ.align = int16(elemSize)
     else:
-      typ.size = toInt64(lengthOrd(conf, typ.sons[0]) * int32(elemSize))
+      typ.size = toInt64Checked(lengthOrd(conf, typ.sons[0]) * int32(elemSize), szTooBigSize)
       typ.align = typ.sons[1].align
 
   of tyUncheckedArray:

compiler/types.nim

@@ -814,11 +814,8 @@ proc floatRangeCheck*(x: BiggestFloat, t: PType): bool =
     false
 
 proc lengthOrd*(conf: ConfigRef; t: PType): Int128 =
-  case t.skipTypes(tyUserTypeClasses).kind
-  of tyInt64, tyInt32, tyInt:
-    # XXX: this is just wrong
-    result = lastOrd(conf, t)
-  of tyDistinct: result = lengthOrd(conf, t.sons[0])
+  if t.skipTypes(tyUserTypeClasses).kind == tyDistinct:
+    result = lengthOrd(conf, t.sons[0])
   else:
     let last = lastOrd(conf, t)
     let first = firstOrd(conf, t)