Skip to content

Commit

Permalink
Allow Int64.MinValue as valid nativeint literal, which it is (fixes d…
Browse files Browse the repository at this point in the history 
…otnet#9524) (dotnet#9632)

* Fix lexer for -9223372036854775808n

* Add tests for nativeint border-values

* int32 -9223372036854775808L == 0

* Cleanup/improve code of lexing of integers

* Cleanup/improve code of lexing of integers

* Oops

* Ensure one-time eval of int-to-string
  • Loading branch information
@abelbraaksma
abelbraaksma committed Aug 9, 2020
1 parent 65f1dce commit fc2c857
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 31 deletions.
2 changes: 1 addition & 1 deletion src/fsharp/LexFilter.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2298,7 +2298,7 @@ type LexFilterImpl (lightSyntaxStatus: LightSyntaxStatus, compilingFsLib, lexer,
| INT32(v, bad) -> delayMergedToken(INT32((if plus then v else -v), (plus && bad))) // note: '-' makes a 'bad' max int 'good'. '+' does not
| INT32_DOT_DOT(v, bad) -> delayMergedToken(INT32_DOT_DOT((if plus then v else -v), (plus && bad))) // note: '-' makes a 'bad' max int 'good'. '+' does not
| INT64(v, bad) -> delayMergedToken(INT64((if plus then v else -v), (plus && bad))) // note: '-' makes a 'bad' max int 'good'. '+' does not
| NATIVEINT v -> delayMergedToken(NATIVEINT(if plus then v else -v))
| NATIVEINT(v, bad) -> delayMergedToken(NATIVEINT((if plus then v else -v), (plus && bad))) // note: '-' makes a 'bad' max int 'good'. '+' does not
| IEEE32 v -> delayMergedToken(IEEE32(if plus then v else -v))
| IEEE64 v -> delayMergedToken(IEEE64(if plus then v else -v))
| DECIMAL v -> delayMergedToken(DECIMAL(if plus then v else System.Decimal.op_UnaryNegation v))
Expand Down
79 changes: 51 additions & 28 deletions src/fsharp/lex.fsl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,20 +30,39 @@ open FSharp.Compiler.ParseHelpers
open FSharp.Compiler.Parser
open FSharp.Compiler.SyntaxTree

module Ranges =
/// Whether valid as signed int8 when a minus sign is prepended, compares true to 0x80
let isInt8BadMax x = 1 <<< 7 = x

/// Whether valid as signed int16 when a minus sign is prepended, compares true to 0x8000
let isInt16BadMax x = 1 <<< 15 = x

/// Whether valid as signed int32 when a minus sign is prepended, compares as string against "2147483648".
let isInt32BadMax = let max = string(1UL <<< 31) in fun s -> max = s

/// Whether valid as signed int64 when a minus sign is prepended, compares as string against "9223372036854775808".
let isInt64BadMax = let max = string(1UL <<< 63) in fun s -> max = s

/// Get string from lexbuf
let lexeme (lexbuf : UnicodeLexing.Lexbuf) = UnicodeLexing.Lexbuf.LexemeString lexbuf

/// Trim n chars from both side of a string
let trimBoth (s:string) n m = s.Substring(n, s.Length - (n+m))

/// Trim n chars from both sides of lexbuf, return string
let lexemeTrimBoth lexbuf n m = trimBoth (lexeme lexbuf) n m

/// Trim n chars from the right of lexbuf, return string
let lexemeTrimRight lexbuf n = lexemeTrimBoth lexbuf 0 n

/// Trim n chars from the left of lexbuf, return string
let lexemeTrimLeft lexbuf n = lexemeTrimBoth lexbuf n 0

/// Throw a lexing error with a message
let fail args (lexbuf:UnicodeLexing.Lexbuf) msg dflt =
let m = lexbuf.LexemeRange
args.errorLogger.ErrorR(Error(msg,m))
dflt
dflt

//--------------------------
// Integer parsing
Expand Down Expand Up @@ -313,88 +332,88 @@ rule token args skip = parse

| int8
{ let n = lexemeTrimRightToInt32 args lexbuf 1
if n > 0x80 || n < -0x80 then fail args lexbuf (FSComp.SR.lexOutsideEightBitSigned()) (INT8(0y,false))
// Allow <max_int+1> to parse as min_int. Allowed only because we parse '-' as an operator.
else if n = 0x80 then INT8(sbyte(-0x80), true (* 'true' = 'bad'*) )
else INT8(sbyte n,false) }
if Ranges.isInt8BadMax n then INT8(SByte.MinValue, true (* 'true' = 'bad'*) )
else if n > int SByte.MaxValue || n < int SByte.MinValue then fail args lexbuf (FSComp.SR.lexOutsideEightBitSigned()) (INT8(0y, false))
else INT8(sbyte n, false) }

| xint8
{ let n = lexemeTrimRightToInt32 args lexbuf 1
if n > 0xFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideEightBitSignedHex()) (INT8(0y,false))
else INT8(sbyte(byte(n)),false) }
if n > int Byte.MaxValue || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideEightBitSignedHex()) (INT8(0y, false))
else INT8(sbyte(byte(n)), false) }

| uint8
{ let n = lexemeTrimRightToInt32 args lexbuf 2
if n > 0xFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideEightBitUnsigned()) (UINT8(0uy))
else UINT8(byte n) }
if n > int Byte.MaxValue || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideEightBitUnsigned()) (UINT8(0uy))
else UINT8(byte n) }

| int16
{ let n = lexemeTrimRightToInt32 args lexbuf 1
if n > 0x8000 || n < -0x8000 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitSigned()) (INT16(0s,false))
// Allow <max_int+1> to parse as min_int. Allowed only because we parse '-' as an operator.
else if n = 0x8000 then INT16(-0x8000s,true)
else INT16(int16 n,false) }
if Ranges.isInt16BadMax n then INT16(Int16.MinValue, true (* 'true' = 'bad'*) )
else if n > int Int16.MaxValue || n < int Int16.MinValue then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitSigned()) (INT16(0s, false))
else INT16(int16 n, false) }

| xint16
{ let n = lexemeTrimRightToInt32 args lexbuf 1
if n > 0xFFFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitSigned()) (INT16(0s,false))
else INT16(int16(uint16(n)),false) }
if n > int UInt16.MaxValue || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitSigned()) (INT16(0s,false))
else INT16(int16(uint16(n)), false) }

| uint16
{ let n = lexemeTrimRightToInt32 args lexbuf 2
if n > 0xFFFF || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitUnsigned()) (UINT16(0us))
if n > int UInt16.MaxValue || n < 0 then fail args lexbuf (FSComp.SR.lexOutsideSixteenBitUnsigned()) (UINT16(0us))
else UINT16(uint16 n) }

| int '.' '.'
{ let s = removeUnderscores (lexemeTrimRight lexbuf 2)
// Allow <max_int+1> to parse as min_int. Allowed only because we parse '-' as an operator.
if s = "2147483648" then INT32_DOT_DOT(-2147483648,true) else
if Ranges.isInt32BadMax s then INT32_DOT_DOT(Int32.MinValue, true (* 'true' = 'bad'*) ) else
let n = try int32 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitSigned()) 0
INT32_DOT_DOT(n,false)
INT32_DOT_DOT(n, false)
}

| xint
| int
{ let s = removeUnderscores (lexeme lexbuf)
// Allow <max_int+1> to parse as min_int. Allowed only because we parse '-' as an operator.
if s = "2147483648" then INT32(-2147483648,true) else
if Ranges.isInt32BadMax s then INT32(Int32.MinValue, true (* 'true' = 'bad'*) ) else
let n =
try int32 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitSigned()) 0
INT32(n,false)
INT32(n, false)
}

| xint32
| int32
{ let s = removeUnderscores (lexemeTrimRight lexbuf 1)
// Allow <max_int+1> to parse as min_int. Allowed only because we parse '-' as an operator.
if s = "2147483648" then INT32(-2147483648,true) else
let n =
if Ranges.isInt32BadMax s then INT32(Int32.MinValue, true (* 'true' = 'bad'*) ) else
let n =
try int32 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitSigned()) 0
INT32(n,false)
INT32(n, false)
}

| uint32
{
let s = removeUnderscores (lexemeTrimRight lexbuf 1)
let n =
try int64 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) 0L
if n > 0xFFFFFFFFL || n < 0L then fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) (UINT32(0u)) else
if n > int64 UInt32.MaxValue || n < 0L then fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) (UINT32(0u)) else
UINT32(uint32 (uint64 n)) }

| uint32l
{
let s = removeUnderscores (lexemeTrimRight lexbuf 2)
let n =
try int64 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) 0L
if n > 0xFFFFFFFFL || n < 0L then fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) (UINT32(0u)) else
if n > int64 UInt32.MaxValue || n < 0L then fail args lexbuf (FSComp.SR.lexOutsideThirtyTwoBitUnsigned()) (UINT32(0u)) else
UINT32(uint32 (uint64 n)) }

| int64
{ let s = removeUnderscores (lexemeTrimRight lexbuf 1)
// Allow <max_int+1> to parse as min_int. Stupid but allowed because we parse '-' as an operator.
if s = "9223372036854775808" then INT64(-9223372036854775808L,true) else
if Ranges.isInt64BadMax s then INT64(Int64.MinValue, true (* 'true' = 'bad'*) ) else
let n =
try int64 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideSixtyFourBitSigned()) 0L
try int64 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideSixtyFourBitSigned()) 0L
INT64(n,false)
}

Expand All @@ -405,9 +424,13 @@ rule token args skip = parse
UINT64(n) }

| nativeint
{ try
NATIVEINT(int64 (removeUnderscores (lexemeTrimRight lexbuf 1)))
with _ -> fail args lexbuf (FSComp.SR.lexOutsideNativeSigned()) (NATIVEINT(0L)) }
{ let s = removeUnderscores (lexemeTrimRight lexbuf 1)
// Allow <max_nativeint+1> to parse as min_nativeint. Stupid but allowed because we parse '-' as an operator.
if Ranges.isInt64BadMax s then NATIVEINT(Int64.MinValue, true) else
let n =
try int64 s with _ -> fail args lexbuf (FSComp.SR.lexOutsideNativeSigned()) 0L
NATIVEINT(n,false)
}

| unativeint
{ try
Expand Down
5 changes: 3 additions & 2 deletions src/fsharp/pars.fsy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,13 +176,13 @@ let rangeOfLongIdent(lid:LongIdent) =
%token <int16 * bool> INT16
%token <int32 * bool> INT32 INT32_DOT_DOT
%token <int64 * bool> INT64
%token <int64 * bool> NATIVEINT

%token <byte> UINT8
%token <uint16> UINT16
%token <uint32> UINT32
%token <uint64> UINT64
%token <uint64> UNATIVEINT
%token <int64> NATIVEINT
%token <single> IEEE32
%token <double> IEEE64
%token <char> CHAR
Expand Down Expand Up @@ -2809,7 +2809,8 @@ rawConstant:
{ SynConst.UInt64 $1 }

| NATIVEINT
{ SynConst.IntPtr $1 }
{ if snd $1 then errorR(Error(FSComp.SR.lexOutsideNativeSigned(), lhs parseState))
SynConst.IntPtr (fst $1) }

| UNATIVEINT
{ SynConst.UIntPtr $1 }
Expand Down
14 changes: 14 additions & 0 deletions tests/FSharp.Core.UnitTests/FSharp.Core/OperatorsModule2.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@ type OperatorsModule2() =
let result = Operators.int64 UInt64.MaxValue
Assert.AreEqual(-1L, result)

// max and min value as literals (this breaks compilation if the lexer fails)
Assert.AreEqual(-9223372036854775808L, Int64.MinValue)
Assert.AreEqual(9223372036854775807L, Int64.MaxValue)

// OverflowException, from decimal is always checked
CheckThrowsOverflowException(fun() -> Operators.int64 Decimal.MinValue |> ignore)

Expand Down Expand Up @@ -355,6 +359,16 @@ type OperatorsModule2() =
// Cannot express this as a literal, see https://github.com/dotnet/fsharp/issues/9524
Assert.AreEqual("-9223372036854775808", string result)

// Max and min value as literals (this breaks compilation if the lexer fails).
// The following tests ensure that the proper value is parsed, which is similar to `nativeint Int64.MaxValue` etc.
if Info.isX86Runtime then
Assert.AreEqual("0", string -9223372036854775808n) // same as int32 -9223372036854775808L
Assert.AreEqual("-1", string 9223372036854775807n) // same as int32 9223372036854775807L
else
Assert.AreEqual("-9223372036854775808", string -9223372036854775808n)
Assert.AreEqual("9223372036854775807", string 9223372036854775807n)


[<Test>]
member _.not() =
let result = Operators.not true
Expand Down

0 comments on commit fc2c857

Please sign in to comment.