Merge pull request #15228 from eschnett/eschnett/gcd

Check for overflow in gcd algorithm

base/intfuncs.jl

@@ -8,7 +8,7 @@ function gcd{T<:Integer}(a::T, b::T)
         b = rem(a, b)
         a = t
     end
-    abs(a)
+    checked_abs(a)
 end
 
 # binary GCD (aka Stein's) algorithm
@@ -19,20 +19,23 @@ function gcd{T<:Union{Int64,UInt64,Int128,UInt128}}(a::T, b::T)
     za = trailing_zeros(a)
     zb = trailing_zeros(b)
     k = min(za, zb)
-    u = abs(a >> za)
-    v = abs(b >> zb)
+    u = unsigned(abs(a >> za))
+    v = unsigned(abs(b >> zb))
     while u != v
         if u > v
             u, v = v, u
         end
         v -= u
         v >>= trailing_zeros(v)
     end
-    u << k
+    r = u << k
+    # T(r) would throw InexactError; we want OverflowError instead
+    r > typemax(T) && throw(OverflowError())
+    r % T
 end
 
 # explicit a==0 test is to handle case of lcm(0,0) correctly
-lcm{T<:Integer}(a::T, b::T) = a == 0 ? a : abs(a * div(b, gcd(b,a)))
+lcm{T<:Integer}(a::T, b::T) = a == 0 ? a : checked_abs(a * div(b, gcd(b,a)))
 
 gcd(a::Integer) = a
 lcm(a::Integer) = a

test/intfuncs.jl

@@ -1,26 +1,35 @@
 # This file is a part of Julia. License is MIT: http://julialang.org/license
 
-@test gcd(3, 5) == 1
-@test gcd(3, 15) == 3
-@test gcd(0, 15) == 15
-@test gcd(3, -15) == 3
-@test gcd(-3, -15) == 3
-@test gcd(0, 0) == 0
-
-@test gcd(2, 4, 6) == 2
-
-@test typeof(gcd(Int32(3), Int32(15))) == Int32
-
-@test lcm(2, 3) == 6
-@test lcm(4, 6) == 12
-@test lcm(3, 0) == 0
-@test lcm(0, 0) == 0
-@test lcm(4, -6) == 12
-@test lcm(-4, -6) == 12
-
-@test lcm(2, 4, 6) == 12
-
-@test typeof(lcm(Int32(2), Int32(3))) == Int32
+# Int32 and Int64 take different code paths -- test both
+for T in (Int32, Int64)
+    @test gcd(T(3), T(5)) === T(1)
+    @test gcd(T(3), T(15)) === T(3)
+    @test gcd(T(0), T(15)) === T(15)
+    @test gcd(T(3), T(-15)) === T(3)
+    @test gcd(T(-3), T(-15)) === T(3)
+    @test gcd(T(0), T(0)) === T(0)
+
+    @test gcd(T(2), T(4), T(6)) === T(2)
+
+    @test gcd(typemax(T), T(1)) === T(1)
+    @test gcd(-typemax(T), T(1)) === T(1)
+    @test gcd(typemin(T), T(1)) === T(1)
+    @test_throws OverflowError gcd(typemin(T), typemin(T))
+
+    @test lcm(T(2), T(3)) === T(6)
+    @test lcm(T(4), T(6)) === T(12)
+    @test lcm(T(3), T(0)) === T(0)
+    @test lcm(T(0), T(0)) === T(0)
+    @test lcm(T(4), T(-6)) === T(12)
+    @test lcm(T(-4), T(-6)) === T(12)
+
+    @test lcm(T(2), T(4), T(6)) === T(12)
+
+    @test lcm(typemax(T), T(1)) === typemax(T)
+    @test lcm(-typemax(T), T(1)) === typemax(T)
+    @test_throws OverflowError lcm(typemin(T), T(1))
+    @test_throws OverflowError lcm(typemin(T), typemin(T))
+end
 
 @test gcdx(5, 12) == (1, 5, -2)
 @test gcdx(5, -12) == (1, 5, 2)