From 586acfbe09104c50eb76e4eb50afde6c58843395 Mon Sep 17 00:00:00 2001
From: Karsten Kretschmer <kkretschmer@gmail.com>
Date: Mon, 19 Jul 2021 21:22:47 +0200
Subject: [PATCH] Avoid potential integer overflow in Mmap.mmap (#41186)

There is a potential integer overflow in Mmap.mmap which can lead to an
out-of-bounds access. The size of the memory-mapped array `len` is
calculated as `prod(dims)`. If this multiplication overflows, the
allocated size will be too small and accesses towards the end of the
array will fail with e.g. a segfault or other errors. I noticed this
when using `dims` taken from a binary file header in UInt32 format.


To fix, use overflow-aware multiplication to determine the size of the
mmapped array and throw an error in case of overflow.
---
 stdlib/Mmap/src/Mmap.jl      | 6 +++++-
 stdlib/Mmap/test/runtests.jl | 1 +
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl
index 6919421b92277..629f53e8371ed 100644
--- a/stdlib/Mmap/src/Mmap.jl
+++ b/stdlib/Mmap/src/Mmap.jl
@@ -191,7 +191,11 @@ function mmap(io::IO,
     isopen(io) || throw(ArgumentError("$io must be open to mmap"))
     isbitstype(T)  || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true"))
 
-    len = prod(dims) * sizeof(T)
+    len = sizeof(T)
+    for l in dims
+        len, overflow = Base.Checked.mul_with_overflow(promote(len, l)...)
+        overflow && throw(ArgumentError("requested size prod($((sizeof(T), dims...))) too large, would overflow typeof(size(T)) == $(typeof(len))"))
+    end
     len >= 0 || throw(ArgumentError("requested size must be ≥ 0, got $len"))
     len == 0 && return Array{T}(undef, ntuple(x->0,Val(N)))
     len < typemax(Int) - PAGESIZE || throw(ArgumentError("requested size must be < $(typemax(Int)-PAGESIZE), got $len"))
diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl
index 51bf898e94b48..0b3cb0b9f1a42 100644
--- a/stdlib/Mmap/test/runtests.jl
+++ b/stdlib/Mmap/test/runtests.jl
@@ -133,6 +133,7 @@ c = mmap(s, Vector{UInt8}, (UInt16(11),))
 finalize(c); c=nothing; GC.gc()
 @test_throws ArgumentError mmap(s, Vector{UInt8}, (Int16(-11),))
 @test_throws ArgumentError mmap(s, Vector{UInt8}, (typemax(UInt),))
+@test_throws ArgumentError mmap(s, Matrix{UInt8}, (typemax(Int) - Mmap.PAGESIZE - 1, 2)) # overflow
 close(s)
 s = open(file, "r+")
 @test isreadonly(s) == false