From 4f5742ba64f94554f6bf3659ad3047336308a797 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 14 Jan 2024 20:03:48 +0100 Subject: [PATCH 01/18] add exec kw to mmap --- stdlib/Mmap/src/Mmap.jl | 51 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 6d328c40cd7b3..f20cdb5af051a 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -54,6 +54,7 @@ if Sys.isunix() const PROT_READ = Cint(1) const PROT_WRITE = Cint(2) +const PROT_EXEC = Cint(4) const MAP_SHARED = Cint(1) const MAP_PRIVATE = Cint(2) const MAP_ANONYMOUS = Cint(Sys.isbsd() ? 0x1000 : 0x20) @@ -62,16 +63,18 @@ const F_GETFL = Cint(3) gethandle(io::IO) = RawFD(fd(io)) # Determine a stream's read/write mode, and return prot & flags appropriate for mmap -function settings(s::RawFD, shared::Bool) +function settings(s::RawFD, shared::Bool, exec::Bool=false) flags = shared ? MAP_SHARED : MAP_PRIVATE if s == INVALID_OS_HANDLE flags |= MAP_ANONYMOUS prot = PROT_READ | PROT_WRITE + exec && (prot |= PROT_EXEC) else mode = ccall(:fcntl, Cint, (RawFD, Cint, Cint...), s, F_GETFL) systemerror("fcntl F_GETFL", mode == -1) mode = mode & 3 prot = (mode == 0) ? PROT_READ : ((mode == 1) ? PROT_WRITE : (PROT_READ | PROT_WRITE)) + exec && (prot |= PROT_EXEC) if prot & PROT_READ == 0 throw(ArgumentError("mmap requires read permissions on the file (open with \"r+\" mode to override)")) end @@ -125,7 +128,7 @@ end # os-test # core implementation of mmap """ - mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) + mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true, exec::Bool=false) mmap(type::Type{Array{T,N}}, dims) Create an `Array` whose values are linked to a file, using memory-mapping. This provides a @@ -155,6 +158,8 @@ privileges are required to grow the file. The `shared` keyword argument specifies whether the resulting `Array` and changes made to it will be visible to other processes mapping the same file. +The `exec` keyword argument specifies whether the underlying mmap data will be executale. + For example, the following code ```julia @@ -186,7 +191,8 @@ like HDF5 (which can be used with memory-mapping). function mmap(io::IO, ::Type{Array{T,N}}=Vector{UInt8}, dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), - offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T,N} + offset::Integer=position(io); grow::Bool=true, shared::Bool=true, + exec::Bool=false) where {T,N} # check inputs isopen(io) || throw(ArgumentError("$io must be open to mmap")) isbitstype(T) || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true")) @@ -217,7 +223,7 @@ function mmap(io::IO, end # platform-specific mmapping @static if Sys.isunix() - prot, flags, iswrite = settings(file_desc, shared) + prot, flags, iswrite = settings(file_desc, shared, exec) if requestedSizeLarger if iswrite if grow @@ -237,6 +243,7 @@ function mmap(io::IO, C_NULL, mmaplen, prot, flags, file_desc, offset_page) systemerror("memory mapping failed", reinterpret(Int, ptr) == -1) else + # TODO settings() here only takes one arg, what to do with PROT_EXEC? name, readonly, create = settings(io) if requestedSizeLarger if readonly @@ -271,18 +278,18 @@ end mmap(file::AbstractString, ::Type{T}=Vector{UInt8}, dims::NTuple{N,Integer}=(div(filesize(file),sizeof(eltype(T))),), - offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:Array,N} = - open(io->mmap(io, T, dims, offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::Array{eltype(T),N} + offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:Array,N} = + open(io->mmap(io, T, dims, offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::Array{eltype(T),N} # using a length argument instead of dims -mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T<:Array} = - mmap(io, T, (len,), offset; grow=grow, shared=shared) -mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:Array} = - open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::Vector{eltype(T)} +mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:Array} = + mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec) +mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:Array} = + open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::Vector{eltype(T)} # constructors for non-file-backed (anonymous) mmaps -mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true) where {T<:Array,N} = mmap(Anonymous(), T, dims, Int64(0); shared=shared) -mmap(::Type{T}, i::Integer...; shared::Bool=true) where {T<:Array} = mmap(Anonymous(), T, convert(Tuple{Vararg{Int}},i), Int64(0); shared=shared) +mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true, exec::Bool=false) where {T<:Array,N} = mmap(Anonymous(), T, dims, Int64(0); shared=shared, exec=exec) +mmap(::Type{T}, i::Integer...; shared::Bool=true, exec::Bool=false) where {T<:Array} = mmap(Anonymous(), T, convert(Tuple{Vararg{Int}},i), Int64(0); shared=shared, exec=exec) """ mmap(io, BitArray, [dims, offset]) @@ -322,10 +329,10 @@ julia> rm("mmap.bin") This creates a 25-by-30000 `BitArray`, linked to the file associated with stream `io`. """ function mmap(io::IOStream, ::Type{<:BitArray}, dims::NTuple{N,Integer}, - offset::Int64=position(io); grow::Bool=true, shared::Bool=true) where N + offset::Int64=position(io); grow::Bool=true, shared::Bool=true, exec::Bool=false) where N n = prod(dims) nc = Base.num_bit_chunks(n) - chunks = mmap(io, Vector{UInt64}, (nc,), offset; grow=grow, shared=shared) + chunks = mmap(io, Vector{UInt64}, (nc,), offset; grow=grow, shared=shared, exec=exec) if !isreadonly(io) chunks[end] &= Base._msk_end(n) else @@ -342,18 +349,18 @@ function mmap(io::IOStream, ::Type{<:BitArray}, dims::NTuple{N,Integer}, return B end -mmap(file::AbstractString, ::Type{T}, dims::NTuple{N,Integer}, offset::Integer=Int64(0);grow::Bool=true, shared::Bool=true) where {T<:BitArray,N} = - open(io->mmap(io, T, dims, offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::BitArray{N} +mmap(file::AbstractString, ::Type{T}, dims::NTuple{N,Integer}, offset::Integer=Int64(0);grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:BitArray,N} = + open(io->mmap(io, T, dims, offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::BitArray{N} # using a length argument instead of dims -mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T<:BitArray} = - mmap(io, T, (len,), offset; grow=grow, shared=shared) -mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:BitArray} = - open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::BitVector +mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:BitArray} = + mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec) +mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:BitArray} = + open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::BitVector # constructors for non-file-backed (anonymous) mmaps -mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true) where {T<:BitArray,N} = mmap(Anonymous(), T, dims, Int64(0); shared=shared) -mmap(::Type{T}, i::Integer...; shared::Bool=true) where {T<:BitArray} = mmap(Anonymous(), T, convert(Tuple{Vararg{Int}},i), Int64(0); shared=shared) +mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true, exec::Bool=false) where {T<:BitArray,N} = mmap(Anonymous(), T, dims, Int64(0); shared=shared, exec=exec) +mmap(::Type{T}, i::Integer...; shared::Bool=true, exec::Bool=false) where {T<:BitArray} = mmap(Anonymous(), T, convert(Tuple{Vararg{Int}},i), Int64(0); shared=shared, exec=exec) # msync flags for unix const MS_ASYNC = 1 From 0c2cd386fe8db48b48e098371e78f75e2b9fb805 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 17 Feb 2024 23:43:06 +0100 Subject: [PATCH 02/18] implement exec for windows --- stdlib/Mmap/src/Mmap.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index f20cdb5af051a..f9888a7a809f4 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -68,17 +68,16 @@ function settings(s::RawFD, shared::Bool, exec::Bool=false) if s == INVALID_OS_HANDLE flags |= MAP_ANONYMOUS prot = PROT_READ | PROT_WRITE - exec && (prot |= PROT_EXEC) else mode = ccall(:fcntl, Cint, (RawFD, Cint, Cint...), s, F_GETFL) systemerror("fcntl F_GETFL", mode == -1) mode = mode & 3 prot = (mode == 0) ? PROT_READ : ((mode == 1) ? PROT_WRITE : (PROT_READ | PROT_WRITE)) - exec && (prot |= PROT_EXEC) if prot & PROT_READ == 0 throw(ArgumentError("mmap requires read permissions on the file (open with \"r+\" mode to override)")) end end + exec && (prot |= PROT_EXEC) return prot, flags, (prot & PROT_WRITE) > 0 end @@ -243,7 +242,6 @@ function mmap(io::IO, C_NULL, mmaplen, prot, flags, file_desc, offset_page) systemerror("memory mapping failed", reinterpret(Int, ptr) == -1) else - # TODO settings() here only takes one arg, what to do with PROT_EXEC? name, readonly, create = settings(io) if requestedSizeLarger if readonly @@ -252,13 +250,16 @@ function mmap(io::IO, throw(ArgumentError("requested size $szfile larger than file size $(filesize(io)), but requested not to grow")) end end + page_flag = exec ? (readonly ? PAGE_EXECUTE_READ : PAGE_EXECUTE_READWRITE) : (readonly ? PAGE_READONLY : PAGE_READWRITE) + file_flag = readonly ? FILE_MAP_READ : FILE_MAP_WRITE + exec && (file_flag |= FILE_MAP_EXECUTE) handle = create ? ccall(:CreateFileMappingW, stdcall, Ptr{Cvoid}, (OS_HANDLE, Ptr{Cvoid}, DWORD, DWORD, DWORD, Cwstring), - file_desc, C_NULL, readonly ? PAGE_READONLY : PAGE_READWRITE, szfile >> 32, szfile & typemax(UInt32), name) : + file_desc, C_NULL, page_flag, szfile >> 32, szfile & typemax(UInt32), name) : ccall(:OpenFileMappingW, stdcall, Ptr{Cvoid}, (DWORD, Cint, Cwstring), - readonly ? FILE_MAP_READ : FILE_MAP_WRITE, true, name) + page_flag, true, name) Base.windowserror(:mmap, handle == C_NULL) ptr = ccall(:MapViewOfFile, stdcall, Ptr{Cvoid}, (Ptr{Cvoid}, DWORD, DWORD, DWORD, Csize_t), - handle, readonly ? FILE_MAP_READ : FILE_MAP_WRITE, offset_page >> 32, offset_page & typemax(UInt32), mmaplen) + handle, file_flag, offset_page >> 32, offset_page & typemax(UInt32), mmaplen) Base.windowserror(:mmap, ptr == C_NULL) end # os-test # convert mmapped region to Julia Array at `ptr + (offset - offset_page)` since file was mapped at offset_page From 0ac80b258735ef5364ba7e678bc7af07a098812e Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 25 Feb 2024 15:17:47 +0100 Subject: [PATCH 03/18] add tests --- stdlib/Mmap/test/runtests.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index ebd16a45ba0ed..3fcc736fcb943 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -28,10 +28,12 @@ finalize(m); m=nothing; GC.gc() @test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0)) == 12 @test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; grow=false)) == 12 @test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; shared=false)) == 12 +@test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; exec=true)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12, 0)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12, 0; grow=false)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12, 0; shared=false)) == 12 +@test length(@inferred mmap(file, Vector{Int8}, 12, 0; exec=true)) == 12 s = open(file) @test length(@inferred mmap(s)) == 12 @test length(@inferred mmap(s, Vector{Int8})) == 12 @@ -39,10 +41,12 @@ s = open(file) @test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0)) == 12 @test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; grow=false)) == 12 @test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; shared=false)) == 12 +@test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; exec=true)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0; grow=false)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0; shared=false)) == 12 +@test length(@inferred mmap(s, Vector{Int8}, 12, 0; exec=true)) == 12 close(s) @test_throws ErrorException mmap(file, Vector{Ref}) # must be bit-type GC.gc(); GC.gc() @@ -292,6 +296,7 @@ m[1] = 0x0a Mmap.sync!(m) @test m[1] === 0x0a m = mmap(Vector{UInt8}, 12; shared=false) +m = mmap(Vector{UInt8}, 12; exec=true) m = mmap(Vector{Int}, 12) @test length(m) == 12 @test all(m .== 0) From f0e93930591a52eb3f1e970e2de4c250c78ff742 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 25 Feb 2024 23:49:11 +0100 Subject: [PATCH 04/18] fix windows flags --- stdlib/Mmap/src/Mmap.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index f9888a7a809f4..a5d82a7017279 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -256,7 +256,7 @@ function mmap(io::IO, handle = create ? ccall(:CreateFileMappingW, stdcall, Ptr{Cvoid}, (OS_HANDLE, Ptr{Cvoid}, DWORD, DWORD, DWORD, Cwstring), file_desc, C_NULL, page_flag, szfile >> 32, szfile & typemax(UInt32), name) : ccall(:OpenFileMappingW, stdcall, Ptr{Cvoid}, (DWORD, Cint, Cwstring), - page_flag, true, name) + file_flag, true, name) Base.windowserror(:mmap, handle == C_NULL) ptr = ccall(:MapViewOfFile, stdcall, Ptr{Cvoid}, (Ptr{Cvoid}, DWORD, DWORD, DWORD, Csize_t), handle, file_flag, offset_page >> 32, offset_page & typemax(UInt32), mmaplen) From c641d855d0b79d9643be534fae4647afd4e1c8ba Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 2 Mar 2024 13:15:44 -0800 Subject: [PATCH 05/18] restrict exec option to non-file-backed (anonymous) mmaps --- stdlib/Mmap/src/Mmap.jl | 49 ++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index a5d82a7017279..8a484c4947b38 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -127,7 +127,8 @@ end # os-test # core implementation of mmap """ - mmap(io::Union{IOStream,AbstractString,Mmap.AnonymousMmap}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true, exec::Bool=false) + mmap(io::Union{IOStream,AbstractString}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) + mmap(io::Anonymous[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true, exec::Bool=false) mmap(type::Type{Array{T,N}}, dims) Create an `Array` whose values are linked to a file, using memory-mapping. This provides a @@ -188,6 +189,21 @@ information in the header. In practice, consider encoding binary data using stan like HDF5 (which can be used with memory-mapping). """ function mmap(io::IO, + type::Type{Array{T,N}}=Vector{UInt8}, + dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), + offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T,N} + _mmap(io, type, dims, offset; grow, shared) +end +function mmap(io::Anonymous, + type::Type{Array{T,N}}=Vector{UInt8}, + dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), + offset::Integer=position(io); grow::Bool=true, shared::Bool=true, + exec::Bool=true) where {T,N} + _mmap(io, type, dims, offset; grow, shared, exec) +end + + +function _mmap(io::IO, ::Type{Array{T,N}}=Vector{UInt8}, dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), offset::Integer=position(io); grow::Bool=true, shared::Bool=true, @@ -195,6 +211,9 @@ function mmap(io::IO, # check inputs isopen(io) || throw(ArgumentError("$io must be open to mmap")) isbitstype(T) || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true")) + if exec && !(io isa Anonymous) + throw(ArgumentError("unable to mmap a file with exec=true")) + end len = sizeof(T) for l in dims @@ -279,14 +298,14 @@ end mmap(file::AbstractString, ::Type{T}=Vector{UInt8}, dims::NTuple{N,Integer}=(div(filesize(file),sizeof(eltype(T))),), - offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:Array,N} = - open(io->mmap(io, T, dims, offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::Array{eltype(T),N} + offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:Array,N} = + open(io->mmap(io, T, dims, offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::Array{eltype(T),N} # using a length argument instead of dims -mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:Array} = - mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec) -mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:Array} = - open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::Vector{eltype(T)} +mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T<:Array} = + mmap(io, T, (len,), offset; grow=grow, shared=shared) +mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:Array} = + open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::Vector{eltype(T)} # constructors for non-file-backed (anonymous) mmaps mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true, exec::Bool=false) where {T<:Array,N} = mmap(Anonymous(), T, dims, Int64(0); shared=shared, exec=exec) @@ -330,10 +349,10 @@ julia> rm("mmap.bin") This creates a 25-by-30000 `BitArray`, linked to the file associated with stream `io`. """ function mmap(io::IOStream, ::Type{<:BitArray}, dims::NTuple{N,Integer}, - offset::Int64=position(io); grow::Bool=true, shared::Bool=true, exec::Bool=false) where N + offset::Int64=position(io); grow::Bool=true, shared::Bool=true) where N n = prod(dims) nc = Base.num_bit_chunks(n) - chunks = mmap(io, Vector{UInt64}, (nc,), offset; grow=grow, shared=shared, exec=exec) + chunks = mmap(io, Vector{UInt64}, (nc,), offset; grow=grow, shared=shared) if !isreadonly(io) chunks[end] &= Base._msk_end(n) else @@ -350,14 +369,14 @@ function mmap(io::IOStream, ::Type{<:BitArray}, dims::NTuple{N,Integer}, return B end -mmap(file::AbstractString, ::Type{T}, dims::NTuple{N,Integer}, offset::Integer=Int64(0);grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:BitArray,N} = - open(io->mmap(io, T, dims, offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::BitArray{N} +mmap(file::AbstractString, ::Type{T}, dims::NTuple{N,Integer}, offset::Integer=Int64(0);grow::Bool=true, shared::Bool=true) where {T<:BitArray,N} = + open(io->mmap(io, T, dims, offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::BitArray{N} # using a length argument instead of dims -mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:BitArray} = - mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec) -mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true, exec::Bool=false) where {T<:BitArray} = - open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared, exec=exec), file, isfile(file) ? "r" : "w+")::BitVector +mmap(io::IO, ::Type{T}, len::Integer, offset::Integer=position(io); grow::Bool=true, shared::Bool=true) where {T<:BitArray} = + mmap(io, T, (len,), offset; grow=grow, shared=shared) +mmap(file::AbstractString, ::Type{T}, len::Integer, offset::Integer=Int64(0); grow::Bool=true, shared::Bool=true) where {T<:BitArray} = + open(io->mmap(io, T, (len,), offset; grow=grow, shared=shared), file, isfile(file) ? "r" : "w+")::BitVector # constructors for non-file-backed (anonymous) mmaps mmap(::Type{T}, dims::NTuple{N,Integer}; shared::Bool=true, exec::Bool=false) where {T<:BitArray,N} = mmap(Anonymous(), T, dims, Int64(0); shared=shared, exec=exec) From 9c0ba10ab7d3a1f8d834cc088654c7a31ab38d9b Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 2 Mar 2024 13:19:40 -0800 Subject: [PATCH 06/18] update tests --- stdlib/Mmap/test/runtests.jl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index 3fcc736fcb943..02df6f451132d 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -28,12 +28,10 @@ finalize(m); m=nothing; GC.gc() @test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0)) == 12 @test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; grow=false)) == 12 @test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; shared=false)) == 12 -@test length(@inferred mmap(file, Matrix{Int8}, (12,1), 0; exec=true)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12, 0)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12, 0; grow=false)) == 12 @test length(@inferred mmap(file, Vector{Int8}, 12, 0; shared=false)) == 12 -@test length(@inferred mmap(file, Vector{Int8}, 12, 0; exec=true)) == 12 s = open(file) @test length(@inferred mmap(s)) == 12 @test length(@inferred mmap(s, Vector{Int8})) == 12 @@ -41,12 +39,10 @@ s = open(file) @test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0)) == 12 @test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; grow=false)) == 12 @test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; shared=false)) == 12 -@test length(@inferred mmap(s, Matrix{Int8}, (12,1), 0; exec=true)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0; grow=false)) == 12 @test length(@inferred mmap(s, Vector{Int8}, 12, 0; shared=false)) == 12 -@test length(@inferred mmap(s, Vector{Int8}, 12, 0; exec=true)) == 12 close(s) @test_throws ErrorException mmap(file, Vector{Ref}) # must be bit-type GC.gc(); GC.gc() @@ -295,8 +291,10 @@ m = mmap(Vector{UInt8}, 12) m[1] = 0x0a Mmap.sync!(m) @test m[1] === 0x0a -m = mmap(Vector{UInt8}, 12; shared=false) -m = mmap(Vector{UInt8}, 12; exec=true) +m = mmap(Vector{UInt8}, 12; shared=false, exec=false) +m = mmap(Vector{UInt8}, 12; shared=true, exec=false) +m = mmap(Vector{UInt8}, 12; shared=false, exec=true) +m = mmap(Vector{UInt8}, 12; shared=true, exec=true) m = mmap(Vector{Int}, 12) @test length(m) == 12 @test all(m .== 0) From 0159a5acd64fd5988ed5bbc3ef44ddee2581008e Mon Sep 17 00:00:00 2001 From: Florian Date: Sat, 2 Mar 2024 22:34:55 +0100 Subject: [PATCH 07/18] Fix typo Co-authored-by: Kristoffer Carlsson --- stdlib/Mmap/src/Mmap.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 8a484c4947b38..11786392962b9 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -158,7 +158,7 @@ privileges are required to grow the file. The `shared` keyword argument specifies whether the resulting `Array` and changes made to it will be visible to other processes mapping the same file. -The `exec` keyword argument specifies whether the underlying mmap data will be executale. +The `exec` keyword argument specifies whether the underlying mmap data will be executable. For example, the following code From 5ecb880ec995dbeac82fcc35a85372c0b824e604 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 3 Mar 2024 16:48:49 +0100 Subject: [PATCH 08/18] add `MAP_JIT` for apple --- stdlib/Mmap/src/Mmap.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 11786392962b9..47f6c26b07766 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -77,7 +77,13 @@ function settings(s::RawFD, shared::Bool, exec::Bool=false) throw(ArgumentError("mmap requires read permissions on the file (open with \"r+\" mode to override)")) end end - exec && (prot |= PROT_EXEC) + if exec + prot |= PROT_EXEC + @static if Sys.isapple() + MAP_JIT = Cint(0x0800) + flags |= MAP_JIT + end + end return prot, flags, (prot & PROT_WRITE) > 0 end From cb0694ae1e687b4cd0fa05d0b1bc1b5d241df231 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 3 Mar 2024 17:21:50 +0100 Subject: [PATCH 09/18] fixup error check --- stdlib/Mmap/src/Mmap.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 47f6c26b07766..e9fe6dc6f0a0c 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -217,8 +217,8 @@ function _mmap(io::IO, # check inputs isopen(io) || throw(ArgumentError("$io must be open to mmap")) isbitstype(T) || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true")) - if exec && !(io isa Anonymous) - throw(ArgumentError("unable to mmap a file with exec=true")) + if exec && !iswritable(io) + throw(ArgumentError("$io must be writeable to mmap with exec = true")) end len = sizeof(T) From 4907cbf237a11c7b327f00d0421f5242624ed132 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 3 Mar 2024 17:24:11 +0100 Subject: [PATCH 10/18] use exec=false as default --- stdlib/Mmap/src/Mmap.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index e9fe6dc6f0a0c..0f4c03d290dca 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -204,7 +204,7 @@ function mmap(io::Anonymous, type::Type{Array{T,N}}=Vector{UInt8}, dims::NTuple{N,Integer}=(div(filesize(io)-position(io),sizeof(T)),), offset::Integer=position(io); grow::Bool=true, shared::Bool=true, - exec::Bool=true) where {T,N} + exec::Bool=false) where {T,N} _mmap(io, type, dims, offset; grow, shared, exec) end From 21a2a0c410d0da1dbd25e34fc904af53cf3e9bf7 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 3 Mar 2024 19:14:25 +0100 Subject: [PATCH 11/18] on MacOS, don't share when exec is requested --- stdlib/Mmap/src/Mmap.jl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 0f4c03d290dca..44f36bbe9933f 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -220,7 +220,12 @@ function _mmap(io::IO, if exec && !iswritable(io) throw(ArgumentError("$io must be writeable to mmap with exec = true")) end - + @static if Sys.isapple() + # on MacOS each thread has its own access permissions, so we can't share when exec=true + # https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon#Disable-Write-Protections-Before-You-Generate-Instructions + exec && (shared = false) + end + len = sizeof(T) for l in dims len, overflow = Base.Checked.mul_with_overflow(promote(len, l)...) From b0502e41165a2ad4bc424c4890dc0ebd8e4cdc3f Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 3 Mar 2024 19:14:44 +0100 Subject: [PATCH 12/18] add note about macOS share and exec policy fix whitespaces --- stdlib/Mmap/src/Mmap.jl | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 44f36bbe9933f..27c7e0deaaab7 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -166,6 +166,10 @@ will be visible to other processes mapping the same file. The `exec` keyword argument specifies whether the underlying mmap data will be executable. +!!! note + On MacOS `exec=true` implies `shared=false`, because each thread has its own access permissions to `mmap` regions. + + For example, the following code ```julia @@ -220,12 +224,12 @@ function _mmap(io::IO, if exec && !iswritable(io) throw(ArgumentError("$io must be writeable to mmap with exec = true")) end - @static if Sys.isapple() - # on MacOS each thread has its own access permissions, so we can't share when exec=true - # https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon#Disable-Write-Protections-Before-You-Generate-Instructions - exec && (shared = false) - end - + @static if Sys.isapple() + # on MacOS each thread has its own access permissions, so we can't share when exec=true + # https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon#Disable-Write-Protections-Before-You-Generate-Instructions + exec && (shared = false) + end + len = sizeof(T) for l in dims len, overflow = Base.Checked.mul_with_overflow(promote(len, l)...) From 0be41d131b4f36c3421e8714f87bfff990722816 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 5 Mar 2024 01:16:26 +0100 Subject: [PATCH 13/18] update docstring --- stdlib/Mmap/src/Mmap.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 27c7e0deaaab7..760f701d46f2f 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -135,7 +135,7 @@ end # os-test """ mmap(io::Union{IOStream,AbstractString}[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true) mmap(io::Anonymous[, type::Type{Array{T,N}}, dims, offset]; grow::Bool=true, shared::Bool=true, exec::Bool=false) - mmap(type::Type{Array{T,N}}, dims) + mmap(type::Type{Array{T,N}}, dims; shared::Bool=true, exec::Bool=false) Create an `Array` whose values are linked to a file, using memory-mapping. This provides a convenient way of working with data too large to fit in the computer's memory. From c2f030183e5cda3bc0f1ef9f6bc417d1a4f64d85 Mon Sep 17 00:00:00 2001 From: Florian Date: Tue, 5 Mar 2024 01:16:50 +0100 Subject: [PATCH 14/18] remove default value --- stdlib/Mmap/src/Mmap.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 760f701d46f2f..d47a3ef57ab23 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -63,7 +63,7 @@ const F_GETFL = Cint(3) gethandle(io::IO) = RawFD(fd(io)) # Determine a stream's read/write mode, and return prot & flags appropriate for mmap -function settings(s::RawFD, shared::Bool, exec::Bool=false) +function settings(s::RawFD, shared::Bool, exec::Bool) flags = shared ? MAP_SHARED : MAP_PRIVATE if s == INVALID_OS_HANDLE flags |= MAP_ANONYMOUS From c3c93e3a61da709191cf457de5b5a61b087f46a5 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 10 Mar 2024 21:23:01 +0100 Subject: [PATCH 15/18] update comments as to why exec=true implies shared=false on MacOS --- stdlib/Mmap/src/Mmap.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index d47a3ef57ab23..c33df93cb6357 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -167,7 +167,7 @@ will be visible to other processes mapping the same file. The `exec` keyword argument specifies whether the underlying mmap data will be executable. !!! note - On MacOS `exec=true` implies `shared=false`, because each thread has its own access permissions to `mmap` regions. + On MacOS `exec=true` implies `shared=false`. For example, the following code @@ -225,8 +225,9 @@ function _mmap(io::IO, throw(ArgumentError("$io must be writeable to mmap with exec = true")) end @static if Sys.isapple() - # on MacOS each thread has its own access permissions, so we can't share when exec=true - # https://developer.apple.com/documentation/apple-silicon/porting-just-in-time-compilers-to-apple-silicon#Disable-Write-Protections-Before-You-Generate-Instructions + # on MacOS exec=true requires the MAP_JIT flag to bypass W^X protections + # but combining MAP_JIT with MAP_SHARED is disallowed on MacOS, although its undocumented + # https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/bsd/kern/kern_mman.c#L328-L337 exec && (shared = false) end From b77258d693c2d8461ca3bbdcd5aef1e61004b788 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 10 Mar 2024 21:26:46 +0100 Subject: [PATCH 16/18] Remove senseless argument error Co-authored-by: Jameson Nash --- stdlib/Mmap/src/Mmap.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index c33df93cb6357..fc37218214d89 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -221,9 +221,6 @@ function _mmap(io::IO, # check inputs isopen(io) || throw(ArgumentError("$io must be open to mmap")) isbitstype(T) || throw(ArgumentError("unable to mmap $T; must satisfy isbitstype(T) == true")) - if exec && !iswritable(io) - throw(ArgumentError("$io must be writeable to mmap with exec = true")) - end @static if Sys.isapple() # on MacOS exec=true requires the MAP_JIT flag to bypass W^X protections # but combining MAP_JIT with MAP_SHARED is disallowed on MacOS, although its undocumented From d199c5d77b7dbbab96d8b29a3ac716e3dfedd2e2 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 10 Mar 2024 21:27:26 +0100 Subject: [PATCH 17/18] Update docstring mmap Co-authored-by: Jameson Nash --- stdlib/Mmap/src/Mmap.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index fc37218214d89..d8be87fc687b7 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -165,6 +165,7 @@ The `shared` keyword argument specifies whether the resulting `Array` and change will be visible to other processes mapping the same file. The `exec` keyword argument specifies whether the underlying mmap data will be executable. +Warning: on most CPUs, you must call a platform-specific function after creating this region and after writing to it, before executing it, or risk unpredictable behavior and corruption. !!! note On MacOS `exec=true` implies `shared=false`. From 263e5114b488b5fb5768cd23c697db44e605c7d0 Mon Sep 17 00:00:00 2001 From: Florian Date: Sun, 10 Mar 2024 21:28:24 +0100 Subject: [PATCH 18/18] fixup MAP_JIT usage on MacOS Co-authored-by: Jameson Nash --- stdlib/Mmap/src/Mmap.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index d8be87fc687b7..09a5f6569d9ae 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -81,7 +81,8 @@ function settings(s::RawFD, shared::Bool, exec::Bool) prot |= PROT_EXEC @static if Sys.isapple() MAP_JIT = Cint(0x0800) - flags |= MAP_JIT + # Bypassing W^X protections requires the MAP_JIT permission + ((prot & PROT_WRITE) > 0) && (flags |= MAP_JIT) end end return prot, flags, (prot & PROT_WRITE) > 0