use read, not mmap, for crc

JuliaLang · stevengj · Jun 13, 2017 · Jun 7, 2017 · Jun 7, 2017 · Jun 7, 2017
commit 184acd1b0d9d9faa020424da40920180f8aa7e93
diff --git a/base/loading.jl b/base/loading.jl
@@ -682,10 +682,8 @@ function compilecache(name::String)
     end
     if success(create_expr_cache(path, cachefile, concrete_deps))
         # append checksum to the end of the .ji file:
-        open(cachefile, "a+") do f
-            data = Mmap.mmap(f, Vector{UInt8}, filesize(f), 0)
-            checksum = crc32c(data)
-            finalize(data)
+        checksum = crc32c(cachefile)
+        open(cachefile, "a") do f
             write(f, hton(checksum))
         end
     else
@@ -809,12 +807,8 @@ function stale_cachefile(modpath::String, cachefile::String)
         end
 
         # finally, verify that the cache file has a valid checksum
-        data = Mmap.mmap(io, Vector{UInt8}, filesize(io), 0)
-        # checksum = UInt32 read in bigendian format from the last 4 bytes:
-        checksum = UInt32(data[end]) + UInt32(data[end-1])<<8 + UInt32(data[end-2])<<16 + UInt32(data[end-3])<<24
-        crc = crc32c(@view(data[1:end-4]))
-        finalize(data)
-        if checksum != crc
+        crc = crc32c(seekstart(io), filesize(io)-4)
+        if crc != ntoh(read(io, UInt32))
             DEBUG_LOADING[] && info("JL_DEBUG_LOADING: Rejecting cache file $cachefile because it has an invalid checksum.")
             return true
         end

diff --git a/base/util.jl b/base/util.jl
@@ -769,7 +769,7 @@ end
     crc32c(data, crc::UInt32=0x00000000)
 
 Compute the CRC-32c checksum of the given `data`, which can be
-an `Array{UInt8}`, a contiguous subarray thereof, an `IOBuffer`, or
+an `Array{UInt8}`, a contiguous subarray thereof, or an `IOBuffer`, or
 a filename (whose contents will be checksummed).  Optionally, you can pass
 a starting `crc` integer to be mixed in with the checksum.  The `crc` parameter
 can be used to compute a checksum on data divided into chunks: performing
@@ -788,15 +788,27 @@ crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} wher
 
 crc32c(buf::IOBuffer, crc::UInt32=0x00000000) = crc32c(buf.data, crc)
 
-function crc32c(filename::AbstractString, crc::UInt32=0x00000000)
-    open(filename, "r") do f
-        data = Mmap.mmap(f, Vector{UInt8}, filesize(f), 0)
-        checksum = crc32c(data, crc)
-        finalize(data)
-        checksum
+"""
+    crc32c(f::IO, nb::Integer, crc::UInt32=0x00000000)
+
+Read up to `nb` bytes from `f` and return the CRC-32c checksum, optionally
+mixed with a starting `crc` integer.
+"""
+function crc32c(f::IO, nb::Integer, crc::UInt32=0x00000000)
+    buf = Array{UInt8}(min(nb, 16384))
+    while !eof(f) && nb > 0
+        n = readbytes!(f, buf, nb)
+        crc = ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, buf, n)
+        nb -= n
     end
+    return crc
 end
 
+crc32c(filename::AbstractString, crc::UInt32=0x00000000) =
+    open(filename, "r") do f
+        crc32c(f, filesize(f), crc)
+    end
+
 
 """
     @kwdef typedef

diff --git a/doc/src/stdlib/io-network.md b/doc/src/stdlib/io-network.md
@@ -12,6 +12,7 @@ Base.take!(::Base.AbstractIOBuffer)
 Base.fdio
 Base.flush
 Base.close
+Base.crc32(::IO, ::Integer, ::UInt32)
 Base.write
 Base.read
 Base.read!

diff --git a/test/misc.jl b/test/misc.jl
@@ -566,21 +566,25 @@ end
 for force_software_crc in (1,0)
     ccall(:jl_crc32c_init, Void, (Cint,), force_software_crc)
     for (n,crc) in [(0,0x00000000),(1,0xa016d052),(2,0x03f89f52),(3,0xf130f21e),(4,0x29308cf4),(5,0x53518fab),(6,0x4f4dfbab),(7,0xbd3a64dc),(8,0x46891f81),(9,0x5a14b9f9),(10,0xb219db69),(11,0xd232a91f),(12,0x51a15563),(13,0x9f92de41),(14,0x4d8ae017),(15,0xc8b74611),(16,0xa0de6714),(17,0x672c992a),(18,0xe8206eb6),(19,0xc52fd285),(20,0x327b0397),(21,0x318263dd),(22,0x08485ccd),(23,0xea44d29e),(24,0xf6c0cb13),(25,0x3969bba2),(26,0x6a8810ec),(27,0x75b3d0df),(28,0x82d535b1),(29,0xbdf7fc12),(30,0x1f836b7d),(31,0xd29f33af),(32,0x8e4acb3e),(33,0x1cbee2d1),(34,0xb25f7132),(35,0xb0fa484c),(36,0xb9d262b4),(37,0x3207fe27),(38,0xa024d7ac),(39,0x49a2e7c5),(40,0x0e2c157f),(41,0x25f7427f),(42,0x368c6adc),(43,0x75efd4a5),(44,0xa84c5c31),(45,0x0fc817b2),(46,0x8d99a881),(47,0x5cc3c078),(48,0x9983d5e2),(49,0x9267c2db),(50,0xc96d4745),(51,0x058d8df3),(52,0x453f9cf3),(53,0xb714ade1),(54,0x55d3c2bc),(55,0x495710d0),(56,0x3bddf494),(57,0x4f2577d0),(58,0xdae0f604),(59,0x3c57c632),(60,0xfe39bbb0),(61,0x6f5d1d41),(62,0x7d996665),(63,0x68c738dc),(64,0x8dfea7ae)]
-        @test Base.crc32c(UInt8[1:n;]) == crc
+        @test crc32c(UInt8[1:n;]) == crc
     end
     # test that crc parameter is equivalent to checksum of concatenated data,
     # and test crc of subarrays:
     a = UInt8[1:255;]
     crc_256 = crc32c(a)
     @views for n = 1:255
-        @test crc32c(a[n+1:end], Base.crc32c(a[1:n])) == crc_256
+        @test crc32c(a[n+1:end], crc32c(a[1:n])) == crc_256
     end
 
     @test crc32c(IOBuffer(a)) == crc_256
     let f = tempname()
         try
             write(f, a)
             @test crc32c(f) == crc_256
+            open(f, "r") do io
+                @test crc32c(io, 16) == crc32c(a[1:16])
+                @test crc32c(io, 16) == crc32c(a[17:32])
+            end
         finally
             rm(f, force=true)
         end