clean up and export crc32c function

JuliaLang · stevengj · Jun 13, 2017 · Jun 7, 2017 · Jun 7, 2017 · Jun 7, 2017
commit 643b1e31de79cf876def6ad4cb3b5dd0211b65c8
diff --git a/NEWS.md b/NEWS.md
@@ -56,6 +56,8 @@ Library improvements
   * `resize!` and `sizehint!` methods no longer over-reserve memory when the
     requested array size is more than double of its current size ([#22038]).
 
+  * The `crc32c` function for CRC-32c checksums is now exported.
+
   * The output of `versioninfo()` is now controlled with keyword arguments ([#21974]).
 
 Compiler/Runtime improvements

diff --git a/base/exports.jl b/base/exports.jl
@@ -1047,6 +1047,7 @@ export
     atexit,
     atreplinit,
     clipboard,
+    crc32c,
     exit,
     ntuple,
     quit,

diff --git a/base/util.jl b/base/util.jl
@@ -765,22 +765,38 @@ if is_windows()
 
 end
 
-# compute sizeof correctly for strings, arrays, and subarrays of bytes
-_sizeof(a) = sizeof(a)
-_sizeof(a::FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N) = length(a)
-
 """
     crc32c(data, crc::UInt32=0x00000000)
 
 Compute the CRC-32c checksum of the given `data`, which can be
-an `Array{UInt8}`, a contiguous subarray thereof, or a `String`.  Optionally, you can pass
+an `Array{UInt8}`, a contiguous subarray thereof, an `IOBuffer`, or
+a filename (whose contents will be checksummed).  Optionally, you can pass
 a starting `crc` integer to be mixed in with the checksum.  The `crc` parameter
 can be used to compute a checksum on data divided into chunks: performing
 `crc32c(data2, crc32c(data1))` is equivalent to the checksum of `[data1; data2]`.
 (Technically, a little-endian checksum is computed.)
+
+To checksum `s::String`, you can do `crc32c(Vector{UInt8}(s))`; note
+that the result is specific to the UTF-8 encoding of `String`.  To checksum
+an `a::Array` of some other bitstype, you can do `crc32c(reinterpret(UInt8,a))`;
+note that the result is endian-dependent.
 """
-crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N,String}, crc::UInt32=0x00000000) =
-    ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, _sizeof(a))
+function crc32c end
+
+crc32c(a::Union{Array{UInt8},FastContiguousSubArray{UInt8,N,<:Array{UInt8}} where N}, crc::UInt32=0x00000000) =
+    ccall(:jl_crc32c, UInt32, (UInt32, Ptr{UInt8}, Csize_t), crc, a, length(a))
+
+crc32c(buf::IOBuffer, crc::UInt32=0x00000000) = crc32c(buf.data, crc)
+
+function crc32c(filename::AbstractString, crc::UInt32=0x00000000)
+    open(filename, "r") do f
+        data = Mmap.mmap(f, Vector{UInt8}, filesize(f), 0)
+        checksum = crc32c(data, crc)
+        finalize(data)
+        checksum
+    end
+end
+
 
 """
     @kwdef typedef

diff --git a/doc/src/stdlib/arrays.md b/doc/src/stdlib/arrays.md
@@ -131,6 +131,7 @@ Base.cumprod!
 Base.cumsum
 Base.cumsum!
 Base.cumsum_kbn
+Base.crc32c
 Base.LinAlg.diff
 Base.LinAlg.gradient
 Base.rot180

diff --git a/test/misc.jl b/test/misc.jl
@@ -571,9 +571,19 @@ for force_software_crc in (1,0)
     # test that crc parameter is equivalent to checksum of concatenated data,
     # and test crc of subarrays:
     a = UInt8[1:255;]
-    crc_256 = Base.crc32c(UInt8[1:255;])
+    crc_256 = crc32c(a)
     @views for n = 1:255
-        @test Base.crc32c(a[n+1:end], Base.crc32c(a[1:n])) == crc_256
+        @test crc32c(a[n+1:end], Base.crc32c(a[1:n])) == crc_256
+    end
+
+    @test crc32c(IOBuffer(a)) == crc_256
+    let f = tempname()
+        try
+            write(f, a)
+            @test crc32c(f) == crc_256
+        finally
+            rm(f, force=true)
+        end
     end
 end
-Original file line number
+Diff line change
@@ Expand Up / @@ -1047,6 +1047,7 @@ export @@
         atexit,
         atreplinit,
         clipboard,
+        crc32c,
         exit,
         ntuple,
         quit,
@@ Expand Down @@