From d149055649bec3871a3cd76d1a9989781c93abb8 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 7 Oct 2016 17:30:25 -0400 Subject: [PATCH 1/3] return `false` for deprecated bindings in `is_exported_from_stdlib` --- base/show.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index f3b8af0f9909e..1b76b80313cef 100644 --- a/base/show.jl +++ b/base/show.jl @@ -156,7 +156,7 @@ function is_exported_from_stdlib(name::Symbol, mod::Module) end mod = parent end - return isexported(mod, name) && isdefined(mod, name) && getfield(mod, name) === orig + return isexported(mod, name) && isdefined(mod, name) && !isdeprecated(mod, name) && getfield(mod, name) === orig end function show(io::IO, f::Function) From fe3b08121d4ab3d929a1921e3beb0e360d2d329e Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Mon, 10 Oct 2016 15:06:03 -0400 Subject: [PATCH 2/3] if a binding exists in multiple `using`s, prefer a non-deprecated one This removes a spurious warning if Base.X is deprecated to Mod.X, and one does `using Mod`. `X` is available from both `Base` and `Mod`, but we asked for the non-deprecated one so there should be no warning. --- src/module.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 651ba648c36b8..9c0492a99b3ee 100644 --- a/src/module.c +++ b/src/module.c @@ -200,8 +200,10 @@ static jl_binding_t *jl_get_binding_(jl_module_t *m, jl_sym_t *var, modstack_t * (void)jl_get_binding_wr(m, var); return NULL; } - owner = imp; - b = tempb; + if (owner == NULL || !tempb->deprecated) { + owner = imp; + b = tempb; + } } } if (owner != NULL) { From 7cc22c51191d3ee91f3de428e07f715a103b61a1 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 7 Oct 2016 17:31:29 -0400 Subject: [PATCH 3/3] add `Iterators` module around extra iterator functionality --- NEWS.md | 4 + base/REPLCompletions.jl | 4 +- base/asyncmap.jl | 1 + base/broadcast.jl | 2 +- base/client.jl | 2 +- base/dates/Dates.jl | 2 + base/deprecated.jl | 10 + base/exports.jl | 12 +- base/generator.jl | 8 - base/{iterator.jl => iterators.jl} | 56 ++-- base/multidimensional.jl | 2 +- base/pkg/resolve.jl | 2 +- base/pmap.jl | 44 +++- base/promotion.jl | 2 +- base/sysimg.jl | 4 +- doc/stdlib/collections.rst | 133 ---------- doc/stdlib/index.rst | 1 + doc/stdlib/iterators.rst | 185 ++++++++++++++ src/julia-syntax.scm | 2 +- test/choosetests.jl | 2 +- test/functional.jl | 398 +---------------------------- test/iterators.jl | 393 ++++++++++++++++++++++++++++ 22 files changed, 671 insertions(+), 598 deletions(-) rename base/{iterator.jl => iterators.jl} (96%) create mode 100644 doc/stdlib/iterators.rst create mode 100644 test/iterators.jl diff --git a/NEWS.md b/NEWS.md index e68bffbf055d4..2e8821d1178e1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -49,6 +49,10 @@ Library improvements One way of doing this is by adding `ENV["JULIA_INFO_COLOR"] = :blue` to the `.juliarc.jl` file. For more information regarding customizing colors in the REPL, see this [manual section]( http://docs.julialang.org/en/latest/manual/interacting-with-julia/#customizing-colors). + * Iteration utilities that wrap iterators and return other iterators (`enumerate`, `zip`, `rest`, + `countfrom`, `take`, `drop`, `cycle`, `repeated`, `product`, `flatten`, `partition`) have been + moved to the module `Base.Iterators` ([#18839]). + Compiler/Runtime improvements ----------------------------- diff --git a/base/REPLCompletions.jl b/base/REPLCompletions.jl index f07b8d454cce4..c7e2897538f19 100644 --- a/base/REPLCompletions.jl +++ b/base/REPLCompletions.jl @@ -376,10 +376,10 @@ function bslash_completions(string, pos) # return possible matches; these cannot be mixed with regular # Julian completions as only latex / emoji symbols contain the leading \ if startswith(s, "\\:") # emoji - emoji_names = filter(k -> startswith(k, s), keys(emoji_symbols)) + emoji_names = Iterators.filter(k -> startswith(k, s), keys(emoji_symbols)) return (true, (sort!(collect(emoji_names)), slashpos:pos, true)) else # latex - latex_names = filter(k -> startswith(k, s), keys(latex_symbols)) + latex_names = Iterators.filter(k -> startswith(k, s), keys(latex_symbols)) return (true, (sort!(collect(latex_names)), slashpos:pos, true)) end end diff --git a/base/asyncmap.jl b/base/asyncmap.jl index 9490f713267a5..a078021fd66ce 100644 --- a/base/asyncmap.jl +++ b/base/asyncmap.jl @@ -1,5 +1,6 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +using Base.Iterators.Enumerate """ AsyncCollector(f, results, c...; ntasks=0) -> iterator diff --git a/base/broadcast.jl b/base/broadcast.jl index 917a9be9f3a5a..fca4df5a09878 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -226,7 +226,7 @@ end else R = typejoin(eltype(B), S) new = similar(B, R) - for II in take(iter, count) + for II in Iterators.take(iter, count) new[II] = B[II] end new[I] = V diff --git a/base/client.jl b/base/client.jl index 28636b43ef341..ce83ff0f8ec5d 100644 --- a/base/client.jl +++ b/base/client.jl @@ -23,7 +23,7 @@ end # Create a docstring with an automatically generated list # of colors. const possible_formatting_symbols = [:normal, :bold] -available_text_colors = collect(filter(x -> !isa(x, Integer), keys(text_colors))) +available_text_colors = collect(Iterators.filter(x -> !isa(x, Integer), keys(text_colors))) available_text_colors = cat(1, sort(intersect(available_text_colors, possible_formatting_symbols), rev=true), sort(setdiff( available_text_colors, possible_formatting_symbols))) diff --git a/base/dates/Dates.jl b/base/dates/Dates.jl index ff2165de586d0..1e764e67d93a3 100644 --- a/base/dates/Dates.jl +++ b/base/dates/Dates.jl @@ -4,6 +4,8 @@ module Dates importall ..Base.Operators +using Base.Iterators + include("types.jl") include("periods.jl") include("accessors.jl") diff --git a/base/deprecated.jl b/base/deprecated.jl index ccc15d183442f..a22b7add42146 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1023,4 +1023,14 @@ end)) @deprecate is (===) +@deprecate_binding Filter Iterators.Filter +@deprecate_binding Zip Iterators.Zip +@deprecate filter(flt, itr) Iterators.filter(flt, itr) +@deprecate_binding rest Iterators.rest +@deprecate_binding countfrom Iterators.countfrom +@deprecate_binding take Iterators.take +@deprecate_binding drop Iterators.drop +@deprecate_binding cycle Iterators.cycle +@deprecate_binding repeated Iterators.repeated + # End deprecations scheduled for 0.6 diff --git a/base/exports.jl b/base/exports.jl index e6d6dabbf8925..306697bbf229f 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -23,6 +23,7 @@ export Docs, Markdown, Threads, + Iterators, # Types AbstractChannel, @@ -61,7 +62,6 @@ export Enumerate, Factorization, FileMonitor, - Filter, FloatRange, Future, Hermitian, @@ -124,7 +124,6 @@ export VersionNumber, WeakKeyDict, WorkerConfig, - Zip, # Ccall types Cchar, @@ -958,16 +957,11 @@ export # iteration done, - enumerate, next, start, + + enumerate, # re-exported from Iterators zip, - rest, - countfrom, - take, - drop, - cycle, - repeated, # object identity and equality copy, diff --git a/base/generator.jl b/base/generator.jl index fce99b73e31b7..9c1b8e0a71ab3 100644 --- a/base/generator.jl +++ b/base/generator.jl @@ -55,11 +55,6 @@ result, and algorithms that resize their result incrementally. iteratorsize(x) = iteratorsize(typeof(x)) iteratorsize(::Type) = HasLength() # HasLength is the default -and_iteratorsize{T}(isz::T, ::T) = isz -and_iteratorsize(::HasLength, ::HasShape) = HasLength() -and_iteratorsize(::HasShape, ::HasLength) = HasLength() -and_iteratorsize(a, b) = SizeUnknown() - abstract IteratorEltype immutable EltypeUnknown <: IteratorEltype end immutable HasEltype <: IteratorEltype end @@ -81,9 +76,6 @@ values. iteratoreltype(x) = iteratoreltype(typeof(x)) iteratoreltype(::Type) = HasEltype() # HasEltype is the default -and_iteratoreltype{T}(iel::T, ::T) = iel -and_iteratoreltype(a, b) = EltypeUnknown() - iteratorsize{T<:AbstractArray}(::Type{T}) = HasShape() iteratorsize{I,F}(::Type{Generator{I,F}}) = iteratorsize(I) length(g::Generator) = length(g.iter) diff --git a/base/iterator.jl b/base/iterators.jl similarity index 96% rename from base/iterator.jl rename to base/iterators.jl index 8d581337ec2bd..d3e532b1d0a82 100644 --- a/base/iterator.jl +++ b/base/iterators.jl @@ -1,5 +1,13 @@ # This file is a part of Julia. License is MIT: http://julialang.org/license +module Iterators + +import Base: start, done, next, isempty, length, size, eltype, iteratorsize, iteratoreltype, indices, ndims + +using Base: tuple_type_cons, SizeUnknown, HasLength, HasShape, IsInfinite, EltypeUnknown, HasEltype, OneTo + +export enumerate, zip, rest, countfrom, take, drop, cycle, repeated, product, flatten, partition + _min_length(a, b, ::IsInfinite, ::IsInfinite) = min(length(a),length(b)) # inherit behaviour, error _min_length(a, b, A, ::IsInfinite) = length(a) _min_length(a, b, ::IsInfinite, B) = length(b) @@ -10,6 +18,14 @@ _diff_length(a, b, ::IsInfinite, ::IsInfinite) = 0 _diff_length(a, b, ::IsInfinite, B) = length(a) # inherit behaviour, error _diff_length(a, b, A, B) = max(length(a)-length(b), 0) +and_iteratorsize{T}(isz::T, ::T) = isz +and_iteratorsize(::HasLength, ::HasShape) = HasLength() +and_iteratorsize(::HasShape, ::HasLength) = HasLength() +and_iteratorsize(a, b) = SizeUnknown() + +and_iteratoreltype{T}(iel::T, ::T) = iel +and_iteratoreltype(a, b) = EltypeUnknown() + # enumerate immutable Enumerate{I} @@ -238,42 +254,6 @@ rest_iteratorsize(::IsInfinite) = IsInfinite() iteratorsize{I,S}(::Type{Rest{I,S}}) = rest_iteratorsize(iteratorsize(I)) -""" - head_and_tail(c, n) -> head, tail - -Returns `head`: the first `n` elements of `c`; -and `tail`: an iterator over the remaining elements. - -```jldoctest -julia> a = 1:10 -1:10 - -julia> b, c = Base.head_and_tail(a, 3) -([1,2,3],Base.Rest{UnitRange{Int64},Int64}(1:10,4)) - -julia> collect(c) -7-element Array{Any,1}: - 4 - 5 - 6 - 7 - 8 - 9 - 10 -``` -""" -function head_and_tail(c, n) - head = Vector{eltype(c)}(n) - s = start(c) - i = 0 - while i < n && !done(c, s) - i += 1 - head[i], s = next(c, s) - end - return resize!(head, i), rest(c, s) -end - - # Count -- infinite counting immutable Count{S<:Number} @@ -677,7 +657,7 @@ end """ - partition(collection, n) -> iterator + partition(collection, n) Iterate over a collection `n` elements at a time. @@ -723,3 +703,5 @@ function next(itr::PartitionIterator, state) end return resize!(v, i), state end + +end diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 69b6ef1a9887b..ddf5263c5bd68 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -381,7 +381,7 @@ end # and ensure the value to set is either an AbstractArray or a Repeated scalar # before redispatching to the _unsafe_batchsetindex! _iterable(v::AbstractArray) = v -_iterable(v) = repeated(v) +_iterable(v) = Iterators.repeated(v) @inline function _setindex!{T,N}(l::LinearIndexing, A::AbstractArray{T,N}, x, J::Vararg{Union{Real,AbstractArray,Colon},N}) @boundscheck checkbounds(A, J...) _unsafe_setindex!(l, A, x, J...) diff --git a/base/pkg/resolve.jl b/base/pkg/resolve.jl index 1f07d66fb98e3..4b5f9c66bf7be 100644 --- a/base/pkg/resolve.jl +++ b/base/pkg/resolve.jl @@ -66,7 +66,7 @@ function sanity_check(deps::Dict{String,Dict{VersionNumber,Available}}, vers = Array{Tuple{String,VersionNumber,VersionNumber}}(0) for (p,d) in deps, vn in keys(d) - lvns = VersionNumber[filter(vn2->(vn2>vn), keys(d))...] + lvns = VersionNumber[Iterators.filter(vn2->(vn2>vn), keys(d))...] nvn = isempty(lvns) ? typemax(VersionNumber) : minimum(lvns) push!(vers, (p,vn,nvn)) end diff --git a/base/pmap.jl b/base/pmap.jl index 9ea825067df32..be0ff63c87680 100644 --- a/base/pmap.jl +++ b/base/pmap.jl @@ -24,7 +24,7 @@ function pgenerate(p::WorkerPool, f, c) return AsyncGenerator(f, c; ntasks=()->nworkers(p)) end batches = batchsplit(c, min_batch_count = length(p) * 3) - return flatten(AsyncGenerator(remote(p, b -> asyncmap(f, b)), batches)) + return Iterators.flatten(AsyncGenerator(remote(p, b -> asyncmap(f, b)), batches)) end pgenerate(p::WorkerPool, f, c1, c...) = pgenerate(p, a->f(a...), zip(c1, c...)) pgenerate(f, c) = pgenerate(default_worker_pool(), f, c) @@ -133,7 +133,7 @@ function pmap(p::AbstractWorkerPool, f, c; distributed=true, batch_size=1, on_er f = wrap_on_error(f, (x,e)->BatchProcessingError(x,e); capture_data=true) end f = wrap_batch(f, p, on_error) - results = collect(flatten(AsyncGenerator(f, batches; ntasks=()->nworkers(p)))) + results = collect(Iterators.flatten(AsyncGenerator(f, batches; ntasks=()->nworkers(p)))) if (on_error !== nothing) || (retry_n > 0) process_batch_errors!(p, f_orig, results, on_error, retry_on, retry_n, retry_max_delay) end @@ -213,6 +213,40 @@ function process_batch_errors!(p, f, results, on_error, retry_on, retry_n, retry nothing end +""" + head_and_tail(c, n) -> head, tail + +Returns `head`: the first `n` elements of `c`; +and `tail`: an iterator over the remaining elements. + +```jldoctest +julia> a = 1:10 +1:10 + +julia> b, c = Base.head_and_tail(a, 3) +([1,2,3],Base.Iterators.Rest{UnitRange{Int64},Int64}(1:10,4)) + +julia> collect(c) +7-element Array{Any,1}: + 4 + 5 + 6 + 7 + 8 + 9 + 10 +``` +""" +function head_and_tail(c, n) + head = Vector{eltype(c)}(n) + s = start(c) + i = 0 + while i < n && !done(c, s) + i += 1 + head[i], s = next(c, s) + end + return resize!(head, i), Iterators.rest(c, s) +end """ batchsplit(c; min_batch_count=1, max_batch_size=100) -> iterator @@ -231,14 +265,14 @@ function batchsplit(c; min_batch_count=1, max_batch_size=100) end # Split collection into batches, then peek at the first few batches - batches = partition(c, max_batch_size) + batches = Iterators.partition(c, max_batch_size) head, tail = head_and_tail(batches, min_batch_count) # If there are not enough batches, use a smaller batch size if length(head) < min_batch_count batch_size = max(1, div(sum(length, head), min_batch_count)) - return partition(collect(flatten(head)), batch_size) + return Iterators.partition(collect(Iterators.flatten(head)), batch_size) end - return flatten((head, tail)) + return Iterators.flatten((head, tail)) end diff --git a/base/promotion.jl b/base/promotion.jl index bb1d343f8a79f..37e0d6a1373ef 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -226,7 +226,7 @@ if isdefined(Core, :Inference) end function _promote_op(op, R::ANY, S::ANY) F = typeof(a -> op(a...)) - G = Tuple{Generator{Zip2{Tuple{R},Tuple{S}},F}} + G = Tuple{Generator{Iterators.Zip2{Tuple{R},Tuple{S}},F}} return Core.Inference.return_type(first, G) end else diff --git a/base/sysimg.jl b/base/sysimg.jl index 6c739d9c8f36c..82b4d45cc0a90 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -129,7 +129,9 @@ include("intset.jl") include("associative.jl") include("dict.jl") include("set.jl") -include("iterator.jl") +include("iterators.jl") +using .Iterators: zip, enumerate +using .Iterators: Flatten, product # for generators # Definition of StridedArray typealias StridedReshapedArray{T,N,A<:DenseArray} ReshapedArray{T,N,A} diff --git a/doc/stdlib/collections.rst b/doc/stdlib/collections.rst index 8b9628b85336f..0d3d6452ecf68 100644 --- a/doc/stdlib/collections.rst +++ b/doc/stdlib/collections.rst @@ -47,139 +47,6 @@ type. For a given iterable object and iteration state, return the current item and the next iteration state. -.. function:: zip(iters...) - - .. Docstring generated from Julia source - - For a set of iterable objects, returns an iterable of tuples, where the ``i``\ th tuple contains the ``i``\ th component of each input iterable. - - Note that :func:`zip` is its own inverse: ``collect(zip(zip(a...)...)) == collect(a)``\ . - - .. doctest:: - - julia> a = 1:5 - 1:5 - - julia> b = ["e","d","b","c","a"] - 5-element Array{String,1}: - "e" - "d" - "b" - "c" - "a" - - julia> c = zip(a,b) - Base.Zip2{UnitRange{Int64},Array{String,1}}(1:5,String["e","d","b","c","a"]) - - julia> length(c) - 5 - - julia> first(c) - (1,"e") - -.. function:: enumerate(iter) - - .. Docstring generated from Julia source - - An iterator that yields ``(i, x)`` where ``i`` is a counter starting at 1, and ``x`` is the ``i``\ th value from the given iterator. It's useful when you need not only the values ``x`` over which you are iterating, but also the number of iterations so far. Note that ``i`` may not be valid for indexing ``iter``\ ; it's also possible that ``x != iter[i]``\ , if ``iter`` has indices that do not start at 1. - - .. doctest:: - - julia> a = ["a", "b", "c"]; - - julia> for (index, value) in enumerate(a) - println("$index $value") - end - 1 a - 2 b - 3 c - -.. function:: rest(iter, state) - - .. Docstring generated from Julia source - - An iterator that yields the same elements as ``iter``\ , but starting at the given ``state``\ . - -.. function:: countfrom(start=1, step=1) - - .. Docstring generated from Julia source - - An iterator that counts forever, starting at ``start`` and incrementing by ``step``\ . - -.. function:: take(iter, n) - - .. Docstring generated from Julia source - - An iterator that generates at most the first ``n`` elements of ``iter``\ . - - .. doctest:: - - julia> a = 1:2:11 - 1:2:11 - - julia> collect(a) - 6-element Array{Int64,1}: - 1 - 3 - 5 - 7 - 9 - 11 - - julia> collect(take(a,3)) - 3-element Array{Int64,1}: - 1 - 3 - 5 - -.. function:: drop(iter, n) - - .. Docstring generated from Julia source - - An iterator that generates all but the first ``n`` elements of ``iter``\ . - - .. doctest:: - - julia> a = 1:2:11 - 1:2:11 - - julia> collect(a) - 6-element Array{Int64,1}: - 1 - 3 - 5 - 7 - 9 - 11 - - julia> collect(drop(a,4)) - 2-element Array{Int64,1}: - 9 - 11 - -.. function:: cycle(iter) - - .. Docstring generated from Julia source - - An iterator that cycles through ``iter`` forever. - -.. function:: repeated(x[, n::Int]) - - .. Docstring generated from Julia source - - An iterator that generates the value ``x`` forever. If ``n`` is specified, generates ``x`` that many times (equivalent to ``take(repeated(x), n)``\ ). - - .. doctest:: - - julia> a = repeated([1 2], 4); - - julia> collect(a) - 4-element Array{Array{Int64,2},1}: - [1 2] - [1 2] - [1 2] - [1 2] - .. function:: iteratorsize(itertype::Type) -> IteratorSize .. Docstring generated from Julia source diff --git a/doc/stdlib/index.rst b/doc/stdlib/index.rst index b78baac701d79..1152cc48f695f 100644 --- a/doc/stdlib/index.rst +++ b/doc/stdlib/index.rst @@ -22,6 +22,7 @@ sort pkg dates + iterators test c libc diff --git a/doc/stdlib/iterators.rst b/doc/stdlib/iterators.rst new file mode 100644 index 0000000000000..393e3d2c05740 --- /dev/null +++ b/doc/stdlib/iterators.rst @@ -0,0 +1,185 @@ +.. module:: Base.Iterators + +********************* + Iteration utilities +********************* + +.. function:: zip(iters...) + + .. Docstring generated from Julia source + + For a set of iterable objects, returns an iterable of tuples, where the ``i``\ th tuple contains the ``i``\ th component of each input iterable. + + Note that :func:`zip` is its own inverse: ``collect(zip(zip(a...)...)) == collect(a)``\ . + + .. doctest:: + + julia> a = 1:5 + 1:5 + + julia> b = ["e","d","b","c","a"] + 5-element Array{String,1}: + "e" + "d" + "b" + "c" + "a" + + julia> c = zip(a,b) + Base.Zip2{UnitRange{Int64},Array{String,1}}(1:5,String["e","d","b","c","a"]) + + julia> length(c) + 5 + + julia> first(c) + (1,"e") + +.. function:: enumerate(iter) + + .. Docstring generated from Julia source + + An iterator that yields ``(i, x)`` where ``i`` is a counter starting at 1, and ``x`` is the ``i``\ th value from the given iterator. It's useful when you need not only the values ``x`` over which you are iterating, but also the number of iterations so far. Note that ``i`` may not be valid for indexing ``iter``\ ; it's also possible that ``x != iter[i]``\ , if ``iter`` has indices that do not start at 1. + + .. doctest:: + + julia> a = ["a", "b", "c"]; + + julia> for (index, value) in enumerate(a) + println("$index $value") + end + 1 a + 2 b + 3 c + +.. function:: rest(iter, state) + + .. Docstring generated from Julia source + + An iterator that yields the same elements as ``iter``\ , but starting at the given ``state``\ . + +.. function:: countfrom(start=1, step=1) + + .. Docstring generated from Julia source + + An iterator that counts forever, starting at ``start`` and incrementing by ``step``\ . + +.. function:: take(iter, n) + + .. Docstring generated from Julia source + + An iterator that generates at most the first ``n`` elements of ``iter``\ . + + .. doctest:: + + julia> a = 1:2:11 + 1:2:11 + + julia> collect(a) + 6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + + julia> collect(take(a,3)) + 3-element Array{Int64,1}: + 1 + 3 + 5 + +.. function:: drop(iter, n) + + .. Docstring generated from Julia source + + An iterator that generates all but the first ``n`` elements of ``iter``\ . + + .. doctest:: + + julia> a = 1:2:11 + 1:2:11 + + julia> collect(a) + 6-element Array{Int64,1}: + 1 + 3 + 5 + 7 + 9 + 11 + + julia> collect(drop(a,4)) + 2-element Array{Int64,1}: + 9 + 11 + +.. function:: cycle(iter) + + .. Docstring generated from Julia source + + An iterator that cycles through ``iter`` forever. + +.. function:: repeated(x[, n::Int]) + + .. Docstring generated from Julia source + + An iterator that generates the value ``x`` forever. If ``n`` is specified, generates ``x`` that many times (equivalent to ``take(repeated(x), n)``\ ). + + .. doctest:: + + julia> a = repeated([1 2], 4); + + julia> collect(a) + 4-element Array{Array{Int64,2},1}: + [1 2] + [1 2] + [1 2] + [1 2] + +.. function:: product(iters...) + + .. Docstring generated from Julia source + + Returns an iterator over the product of several iterators. Each generated element is a tuple whose ``i``\ th element comes from the ``i``\ th argument iterator. The first iterator changes the fastest. Example: + + .. doctest:: + + julia> collect(product(1:2,3:5)) + 6-element Array{Tuple{Int64,Int64},1}: + (1,3) + (2,3) + (1,4) + (2,4) + (1,5) + (2,5) + +.. function:: flatten(iter) + + .. Docstring generated from Julia source + + Given an iterator that yields iterators, return an iterator that yields the elements of those iterators. Put differently, the elements of the argument iterator are concatenated. Example: + + .. doctest:: + + julia> collect(flatten((1:2, 8:9))) + 4-element Array{Int64,1}: + 1 + 2 + 8 + 9 + +.. function:: partition(collection, n) + + .. Docstring generated from Julia source + + Iterate over a collection ``n`` elements at a time. + + .. doctest:: + + julia> collect(partition([1,2,3,4,5], 2)) + 3-element Array{Array{Int64,1},1}: + [1,2] + [3,4] + [5] + diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 64325c8dd66a7..fc8c98b7f0ba4 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2156,7 +2156,7 @@ (car ranges) `(call (top product) ,@ranges))) (iter (if filt? - `(call (top Filter) + `(call (|.| (top Iterators) 'Filter) ,(func-for-generator-ranges (cadr (caddr e)) range-exprs) ,iter) iter))) diff --git a/test/choosetests.jl b/test/choosetests.jl index 03b91553e40d8..bc6058b386ea1 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -20,7 +20,7 @@ function choosetests(choices = []) "dates", "dict", "hashing", "iobuffer", "staged", "offsetarray", "arrayops", "tuple", "reduce", "reducedim", "random", "abstractarray", "intfuncs", "simdloop", "vecelement", "blas", "sparse", - "bitarray", "copy", "math", "fastmath", "functional", + "bitarray", "copy", "math", "fastmath", "functional", "iterators", "operators", "path", "ccall", "parse", "loading", "bigint", "bigfloat", "sorting", "statistics", "spawn", "backtrace", "priorityqueue", "file", "read", "mmap", "version", "resolve", diff --git a/test/functional.jl b/test/functional.jl index f2b7a9ca2fcb2..91da29567333c 100644 --- a/test/functional.jl +++ b/test/functional.jl @@ -45,362 +45,8 @@ end # numbers @test size(collect(1)) == size(1) -# zip and filter iterators -# issue #4718 -@test collect(filter(x->x[1], zip([true, false, true, false],"abcd"))) == [(true,'a'),(true,'c')] - -let z = zip(1:2) - @test collect(z) == [(1,), (2,)] - # Issue #13979 - @test eltype(z) == Tuple{Int} -end - -let z = zip(1:2, 3:4) - @test collect(z) == [(1,3), (2,4)] - @test eltype(z) == Tuple{Int,Int} -end - -let z = zip(1:2, 3:4, 5:6) - @test collect(z) == [(1,3,5), (2,4,6)] - @test eltype(z) == Tuple{Int,Int,Int} -end - -@test eltype(Filter(isodd, 1:5)) == Int - -# typed `collect` -@test collect(Float64, Filter(isodd, [1,2,3,4]))[1] === 1.0 - @test isa(collect(Any, [1,2]), Vector{Any}) -# enumerate (issue #6284) -let b = IOBuffer("1\n2\n3\n"), a = [] - for (i,x) in enumerate(eachline(b)) - push!(a, (i,x)) - end - @test a == [(1,"1\n"),(2,"2\n"),(3,"3\n")] -end - -# zip eachline (issue #7369) -let zeb = IOBuffer("1\n2\n3\n4\n5\n"), - letters = ['a', 'b', 'c', 'd', 'e'], - res = [] - for (number, letter) in zip(eachline(zeb), letters) - push!(res, (parse(Int,strip(number)), letter)) - end - @test res == [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')] -end - -@test length(zip(cycle(1:3), 1:7)) == 7 -@test length(zip(cycle(1:3), 1:7, cycle(1:3))) == 7 -@test length(zip(1:3,Base.product(1:7,cycle(1:3)))) == 3 -@test length(zip(1:3,Base.product(1:7,cycle(1:3)),8)) == 1 - -# rest -# ---- -let s = "hello" - _, st = next(s, start(s)) - @test collect(rest(s, st)) == ['e','l','l','o'] -end - -@test_throws MethodError collect(rest(countfrom(1), 5)) - -# countfrom -# --------- - -let i = 0 - for j = countfrom(0, 2) - @test j == i*2 - i += 1 - i <= 10 || break - end -end - -# take -# ---- - -let t = take(0:2:8, 10), i = 0 - @test length(collect(t)) == 5 - - for j = t - @test j == i*2 - i += 1 - end - @test i == 5 -end - -let i = 0 - for j = take(0:2:100, 10) - @test j == i*2 - i += 1 - end - @test i == 10 -end - -@test length(take(1:3,typemax(Int))) == 3 -@test length(take(countfrom(1),3)) == 3 -@test length(take(1:6,3)) == 3 - -# drop -# ---- - -let i = 0 - for j = drop(0:2:10, 2) - @test j == (i+2)*2 - i += 1 - end - @test i == 4 -end - -@test length(drop(1:3,typemax(Int))) == 0 -@test Base.iteratorsize(drop(countfrom(1),3)) == Base.IsInfinite() -@test_throws MethodError length(drop(countfrom(1), 3)) - -# double take -# and take/drop canonicalization -# ----------- - -for xs in Any["abc", [1, 2, 3]] - @test take(take(xs, 2), 3) === take(xs, 2) - @test take(take(xs, 4), 2) === take(xs, 2) - @test drop(drop(xs, 1), 1) === drop(xs, 2) - @test take(drop(xs, 1), 1) === drop(take(xs, 2), 1) - @test take(drop(xs, 3), 0) === drop(take(xs, 2), 3) - @test isempty(drop(drop(xs, 2), 2)) - @test drop(take(drop(xs, 1), 2), 1) === take(drop(xs, 2), 1) - @test take(drop(take(xs, 3), 1), 1) === take(drop(xs, 1), 1) -end - -# cycle -# ----- - -let i = 0 - for j = cycle(0:3) - @test j == i % 4 - i += 1 - i <= 10 || break - end -end - -# repeated -# -------- - -let i = 0 - for j = repeated(1, 10) - @test j == 1 - i += 1 - end - @test i == 10 -end -let i = 0 - for j = repeated(1) - @test j == 1 - i += 1 - i <= 10 || break - end -end -@test eltype(repeated(0)) == Int -@test eltype(repeated(0, 5)) == Int -@test Base.iteratorsize(repeated(0)) == Base.IsInfinite() -@test Base.iteratorsize(repeated(0, 5)) == Base.HasLength() -@test Base.iteratoreltype(repeated(0)) == Base.HasEltype() -@test Base.iteratoreltype(repeated(0, 5)) == Base.HasEltype() -@test Base.iteratorsize(zip(repeated(0), repeated(0))) == Base.IsInfinite() - - -# product -# ------- - -# empty? -for itr in [Base.product(1:0), - Base.product(1:2, 1:0), - Base.product(1:0, 1:2), - Base.product(1:0, 1:1, 1:2), - Base.product(1:1, 1:0, 1:2), - Base.product(1:1, 1:2 ,1:0)] - @test isempty(itr) - @test isempty(collect(itr)) -end - -# collect a product - first iterators runs faster -@test collect(Base.product(1:2)) == [(i,) for i=1:2] -@test collect(Base.product(1:2, 3:4)) == [(i, j) for i=1:2, j=3:4] -@test collect(Base.product(1:2, 3:4, 5:6)) == [(i, j, k) for i=1:2, j=3:4, k=5:6] - -# iteration order -let - expected = [(1,3,5), (2,3,5), (1,4,5), (2,4,5), (1,3,6), (2,3,6), (1,4,6), (2,4,6)] - actual = Base.product(1:2, 3:4, 5:6) - for (exp, act) in zip(expected, actual) - @test exp == act - end -end - -# collect multidimensional array -let - a, b = 1:3, [4 6; - 5 7] - p = Base.product(a, b) - @test size(p) == (3, 2, 2) - @test length(p) == 12 - @test ndims(p) == 3 - @test eltype(p) == NTuple{2, Int} - cp = collect(p) - for i = 1:3 - @test cp[i, :, :] == [(i, 4) (i, 6); - (i, 5) (i, 7)] - end -end - -# with 1D inputs -let - a, b, c = 1:2, 1.0:10.0, Int32(1):Int32(0) - - # length - @test length(Base.product(a)) == 2 - @test length(Base.product(a, b)) == 20 - @test length(Base.product(a, b, c)) == 0 - - # size - @test size(Base.product(a)) == (2, ) - @test size(Base.product(a, b)) == (2, 10) - @test size(Base.product(a, b, c)) == (2, 10, 0) - - # eltype - @test eltype(Base.product(a)) == Tuple{Int} - @test eltype(Base.product(a, b)) == Tuple{Int, Float64} - @test eltype(Base.product(a, b, c)) == Tuple{Int, Float64, Int32} - - # ndims - @test ndims(Base.product(a)) == 1 - @test ndims(Base.product(a, b)) == 2 - @test ndims(Base.product(a, b, c)) == 3 -end - -# with multidimensional inputs -let - a, b, c = randn(4, 4), randn(3, 3, 3), randn(2, 2, 2, 2) - args = Any[(a,), - (a, a), - (a, b), - (a, a, a), - (a, b, c)] - sizes = Any[(4, 4), - (4, 4, 4, 4), - (4, 4, 3, 3, 3), - (4, 4, 4, 4, 4, 4), - (4, 4, 3, 3, 3, 2, 2, 2, 2)] - for (method, fun) in zip([size, ndims, length], [x->x, length, prod]) - for i in 1:length(args) - @test method(Base.product(args[i]...)) == method(collect(Base.product(args[i]...))) == fun(sizes[i]) - end - end -end - -# more tests on product with iterators of various type -let - iters = (1:2, - rand(2, 2, 2), - take(1:4, 2), - Base.product(1:2, 1:3), - Base.product(rand(2, 2), rand(1, 1, 1)) - ) - for method in [size, length, ndims, eltype] - for i = 1:length(iters) - args = iters[i] - @test method(Base.product(args...)) == method(collect(Base.product(args...))) - for j = 1:length(iters) - args = iters[i], iters[j] - @test method(Base.product(args...)) == method(collect(Base.product(args...))) - for k = 1:length(iters) - args = iters[i], iters[j], iters[k] - @test method(Base.product(args...)) == method(collect(Base.product(args...))) - end - end - end - end -end - -# product of finite length and infinite length iterators -let - a = 1:2 - b = countfrom(1) - ab = Base.product(a, b) - ba = Base.product(b, a) - abexp = [(1, 1), (2, 1), (1, 2), (2, 2), (1, 3), (2, 3)] - baexp = [(1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1)] - for (expected, actual) in zip([abexp, baexp], [ab, ba]) - for (i, el) in enumerate(actual) - @test el == expected[i] - i == length(expected) && break - end - @test_throws ArgumentError length(actual) - @test_throws ArgumentError size(actual) - @test_throws ArgumentError ndims(actual) - end - - # size infinite or unknown raises an error - for itr in Any[countfrom(1), Filter(i->0, 1:10)] - @test_throws ArgumentError length(Base.product(itr)) - @test_throws ArgumentError size(Base.product(itr)) - @test_throws ArgumentError ndims(Base.product(itr)) - end -end - -# iteratorsize trait business -let f1 = Filter(i->i>0, 1:10) - @test Base.iteratorsize(Base.product(f1)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(1:2, f1)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(f1, 1:2)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(f1, f1)) == Base.SizeUnknown() - @test Base.iteratorsize(Base.product(f1, countfrom(1))) == Base.IsInfinite() - @test Base.iteratorsize(Base.product(countfrom(1), f1)) == Base.IsInfinite() -end -@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite() -@test Base.iteratorsize(Base.product(countfrom(2), countfrom(1))) == Base.IsInfinite() -@test Base.iteratorsize(Base.product(countfrom(1), 1:2)) == Base.IsInfinite() -@test Base.iteratorsize(Base.product(1:2)) == Base.HasShape() -@test Base.iteratorsize(Base.product(1:2, 1:2)) == Base.HasShape() -@test Base.iteratorsize(Base.product(take(1:2, 1), take(1:2, 1))) == Base.HasShape() -@test Base.iteratorsize(Base.product(take(1:2, 2))) == Base.HasLength() -@test Base.iteratorsize(Base.product([1 2; 3 4])) == Base.HasShape() - -# iteratoreltype trait business -let f1 = Filter(i->i>0, 1:10) - @test Base.iteratoreltype(Base.product(f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(1:2, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(f1, 1:2)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(f1, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(f1, countfrom(1))) == Base.HasEltype() # FIXME? eltype(f1) is Any - @test Base.iteratoreltype(Base.product(countfrom(1), f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any -end -@test Base.iteratoreltype(Base.product(1:2, countfrom(1))) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(countfrom(1), 1:2)) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(1:2)) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(1:2, 1:2)) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(take(1:2, 1), take(1:2, 1))) == Base.HasEltype() -@test Base.iteratoreltype(Base.product(take(1:2, 2))) == Base.HasEltype() -@test Base.iteratoreltype(Base.product([1 2; 3 4])) == Base.HasEltype() - -@test collect(Base.product(1:2,3:4)) == [(1,3) (1,4); (2,3) (2,4)] -@test isempty(collect(Base.product(1:0,1:2))) -@test length(Base.product(1:2,1:10,4:6)) == 60 -@test Base.iteratorsize(Base.product(1:2, countfrom(1))) == Base.IsInfinite() - -# flatten -# ------- - -import Base.flatten - -@test collect(flatten(Any[1:2, 4:5])) == Any[1,2,4,5] -@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[10:7, 10:9])])) == Any[1,2] -@test collect(flatten(Any[flatten(Any[1:2, 4:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,4,5,6,7,8,9] -@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,6,7,8,9] -@test collect(flatten(Any[2:1])) == Any[] -@test eltype(flatten(UnitRange{Int8}[1:2, 3:4])) == Int8 -@test_throws ArgumentError collect(flatten(Any[])) - -@test Base.iteratoreltype(Base.Flatten((i for i=1:2) for j=1:1)) == Base.EltypeUnknown() - # foreach let a = [] @@ -478,13 +124,13 @@ let gen = Base.Generator(+, 1:10, 1:10, 1:10) @test collect(gen) == collect(3:3:30) end -let gen = (x for x in 1:10 if x % 2 == 0), gen2 = Filter(x->x % 2 == 0, x for x in 1:10) +let gen = (x for x in 1:10 if x % 2 == 0), gen2 = Iterators.filter(x->x % 2 == 0, x for x in 1:10) @test collect(gen) == collect(gen2) @test collect(gen) == collect(2:2:10) end let gen = ((x,y) for x in 1:10, y in 1:10 if x % 2 == 0 && y % 2 == 0), - gen2 = Filter(x->x[1] % 2 == 0 && x[2] % 2 == 0, (x,y) for x in 1:10, y in 1:10) + gen2 = Iterators.filter(x->x[1] % 2 == 0 && x[2] % 2 == 0, (x,y) for x in 1:10, y in 1:10) @test collect(gen) == collect(gen2) end @@ -500,43 +146,3 @@ end for n = 0:5:100-q-d for p = 100-q-d-n if p < n < d < q] == [(50,30,15,5), (50,30,20,0), (50,40,10,0), (75,20,5,0)] - -# partition(c, n) -let v = collect(Base.partition([1,2,3,4,5], 1)) - @test all(i->v[i][1] == i, v) -end - -let v = collect(Base.partition([1,2,3,4,5], 2)) - @test v[1] == [1,2] - @test v[2] == [3,4] - @test v[3] == [5] -end - -let v = collect(Base.partition(enumerate([1,2,3,4,5]), 3)) - @test v[1] == [(1,1),(2,2),(3,3)] - @test v[2] == [(4,4),(5,5)] -end - -for n in [5,6] - @test collect(Base.partition([1,2,3,4,5], n))[1] == [1,2,3,4,5] - @test collect(Base.partition(enumerate([1,2,3,4,5]), n))[1] == - [(1,1),(2,2),(3,3),(4,4),(5,5)] -end - - -@test join(map(x->string(x...), Base.partition("Hello World!", 5)), "|") == - "Hello| Worl|d!" - -let s = "Monkey 🙈🙊🙊" - tf = (n)->join(map(x->string(x...), Base.partition(s,n)), "|") - @test tf(10) == s - @test tf(9) == "Monkey 🙈🙊|🙊" - @test tf(8) == "Monkey 🙈|🙊🙊" - @test tf(7) == "Monkey |🙈🙊🙊" - @test tf(6) == "Monkey| 🙈🙊🙊" - @test tf(5) == "Monke|y 🙈🙊🙊" - @test tf(4) == "Monk|ey 🙈|🙊🙊" - @test tf(3) == "Mon|key| 🙈🙊|🙊" - @test tf(2) == "Mo|nk|ey| 🙈|🙊🙊" - @test tf(1) == "M|o|n|k|e|y| |🙈|🙊|🙊" -end diff --git a/test/iterators.jl b/test/iterators.jl new file mode 100644 index 0000000000000..fb5c10411f218 --- /dev/null +++ b/test/iterators.jl @@ -0,0 +1,393 @@ +using Base.Iterators + +# zip and filter iterators +# issue #4718 +@test collect(Iterators.filter(x->x[1], zip([true, false, true, false],"abcd"))) == [(true,'a'),(true,'c')] + +let z = zip(1:2) + @test collect(z) == [(1,), (2,)] + # Issue #13979 + @test eltype(z) == Tuple{Int} +end + +let z = zip(1:2, 3:4) + @test collect(z) == [(1,3), (2,4)] + @test eltype(z) == Tuple{Int,Int} +end + +let z = zip(1:2, 3:4, 5:6) + @test collect(z) == [(1,3,5), (2,4,6)] + @test eltype(z) == Tuple{Int,Int,Int} +end + +@test eltype(Iterators.filter(isodd, 1:5)) == Int + +# typed `collect` +@test collect(Float64, Iterators.filter(isodd, [1,2,3,4]))[1] === 1.0 + +# enumerate (issue #6284) +let b = IOBuffer("1\n2\n3\n"), a = [] + for (i,x) in enumerate(eachline(b)) + push!(a, (i,x)) + end + @test a == [(1,"1\n"),(2,"2\n"),(3,"3\n")] +end + +# zip eachline (issue #7369) +let zeb = IOBuffer("1\n2\n3\n4\n5\n"), + letters = ['a', 'b', 'c', 'd', 'e'], + res = [] + for (number, letter) in zip(eachline(zeb), letters) + push!(res, (parse(Int,strip(number)), letter)) + end + @test res == [(1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e')] +end + +@test length(zip(cycle(1:3), 1:7)) == 7 +@test length(zip(cycle(1:3), 1:7, cycle(1:3))) == 7 +@test length(zip(1:3,product(1:7,cycle(1:3)))) == 3 +@test length(zip(1:3,product(1:7,cycle(1:3)),8)) == 1 + +# rest +# ---- +let s = "hello" + _, st = next(s, start(s)) + @test collect(rest(s, st)) == ['e','l','l','o'] +end + +@test_throws MethodError collect(rest(countfrom(1), 5)) + +# countfrom +# --------- + +let i = 0 + for j = countfrom(0, 2) + @test j == i*2 + i += 1 + i <= 10 || break + end +end + +# take +# ---- + +let t = take(0:2:8, 10), i = 0 + @test length(collect(t)) == 5 + + for j = t + @test j == i*2 + i += 1 + end + @test i == 5 +end + +let i = 0 + for j = take(0:2:100, 10) + @test j == i*2 + i += 1 + end + @test i == 10 +end + +@test length(take(1:3,typemax(Int))) == 3 +@test length(take(countfrom(1),3)) == 3 +@test length(take(1:6,3)) == 3 + +# drop +# ---- + +let i = 0 + for j = drop(0:2:10, 2) + @test j == (i+2)*2 + i += 1 + end + @test i == 4 +end + +@test length(drop(1:3,typemax(Int))) == 0 +@test Base.iteratorsize(drop(countfrom(1),3)) == Base.IsInfinite() +@test_throws MethodError length(drop(countfrom(1), 3)) + +# double take +# and take/drop canonicalization +# ----------- + +for xs in Any["abc", [1, 2, 3]] + @test take(take(xs, 2), 3) === take(xs, 2) + @test take(take(xs, 4), 2) === take(xs, 2) + @test drop(drop(xs, 1), 1) === drop(xs, 2) + @test take(drop(xs, 1), 1) === drop(take(xs, 2), 1) + @test take(drop(xs, 3), 0) === drop(take(xs, 2), 3) + @test isempty(drop(drop(xs, 2), 2)) + @test drop(take(drop(xs, 1), 2), 1) === take(drop(xs, 2), 1) + @test take(drop(take(xs, 3), 1), 1) === take(drop(xs, 1), 1) +end + +# cycle +# ----- + +let i = 0 + for j = cycle(0:3) + @test j == i % 4 + i += 1 + i <= 10 || break + end +end + +# repeated +# -------- + +let i = 0 + for j = repeated(1, 10) + @test j == 1 + i += 1 + end + @test i == 10 +end +let i = 0 + for j = repeated(1) + @test j == 1 + i += 1 + i <= 10 || break + end +end +@test eltype(repeated(0)) == Int +@test eltype(repeated(0, 5)) == Int +@test Base.iteratorsize(repeated(0)) == Base.IsInfinite() +@test Base.iteratorsize(repeated(0, 5)) == Base.HasLength() +@test Base.iteratoreltype(repeated(0)) == Base.HasEltype() +@test Base.iteratoreltype(repeated(0, 5)) == Base.HasEltype() +@test Base.iteratorsize(zip(repeated(0), repeated(0))) == Base.IsInfinite() + + +# product +# ------- + +# empty? +for itr in [product(1:0), + product(1:2, 1:0), + product(1:0, 1:2), + product(1:0, 1:1, 1:2), + product(1:1, 1:0, 1:2), + product(1:1, 1:2 ,1:0)] + @test isempty(itr) + @test isempty(collect(itr)) +end + +# collect a product - first iterators runs faster +@test collect(product(1:2)) == [(i,) for i=1:2] +@test collect(product(1:2, 3:4)) == [(i, j) for i=1:2, j=3:4] +@test collect(product(1:2, 3:4, 5:6)) == [(i, j, k) for i=1:2, j=3:4, k=5:6] + +# iteration order +let + expected = [(1,3,5), (2,3,5), (1,4,5), (2,4,5), (1,3,6), (2,3,6), (1,4,6), (2,4,6)] + actual = product(1:2, 3:4, 5:6) + for (exp, act) in zip(expected, actual) + @test exp == act + end +end + +# collect multidimensional array +let + a, b = 1:3, [4 6; + 5 7] + p = product(a, b) + @test size(p) == (3, 2, 2) + @test length(p) == 12 + @test ndims(p) == 3 + @test eltype(p) == NTuple{2, Int} + cp = collect(p) + for i = 1:3 + @test cp[i, :, :] == [(i, 4) (i, 6); + (i, 5) (i, 7)] + end +end + +# with 1D inputs +let + a, b, c = 1:2, 1.0:10.0, Int32(1):Int32(0) + + # length + @test length(product(a)) == 2 + @test length(product(a, b)) == 20 + @test length(product(a, b, c)) == 0 + + # size + @test size(product(a)) == (2, ) + @test size(product(a, b)) == (2, 10) + @test size(product(a, b, c)) == (2, 10, 0) + + # eltype + @test eltype(product(a)) == Tuple{Int} + @test eltype(product(a, b)) == Tuple{Int, Float64} + @test eltype(product(a, b, c)) == Tuple{Int, Float64, Int32} + + # ndims + @test ndims(product(a)) == 1 + @test ndims(product(a, b)) == 2 + @test ndims(product(a, b, c)) == 3 +end + +# with multidimensional inputs +let + a, b, c = randn(4, 4), randn(3, 3, 3), randn(2, 2, 2, 2) + args = Any[(a,), + (a, a), + (a, b), + (a, a, a), + (a, b, c)] + sizes = Any[(4, 4), + (4, 4, 4, 4), + (4, 4, 3, 3, 3), + (4, 4, 4, 4, 4, 4), + (4, 4, 3, 3, 3, 2, 2, 2, 2)] + for (method, fun) in zip([size, ndims, length], [x->x, length, prod]) + for i in 1:length(args) + @test method(product(args[i]...)) == method(collect(product(args[i]...))) == fun(sizes[i]) + end + end +end + +# more tests on product with iterators of various type +let + iters = (1:2, + rand(2, 2, 2), + take(1:4, 2), + product(1:2, 1:3), + product(rand(2, 2), rand(1, 1, 1)) + ) + for method in [size, length, ndims, eltype] + for i = 1:length(iters) + args = iters[i] + @test method(product(args...)) == method(collect(product(args...))) + for j = 1:length(iters) + args = iters[i], iters[j] + @test method(product(args...)) == method(collect(product(args...))) + for k = 1:length(iters) + args = iters[i], iters[j], iters[k] + @test method(product(args...)) == method(collect(product(args...))) + end + end + end + end +end + +# product of finite length and infinite length iterators +let + a = 1:2 + b = countfrom(1) + ab = product(a, b) + ba = product(b, a) + abexp = [(1, 1), (2, 1), (1, 2), (2, 2), (1, 3), (2, 3)] + baexp = [(1, 1), (2, 1), (3, 1), (4, 1), (5, 1), (6, 1)] + for (expected, actual) in zip([abexp, baexp], [ab, ba]) + for (i, el) in enumerate(actual) + @test el == expected[i] + i == length(expected) && break + end + @test_throws ArgumentError length(actual) + @test_throws ArgumentError size(actual) + @test_throws ArgumentError ndims(actual) + end + + # size infinite or unknown raises an error + for itr in Any[countfrom(1), Iterators.filter(i->0, 1:10)] + @test_throws ArgumentError length(product(itr)) + @test_throws ArgumentError size(product(itr)) + @test_throws ArgumentError ndims(product(itr)) + end +end + +# iteratorsize trait business +let f1 = Iterators.filter(i->i>0, 1:10) + @test Base.iteratorsize(product(f1)) == Base.SizeUnknown() + @test Base.iteratorsize(product(1:2, f1)) == Base.SizeUnknown() + @test Base.iteratorsize(product(f1, 1:2)) == Base.SizeUnknown() + @test Base.iteratorsize(product(f1, f1)) == Base.SizeUnknown() + @test Base.iteratorsize(product(f1, countfrom(1))) == Base.IsInfinite() + @test Base.iteratorsize(product(countfrom(1), f1)) == Base.IsInfinite() +end +@test Base.iteratorsize(product(1:2, countfrom(1))) == Base.IsInfinite() +@test Base.iteratorsize(product(countfrom(2), countfrom(1))) == Base.IsInfinite() +@test Base.iteratorsize(product(countfrom(1), 1:2)) == Base.IsInfinite() +@test Base.iteratorsize(product(1:2)) == Base.HasShape() +@test Base.iteratorsize(product(1:2, 1:2)) == Base.HasShape() +@test Base.iteratorsize(product(take(1:2, 1), take(1:2, 1))) == Base.HasShape() +@test Base.iteratorsize(product(take(1:2, 2))) == Base.HasLength() +@test Base.iteratorsize(product([1 2; 3 4])) == Base.HasShape() + +# iteratoreltype trait business +let f1 = Iterators.filter(i->i>0, 1:10) + @test Base.iteratoreltype(product(f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(1:2, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(f1, 1:2)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(f1, f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(f1, countfrom(1))) == Base.HasEltype() # FIXME? eltype(f1) is Any + @test Base.iteratoreltype(product(countfrom(1), f1)) == Base.HasEltype() # FIXME? eltype(f1) is Any +end +@test Base.iteratoreltype(product(1:2, countfrom(1))) == Base.HasEltype() +@test Base.iteratoreltype(product(countfrom(1), 1:2)) == Base.HasEltype() +@test Base.iteratoreltype(product(1:2)) == Base.HasEltype() +@test Base.iteratoreltype(product(1:2, 1:2)) == Base.HasEltype() +@test Base.iteratoreltype(product(take(1:2, 1), take(1:2, 1))) == Base.HasEltype() +@test Base.iteratoreltype(product(take(1:2, 2))) == Base.HasEltype() +@test Base.iteratoreltype(product([1 2; 3 4])) == Base.HasEltype() + +@test collect(product(1:2,3:4)) == [(1,3) (1,4); (2,3) (2,4)] +@test isempty(collect(product(1:0,1:2))) +@test length(product(1:2,1:10,4:6)) == 60 +@test Base.iteratorsize(product(1:2, countfrom(1))) == Base.IsInfinite() + +# flatten +# ------- + +@test collect(flatten(Any[1:2, 4:5])) == Any[1,2,4,5] +@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[10:7, 10:9])])) == Any[1,2] +@test collect(flatten(Any[flatten(Any[1:2, 4:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,4,5,6,7,8,9] +@test collect(flatten(Any[flatten(Any[1:2, 6:5]), flatten(Any[6:7, 8:9])])) == Any[1,2,6,7,8,9] +@test collect(flatten(Any[2:1])) == Any[] +@test eltype(flatten(UnitRange{Int8}[1:2, 3:4])) == Int8 +@test_throws ArgumentError collect(flatten(Any[])) + +@test Base.iteratoreltype(Base.Flatten((i for i=1:2) for j=1:1)) == Base.EltypeUnknown() + +# partition(c, n) +let v = collect(partition([1,2,3,4,5], 1)) + @test all(i->v[i][1] == i, v) +end + +let v = collect(partition([1,2,3,4,5], 2)) + @test v[1] == [1,2] + @test v[2] == [3,4] + @test v[3] == [5] +end + +let v = collect(partition(enumerate([1,2,3,4,5]), 3)) + @test v[1] == [(1,1),(2,2),(3,3)] + @test v[2] == [(4,4),(5,5)] +end + +for n in [5,6] + @test collect(partition([1,2,3,4,5], n))[1] == [1,2,3,4,5] + @test collect(partition(enumerate([1,2,3,4,5]), n))[1] == + [(1,1),(2,2),(3,3),(4,4),(5,5)] +end + + +@test join(map(x->string(x...), partition("Hello World!", 5)), "|") == + "Hello| Worl|d!" + +let s = "Monkey 🙈🙊🙊" + tf = (n)->join(map(x->string(x...), partition(s,n)), "|") + @test tf(10) == s + @test tf(9) == "Monkey 🙈🙊|🙊" + @test tf(8) == "Monkey 🙈|🙊🙊" + @test tf(7) == "Monkey |🙈🙊🙊" + @test tf(6) == "Monkey| 🙈🙊🙊" + @test tf(5) == "Monke|y 🙈🙊🙊" + @test tf(4) == "Monk|ey 🙈|🙊🙊" + @test tf(3) == "Mon|key| 🙈🙊|🙊" + @test tf(2) == "Mo|nk|ey| 🙈|🙊🙊" + @test tf(1) == "M|o|n|k|e|y| |🙈|🙊|🙊" +end