Skip to content

Commit

Permalink
precompile: fix performance issues with IO (#56370)
Browse files Browse the repository at this point in the history 
The string API here rapidly becomes unusably slow if dumping much debug
output during precompile. Fix the design here to use an intermediate IO
instead to prevent that.

(cherry picked from commit e4dc9d3)
  • Loading branch information
@vtjnash @KristofferC
vtjnash authored and KristofferC committed Nov 11, 2024
1 parent fac5efc commit b3c3460
Showing 1 changed file with 30 additions and 21 deletions.
51 changes: 30 additions & 21 deletions base/precompilation.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,7 @@ function _precompilepkgs(pkgs::Vector{String},
return false
end
end
std_outputs = Dict{PkgConfig,String}()
std_outputs = Dict{PkgConfig,IOBuffer}()
taskwaiting = Set{PkgConfig}()
pkgspidlocked = Dict{PkgConfig,String}()
pkg_liveprinted = nothing
Expand All @@ -655,7 +655,7 @@ function _precompilepkgs(pkgs::Vector{String},
print(io, ansi_cleartoendofline, str)
end
end
std_outputs[pkg_config] = string(get(std_outputs, pkg_config, ""), str)
write(get!(IOBuffer, std_outputs, pkg_config), str)
if !in(pkg_config, taskwaiting) && occursin("waiting for IO to finish", str)
!fancyprint && lock(print_lock) do
println(io, pkg.name, color_string(" Waiting for background task / IO / timer.", Base.warn_color()))
Expand Down Expand Up @@ -850,8 +850,9 @@ function _precompilepkgs(pkgs::Vector{String},
close(std_pipe.in) # close pipe to end the std output monitor
wait(t_monitor)
if err isa ErrorException || (err isa ArgumentError && startswith(err.msg, "Invalid header in cache file"))
failed_deps[pkg_config] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(get(std_outputs, pkg_config, ""))) : ""
errmsg = String(take!(get(IOBuffer, std_outputs, pkg_config)))
delete!(std_outputs, pkg_config) # so it's not shown as warnings, given error report
failed_deps[pkg_config] = (strict || is_direct_dep) ? string(sprint(showerror, err), "\n", strip(errmsg)) : ""
!fancyprint && lock(print_lock) do
println(io, " "^9, color_string(" ✗ ", Base.error_color()), name)
end
Expand Down Expand Up @@ -924,20 +925,22 @@ function _precompilepkgs(pkgs::Vector{String},
end
# show any stderr output, even if Pkg.precompile has been interrupted (quick_exit=true), given user may be
# interrupting a hanging precompile job with stderr output. julia#48371
filter!(kv -> !isempty(strip(last(kv))), std_outputs) # remove empty output
if !isempty(std_outputs)
plural1 = length(std_outputs) == 1 ? "y" : "ies"
plural2 = length(std_outputs) == 1 ? "" : "s"
print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:")
for (pkg_config, err) in std_outputs
pkg, config = pkg_config
err = if pkg == pkg_liveprinted
"[Output was shown above]"
else
join(split(strip(err), "\n"), color_string("\n│ ", Base.warn_color()))
let std_outputs = Tuple{PkgConfig,SubString{String}}[(pkg_config, strip(String(take!(io)))) for (pkg_config,io) in std_outputs]
filter!(kv -> !isempty(last(kv)), std_outputs)
if !isempty(std_outputs)
plural1 = length(std_outputs) == 1 ? "y" : "ies"
plural2 = length(std_outputs) == 1 ? "" : "s"
print(iostr, "\n ", color_string("$(length(std_outputs))", Base.warn_color()), " dependenc$(plural1) had output during precompilation:")
for (pkg_config, err) in std_outputs
pkg, config = pkg_config
err = if pkg == pkg_liveprinted
"[Output was shown above]"
else
join(split(err, "\n"), color_string("\n│ ", Base.warn_color()))
end
name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name
print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color()))
end
name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name
print(iostr, color_string("\n┌ ", Base.warn_color()), name, color_string("\n│ ", Base.warn_color()), err, color_string("\n└ ", Base.warn_color()))
end
end
end
Expand All @@ -947,20 +950,26 @@ function _precompilepkgs(pkgs::Vector{String},
end
end
quick_exit && return
err_str = ""
err_str = IOBuffer()
n_direct_errs = 0
for (pkg_config, err) in failed_deps
dep, config = pkg_config
if strict || (dep in direct_deps)
config_str = isempty(config[1]) ? "" : "$(join(config[1], " ")) "
err_str = string(err_str, "\n$(dep.name) $config_str\n\n$err", (n_direct_errs > 0 ? "\n" : ""))
print(err_str, "\n", dep.name, " ")
for cfg in config[1]
print(err_str, cfg, " ")
end
print(err_str, "\n\n", err)
n_direct_errs > 0 && write(err_str, "\n")
n_direct_errs += 1
end
end
if err_str != ""
if position(err_str) > 0
skip(err_str, -1)
truncate(err_str, position(err_str))
pluralde = n_direct_errs == 1 ? "y" : "ies"
direct = strict ? "" : "direct "
err_msg = "The following $n_direct_errs $(direct)dependenc$(pluralde) failed to precompile:\n$(err_str[1:end-1])"
err_msg = "The following $n_direct_errs $(direct)dependenc$(pluralde) failed to precompile:\n$(String(take!(err_str)))"
if internal_call # aka. auto-precompilation
if isinteractive() && !get(ENV, "CI", false)
plural1 = length(failed_deps) == 1 ? "y" : "ies"
Expand Down

0 comments on commit b3c3460

Please sign in to comment.