Rock in parallel

Implement parallel processing of the files. Option 'parallel' takes pos integer as a number of processes allowed to run in parallel, default to 1, to keep current behaviour. Drop most of the file section from the result, and split loading phase to reduce memory consumption and speed up gathering of the results.
inaka · Apr 22, 2019 · 3fc1fe2 · 3fc1fe2
1 parent 00b4372
commit 3fc1fe2
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 27 deletions.
diff --git a/src/elvis_core.erl b/src/elvis_core.erl
@@ -37,7 +37,7 @@ rock() ->
 rock(Config) ->
     ok = elvis_config:validate(Config),
     NewConfig = elvis_config:normalize(Config),
-    Results = lists:map(fun do_rock/1, NewConfig),
+    Results = lists:map(fun do_parallel_rock/1, NewConfig),
     lists:foldl(fun combine_results/2, ok, Results).
 
 -spec rock_this(target()) ->
@@ -74,23 +74,88 @@ rock_this(Path, Config) ->
             elvis_utils:info("Skipping ~s", [Path]);
         FilteredConfig ->
             LoadedFile = load_file_data(FilteredConfig, File),
-            ApplyRulesFun = fun(Cfg) -> apply_rules(Cfg, LoadedFile) end,
+            ApplyRulesFun = fun(Cfg) -> apply_rules_and_print(Cfg, LoadedFile) end,
             Results = lists:map(ApplyRulesFun, FilteredConfig),
             elvis_result_status(Results)
     end.
 
 %% @private
--spec do_rock(map()) -> ok | {fail, [elvis_result:file() | elvis_result:rule()]}.
-do_rock(Config0) ->
-    elvis_utils:info("Loading files..."),
+-spec do_parallel_rock(map()) -> ok | {fail, [elvis_result:file() | elvis_result:rule()]}.
+do_parallel_rock(Config0) ->
+    Parallel = application:get_env(elvis, parallel, 1),
     Config = elvis_config:resolve_files(Config0),
     Files = elvis_config:files(Config),
-    Fun = fun (File) -> load_file_data(Config, File) end,
-    LoadedFiles = lists:map(Fun, Files),
-    elvis_utils:info("Applying rules..."),
-    Results = [apply_rules(Config, File) || File <- LoadedFiles],
+
+    Results = do_parallel_rock0(Config, Files, Parallel),
     elvis_result_status(Results).
 
+do_parallel_rock0(Config, Files, N) ->
+    do_parallel_rock1(Config, Files, N, N, [], []).
+
+do_parallel_rock1(_Config, [], _MaxW, _RemainW, AccR, AccG) ->
+    gather_all_results(AccR, AccG);
+do_parallel_rock1(Config, FilesList, MaxW, 0, AccR, AccG) ->
+    {AccR1, AccG1, N} = gather_results(AccR, AccG),
+    do_parallel_rock1(Config, FilesList, MaxW, erlang:min(N, MaxW), AccR1, AccG1);
+do_parallel_rock1(Config, FilesList, MaxW, RemainW, AccR, AccG) ->
+    {WorkToBeDone, FilesRemain} =
+        try lists:split(RemainW, FilesList) of
+            Res -> Res
+        catch error:badarg -> {FilesList, []}
+        end,
+
+    Gather = [do_rock_worker(Config, File) || File <- WorkToBeDone],
+    do_parallel_rock1(Config, FilesRemain, MaxW, 0, AccR, Gather ++ AccG).
+
+do_rock_worker(Config, #{path := Path} = File) ->
+    Parent = self(),
+    Key = spawn_monitor(fun() -> do_rock(Parent, Config, File) end),
+    {Key, Path}.
+
+-spec do_rock(pid(), elvis_config:config(), elvis_result:file()) -> no_return().
+do_rock(Parent, Config, File) ->
+    try
+        LoadedFile = load_file_data(Config, File),
+        apply_rules(Config, LoadedFile)
+    of
+        Results ->
+            exit({Parent, {ok, Results}})
+    catch T:E ->
+            exit({Parent, {error, {T,E}}})
+    end.
+
+gather_all_results(AccR, Remain) ->
+    {AccR1, _, _} = gather_results0(AccR, Remain, 0, infinity),
+    AccR1.
+
+gather_results(AccR, AccG) ->
+    {Key, Res0} = gather(infinity),
+    gather_results0([Res0 | AccR], lists:keydelete(Key, 1, AccG), 1, 0).
+
+gather_results0(AccR, [], N, _Timeout) ->
+    {AccR, [], N};
+gather_results0(AccR, AccG, N, Timeout) ->
+    case gather(Timeout) of
+        timeout -> {AccR, AccG, N};
+        {Key, Res0} ->
+            gather_results0([Res0 | AccR], lists:keydelete(Key, 1, AccG), N + 1, Timeout)
+    end.
+
+gather(Timeout) ->
+    Self = self(),
+    receive
+        {'DOWN', MonRef, process, Pid, {Self, Res}} ->
+            case Res of
+                {ok, Res0} ->
+                    elvis_result:print_results(Res0),
+                    {{Pid, MonRef}, Res0};
+                {error, {T,E}} ->
+                    erlang:T(E)
+            end
+    after Timeout ->
+            timeout
+    end.
+
 %% @private
 -spec load_file_data(map() | [map()], elvis_file:file()) -> elvis_file:file().
 load_file_data(Config, File) ->
@@ -119,16 +184,18 @@ combine_results(Item, ok) ->
 combine_results({fail, ItemResults}, {fail, AccResults}) ->
     {fail, ItemResults ++ AccResults}.
 
+apply_rules_and_print(Config, File) ->
+    Results = apply_rules(Config, File),
+    elvis_result:print_results(Results),
+    Results.
+
 -spec apply_rules(map(), File::elvis_file:file()) ->
     elvis_result:file().
 apply_rules(Config, File) ->
     Rules = elvis_config:rules(Config),
     Acc = {[], Config, File},
     {RulesResults, _, _} = lists:foldl(fun apply_rule/2, Acc, Rules),
-
-    Results = elvis_result:new(file, File, RulesResults),
-    elvis_result:print_results(Results),
-    Results.
+    elvis_result:new(file, File, RulesResults).
 
 apply_rule({Module, Function}, {Result, Config, File}) ->
     apply_rule({Module, Function, #{}}, {Result, Config, File});

diff --git a/src/elvis_result.erl b/src/elvis_result.erl
@@ -12,7 +12,7 @@
         ]).
 
 -export([
-         get_file/1,
+         get_path/1,
          get_rules/1,
          get_name/1,
          get_items/1,
@@ -45,7 +45,7 @@
          }.
 -type file() ::
         #{
-           file => elvis_file:file(),
+           file => string(),
            rules => [rule()]
          }.
 -type elvis_error() ::
@@ -68,8 +68,8 @@ new(item, Msg, Info) ->
     new(item, Msg, Info, 0);
 new(rule, Name, Results) ->
     #{name => Name, items => Results};
-new(file, File, Rules) ->
-    #{file => File, rules => Rules};
+new(file, #{path := Path}, Rules) ->
+    #{file => Path, rules => Rules};
 new(error, Msg, Info) ->
     #{error_msg => Msg, info => Info}.
 
@@ -81,8 +81,8 @@ new(item, Msg, Info, LineNum) ->
 
 %% Getters
 
--spec get_file(file()) -> elvis_file:file().
-get_file(#{file := File}) -> File.
+-spec get_path(file()) -> string().
+get_path(#{file := File}) -> File.
 
 -spec get_rules(file()) -> [rule()].
 get_rules(#{rules := Rules}) -> Rules.
@@ -117,8 +117,7 @@ print(Format, [Result | Results]) ->
     print(Format, Result),
     print(Format, Results);
 %% File
-print(Format, #{file := File, rules := Rules}) ->
-    Path = elvis_file:path(File),
+print(Format, #{file := Path, rules := Rules}) ->
     case Format of
         parsable -> ok;
         _ ->
@@ -193,12 +192,9 @@ clean([], Result) ->
     lists:reverse(Result);
 clean([#{rules := []} | Files], Result) ->
     clean(Files, Result);
-clean([File = #{rules := Rules, file := FileInfo} | Files], Result) ->
+clean([File = #{rules := Rules} | Files], Result) ->
     CleanRules = clean(Rules),
-    FileInfo1 = maps:remove(content, FileInfo),
-    FileInfo2 = maps:remove(parse_tree, FileInfo1),
-    NewFile = File#{rules => CleanRules,
-                    file => FileInfo2},
+    NewFile = File#{rules => CleanRules},
     clean(Files, [NewFile | Result]);
 clean([#{items := []} | Rules], Result) ->
     clean(Rules, Result);

diff --git a/src/elvis_utils.erl b/src/elvis_utils.erl
@@ -191,7 +191,8 @@ parse_colors(Message) ->
                "reset" => "\e[0m"},
     Opts = [global, {return, list}],
     case application:get_env(elvis, output_format, colors) of
-        plain ->
+        P when P =:= plain orelse
+               P =:= parsable ->
             re:replace(Message, "{{.*?}}", "", Opts);
         colors ->
             Fun = fun(Key, Acc) ->