Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve async runtime scaling #946

Merged
merged 18 commits into from
Aug 23, 2023
Merged

Improve async runtime scaling #946

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
d5f4aa7
Stub: async scheduler
jaspervdj Aug 3, 2022
912e36b
Stub: async scheduler
jaspervdj Aug 3, 2022
4c86f66
Stub: async scheduler
jaspervdj Aug 3, 2022
ecafce9
Stub: async scheduler
jaspervdj Aug 3, 2022
ef9fcc8
Stub: async scheduler
jaspervdj Aug 3, 2022
7cc5e9d
Stub: async scheduler
jaspervdj Aug 3, 2022
d022b27
Stub: async scheduler
jaspervdj Aug 4, 2022
7f411cb
Stub: async scheduler
jaspervdj Aug 4, 2022
1ea25de
Stub: async scheduler
jaspervdj Aug 4, 2022
d3274cc
Stub: async scheduler
jaspervdj Aug 4, 2022
1f9ba8b
Stub: async scheduler
jaspervdj Aug 4, 2022
866202c
Stub: async scheduler
jaspervdj Aug 4, 2022
04e9f87
Stub: async scheduler
jaspervdj Aug 4, 2022
8266a73
Stub: async scheduler
jaspervdj Aug 4, 2022
0e1dcf2
Stub: async scheduler
jaspervdj Aug 4, 2022
5c70f28
Make logger more testable
jaspervdj Aug 12, 2022
3a54b21
Add better error for cycles and test for this
jaspervdj Aug 12, 2022
8f8f578
Merge branch 'master' into async-scheduler
jaspervdj Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Stub: async scheduler
  • Loading branch information
@jaspervdj
jaspervdj committed Aug 4, 2022
commit d3274cc1d9e85bf394965f05f1a69d13a78f4298
37 changes: 20 additions & 17 deletions lib/Hakyll/Core/Runtime.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,22 @@ module Hakyll.Core.Runtime


--------------------------------------------------------------------------------
import Control.Concurrent (getNumCapabilities, forkIO)
import Control.Concurrent (forkIO, getNumCapabilities,
rtsSupportsBoundThreads)
import Control.Concurrent.Async.Lifted (forConcurrently)
import Control.Concurrent.MVar (modifyMVar_, readMVar, newMVar, MVar)
import Control.Concurrent.MVar (MVar, modifyMVar_, newMVar,
readMVar)
import qualified Control.Concurrent.MVar as MVar
import Control.Monad (join, replicateM_, unless, void, when)
import Control.Monad.Except (ExceptT, runExceptT, throwError)
import Control.Monad (join, replicateM_, unless,
void, when)
import Control.Monad.Except (ExceptT, runExceptT,
throwError)
import Control.Monad.Reader (ReaderT, ask, runReaderT)
import Control.Monad.Trans (liftIO)
import Data.Foldable (for_, traverse_)
import Data.List (foldl')
import Data.IORef (IORef)
import qualified Data.IORef as IORef
import Data.List (intercalate)
import Data.List (foldl', intercalate)
import Data.Map (Map)
import qualified Data.Map as Map
import Data.Maybe (fromMaybe)
Expand All @@ -40,13 +43,13 @@ import Hakyll.Core.Dependencies
import Hakyll.Core.Identifier
import Hakyll.Core.Item
import Hakyll.Core.Item.SomeItem
import Hakyll.Core.Logger (Logger)
import qualified Hakyll.Core.Logger as Logger
import Hakyll.Core.Logger (Logger)
import qualified Hakyll.Core.Logger as Logger
import Hakyll.Core.Provider
import Hakyll.Core.Routes
import Hakyll.Core.Rules.Internal
import Hakyll.Core.Store (Store)
import qualified Hakyll.Core.Store as Store
import Hakyll.Core.Store (Store)
import qualified Hakyll.Core.Store as Store
import Hakyll.Core.Util.File
import Hakyll.Core.Writable

Expand Down Expand Up @@ -391,7 +394,7 @@ build2 mode = do
case mode of
RunModeNormal -> do
Logger.header logger "Compiling"
pickAndChaseAsync
if rtsSupportsBoundThreads then pickAndChaseAsync else pickAndChase2
Logger.header logger "Success"
facts <- liftIO $ schedulerFacts <$> IORef.readIORef schedulerRef
store <- runtimeStore <$> ask
Expand Down Expand Up @@ -454,10 +457,10 @@ pickAndChase = do
unless (null todo) $ do
acted <- mconcat <$> forConcurrently (Map.keys todo) chase
when (acted == Idled) $ do
-- This clause happens when chasing *every item* in `todo` resulted in
-- idling because tasks are all waiting on something: a dependency cycle
-- This clause happens when chasing *every item* in `todo` resulted in
-- idling because tasks are all waiting on something: a dependency cycle
deps <- runtimeDependencies <$> getRuntimeState
throwError $ "Hakyll.Core.Runtime.pickAndChase: Dependency cycle detected: " ++
throwError $ "Hakyll.Core.Runtime.pickAndChase: Dependency cycle detected: " ++
intercalate ", " [show k ++ " depends on " ++ show (Set.toList v) | (k, v) <- Map.toList deps]
pickAndChase

Expand Down Expand Up @@ -690,7 +693,7 @@ chase id' = do
(depId, depSnapshot) `Set.member` snapshots
actualDep = [(depId, depSnapshot) | not depDone]

return actualDep
return actualDep

modifyRuntimeState $ \s -> s
{ runtimeTodo = Map.insert id'
Expand All @@ -700,9 +703,9 @@ chase id' = do
, runtimeDependencies = Map.insertWith Set.union id' (Set.fromList deps) (runtimeDependencies s)
}

-- Progress has been made if at least one of the
-- Progress has been made if at least one of the
-- requirements can move forwards at the next pass
-- In some cases, dependencies have been processed in parallel in which case `deps`
-- In some cases, dependencies have been processed in parallel in which case `deps`
-- can be empty, and we can progress to the next stage. See issue #907
let progress | null deps = Progressed
| deps == reqs = Idled
Expand Down