Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve async runtime scaling #946

Merged
merged 18 commits into from
Aug 23, 2023
Merged

Improve async runtime scaling #946

Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
d5f4aa7
Stub: async scheduler
jaspervdj Aug 3, 2022
912e36b
Stub: async scheduler
jaspervdj Aug 3, 2022
4c86f66
Stub: async scheduler
jaspervdj Aug 3, 2022
ecafce9
Stub: async scheduler
jaspervdj Aug 3, 2022
ef9fcc8
Stub: async scheduler
jaspervdj Aug 3, 2022
7cc5e9d
Stub: async scheduler
jaspervdj Aug 3, 2022
d022b27
Stub: async scheduler
jaspervdj Aug 4, 2022
7f411cb
Stub: async scheduler
jaspervdj Aug 4, 2022
1ea25de
Stub: async scheduler
jaspervdj Aug 4, 2022
d3274cc
Stub: async scheduler
jaspervdj Aug 4, 2022
1f9ba8b
Stub: async scheduler
jaspervdj Aug 4, 2022
866202c
Stub: async scheduler
jaspervdj Aug 4, 2022
04e9f87
Stub: async scheduler
jaspervdj Aug 4, 2022
8266a73
Stub: async scheduler
jaspervdj Aug 4, 2022
0e1dcf2
Stub: async scheduler
jaspervdj Aug 4, 2022
5c70f28
Make logger more testable
jaspervdj Aug 12, 2022
3a54b21
Add better error for cycles and test for this
jaspervdj Aug 12, 2022
8f8f578
Merge branch 'master' into async-scheduler
jaspervdj Aug 23, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Stub: async scheduler
  • Loading branch information
@jaspervdj
jaspervdj committed Aug 4, 2022
commit d022b277c2d5d849d91356a6428bc170f5a11b89
29 changes: 24 additions & 5 deletions lib/Hakyll/Core/Runtime.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,13 @@ schedulerPop scheduler@Scheduler {..} = case Seq.viewl schedulerQueue of
(scheduler {schedulerStarved = schedulerStarved + 1}, PopStarve)
x Seq.:< xs
| x `Set.member` schedulerDone ->
trace ("ignoring identifier " <> show x <> " (done)") $
schedulerPop scheduler {schedulerQueue = xs}
| x `Set.member` schedulerWorking ->
trace ("ignoring identifier " <> show x <> " (working)") $
schedulerPop scheduler {schedulerQueue = xs}
| x `Set.member` schedulerBlocked ->
trace ("ignoring identifier " <> show x <> " (blocked)") $
schedulerPop scheduler {schedulerQueue = xs}
| otherwise -> case Map.lookup x schedulerTodo of
Nothing ->
Expand Down Expand Up @@ -257,7 +260,15 @@ schedulerBlock
schedulerBlock identifier deps0 compiler scheduler@Scheduler {..}
| null deps1 =
trace ("done for identifier " <> show identifier <> ", " <> show deps1 <> ", " <> show deps0) $
(scheduler, BlockContinue)
( scheduler
-- TODO: Not needed? Should we just continue directly?
{ schedulerBlocked = Set.delete identifier schedulerBlocked
, schedulerQueue = Seq.singleton identifier <> schedulerQueue
, schedulerWorking = Set.delete identifier schedulerWorking
, schedulerTodo = Map.insert identifier compiler schedulerTodo
}
, BlockContinue
)
| otherwise =
( scheduler
{ schedulerQueue =
Expand All @@ -267,11 +278,14 @@ schedulerBlock identifier deps0 compiler scheduler@Scheduler {..}
Seq.singleton identifier
, schedulerTodo =
trace ("insert for identifier " <> show identifier) $
Map.insert identifier compiler schedulerTodo
Map.insert identifier
(Compiler $ \_ -> pure $ CompilerRequire deps0 compiler)
schedulerTodo
, schedulerWorking = Set.delete identifier schedulerWorking
, schedulerBlocked = Set.insert identifier schedulerBlocked
, schedulerTriggers = foldl'
(\acc (depId, _) ->
trace ("identifier " <> show identifier <> " blocked on " <> show depId) $
Map.insertWith Set.union depId (Set.singleton identifier) acc)
schedulerTriggers
deps1
Expand All @@ -293,10 +307,12 @@ schedulerBlock identifier deps0 compiler scheduler@Scheduler {..}
schedulerUnblock :: Identifier -> Scheduler -> (Scheduler, Int)
schedulerUnblock identifier scheduler@Scheduler {..} =
( scheduler
{ schedulerQueue = schedulerQueue <>
Seq.fromList (Set.toList triggered)
{ schedulerQueue =
trace ("identifier " <> show identifier <> " triggered " <> show triggered) $
schedulerQueue <> Seq.fromList (Set.toList triggered)
, schedulerStarved = 0
, schedulerBlocked = schedulerBlocked `Set.difference` triggered
, schedulerBlocked = Set.delete identifier $
schedulerBlocked `Set.difference` triggered
, schedulerTriggers = Map.delete identifier schedulerTriggers
}
, schedulerStarved
Expand Down Expand Up @@ -540,8 +556,11 @@ chase2 = do

CompilerRequire reqs c -> do
block <- liftIO . IORef.atomicModifyIORef' scheduler $
schedulerBlock id' reqs c
{-
schedulerBlock id' reqs $ Compiler $
\_ -> pure $ CompilerRequire reqs c
-}
case block of
BlockContinue -> pure (True, 0)
BlockBlocked -> pure (True, 0)
Expand Down