Skip to content

Commit

Permalink
Add an experimental DAML script REPL (digital-asset#4660)
Browse files Browse the repository at this point in the history
As mentioned in the title, this is still very experimental and needs
more work before we want to advertise it. However, the code is in a
somewhat reasonable shape, there are tests and I think even in the
current state it is already useful. Also this PR is already getting
very large so I don’t want to hold off much longer before merging this.

It is included in the SDK but hidden from `damlc --help` and `daml
--help` until the most pressing issues are addressed (primarily around
making sure that it doesn’t just shut down if you have a type error
and better error messages in general).

changelog_begin
changelog_end
  • Loading branch information
cocreature authored Feb 24, 2020
1 parent b38ec15 commit 8d81399
Show file tree
Hide file tree
Showing 21 changed files with 899 additions and 29 deletions.
2 changes: 1 addition & 1 deletion bazel_tools/client_server/runner/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@ da_haskell_binary(
"split",
],
visibility = ["//visibility:public"],
deps = [],
deps = ["//libs-haskell/da-hs-base"],
)
26 changes: 1 addition & 25 deletions bazel_tools/client_server/runner/Main.hs
Original file line number Diff line number Diff line change
Expand Up @@ -3,36 +3,12 @@

module Main(main) where

import Control.Concurrent (threadDelay)
import qualified Data.Text.IO as T
import DA.PortFile
import System.Environment
import System.Process
import System.IO
import System.IO.Extra (withTempFile)
import System.Exit
import Safe
import Data.List.Split (splitOn)

retryDelayMillis :: Int
retryDelayMillis = 100

-- Wait up to 60s for the port file to be written to. A long timeout is used to
-- avoid flaky results under high system load.
maxRetries :: Int
maxRetries = 60 * (1000 `div` retryDelayMillis)

readPortFile :: Int -> String -> IO Int
readPortFile 0 _file = do
T.hPutStrLn stderr "Port file was not written to in time."
exitFailure

readPortFile n file =
readMay <$> readFile file >>= \case
Nothing -> do
threadDelay (1000 * retryDelayMillis)
readPortFile (n-1) file
Just p -> pure p

main :: IO ()
main = do
[clientExe, clientArgs, serverExe, serverArgs] <- getArgs
Expand Down
3 changes: 3 additions & 0 deletions compiler/damlc/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ da_haskell_binary(
"//compiler/damlc:ghcversion",
"//compiler/damlc/pkg-db",
"//compiler/damlc/stable-packages",
"//compiler/repl-service/server:repl_service_jar",
"//compiler/scenario-service/server:scenario_service_jar",
],
hackage_deps = [
Expand Down Expand Up @@ -88,6 +89,7 @@ package_app(
"//compiler/damlc/daml-ide-core:dlint.yaml",
"//compiler/damlc/pkg-db",
"//compiler/damlc/stable-packages",
"//compiler/repl-service/server:repl_service_jar",
"//compiler/scenario-service/server:scenario_service_jar",
"@static_asset_d3plus//:js/d3.min.js",
"@static_asset_d3plus//:js/d3plus.min.js",
Expand Down Expand Up @@ -175,6 +177,7 @@ da_haskell_library(
"//compiler/damlc/daml-opts",
"//compiler/damlc/daml-opts:daml-opts-types",
"//compiler/damlc/daml-visual",
"//compiler/repl-service/client",
"//compiler/scenario-service/client",
"//compiler/scenario-service/protos:scenario_service_haskell_proto",
"//daml-assistant:daml-project-config",
Expand Down
3 changes: 3 additions & 0 deletions compiler/damlc/daml-compiler/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ da_haskell_library(
"filepath",
"ghc-lib",
"ghc-lib-parser",
"ghc-lib-parser-ex",
"ghcide",
"haskell-lsp",
"lens",
Expand Down Expand Up @@ -46,9 +47,11 @@ da_haskell_library(
"//compiler/daml-lf-tools",
"//compiler/damlc/daml-doctest",
"//compiler/damlc/daml-ide-core",
"//compiler/damlc/daml-lf-conversion",
"//compiler/damlc/daml-opts",
"//compiler/damlc/daml-opts:daml-opts-types",
"//compiler/damlc/daml-preprocessor",
"//compiler/repl-service/client",
"//compiler/scenario-service/client",
"//daml-assistant:daml-project-config",
"//libs-haskell/da-hs-base",
Expand Down
140 changes: 140 additions & 0 deletions compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Repl.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
-- Copyright (c) 2020 The DAML Authors. All rights reserved.
-- SPDX-License-Identifier: Apache-2.0

module DA.Daml.Compiler.Repl (runRepl) where

import qualified "zip-archive" Codec.Archive.Zip as Zip
import Control.Exception
import Control.Monad
import qualified DA.Daml.LF.Ast as LF
import qualified DA.Daml.LF.Proto3.Archive as LFArchive
import DA.Daml.LF.Reader (readDalfs, Dalfs(..))
import qualified DA.Daml.LF.ReplClient as ReplClient
import DA.Daml.LFConversion.UtilGHC
import DA.Daml.Options.Types
import qualified Data.ByteString.Lazy as BSL
import Data.Data (toConstr)
import Data.Foldable
import Data.Maybe
import qualified Data.NameMap as NM
import Data.Text (Text)
import qualified Data.Text as T
import Development.IDE.Core.API
import Development.IDE.Core.RuleTypes
import Development.IDE.Core.RuleTypes.Daml
import Development.IDE.Core.Shake
import Development.IDE.GHC.Util
import Development.IDE.Types.Location
import GHC
import HsExpr (Stmt, StmtLR(..), LHsExpr)
import HsExtension (GhcPs, GhcTc)
import HsPat (Pat(..))
import HscTypes (HscEnv(..))
import Language.Haskell.GhclibParserEx.Parse
import Lexer (ParseResult(..))
import OccName (occName, occNameFS)
import Outputable (ppr, showSDoc)
import SrcLoc (unLoc)
import System.Exit
import System.IO.Error
import System.IO.Extra
import Type

-- | Split a statement into the name of the binder (patterns are not supported)
-- and the body. For unsupported statements we return `Nothing`.
splitStmt :: Stmt GhcPs (LHsExpr GhcPs) -> Maybe (Maybe Text, LHsExpr GhcPs)
splitStmt (BodyStmt _ expr _ _) = Just (Nothing, expr)
splitStmt (BindStmt _ pat expr _ _)
-- TODO Support more complex patterns
| VarPat _ (unLoc -> id) <- unLoc pat =
let bind = (fsToText . occNameFS . occName) id
in Just (Just bind, expr)
splitStmt _ = Nothing

runRepl :: Options -> FilePath -> ReplClient.Handle -> IdeState -> IO ()
runRepl opts mainDar replClient ideState = do
Right Dalfs{..} <- readDalfs . Zip.toArchive <$> BSL.readFile mainDar
(_, pkg) <- either (fail . show) pure (LFArchive.decodeArchive LFArchive.DecodeAsMain (BSL.toStrict mainDalf))
let moduleNames = map LF.moduleName (NM.elems (LF.packageModules pkg))
Just pkgs <- runAction ideState (use GeneratePackageMap "Dummy.daml")
Just stablePkgs <- runAction ideState (use GenerateStablePackages "Dummy.daml")
for_ (toList pkgs <> toList stablePkgs) $ \pkg -> do
r <- ReplClient.loadPackage replClient (LF.dalfPackageBytes pkg)
case r of
Left err -> do
hPutStrLn stderr ("Package could not be loaded: " <> show err)
exitFailure
Right _ -> pure ()
go moduleNames 0 []
where
go :: [LF.ModuleName] -> Int -> [(T.Text, Type)] -> IO ()
go moduleNames !i !binds = do
putStr "daml> "
hFlush stdout
l <- catchJust (guard . isEOFError) getLine (const exitSuccess)
dflags <-
hsc_dflags . hscEnv <$>
runAction ideState (use_ GhcSession $ lineFilePath i)
POk _ (unLoc -> stmt) <- pure (parseStatement l dflags)
let !(mbBind, expr) = fromMaybe (fail ("Unsupported statement type: " <> show (toConstr stmt))) (splitStmt stmt)
writeFileUTF8 (fromNormalizedFilePath $ lineFilePath i)
(renderModule dflags moduleNames i binds expr)
-- Useful for debugging, probably best to put it behind a --debug flag
-- rendered <- readFileUTF8 (fromNormalizedFilePath $ lineFilePath i)
-- for_ (lines rendered) $ \line ->
-- hPutStrLn stderr ("> " <> line)

-- TODO Handle failures here cracefully instead of
-- tearing down the whole process.
Just lfMod <- runAction ideState $ use GenerateDalf (lineFilePath i)
Just (tmrModule -> tcMod) <- runAction ideState $ use TypeCheck (lineFilePath i)
-- We need type annotations to avoid things becoming polymorphic.
-- If we end up with a typeclass constraint on `expr` things
-- will go wrong.
Just ty <- pure $ exprTy $ tm_typechecked_source tcMod

r <- ReplClient.runScript replClient (optDamlLfVersion opts) lfMod
case r of
Right _ -> pure ()
Left err -> do
hPutStrLn stderr ("Script produced an error: " <> show err)
-- TODO don’t kill the whole process
exitFailure

-- TODO Allow shadowing
go moduleNames (i + 1 :: Int) (binds <> [(fromMaybe "_" mbBind, ty)])

exprTy :: LHsBinds GhcTc -> Maybe Type
exprTy binds = listToMaybe
[ argTy
| FunBind{..} <- map unLoc (concatMap expand $ toList binds)
, getOccText fun_id == "expr"
, (_, [argTy]) <- [(splitTyConApp . mg_res_ty . mg_ext) fun_matches]
]

expand :: LHsBindLR id id -> [LHsBindLR id id]
expand (unLoc -> AbsBinds{..}) = toList abs_binds
expand bind = [bind]

lineFilePath :: Int -> NormalizedFilePath
lineFilePath i = toNormalizedFilePath $ "Line" <> show i <> ".daml"

lineModuleName :: Int -> String
lineModuleName i = "Line" <> show i

renderModule :: DynFlags -> [LF.ModuleName] -> Int -> [(Text, Type)] -> LHsExpr GhcPs -> String
renderModule dflags imports line binds expr = unlines $
[ "{-# OPTIONS_GHC -Wno-unused-imports -Wno-partial-type-signatures #-}"
, "{-# LANGUAGE PartialTypeSignatures #-}"
, "daml 1.2"
, "module " <> lineModuleName line <> " where"
, "import Prelude hiding (submit)"
, "import Daml.Script"
] <>
map (\moduleName -> T.unpack $ "import " <> LF.moduleNameString moduleName) imports <>
[ "expr : " <> concatMap (renderTy . snd) binds <> "Script _"
, "expr " <> unwords (map renderBind binds) <> " = " <> prettyPrint expr
]
where renderBind (name, ty) = "(" <> T.unpack name <> " : " <> showSDoc dflags (ppr ty) <> ")"
renderTy ty = showSDoc dflags (ppr ty) <> " -> "

51 changes: 51 additions & 0 deletions compiler/damlc/lib/DA/Cli/Damlc.hs
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ import DA.Cli.Damlc.IdeState
import DA.Cli.Damlc.Packaging
import DA.Cli.Damlc.Test
import DA.Daml.Compiler.Dar
import qualified DA.Daml.Compiler.Repl as Repl
import DA.Daml.Compiler.DataDependencies as DataDeps
import DA.Daml.Compiler.DocTest
import DA.Daml.Compiler.Scenario
import qualified DA.Daml.LF.ReplClient as ReplClient
import DA.Daml.Compiler.Upgrade
import qualified DA.Daml.LF.Ast as LF
import qualified DA.Daml.LF.Proto3.Archive as Archive
Expand Down Expand Up @@ -90,6 +92,7 @@ import "ghc-lib" HsDumpAst
import "ghc-lib" HscStats
import "ghc-lib-parser" HscTypes
import qualified "ghc-lib-parser" Outputable as GHC
import qualified SdkVersion

--------------------------------------------------------------------------------
-- Commands
Expand All @@ -114,6 +117,7 @@ data CommandName =
| Package
| Test
| Visual
| Repl
deriving (Ord, Show, Eq)
data Command = Command CommandName (Maybe ProjectOpts) (IO ())

Expand Down Expand Up @@ -246,6 +250,21 @@ cmdBuild numProcessors =
<*> incrementalBuildOpt
<*> initPkgDbOpt

cmdRepl :: Int -> Mod CommandFields Command
cmdRepl numProcessors =
command "repl" $
info (helper <*> cmd) fullDesc
where
cmd =
execRepl
<$> projectOpts "daml build"
<*> optionsParser numProcessors (EnableScenarioService False) (pure Nothing)
<*> strOption (long "script-lib" <> value "daml-script" <> internal)
-- ^ This is useful for tests and `bazel run`.
<*> strArgument (help "DAR to load in the repl")
<*> strOption (long "ledger-host" <> help "Host of the ledger API")
<*> strOption (long "ledger-port" <> help "Port of the ledger API")

cmdClean :: Mod CommandFields Command
cmdClean =
command "clean" $
Expand Down Expand Up @@ -576,6 +595,35 @@ execBuild projectOpts opts mbOutFile incrementalBuild initPkgDb =
where
targetFilePath name = fromMaybe (distDir </> name <.> "dar") mbOutFile

execRepl :: ProjectOpts -> Options -> FilePath -> FilePath -> String -> String -> Command
execRepl projectOpts opts scriptDar mainDar ledgerHost ledgerPort = Command Repl (Just projectOpts) effect
where effect = do
opts <- pure opts
{ optDlintUsage = DlintDisabled
, optScenarioService = EnableScenarioService False
}
logger <- getLogger opts "repl"
runfilesDir <- locateRunfiles (mainWorkspace </> "compiler/repl-service/server")
let jar = runfilesDir </> "repl-service.jar"
ReplClient.withReplClient (ReplClient.Options jar ledgerHost ledgerPort) $ \replHandle ->
withTempDir $ \dir ->
withCurrentDirectory dir $ do
sdkVer <- fromMaybe SdkVersion.sdkVersion <$> lookupEnv sdkVersionEnvVar
writeFileUTF8 "daml.yaml" $ unlines
[ "sdk-version: " <> sdkVer
, "name: repl"
, "version: 0.0.1"
, "source: ."
, "dependencies:"
, "- daml-prim"
, "- daml-stdlib"
, "- " <> show scriptDar
, "- " <> show mainDar
]
initPackageDb opts (InitPkgDb True)
withDamlIdeState opts logger diagnosticsLogger
(Repl.runRepl opts mainDar replHandle)

-- | Remove any build artifacts if they exist.
execClean :: ProjectOpts -> Command
execClean projectOpts =
Expand Down Expand Up @@ -965,6 +1013,9 @@ options numProcessors =
<> cmdClean
<> cmdGenerateSrc numProcessors
<> cmdGenerateGenSrc
<> cmdRepl numProcessors
-- once the repl is a bit more mature, make it non-internal
-- and modify sdk-config.yaml to add a description.
)

parserInfo :: Int -> ParserInfo Command
Expand Down
59 changes: 59 additions & 0 deletions compiler/damlc/tests/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
load("//bazel_tools:haskell.bzl", "da_haskell_binary", "da_haskell_test")
load("@os_info//:os_info.bzl", "is_windows")
load(":util.bzl", "damlc_compile_test", "damlc_integration_test")
load("//rules_daml:daml.bzl", "daml_compile")

# Tests for the lax CLI parser
da_haskell_test(
Expand Down Expand Up @@ -242,6 +243,64 @@ da_haskell_test(
],
)

genrule(
name = "repl-test",
srcs = [
"ReplTest.daml",
"//:VERSION",
],
outs = ["repl-test.dar"],
cmd = """
set -eou pipefail
TMP_DIR=$$(mktemp -d)
mkdir -p $$TMP_DIR/daml
cp -L $(location ReplTest.daml) $$TMP_DIR/daml
cat << EOF > $$TMP_DIR/daml.yaml
sdk-version: $$(cat $(location //:VERSION))
name: repl-test
source: daml
version: 0.1.0
dependencies:
- daml-stdlib
- daml-prim
build-options: ["--ghc-option", "-Werror"]
EOF
$(location //compiler/damlc) build --project-root=$$TMP_DIR -o $$PWD/$(location repl-test.dar)
rm -rf $$TMP_DIR
""",
tools = ["//compiler/damlc"],
visibility = ["//visibility:public"],
)

da_haskell_test(
name = "repl",
srcs = ["src/DA/Test/Repl.hs"],
data = [
":repl-test.dar",
"//compiler/damlc",
"//daml-script/daml:daml-script.dar",
"//ledger/sandbox:sandbox-binary",
],
hackage_deps = [
"base",
"extra",
"filepath",
"process",
"regex-tdfa",
"safe",
"safe-exceptions",
"tasty",
"tasty-hunit",
],
main_function = "DA.Test.Repl.main",
src_strip_prefix = "src",
visibility = ["//visibility:public"],
deps = [
"//libs-haskell/bazel-runfiles",
"//libs-haskell/da-hs-base",
],
)

# Memory tests

da_haskell_binary(
Expand Down
Loading

0 comments on commit 8d81399

Please sign in to comment.