language: introduce data-imports (digital-asset#3399)

* language: introduce data-imports Right now the user experience for importing dalfs and dars from different sdks is quiet confusing. This PR tries to solve this. We add an additional field `data-imports` to daml.yaml. These imports can come from different SDK's and we will generate interface files containing the data types and their Template instances. This also simplifies the migration command, as it now always imports the respective packages as `data-imports`.
kgeonw · Nov 8, 2019 · 6cf3a52 · 6cf3a52
1 parent 1bae7cd
commit 6cf3a52
Show file tree

Hide file tree

Showing 11 changed files with 171 additions and 153 deletions.
diff --git a/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Dar.hs b/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Dar.hs
@@ -88,6 +88,7 @@ data PackageConfigFields = PackageConfigFields
     , pExposedModules :: Maybe [String]
     , pVersion :: Maybe String
     , pDependencies :: [String]
+    , pDataDependencies :: [String]
     , pSdkVersion :: String
     , cliOpts :: Maybe [String]
     }

diff --git a/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Upgrade.hs b/compiler/damlc/daml-compiler/src/DA/Daml/Compiler/Upgrade.hs
@@ -53,13 +53,11 @@ data Env = Env
 newtype DiffSdkVers = DiffSdkVers Bool
 
 -- | Generate non-consuming choices to upgrade all templates defined in the module.
-generateUpgradeModule :: DiffSdkVers -> [String] -> String -> String -> String -> String
-generateUpgradeModule (DiffSdkVers diffSdks) templateNames modName qualA qualB =
+generateUpgradeModule :: [String] -> String -> String -> String -> String
+generateUpgradeModule templateNames modName qualA qualB =
     unlines $ header ++ concatMap upgradeTemplates templateNames
   where
-    header
-      | diffSdks = header0 ++ header1 ++ header2
-      | otherwise = header0 ++ header2
+    header = header0 ++ header1 ++ header2
       -- If we compile with packages from a single sdk version, the instances modules will not be
       -- there and hence we can not include header1.
     header0 =
@@ -69,8 +67,8 @@ generateUpgradeModule (DiffSdkVers diffSdks) templateNames modName qualA qualB =
         , "import " <> modName <> qualB <> " qualified as B"
         ]
     header1 =
-        [ "import " <> modName <> "Instances()"
-        , "import " <> modName <> "Instances()"
+        [ "import " <> modName <> qualA <> "Instances()"
+        , "import " <> modName <> qualB <> "Instances()"
         ]
     header2 = [
         "import DA.Upgrade"

diff --git a/compiler/damlc/daml-opts/daml-opts-types/DA/Daml/Options/Types.hs b/compiler/damlc/daml-opts/daml-opts-types/DA/Daml/Options/Types.hs
@@ -63,8 +63,6 @@ data Options = Options
   -- ^ Information about dlint usage.
   , optIsGenerated :: Bool
     -- ^ Whether we're compiling generated code. Then we allow internal imports.
-  , optAllowDifferentSdks :: Bool
-    -- ^ Whether we're allowing imports from packages compiled with different SDK's.
   , optDflagCheck :: Bool
     -- ^ Whether to check dflags. In some cases we want to turn this check of. For example when
     -- migrating or running the daml doc test.
@@ -169,7 +167,6 @@ defaultOptions mbVersion =
         , optSkipScenarioValidation = SkipScenarioValidation False
         , optDlintUsage = DlintDisabled
         , optIsGenerated = False
-        , optAllowDifferentSdks = False
         , optDflagCheck = True
         , optCoreLinting = False
         , optHaddock = Haddock False

diff --git a/compiler/damlc/lib/DA/Cli/Damlc.hs b/compiler/damlc/lib/DA/Cli/Damlc.hs
diff --git a/compiler/damlc/lib/DA/Cli/Options.hs b/compiler/damlc/lib/DA/Cli/Options.hs
@@ -411,7 +411,6 @@ optionsParser numProcessors enableScenarioService parsePkgName = Options
     <*> pure (optSkipScenarioValidation $ defaultOptions Nothing)
     <*> dlintUsageOpt
     <*> optIsGenerated
-    <*> optAllowDifferentSdks
     <*> optNoDflagCheck
     <*> pure False
     <*> pure (Haddock False)
@@ -497,13 +496,6 @@ optionsParser numProcessors enableScenarioService parsePkgName = Options
         <> help "Set path to CPP."
         <> internal
 
-    optAllowDifferentSdks :: Parser Bool
-    optAllowDifferentSdks =
-        switch $
-        help "Allow the import of packages compiled with different SDKs." <>
-        long "allow-different-sdks" <>
-        internal
-
 optGhcCustomOptions :: Parser [String]
 optGhcCustomOptions =
     fmap concat $ many $

diff --git a/daml-assistant/daml-helper/src/DA/Daml/Helper/Main.hs b/daml-assistant/daml-helper/src/DA/Daml/Helper/Main.hs
@@ -217,7 +217,7 @@ commandParser = subparser $ fold
 runCommand :: Command -> IO ()
 runCommand DamlStudio {..} = runDamlStudio replaceExtension remainingArguments
 runCommand RunJar {..} = runJar jarPath mbLogbackConfig remainingArguments
-runCommand New {..} = runNew targetFolder templateNameM []
+runCommand New {..} = runNew targetFolder templateNameM [] []
 runCommand Migrate {..} = runMigrate targetFolder pkgPathFrom pkgPathTo
 runCommand Init {..} = runInit targetFolderM
 runCommand ListTemplates = runListTemplates

diff --git a/daml-assistant/daml-helper/src/DA/Daml/Helper/Run.hs b/daml-assistant/daml-helper/src/DA/Daml/Helper/Run.hs
@@ -525,8 +525,8 @@ runInit targetFolderM = do
 -- * Creation of a project in existing folder (suggest daml init instead).
 -- * Creation of a project inside another project.
 --
-runNew :: FilePath -> Maybe String -> [String] -> IO ()
-runNew targetFolder templateNameM pkgDeps = do
+runNew :: FilePath -> Maybe String -> [String] -> [String] -> IO ()
+runNew targetFolder templateNameM pkgDeps dataDeps = do
     templatesFolder <- getTemplatesFolder
     let templateName = fromMaybe defaultProjectTemplate templateNameM
         templateFolder = templatesFolder </> templateName
@@ -606,6 +606,7 @@ runNew targetFolder templateNameM pkgDeps = do
         let config = replace "__VERSION__"  sdkVersion
                    . replace "__PROJECT_NAME__" projectName
                    . replace "__DEPENDENCIES__" (unlines ["  - " <> dep | dep <- pkgDeps])
+                   . replace "__DATA_DEPENDENCIES__" (unlines ["  - " <> dep | dep <- dataDeps])
                    $ configTemplate
         writeFileUTF8 configPath config
         removeFile configTemplatePath
@@ -622,7 +623,7 @@ runMigrate targetFolder pkgPath1 pkgPath2
     pkgPath1Abs <- makeAbsolute pkgPath1
     pkgPath2Abs <- makeAbsolute pkgPath2
     -- Create a new project
-    runNew targetFolder (Just "migrate") [pkgPath1Abs, pkgPath2Abs]
+    runNew targetFolder (Just "migrate") [] [pkgPath1Abs, pkgPath2Abs]
 
     -- Call damlc to create the upgrade source files.
     procConfig <- toAssistantCommand

diff --git a/daml-assistant/integration-tests/src/DA/Daml/Assistant/IntegrationTests.hs b/daml-assistant/integration-tests/src/DA/Daml/Assistant/IntegrationTests.hs
@@ -414,15 +414,16 @@ packagingTests = testGroup "packaging"
           , "name: proj"
           , "version: 0.1.0"
           , "source: ."
-          , "dependencies: [daml-prim, daml-stdlib, simple-dalf-0.0.0.dalf]"
+          , "dependencies: [daml-prim, daml-stdlib]"
+          , "data-dependencies: [simple-dalf-0.0.0.dalf]"
           , "build-options:"
           , "- --generated-src"
           ]
         writeFileUTF8 (projDir </> "A.daml") $ unlines
             [ "daml 1.2"
             , "module A where"
             , "import qualified Module"
-            , "import qualified ModuleInstances"
+            , "import qualified ModuleInstances()"
             , "import DA.Internal.Template (toAnyTemplate, fromAnyTemplate)"
             , "newTemplate : Party -> Party -> Module.Template"
             , "newTemplate p1 p2 = Module.Template with Module.this = p1, Module.arg = p2"
@@ -631,6 +632,8 @@ packagingTests = testGroup "packaging"
            , "module Main where"
            , "import MainA qualified as A"
            , "import MainB qualified as B"
+           , "import MainAInstances()"
+           , "import MainBInstances()"
            , "import MainAGenInstances()"
            , "import MainBGenInstances()"
            , "import DA.Upgrade"

diff --git a/docs/source/daml/reference/packages.rst b/docs/source/daml/reference/packages.rst
@@ -37,7 +37,7 @@ DAML package management. To import a package ``Bar`` in project ``Foo``, add the
   dependencies:
     - daml-prim
     - daml-stdlib
-    - /home/johndoe/bar/.daml/dist/bar.dar
+    - /home/johndoe/bar/.daml/dist/bar-1.0.0.dar
 
 The import path needs to be the relative or absolute path pointing to the created DAML archive of
 the ``bar`` project. The archive can reside anywhere on the local file system. Note that the SDK
@@ -53,3 +53,26 @@ Some.Module``.
 Note that all modules of package ``foo`` that should be available as imports of other packages need
 to be exposed by adding them to the ``exposed-modules`` stanza of the `daml.yaml` file. If the
 ``exposed-modules`` stanza is omitted, all modules of the project are exposed by default.
+
+Importing archives compiled with different SDK's
+************************************************
+
+All DAML archive dependencies of a project need to be compiled with the same SDK as the project
+itself. However, it is possible to import templates and data types of an archive compiled with an
+older SDK, by listing them under the ``data-dependencies`` stanza:
+
+.. code-block:: yaml
+
+  sdk-version: 0.0.0
+  name: foo
+  source: daml
+  version: 1.0.0
+  exposed-modules:
+    - Some.Module
+    - Some.Other.Module
+  dependencies:
+    - daml-prim
+    - daml-stdlib
+    - /home/johndoe/bar/.daml/dist/bar.dar
+  data-dependencies:
+    - /home/jondoe/bar-0.0.0/.daml/dist/bar-0.0.0.dar
diff --git a/templates/migrate/daml.yaml.template b/templates/migrate/daml.yaml.template
@@ -8,4 +8,5 @@ version: 0.0.1
 dependencies:
   - daml-prim
   - daml-stdlib
-__DEPENDENCIES__
+data-dependencies:
+__DATA_DEPENDENCIES__
diff --git a/unreleased.rst b/unreleased.rst
@@ -18,6 +18,9 @@ HEAD — ongoing
 - [DAML Compiler] Support for incremental builds in ``daml build`` using the ``--incremental=yes`` flag.
   This is still experimental and disabled by default but will become enabled by default in the future.
   On large codebases, this can significantly improve compile times and reduce memory usage.
+- [DAML Compiler] Support for data dependencies on packages compiled with an older SDK
+  (experimental). To import data dependencies, list the packages under the ``data-dependencies``
+  stanza in the project's daml.yaml file.
 - [JSON API - Experimental] Fix to support Archive choice. See issue #3219
 - [Sandbox] Add the option to start the sandbox with JWT based authentication. See `issue #3363 <https://github.com/digital-asset/daml/issues/3363>`__.
 - [Sandbox] Fixed a bug in the SQL backend that caused the database to be flooded with requests when streaming out transactions.