Ignore missing maven deps on macos (digital-asset#4201)

Currently, pretty much all of our builds are bottlenecked on MacOS (mainly because the builders are slower and have worse caching). The release step adds > 3min to each build which is a bit annoying. It turns out that removing the calls to `bazel query` which are used to check for missing Maven dependencies speeds this up by > 5x. Given that the check is platform independent anyway, we can just disable it on MacOS. changelog_begin changelog_end
iggy-da · Jan 24, 2020 · 1a0fb13 · 1a0fb13
1 parent b87ff49
commit 1a0fb13
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 7 deletions.
diff --git a/ci/release.sh b/ci/release.sh
@@ -27,16 +27,23 @@ step "set up temporary location"
 release_dir="$(mktemp -d)"
 step "temporary release directory is ${release_dir}"
 
+EXTRA_ARGS=""
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    EXTRA_ARGS="--ignore-missing-deps"
+fi
+
+
 if [[ "${BUILD_SOURCEBRANCHNAME:-}" == "master" ]]; then
     # set up bintray credentials
     mkdir -p ~/.jfrog
     echo "$JFROG_CONFIG_CONTENT" > ~/.jfrog/jfrog-cli.conf
     unset JFROG_CONFIG_CONTENT
 
     step "run release script (with --upload)"
-    ./bazel-bin/release/release --artifacts release/artifacts.yaml --upload --log-level debug --release-dir "${release_dir}"
+    ./bazel-bin/release/release --artifacts release/artifacts.yaml --upload --log-level debug --release-dir "${release_dir}" $EXTRA_ARGS
 else
     step "run release script (dry run)"
-    ./bazel-bin/release/release --artifacts release/artifacts.yaml --log-level debug --release-dir "${release_dir}"
+    ./bazel-bin/release/release --artifacts release/artifacts.yaml --log-level debug --release-dir "${release_dir}" $EXTRA_ARGS
     step "release artifacts got stored in ${release_dir}"
 fi
diff --git a/release/src/Main.hs b/release/src/Main.hs
@@ -64,11 +64,14 @@ main = do
           -- originating from genrules that use damlc. This is a bit hacky but
           -- given that it’s fairly unlikely to accidentally introduce a dependency on the scenario
           -- service it doesn’t seem worth fixing properly.
-          let bazelQueryCommand = shell $ "bazel query 'kind(\"(scala|java)_library\", deps(" ++ (T.unpack . getBazelTarget . artTarget) a ++ ")) intersect //... except //compiler/scenario-service/protos:scenario_service_java_proto'"
-          internalDeps <- liftIO $ lines <$> readCreateProcess bazelQueryCommand ""
-          -- check if a dependency is not already a maven target from artifacts.yaml
-          let missingDeps = filter (`Set.notMember` allMavenTargets) internalDeps
-          return (a, missingDeps)
+          if getIgnoreMissingDeps optsIgnoreMissingDeps
+              then pure (a, [])
+              else do
+                let bazelQueryCommand = shell $ "bazel query 'kind(\"(scala|java)_library\", deps(" ++ (T.unpack . getBazelTarget . artTarget) a ++ ")) intersect //... except //compiler/scenario-service/protos:scenario_service_java_proto'"
+                internalDeps <- liftIO $ lines <$> readCreateProcess bazelQueryCommand ""
+                -- check if a dependency is not already a maven target from artifacts.yaml
+                let missingDeps = filter (`Set.notMember` allMavenTargets) internalDeps
+                return (a, missingDeps)
 
       let onlyMissing = filter (not . null . snd) missingDepsForAllArtifacts
       -- now we can report all the missing dependencies per artifact

diff --git a/release/src/Options.hs b/release/src/Options.hs
@@ -26,6 +26,7 @@ data Options = Options
   , optsLogLevel :: LogLevel
   , optsAllArtifacts :: AllArtifacts
   , optsLocallyInstallJars :: Bool
+  , optsIgnoreMissingDeps :: IgnoreMissingDeps
   } deriving (Eq, Show)
 
 optsParser :: Parser Options
@@ -38,6 +39,8 @@ optsParser = Options
   <*> option readLogLevel (long "log-level" <> metavar "debug|info|warn|error (default: info)" <> help "Specify log level during release run" <> value LevelInfo )
   <*> (AllArtifacts <$> switch (long "all-artifacts" <> help "Produce all artifacts including platform-independent artifacts on MacOS"))
   <*> switch (long "install-head-jars" <> help "install jars to ~/.m2")
+  <*> (IgnoreMissingDeps <$> switch (long "ignore-missing-deps" <> help "Do not check for missing Maven dependencies"))
+
   where
     readLogLevel :: ReadM LogLevel
     readLogLevel = do

diff --git a/release/src/Types.hs b/release/src/Types.hs
@@ -11,6 +11,7 @@ module Types (
     BintrayPackage(..),
     GitRev,
     GroupId,
+    IgnoreMissingDeps(..),
     MavenAllowUnsecureTls(..),
     MavenCoords(..),
     MavenUpload(..),
@@ -89,6 +90,12 @@ newtype MavenUpload = MavenUpload { getMavenUpload :: Bool }
 newtype AllArtifacts = AllArtifacts Bool
     deriving (Eq, Show)
 
+-- | If True, we do not check for missing deps.
+-- This is significantly faster and deps checking is platform independent
+-- so we only run it on Linux which is usually faster than the MacOS build.
+newtype IgnoreMissingDeps = IgnoreMissingDeps { getIgnoreMissingDeps :: Bool }
+    deriving (Eq, Show)
+
 -- execution
 type MonadCI m = (MonadIO m, MonadMask m, MonadLogger m,
                   MonadUnliftIO m, MonadBaseControl IO m, Async.Forall (Async.Pure m))