Skip to content

Commit

Permalink
Disable release step on macos (digital-asset#4330)
Browse files Browse the repository at this point in the history 
Originally we ran the release step on both Linux and MacOS to handle
platform dependent artifacts, in particular, damlc.jar. However, we
don’t have any platform dependent artifacts that are uploaded as part
of the release script anymore and I hope we will never have to add any
in the future.

So this PR, removes the code for handling platform dependent artifacts
in the release step and disables the release step on MacOS (while
still setting the variables like we do on Windows).

Currently the release step still costs us ~2 minutes on MacOS which is
already our slowest platform so hopefully this will speed things up a
bit.

changelog_begin
changelog_end
  • Loading branch information
@cocreature
cocreature authored Jan 31, 2020
1 parent ea0bf33 commit 60e32a9
Show file tree
Hide file tree
Showing 6 changed files with 32 additions and 68 deletions.
19 changes: 10 additions & 9 deletions ci/release.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ step() {

cd "$(dirname "$0")"/..

if [[ "$(uname)" == "Darwin" ]]; then
echo "Simulating release step..."
echo "##vso[task.setvariable variable=has_released;isOutput=true]true"
echo "##vso[task.setvariable variable=release_tag]$(cat VERSION)"
exit 0
fi


step "loading dev-env"

eval "$(dev-env/bin/dade assist)"
Expand All @@ -27,23 +35,16 @@ step "set up temporary location"
release_dir="$(mktemp -d)"
step "temporary release directory is ${release_dir}"

EXTRA_ARGS=""

if [[ "$(uname)" == "Darwin" ]]; then
EXTRA_ARGS="--ignore-missing-deps"
fi


if [[ "${BUILD_SOURCEBRANCHNAME:-}" == "master" ]]; then
# set up bintray credentials
mkdir -p ~/.jfrog
echo "$JFROG_CONFIG_CONTENT" > ~/.jfrog/jfrog-cli.conf
unset JFROG_CONFIG_CONTENT

step "run release script (with --upload)"
./bazel-bin/release/release --artifacts release/artifacts.yaml --upload --log-level debug --release-dir "${release_dir}" $EXTRA_ARGS
./bazel-bin/release/release --artifacts release/artifacts.yaml --upload --log-level debug --release-dir "${release_dir}"
else
step "run release script (dry run)"
./bazel-bin/release/release --artifacts release/artifacts.yaml --log-level debug --release-dir "${release_dir}" $EXTRA_ARGS
./bazel-bin/release/release --artifacts release/artifacts.yaml --log-level debug --release-dir "${release_dir}"
step "release artifacts got stored in ${release_dir}"
fi
2 changes: 1 addition & 1 deletion dev-env/bin/daml-sdk-head
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ if [ "$SKIP_JARS" -eq "0" ]; then

bazel build //release:release
tmp=$(mktemp -d)
"$REPO_ROOT/bazel-bin/release/release" --artifacts "$REPO_ROOT/release/artifacts.yaml" --release-dir $tmp --all-artifacts --install-head-jars
"$REPO_ROOT/bazel-bin/release/release" --artifacts "$REPO_ROOT/release/artifacts.yaml" --release-dir $tmp --install-head-jars

mv "$REPO_ROOT/VERSION.head-build-in-progress" "$REPO_ROOT/VERSION"
trap - EXIT
Expand Down
17 changes: 7 additions & 10 deletions release/src/Main.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,11 @@ main = do
-- originating from genrules that use damlc. This is a bit hacky but
-- given that it’s fairly unlikely to accidentally introduce a dependency on the scenario
-- service it doesn’t seem worth fixing properly.
if getIgnoreMissingDeps optsIgnoreMissingDeps
then pure (a, [])
else do
let bazelQueryCommand = shell $ "bazel query 'kind(\"(scala|java)_library\", deps(" ++ (T.unpack . getBazelTarget . artTarget) a ++ ")) intersect //... except //compiler/scenario-service/protos:scenario_service_java_proto'"
internalDeps <- liftIO $ lines <$> readCreateProcess bazelQueryCommand ""
-- check if a dependency is not already a maven target from artifacts.yaml
let missingDeps = filter (`Set.notMember` allMavenTargets) internalDeps
return (a, missingDeps)
let bazelQueryCommand = shell $ "bazel query 'kind(\"(scala|java)_library\", deps(" ++ (T.unpack . getBazelTarget . artTarget) a ++ ")) intersect //... except //compiler/scenario-service/protos:scenario_service_java_proto'"
internalDeps <- liftIO $ lines <$> readCreateProcess bazelQueryCommand ""
-- check if a dependency is not already a maven target from artifacts.yaml
let missingDeps = filter (`Set.notMember` allMavenTargets) internalDeps
return (a, missingDeps)

let onlyMissing = filter (not . null . snd) missingDepsForAllArtifacts
-- now we can report all the missing dependencies per artifact
Expand All @@ -83,11 +80,11 @@ main = do
liftIO exitFailure

files <- fmap concat $ forM artifacts $ \a -> do
fs <- artifactFiles optsAllArtifacts a
fs <- artifactFiles a
pure $ map (a,) fs
mapM_ (\(_, (inp, outp)) -> copyToReleaseDir bazelLocations releaseDir inp outp) files

uploadArtifacts <- concatMapM (mavenArtifactCoords optsAllArtifacts) mavenUploadArtifacts
uploadArtifacts <- concatMapM mavenArtifactCoords mavenUploadArtifacts
validateMavenArtifacts releaseDir uploadArtifacts

-- npm packages we want to publish.
Expand Down
4 changes: 0 additions & 4 deletions release/src/Options.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@ data Options = Options
, optsSlackReleaseMessageFile :: Maybe FilePath
, optsFullLogging :: Bool
, optsLogLevel :: LogLevel
, optsAllArtifacts :: AllArtifacts
, optsLocallyInstallJars :: Bool
, optsIgnoreMissingDeps :: IgnoreMissingDeps
} deriving (Eq, Show)

optsParser :: Parser Options
Expand All @@ -37,9 +35,7 @@ optsParser = Options
<*> option (Just <$> str) (long "slack-release-message" <> help "if present will write out what to write in slack. if there are no releases the file will be empty" <> value Nothing)
<*> switch (long "full-logging" <> help "full logging detail")
<*> option readLogLevel (long "log-level" <> metavar "debug|info|warn|error (default: info)" <> help "Specify log level during release run" <> value LevelInfo )
<*> (AllArtifacts <$> switch (long "all-artifacts" <> help "Produce all artifacts including platform-independent artifacts on MacOS"))
<*> switch (long "install-head-jars" <> help "install jars to ~/.m2")
<*> (IgnoreMissingDeps <$> switch (long "ignore-missing-deps" <> help "Do not check for missing Maven dependencies"))

where
readLogLevel :: ReadM LogLevel
Expand Down
17 changes: 0 additions & 17 deletions release/src/Types.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,20 @@

{-# LANGUAGE ConstraintKinds #-}
module Types (
AllArtifacts(..),
ArtifactId,
ArtifactType,
CIException(..),
Classifier,
BintrayPackage(..),
GitRev,
GroupId,
IgnoreMissingDeps(..),
MavenAllowUnsecureTls(..),
MavenCoords(..),
MavenUpload(..),
MavenUploadConfig(..),
MonadCI,
OS(..),
PerformUpload(..),
PlatformDependent(..),
TextVersion,
Version(..),
VersionChange(..),
Expand Down Expand Up @@ -79,23 +76,9 @@ data MavenCoords = MavenCoords
, artifactType :: !ArtifactType
} deriving Show

newtype PlatformDependent = PlatformDependent{getPlatformDependent :: Bool}
deriving (Eq, Show, FromJSON)

newtype MavenUpload = MavenUpload { getMavenUpload :: Bool }
deriving (Eq, Show, FromJSON)

-- | If this is True, we produce all artifacts even platform independent artifacts on MacOS.
-- This is useful for testing purposes.
newtype AllArtifacts = AllArtifacts Bool
deriving (Eq, Show)

-- | If True, we do not check for missing deps.
-- This is significantly faster and deps checking is platform independent
-- so we only run it on Linux which is usually faster than the MacOS build.
newtype IgnoreMissingDeps = IgnoreMissingDeps { getIgnoreMissingDeps :: Bool }
deriving (Eq, Show)

-- execution
type MonadCI m = (MonadIO m, MonadMask m, MonadLogger m,
MonadUnliftIO m, MonadBaseControl IO m, Async.Forall (Async.Pure m))
Expand Down
41 changes: 14 additions & 27 deletions release/src/Util.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ instance FromJSON ReleaseType where
data Artifact c = Artifact
{ artTarget :: !BazelTarget
, artReleaseType :: !ReleaseType
, artPlatformDependent :: !PlatformDependent
, artBintrayPackage :: !BintrayPackage
-- ^ Defaults to sdk-components if not specified
, artMavenUpload :: MavenUpload
Expand All @@ -117,7 +116,6 @@ instance FromJSON (Artifact (Maybe ArtifactLocation)) where
parseJSON = withObject "Artifact" $ \o -> Artifact
<$> o .: "target"
<*> o .: "type"
<*> (fromMaybe (PlatformDependent False) <$> o .:? "platformDependent")
<*> (fromMaybe PkgSdkComponents <$> o .:? "bintrayPackage")
<*> (fromMaybe (MavenUpload False) <$> o .:? "mavenUpload")
<*> o .:? "javadoc-jar"
Expand Down Expand Up @@ -283,17 +281,16 @@ customJavadocJarName :: Artifact a -> Maybe Text
customJavadocJarName Artifact{..} = T.pack . toFilePath <$> artJavadocJar

-- | Given an artifact, produce a list of pairs of an input file and the corresponding output file.
artifactFiles :: E.MonadThrow m => AllArtifacts -> Artifact PomData -> m [(Path Rel File, Path Rel File)]
artifactFiles allArtifacts art@Artifact{..} = do
artifactFiles :: E.MonadThrow m => Artifact PomData -> m [(Path Rel File, Path Rel File)]
artifactFiles art@Artifact{..} = do
let PomData{..} = artMetadata
outDir <- parseRelDir $ unpack $
T.intercalate "/" pomGroupId #"/"# pomArtifactId #"/"# pomVersion #"/"
let ostxt = if getPlatformDependent artPlatformDependent then "-" <> osName else ""
let (directory, name) = splitBazelTarget artTarget
directory <- parseRelDir $ unpack directory

mainArtifactIn <- parseRelFile $ unpack $ mainFileName artReleaseType name
mainArtifactOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "." # mainExt artReleaseType))
mainArtifactOut <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # "." # mainExt artReleaseType))

pomFileIn <- parseRelFile (unpack (name <> "_pom.xml"))
pomFileOut <- releasePomPath artMetadata
Expand All @@ -310,13 +307,11 @@ artifactFiles allArtifacts art@Artifact{..} = do
(customJavadocJarName art <|> javadocJarName art <|> scaladocJarName art <|> javadocDeployJarName art <|> javadocProtoJarName art)
javadocJarOut <- releaseDocJarPath artMetadata

let shouldReleasePlatInd = shouldRelease allArtifacts (PlatformDependent False)

pure $
[(directory </> mainArtifactIn, outDir </> mainArtifactOut) | shouldRelease allArtifacts artPlatformDependent] <>
[(directory </> pomFileIn, outDir </> pomFileOut) | isJar artReleaseType, shouldReleasePlatInd] <>
[(directory </> sourceJarIn, outDir </> sourceJarOut) | shouldReleasePlatInd, Just sourceJarIn <- pure mbSourceJarIn] <>
[(directory </> javadocJarIn, outDir </> javadocJarOut) | shouldReleasePlatInd, Just javadocJarIn <- pure mbJavadocJarIn]
[(directory </> mainArtifactIn, outDir </> mainArtifactOut)] <>
[(directory </> pomFileIn, outDir </> pomFileOut) | isJar artReleaseType] <>
[(directory </> sourceJarIn, outDir </> sourceJarOut) | Just sourceJarIn <- pure mbSourceJarIn] <>
[(directory </> javadocJarIn, outDir </> javadocJarOut) | Just javadocJarIn <- pure mbJavadocJarIn]
-- ^ Note that the Scaladoc is specified with the "javadoc" classifier.

-- | The file path to the source jar for the given artifact in the release directory.
Expand All @@ -336,31 +331,23 @@ releasePomPath PomData{..} =

-- | Given an artifact, produce a list of pairs of an input file and the Maven coordinates.
-- This corresponds to the files uploaded to Maven Central.
mavenArtifactCoords :: E.MonadThrow m => AllArtifacts -> Artifact PomData -> m [(MavenCoords, Path Rel File)]
mavenArtifactCoords allArtifacts Artifact{..} = do
mavenArtifactCoords :: E.MonadThrow m => Artifact PomData -> m [(MavenCoords, Path Rel File)]
mavenArtifactCoords Artifact{..} = do
let PomData{..} = artMetadata
let jarClassifier = if getPlatformDependent artPlatformDependent then Just osName else Nothing
outDir <- parseRelDir $ unpack $
T.intercalate "/" pomGroupId #"/"# pomArtifactId #"/"# pomVersion #"/"
let ostxt = if getPlatformDependent artPlatformDependent then "-" <> osName else ""

mainArtifactFile <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # ostxt # "." # mainExt artReleaseType))
mainArtifactFile <- parseRelFile (unpack (pomArtifactId #"-"# pomVersion # "." # mainExt artReleaseType))
pomFile <- releasePomPath artMetadata
sourcesFile <- releaseSourceJarPath artMetadata
javadocFile <- releaseDocJarPath artMetadata

let mavenCoords classifier artifactType =
MavenCoords { groupId = pomGroupId, artifactId = pomArtifactId, version = pomVersion, classifier, artifactType }
let shouldReleasePlatInd = shouldRelease allArtifacts (PlatformDependent False)

pure $ [ (mavenCoords jarClassifier $ mainExt artReleaseType, outDir </> mainArtifactFile) | shouldReleasePlatInd] <>
[ (mavenCoords Nothing "pom", outDir </> pomFile) | isJar artReleaseType, shouldReleasePlatInd] <>
[ (mavenCoords (Just "sources") "jar", outDir </> sourcesFile) | isJar artReleaseType, shouldReleasePlatInd] <>
[ (mavenCoords (Just "javadoc") "jar", outDir </> javadocFile) | isJar artReleaseType, shouldReleasePlatInd]

shouldRelease :: AllArtifacts -> PlatformDependent -> Bool
shouldRelease (AllArtifacts allArtifacts) (PlatformDependent platformDependent) =
allArtifacts || platformDependent || osName == "linux"
pure $ [ (mavenCoords Nothing $ mainExt artReleaseType, outDir </> mainArtifactFile)] <>
[ (mavenCoords Nothing "pom", outDir </> pomFile) | isJar artReleaseType] <>
[ (mavenCoords (Just "sources") "jar", outDir </> sourcesFile) | isJar artReleaseType] <>
[ (mavenCoords (Just "javadoc") "jar", outDir </> javadocFile) | isJar artReleaseType]

copyToReleaseDir :: (MonadLogger m, MonadIO m) => BazelLocations -> Path Abs Dir -> Path Rel File -> Path Rel File -> m ()
copyToReleaseDir BazelLocations{..} releaseDir inp out = do
Expand Down

0 comments on commit 60e32a9

Please sign in to comment.