Skip to content

Commit

Permalink
Merge pull request kubernetes#6369 from zmerlynn/i6346
Browse files Browse the repository at this point in the history 
Redo on building platforms in parallel
  • Loading branch information
@brendandburns
brendandburns committed Apr 3, 2015
2 parents fd966b7 + 5b6c75f commit b35e66b
Show file tree
Hide file tree
Showing 2 changed files with 183 additions and 90 deletions.
94 changes: 58 additions & 36 deletions build/common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,11 +505,14 @@ function kube::release::package_tarballs() {
# Clean out any old releases
rm -rf "${RELEASE_DIR}"
mkdir -p "${RELEASE_DIR}"
kube::release::package_client_tarballs
kube::release::package_server_tarballs
kube::release::package_salt_tarball
kube::release::package_test_tarball
kube::release::package_full_tarball
kube::release::package_client_tarballs &
kube::release::package_server_tarballs &
kube::release::package_salt_tarball &
wait || { kube::log::error "previous tarball phase failed"; return 1; }

kube::release::package_full_tarball & # _full depends on all the previous phases
kube::release::package_test_tarball & # _test doesn't depend on anything
wait || { kube::log::error "previous tarball phase failed"; return 1; }
}

# Package up all of the cross compiled clients. Over time this should grow into
Expand All @@ -518,30 +521,35 @@ function kube::release::package_client_tarballs() {
# Find all of the built client binaries
local platform platforms
platforms=($(cd "${LOCAL_OUTPUT_BINPATH}" ; echo */*))
for platform in "${platforms[@]}" ; do
for platform in "${platforms[@]}"; do
local platform_tag=${platform/\//-} # Replace a "/" for a "-"
kube::log::status "Building tarball: client $platform_tag"
kube::log::status "Starting tarball: client $platform_tag"

local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes"
rm -rf "${release_stage}"
mkdir -p "${release_stage}/client/bin"
(
local release_stage="${RELEASE_STAGE}/client/${platform_tag}/kubernetes"
rm -rf "${release_stage}"
mkdir -p "${release_stage}/client/bin"

local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
if [[ "${platform%/*}" == "windows" ]]; then
client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
fi
local client_bins=("${KUBE_CLIENT_BINARIES[@]}")
if [[ "${platform%/*}" == "windows" ]]; then
client_bins=("${KUBE_CLIENT_BINARIES_WIN[@]}")
fi

# This fancy expression will expand to prepend a path
# (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
# KUBE_CLIENT_BINARIES array.
cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/client/bin/"
# This fancy expression will expand to prepend a path
# (${LOCAL_OUTPUT_BINPATH}/${platform}/) to every item in the
# KUBE_CLIENT_BINARIES array.
cp "${client_bins[@]/#/${LOCAL_OUTPUT_BINPATH}/${platform}/}" \
"${release_stage}/client/bin/"

kube::release::clean_cruft
kube::release::clean_cruft

local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
kube::release::create_tarball "${package_name}" "${release_stage}/.."
local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
kube::release::create_tarball "${package_name}" "${release_stage}/.."
) &
done

kube::log::status "Waiting on tarballs"
wait || { kube::log::error "client tarball creation failed"; exit 1; }
}

# Package up all of the server binaries
Expand Down Expand Up @@ -578,27 +586,41 @@ function kube::release::package_server_tarballs() {
done
}

function kube::release::md5() {
if which md5 >/dev/null 2>&1; then
md5 -q "$1"
else
md5sum "$1" | awk '{ print $1 }'
fi
}

# This will take binaries that run on master and creates Docker images
# that wrap the binary in them. (One docker image per binary)
function kube::release::create_docker_images_for_server() {
# Create a sub-shell so that we don't pollute the outer environment
(
local binary_name;
local binary_name
for binary_name in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do
echo "+++ Building docker image: ${binary_name}";
local docker_file_path="$1/Dockerfile";
local binary_file_path="$1/${binary_name}";
if [ -f ${docker_file_path} ]; then
rm ${docker_file_path};
fi;
printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path};
local md5_sum=$(md5sum ${binary_file_path} | awk '{print $1}')
local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum
docker build -t "${docker_image_tag}" ${1};
docker save ${docker_image_tag} > ${1}/${binary_name}.tar;
echo $md5_sum > ${1}/${binary_name}.docker_tag;
rm ${docker_file_path};
kube::log::status "Starting Docker build for image: ${binary_name}"

(
local docker_file_path="$1/${binary_name}.Dockerfile"
local binary_file_path="$1/${binary_name}"
if [ -f ${docker_file_path} ]; then
rm ${docker_file_path}
fi
printf " FROM scratch \n ADD ${binary_name} /${binary_name} \n ENTRYPOINT [ \"/${binary_name}\" ]\n" >> ${docker_file_path}
local md5_sum=$(kube::release::md5 ${binary_file_path})
local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum
docker build -q -f "${docker_file_path}" -t "${docker_image_tag}" ${1} >/dev/null
docker save ${docker_image_tag} > ${1}/${binary_name}.tar
echo $md5_sum > ${1}/${binary_name}.docker_tag
rm ${docker_file_path}
) &
done

wait || { kube::log::error "previous Docker build failed"; return 1; }
kube::log::status "Docker builds done"
)
}

Expand Down
179 changes: 125 additions & 54 deletions hack/lib/golang.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ readonly KUBE_CLIENT_PLATFORMS=(
windows/amd64
)

# Gigabytes desired for parallel platform builds. 8 is fairly
# arbitrary, but is a reasonable splitting point for 2015
# laptops-versus-not.
readonly KUBE_PARALLEL_BUILD_MEMORY=8

readonly KUBE_ALL_TARGETS=(
"${KUBE_SERVER_TARGETS[@]}"
"${KUBE_CLIENT_TARGETS[@]}"
Expand Down Expand Up @@ -263,6 +268,87 @@ kube::golang::exit_if_stdlib_not_installed() {
exit 0;
}

kube::golang::build_binaries_for_platform() {
local platform=$1
local use_go_build=${2-}

local -a statics=()
local -a nonstatics=()
for binary in "${binaries[@]}"; do
if kube::golang::is_statically_linked_library "${binary}"; then
kube::golang::exit_if_stdlib_not_installed;
statics+=($binary)
else
nonstatics+=($binary)
fi
done

if [[ -n ${use_go_build:-} ]]; then
# Try and replicate the native binary placement of go install without
# calling go install. This means we have to iterate each binary.
local output_path="${KUBE_GOPATH}/bin"
if [[ $platform != $host_platform ]]; then
output_path="${output_path}/${platform//\//_}"
fi

for binary in "${binaries[@]}"; do
local bin=$(basename "${binary}")
if [[ ${GOOS} == "windows" ]]; then
bin="${bin}.exe"
fi

if kube::golang::is_statically_linked_library "${binary}"; then
kube::golang::exit_if_stdlib_not_installed;
CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \
"${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${binary}"
else
go build -o "${output_path}/${bin}" \
"${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${binary}"
fi
done
else
# Use go install.
if [[ "${#nonstatics[@]}" != 0 ]]; then
go install "${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${nonstatics[@]:+${nonstatics[@]}}"
fi
if [[ "${#statics[@]}" != 0 ]]; then
CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${statics[@]:+${statics[@]}}"
fi
fi

}

# Return approximate physical memory in gigabytes.
kube::golang::get_physmem() {
local mem

# Linux, in kb
if mem=$(grep MemTotal /proc/meminfo | awk '{ print $2 }'); then
echo $(( ${mem} / 1048576 ))
return
fi

# OS X, in bytes. Note that get_physmem, as used, should only ever
# run in a Linux container (because it's only used in the multiple
# platform case, which is a Dockerized build), but this is provided
# for completeness.
if mem=$(sysctl -n hw.memsize 2>/dev/null); then
echo $(( ${mem} / 1073741824 ))
return
fi

# If we can't infer it, just give up and assume a low memory system
echo 1
}

# Build binaries targets specified
#
# Input:
Expand Down Expand Up @@ -313,62 +399,47 @@ kube::golang::build_binaries() {
local binaries
binaries=($(kube::golang::binaries_from_targets "${targets[@]}"))

local platform
for platform in "${platforms[@]}"; do
kube::golang::set_platform_envs "${platform}"
kube::log::status "Building go targets for ${platform}:" "${targets[@]}"

local -a statics=()
local -a nonstatics=()
for binary in "${binaries[@]}"; do
if kube::golang::is_statically_linked_library "${binary}"; then
kube::golang::exit_if_stdlib_not_installed;
statics+=($binary)
else
nonstatics+=($binary)
fi
done
local parallel=false
if [[ ${#platforms[@]} -gt 1 ]]; then
local gigs
gigs=$(kube::golang::get_physmem)

if [[ -n ${use_go_build:-} ]]; then
# Try and replicate the native binary placement of go install without
# calling go install. This means we have to iterate each binary.
local output_path="${KUBE_GOPATH}/bin"
if [[ $platform != $host_platform ]]; then
output_path="${output_path}/${platform//\//_}"
fi

for binary in "${binaries[@]}"; do
local bin=$(basename "${binary}")
if [[ ${GOOS} == "windows" ]]; then
bin="${bin}.exe"
fi

if kube::golang::is_statically_linked_library "${binary}"; then
kube::golang::exit_if_stdlib_not_installed;
CGO_ENABLED=0 go build -installsuffix cgo -o "${output_path}/${bin}" \
"${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${binary}"
else
go build -o "${output_path}/${bin}" \
"${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${binary}"
fi
done
if [[ ${gigs} -gt ${KUBE_PARALLEL_BUILD_MEMORY} ]]; then
kube::log::status "Multiple platforms requested and available ${gigs}G > threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in parallel"
parallel=true
else
# Use go install.
if [[ "${#nonstatics[@]}" != 0 ]]; then
go install "${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${nonstatics[@]:+${nonstatics[@]}}"
fi
if [[ "${#statics[@]}" != 0 ]]; then
CGO_ENABLED=0 go install -installsuffix cgo "${goflags[@]:+${goflags[@]}}" \
-ldflags "${version_ldflags}" \
"${statics[@]:+${statics[@]}}"
fi
kube::log::status "Multiple platforms requested, but available ${gigs}G < threshold ${KUBE_PARALLEL_BUILD_MEMORY}G, building platforms in serial"
parallel=false
fi
done
fi

if [[ "${parallel}" == "true" ]]; then
kube::log::status "Building go targets for ${platforms[@]} in parallel (output will appear in a burst when complete):" "${targets[@]}"
local platform
for platform in "${platforms[@]}"; do (
kube::golang::set_platform_envs "${platform}"
kube::log::status "${platform}: go build started"
kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
kube::log::status "${platform}: go build finished"
) &> "/tmp//${platform//\//_}.build" &
done

local fails=0
for job in $(jobs -p); do
wait ${job} || let "fails+=1"
done

for platform in "${platforms[@]}"; do
cat "/tmp//${platform//\//_}.build"
done

exit ${fails}
else
for platform in "${platforms[@]}"; do
kube::log::status "Building go targets for ${platform}:" "${targets[@]}"
kube::golang::set_platform_envs "${platform}"
kube::golang::build_binaries_for_platform ${platform} ${use_go_build:-}
done
fi
)
}

0 comments on commit b35e66b

Please sign in to comment.