Syncs common scripts and makefiles from eks-a-build-tooling (aws#1426)

chrisnegus · Nov 11, 2022 · cfd745b · cfd745b
1 parent 2b3a5e6
commit cfd745b
Show file tree

Hide file tree

Showing 13 changed files with 269 additions and 71 deletions.
diff --git a/Common.mk b/Common.mk
diff --git a/Makefile b/Makefile
@@ -17,9 +17,7 @@ RELEASE_GIT_COMMIT_HASH?=$(shell git rev-parse @)
 
 REBUILD_ALL?=false
 
-ALL_PROJECTS=containernetworking_plugins coredns_coredns etcd-io_etcd kubernetes-csi_external-attacher kubernetes-csi_external-resizer \
-	kubernetes-csi_livenessprobe kubernetes-csi_node-driver-registrar kubernetes-sigs_aws-iam-authenticator kubernetes-sigs_metrics-server \
-	kubernetes-csi_external-snapshotter kubernetes-csi_external-provisioner kubernetes_release kubernetes_kubernetes
+ALL_PROJECTS=$(shell build/lib/all_projects.sh $(BASE_DIRECTORY))
 
 ifdef MAKECMDGOALS
 TARGET=$(MAKECMDGOALS)

diff --git a/build/lib/all_projects.sh b/build/lib/all_projects.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+BASE_DIRECTORY="${1?Specify first argument - Base directory of build-tooling repo}"
+MAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+
+cd $MAKE_ROOT
+
+# Iterating over orgs under projects folder
+for org_path in projects/*; do
+    org=$(basename "$org_path")
+    for repo_path in projects/$org/*; do
+        repo=$(basename "$repo_path")
+        # couple odd ball non-builds in this repo
+        if [[ -f $repo_path/DO_NOT_BUILD ]]; then
+            continue
+        fi
+        echo -n "${org}_${repo} "
+    done
+done
+
+echo
diff --git a/build/lib/buildkit.sh b/build/lib/buildkit.sh
@@ -45,5 +45,15 @@ if [ -f "/buildkit.sh" ]; then
     (exit $s)
 else
     # skip retry when running locally
-    $CMD "$@"
+    log_file=$(mktemp)
+    trap "rm -f $log_file" EXIT
+    if ! $CMD "$@" 2>&1 | tee $log_file; then        
+        if grep -q "blobs/uploads/\": EOF" $log_file ; then
+            echo "******************************************************"
+            echo "Ensure container registry and repository exists!!"
+            echo "Try running make create-ecr-repos to create ecr repositories in your aws account."
+            echo "******************************************************"
+        fi
+        exit 1
+    fi
 fi
diff --git a/build/lib/common.sh b/build/lib/common.sh
@@ -83,7 +83,9 @@ function build::gather_licenses() {
 
   # the version of go used here must be the version go-licenses was installed with
   # by default we use 1.16, but due to changes in 1.17, there are some changes that require using 1.17
-  if [ "$golang_version" == "1.18" ]; then
+  if [ "$golang_version" == "1.19" ]; then
+    build::common::use_go_version 1.19
+  elif [ "$golang_version" == "1.18" ]; then
     build::common::use_go_version 1.18
   elif [ "$golang_version" == "1.17" ]; then
     build::common::use_go_version 1.17
@@ -170,7 +172,7 @@ function build::non-golang::copy_licenses(){
   do
     license_dest=$destination_dir/$(dirname $file)
     mkdir -p $license_dest
-    cp "${source_dir}/${file}" $license_dest/$(basename $file)
+    cp -r "${source_dir}/${file}" $license_dest/$(basename $file)
   done
 }
 
@@ -243,8 +245,8 @@ function build::common::wait_for_tag() {
   sleep_interval=20
   for i in {1..60}; do
     echo "Checking for tag ${tag}..."
-    git fetch --tags > /dev/null 2>&1
     git rev-parse --verify --quiet "${tag}" && echo "Tag ${tag} exists!" && break
+    git fetch --tags > /dev/null 2>&1
     echo "Tag ${tag} does not exist!"
     echo "Waiting for tag ${tag}..."
     sleep $sleep_interval
@@ -254,6 +256,24 @@ function build::common::wait_for_tag() {
   done
 }
 
+function build::common::wait_for_tarball() {
+  local -r tarball_url=$1
+  sleep_interval=20
+  for i in {1..60}; do
+    echo "Checking for URL ${tarball_url}..."
+    local -r http_code=$(curl -I -L -s -o /dev/null -w "%{http_code}" $tarball_url)
+    if [[ "$http_code" == "200" ]]; then 
+      echo "Tarball exists!" && break
+    fi
+    echo "Tarball does not exist!"
+    echo "Waiting for tarball to be uploaded to ${tarball_url}"
+    sleep $sleep_interval
+    if [ "$i" = "60" ]; then
+      exit 1
+    fi
+  done
+}
+
 function build::common::get_clone_url() {
   local -r org=$1
   local -r repo=$2
@@ -266,3 +286,24 @@ function build::common::get_clone_url() {
     echo "https://github.com/${org}/${repo}.git"
   fi
 }
+
+function retry() {
+  local n=1
+  local max=120
+  local delay=5
+  while true; do
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+        sleep $delay;
+      else
+        fail "The command has failed after $n attempts."
+      fi
+    }
+  done
+}
+
+function build::docker::retry_pull() {
+  retry docker pull "$@"
+}
diff --git a/build/lib/generate_help_body.sh b/build/lib/generate_help_body.sh
@@ -153,7 +153,7 @@ ${GIT_TARGETS_HELP}${PATCHES_TARGET}${BINARY_TARGETS_HELP}${IMAGE_TARGETS_HELP}$
 ${EXTRA_HELP}
 
 ##@ Build Targets
-build: ## Called via prow presubmit, calls \`${BUILD_TARGETS}\`
+build: ## Called via prow presubmit, calls \`${BUILD_TARGETS//$PROJECT_ROOT\//""}\`
 release: ## Called via prow postsubmit + release jobs, calls \`${RELEASE_TARGETS}\`
 ${FOOTER}
 EOF

diff --git a/build/lib/install_go_versions.sh b/build/lib/install_go_versions.sh
@@ -58,5 +58,4 @@ build::common::use_go_version "1.19"
 GOBIN=${GOPATH}/go1.19/bin go install github.com/google/go-licenses@v1.2.1
 
 # 1.16 is the default so symlink it to /go/bin
-unlink ${GOPATH}/bin/go-licenses
-ln -s ${GOPATH}/go1.16/bin/go-licenses ${GOPATH}/bin
+ln -sf ${GOPATH}/go1.16/bin/go-licenses ${GOPATH}/bin
diff --git a/build/lib/run_target_docker.sh b/build/lib/run_target_docker.sh
@@ -23,27 +23,55 @@ IMAGE_REPO="${3:-}"
 RELEASE_BRANCH="${4:-}"
 ARTIFACTS_BUCKET="${5:-}"
 
+SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 MAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 
+source "${SCRIPT_ROOT}/common.sh"
+
 echo "****************************************************************"
 echo "A docker container with the name eks-d-builder will be launched."
 echo "It will be left running to support running consecutive runs."
 echo "Run 'make stop-docker-builder' when you are done to stop it."
 echo "****************************************************************"
 
 if ! docker ps -f name=eks-d-builder | grep -w eks-d-builder; then
-	docker pull public.ecr.aws/eks-distro-build-tooling/builder-base:latest
-	docker run -d --name eks-d-builder --privileged -e GOPROXY=$GOPROXY --entrypoint sleep \
+	build::docker::retry_pull public.ecr.aws/eks-distro-build-tooling/builder-base:latest
+
+    NETRC=""
+    if [ -f $HOME/.netrc ]; then
+        NETRC="--mount type=bind,source=$HOME/.netrc,target=/root/.netrc"
+    fi
+
+	docker run -d --name eks-d-builder --privileged $NETRC -e GOPROXY=$GOPROXY --entrypoint sleep \
 		public.ecr.aws/eks-distro-build-tooling/builder-base:latest  infinity 
 fi
 
-rsync -e 'docker exec -i' -rm --exclude='.git/***' \
+EXTRA_INCLUDES=""
+PROJECT_DEPENDENCIES=$(make --no-print-directory -C $MAKE_ROOT/projects/$PROJECT var-value-PROJECT_DEPENDENCIES RELEASE_BRANCH=$RELEASE_BRANCH)
+if [ -n "$PROJECT_DEPENDENCIES" ]; then
+	DEPS=(${PROJECT_DEPENDENCIES// / })
+	for dep in "${DEPS[@]}"; do
+		DEP_PRODUCT="$(cut -d/ -f1 <<< $dep)"
+		DEP_ORG="$(cut -d/ -f2 <<< $dep)"
+		DEP_REPO="$(cut -d/ -f3 <<< $dep)"
+
+		if [[ "$DEP_PRODUCT" == "eksd" ]]; then
+			continue
+		fi
+
+		EXTRA_INCLUDES+=" --include=projects/$DEP_ORG/$DEP_REPO/***"
+	done
+fi
+
+rsync -e 'docker exec -i' -t -rm --exclude='.git/***' \
 	--exclude="projects/$PROJECT/_output/***" --exclude="projects/$PROJECT/$(basename $PROJECT)/***" \
-	--include="projects/$PROJECT/***" \
+	--include="projects/$PROJECT/***" $EXTRA_INCLUDES \
 	--include='*/' --exclude='projects/***' $MAKE_ROOT/ eks-d-builder:/eks-distro
 
 # Need so git properly finds the root of the repo
-docker exec -it eks-d-builder mkdir -p /eks-distro/.git/{refs,objects}
+CURRENT_HEAD="$(cat $MAKE_ROOT/.git/HEAD | awk '{print $2}')"
+docker exec -it eks-d-builder mkdir -p /eks-distro/.git/{refs,objects} /eks-distro/.git/$(dirname $CURRENT_HEAD)
 docker cp $MAKE_ROOT/.git/HEAD eks-d-builder:/eks-distro/.git
+docker cp $MAKE_ROOT/.git/$CURRENT_HEAD eks-d-builder:/eks-distro/.git/$CURRENT_HEAD
 
-docker exec -it eks-d-builder make $TARGET -C /eks-distro/projects/$PROJECT RELEASE_BRANCH=$RELEASE_BRANCH IMAGE_REPO=$IMAGE_REPO
+docker exec -it eks-d-builder make $TARGET -C /eks-distro/projects/$PROJECT RELEASE_BRANCH=$RELEASE_BRANCH IMAGE_REPO=$IMAGE_REPO ARTIFACTS_BUCKET=$ARTIFACTS_BUCKET
diff --git a/build/lib/simple_create_binaries.sh b/build/lib/simple_create_binaries.sh
@@ -65,7 +65,8 @@ function build::simple::binaries(){
       index=0
       for source in $SOURCE_PATTERN; do
         if [ "$(basename $source)" != "${TARGET_FILES[$index]}" ]; then
-          mv $TARGET_FILE$(basename $source) $TARGET_FILE${TARGET_FILES[$index]}
+          # in the case of multiple target files but the first is a ., we cant move . but it should already be named correctly
+          [ -f $TARGET_FILE$(basename $source) ] && mv $TARGET_FILE$(basename $source) $TARGET_FILE${TARGET_FILES[$index]}
         fi
         ((index=index+1))
       done

diff --git a/build/lib/update_checksum_docker.sh b/build/lib/update_checksum_docker.sh
@@ -26,14 +26,12 @@ MAKE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 $MAKE_ROOT/build/lib/run_target_docker.sh $PROJECT "binaries attribution checksums" $IMAGE_REPO $RELEASE_BRANCH
 
 PROJECT_CHECKSUM=$PROJECT/CHECKSUMS
-PROJECT_ATTRIBUTION_DIR=$PROJECT
+PROJECT_ATTRIBUTION=$PROJECT/ATTRIBUTION.txt
 
 if [[ ! -z "$RELEASE_BRANCH" ]] && [ -d $MAKE_ROOT/projects/$PROJECT/$RELEASE_BRANCH ]; then
 	PROJECT_CHECKSUM=$PROJECT/$RELEASE_BRANCH/CHECKSUMS
-	PROJECT_ATTRIBUTION_DIR=$PROJECT/$RELEASE_BRANCH
+	PROJECT_ATTRIBUTION=$PROJECT/$RELEASE_BRANCH/ATTRIBUTION.txt
 fi
 
 docker cp eks-d-builder:/eks-distro/projects/$PROJECT_CHECKSUM $MAKE_ROOT/projects/$PROJECT_CHECKSUM
-for file in $(find $MAKE_ROOT/projects/$PROJECT_ATTRIBUTION_DIR -name '*TTRIBUTION.txt'); do
-	docker cp eks-d-builder:/eks-distro/projects/$PROJECT_ATTRIBUTION_DIR/$(basename $file) $file
-done
+docker cp eks-d-builder:/eks-distro/projects/$PROJECT_ATTRIBUTION $MAKE_ROOT/projects/$PROJECT_ATTRIBUTION
diff --git a/build/update-attribution-files/create_pr.sh b/build/update-attribution-files/create_pr.sh
@@ -23,6 +23,8 @@ fi
 
 SCRIPT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 
+source $SCRIPT_ROOT/../lib/common.sh
+
 ORIGIN_ORG="eks-distro-pr-bot"
 UPSTREAM_ORG="aws"
 
@@ -37,7 +39,7 @@ git config remote.upstream.url >&- || git remote add upstream https://github.com
 
 # Files have already changed, stash to perform rebase
 git stash
-git fetch upstream
+retry git fetch upstream
 
 git checkout $MAIN_BRANCH
 # there will be conflicts before we are on the bots fork at this point
@@ -171,7 +173,7 @@ function pr::file:add() {
 }
 
 # Add checksum files
-for FILE in $(find . -type f -name CHECKSUMS); do    
+for FILE in $(find . -type f -name CHECKSUMS); do
     pr::file:add $FILE
 done
 
@@ -189,7 +191,7 @@ if [ "$(git stash list)" != "" ]; then
 fi
 
 # Add attribution files
-for FILE in $(find . -type f \( -name "*ATTRIBUTION.txt" ! -path "*/_output/*" \)); do    
+for FILE in $(find . -type f \( -name "*ATTRIBUTION.txt" ! -path "*/_output/*" \)); do
     pr::file:add $FILE
 done
 
@@ -204,7 +206,7 @@ if [ "$(git stash list)" != "" ]; then
     git stash pop
 fi
 # Add help.mk/Makefile files
-for FILE in $(find . -type f \( -name Help.mk -o -name Makefile \)); do    
+for FILE in $(find . -type f \( -name Help.mk -o -name Makefile \)); do
     pr::file:add $FILE
 done
 

diff --git a/projects/coredns/coredns/Makefile b/projects/coredns/coredns/Makefile
@@ -5,8 +5,6 @@ GOLANG_VERSION?=$(shell cat $(RELEASE_BRANCH)/GOLANG_VERSION)
 REPO=coredns
 REPO_OWNER=coredns
 
-LICENSE_PACKAGE_FILTER?=./coredns.go
-
 BINARY_TARGET_FILES=coredns
 SOURCE_PATTERNS=.
 EXTRA_GO_LDFLAGS=-X github.com/coredns/coredns/coremain.GitCommit=$(shell git -C $(REPO) rev-list -n 1  "${GIT_TAG}")

diff --git a/projects/kubernetes/release/Makefile b/projects/kubernetes/release/Makefile
@@ -16,7 +16,6 @@ IMAGE_NAMES=go-runner kube-proxy-base
 # Name of base image to be used when building the kube-proxy-base image
 KUBE_PROXY_BASE_BASE_IMAGE_NAME=$(if $(IS_BUILDING_MINIMAL),eks-distro-minimal-base-iptables,eks-distro-base)
 
-LICENSE_PACKAGE_FILTER=./go-runner.go
 IPTABLES_WRAPPER=$(OUTPUT_DIR)/iptables-wrapper
 
 BINARY_TARGET_FILES=go-runner