Update curl requests to retry 7x (~127 seconds)

Signed-off-by: Thomas Stromberg <t+github@stromberg.org>
tinkerbell · Oct 29, 2021 · 88190a4 · 88190a4
1 parent ab2d5dd
commit 88190a4
Show file tree

Hide file tree

Showing 14 changed files with 64 additions and 39 deletions.
diff --git a/apps/discover.sh b/apps/discover.sh
@@ -13,9 +13,15 @@ if [ -z "$discover_url" ]; then
 	discover_url="http://$tinkerbell/discover-os"
 fi
 
+# syntax: rcurl --flags
+rcurl() {
+	# TODO(tstromberg): Add --retry-connrefused once we can upgrade to curl 7.52
+	curl --retry 7 "$@"
+}
+
 reason='unknown'
 fail() {
-	curl -H 'Content-Type: application/json' \
+	rcurl -H 'Content-Type: application/json' \
 		-d '{"type":"failure", "reason":"'"$reason"'"}' \
 		"$phone_home_url"
 }
@@ -42,7 +48,7 @@ EOF
 
 mkdir -p /root/.ssh
 
-curl -X POST -vs "$phone_home_url" 2>&1 | logger -t phone_home
+rcurl -X POST -vs "$phone_home_url" 2>&1 | logger -t phone_home
 
 mdadm --assemble --scan || :
 
@@ -56,7 +62,7 @@ done
 
 reason='unable to load discover container image'
 if ! docker images "quay.io/packet/discover-metal" | grep discover >/dev/null; then
-	curl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/discover-metal-$arch.tar.gz" |
+	rcurl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/discover-metal-$arch.tar.gz" |
 		docker load |
 		tee
 fi

diff --git a/apps/osie-installer.sh b/apps/osie-installer.sh
@@ -11,6 +11,12 @@ cleanup() {
 	fi
 }
 
+# syntax: rcurl --flags
+rcurl() {
+	# TODO(tstromberg): Add --retry-connrefused once we can upgrade to curl 7.52
+	curl --retry 7 "$@"
+}
+
 reason='unknown'
 fail() {
 	code=$?
@@ -34,7 +40,7 @@ fail() {
 		fi
 	fi
 
-	curl \
+	rcurl \
 		-H 'Content-Type: application/json' \
 		-d @- \
 		"$phone_home_url" <<-EOF
@@ -55,7 +61,7 @@ ensure_time() {
 	local d hwdate mddate month
 	local months='jan feb mar apr may jun jul aug sep oct nov dec'
 	# *must* be http, not https
-	d=$(curl -sI "http://tinkerbell.$facility.packet.net" | sed -n '/^Date:/ s|Date: ||p')
+	d=$(rcurl -sI "http://tinkerbell.$facility.packet.net" | sed -n '/^Date:/ s|Date: ||p')
 	# shellcheck disable=SC2018 disable=SC2019
 	month=$(echo "$d" | awk '{print $3}' | tr 'A-Z' 'a-z')
 	local i=1
@@ -116,15 +122,15 @@ esac
 
 ensure_time
 
-hardware_id=$(curl -sSL --connect-timeout 60 https://metadata.packet.net/metadata | jq -r .hardware_id)
+hardware_id=$(rcurl -sSL --connect-timeout 60 https://metadata.packet.net/metadata | jq -r .hardware_id)
 statedir=${TMPDIR:-/tmp}/osie-statedir-$hardware_id
 metadata=$statedir/metadata
 userdata=$statedir/userdata
 mkdir -p "$statedir"
 
 reason='unable to fetch metadata'
 echo "metadata:"
-curl -sSL --connect-timeout 60 https://metadata.packet.net/metadata |
+rcurl -sSL --connect-timeout 60 https://metadata.packet.net/metadata |
 	jq -S . |
 	tee "$metadata" |
 	jq .
@@ -145,7 +151,7 @@ jq -S . "$metadata"
 
 reason='unable to fetch userdata'
 echo "userdata:"
-curl -sSL --connect-timeout 60 https://metadata.packet.net/userdata | tee "$userdata"
+rcurl -sSL --connect-timeout 60 https://metadata.packet.net/userdata | tee "$userdata"
 
 # Get values from metadata
 facility=$(jq -r .facility "$metadata")
@@ -182,7 +188,7 @@ docker info
 
 reason='unable to fetch/load osie image'
 if ! docker images "osie:$arch" | grep osie >/dev/null; then
-	curl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/osie-$arch.tar.gz" |
+	rcurl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/osie-$arch.tar.gz" |
 		docker load
 fi
 

diff --git a/apps/rescue-helper.sh b/apps/rescue-helper.sh
@@ -23,10 +23,10 @@ See docs at http://wiki.alpinelinux.org
 EOF
 
 mkdir -p /root/.ssh
-curl -sSLf https://metadata.packet.net/2009-04-04/meta-data/public-keys >/root/.ssh/authorized_keys
+curl -sSLf --retry 7 https://metadata.packet.net/2009-04-04/meta-data/public-keys >/root/.ssh/authorized_keys
 
 tinkerbell=$(sed -nr 's|.*\btinkerbell=(\S+).*|\1|p' /proc/cmdline)
 
-curl -X POST -vs "$tinkerbell/phone-home" 2>&1 | logger -t phone_home
+curl --retry 7 -X POST -vs "$tinkerbell/phone-home" 2>&1 | logger -t phone_home
 
 mdadm --assemble --scan || :
diff --git a/apps/runner.sh b/apps/runner.sh
@@ -2,9 +2,15 @@
 
 # shellcheck shell=dash
 
+# syntax: rcurl --flags
+rcurl() {
+	# TODO(tstromberg): Add --retry-connrefused once we can upgrade to curl 7.52
+	curl --retry 7 "$@"
+}
+
 reason='unknown'
 fail() {
-	curl -H 'Content-Type: application/json' \
+	rcurl -H 'Content-Type: application/json' \
 		-d '{"type":"failure", "reason":"'"$reason"'"}' \
 		"$phone_home_url"
 }
@@ -22,7 +28,7 @@ ensure_time() {
 	local d hwdate mddate month
 	local months='jan feb mar apr may jun jul aug sep oct nov dec'
 	# *must* be http, not https
-	d=$(curl -sI "http://tinkerbell.$facility.packet.net" | sed -n '/^Date:/ s|Date: ||p')
+	d=$(rcurl -sI "http://tinkerbell.$facility.packet.net" | sed -n '/^Date:/ s|Date: ||p')
 	# shellcheck disable=SC2018 disable=SC2019
 	month=$(echo "$d" | awk '{print $3}' | tr 'A-Z' 'a-z')
 	local i=1
@@ -69,13 +75,13 @@ syslog_host=$(sed -nr 's|.*\bsyslog_host=(\S+).*|\1|p' /proc/cmdline)
 
 ensure_time
 
-hardware_id=$(curl -sSL --connect-timeout 60 https://metadata.packet.net/metadata | jq -r .hardware_id)
+hardware_id=$(rcurl -sSL --connect-timeout 60 https://metadata.packet.net/metadata | jq -r .hardware_id)
 statedir=${TMPDIR:-/tmp}/osie-statedir-$hardware_id
 metadata=$statedir/metadata
 mkdir -p "$statedir"
 
 echo "metadata:"
-curl -sSL --connect-timeout 60 https://metadata.packet.net/metadata |
+rcurl -sSL --connect-timeout 60 https://metadata.packet.net/metadata |
 	jq -S . |
 	tee "$metadata" |
 	jq .
@@ -101,14 +107,14 @@ docker info
 
 reason='unable to fetch/load osie-runner image'
 if ! docker images "osie-runner:$arch" | grep osie >/dev/null; then
-	curl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/osie-runner-$arch.tar.gz" |
+	rcurl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/osie-runner-$arch.tar.gz" |
 		docker load |
 		tee
 fi
 
 reason='unable to fetch/load osie image'
 if ! docker images "osie:$arch" | grep osie >/dev/null; then
-	curl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/osie-$arch.tar.gz" |
+	rcurl "${packet_base_url:-http://install.$facility.packet.net/misc/osie/current}/osie-$arch.tar.gz" |
 		docker load |
 		tee
 fi

diff --git a/docker/build-lshw.sh b/docker/build-lshw.sh
@@ -6,7 +6,7 @@ LSHW_RELEASE=B.02.18
 LSHW_SHA512=090b79c144e4a42dd88a0a6d6992183d747ea891850f67403e0b63037b84d4b1891f2ad38f9151393b5ec1ac987b7ae70083c0f6eb8611f89d372a461390e8f3
 LSHW_BASEURL=https://github.com/lyonel/lshw/archive
 
-curl -L "${LSHW_BASEURL}/${LSHW_RELEASE}.tar.gz" >lshw.tar.gz
+curl --retry 7 -L "${LSHW_BASEURL}/${LSHW_RELEASE}.tar.gz" >lshw.tar.gz
 echo "${LSHW_SHA512}  lshw.tar.gz" | sha512sum -c
 tar -zxvf lshw.tar.gz
 cd lshw-${LSHW_RELEASE}/src

diff --git a/docker/build-mstflint.sh b/docker/build-mstflint.sh
@@ -6,7 +6,7 @@ MSTFLINT_RELEASE=4.14.0-1
 MSTFLINT_SHA512=965b25141d1b960bb575fc9fb089e912b0408af72919d23f295c6a8e8650c95c9459cb496171dca7f818252a180bd85bee8ed0f876159279013828478a0c2101
 MSTFLINT_BASEURL=https://github.com/Mellanox/mstflint/releases/download/
 
-curl -L "${MSTFLINT_BASEURL}/v${MSTFLINT_RELEASE}/mstflint-${MSTFLINT_RELEASE}.tar.gz" >mstflint.tar.gz
+curl --retry 7 -L "${MSTFLINT_BASEURL}/v${MSTFLINT_RELEASE}/mstflint-${MSTFLINT_RELEASE}.tar.gz" >mstflint.tar.gz
 apt install -y zlib1g-dev libibmad-dev libssl-dev g++
 echo "${MSTFLINT_SHA512}  mstflint.tar.gz" | sha512sum -c
 tar -zxvf mstflint.tar.gz

diff --git a/docker/build-nvme-cli.sh b/docker/build-nvme-cli.sh
@@ -6,7 +6,7 @@ NVMECLI_RELEASE=1.8.1
 NVMECLI_SHA512=b31690f6dbc1f88ebd461636b452b8dedc6e1f67e2fe9d088b1f1d2ddf634ab6ef8d628d2c7fdc6977587d9565deb816a1df8f4881759a12b030a190af5c9095
 NVMECLI_BASEURL=https://github.com/linux-nvme/nvme-cli/archive
 
-curl -L "${NVMECLI_BASEURL}/v${NVMECLI_RELEASE}.tar.gz" >nvme-cli.tar.gz
+curl --retry 7 -L "${NVMECLI_BASEURL}/v${NVMECLI_RELEASE}.tar.gz" >nvme-cli.tar.gz
 echo "${NVMECLI_SHA512}  nvme-cli.tar.gz" | sha512sum -c
 tar -zxvf nvme-cli.tar.gz
 cd nvme-cli-${NVMECLI_RELEASE}

diff --git a/docker/build-smartmontools.sh b/docker/build-smartmontools.sh
@@ -7,7 +7,7 @@ SMARTMONTOOLS_TARNAME=smartmontools-7.1
 SMARTMONTOOLS_SHA512=440b2a957da10d240a8ef0008bd3358b83adb9eaca0f8d3e049b25d56a139c61dcd0bb4b27898faef6f189a27e159bdca3331e52e445c0eebf35e5d930f9e295
 SMARTMONTOOLS_BASEURL=https://github.com/smartmontools/smartmontools/releases/download/
 
-curl -L "${SMARTMONTOOLS_BASEURL}/${SMARTMONTOOLS_RELEASE}/${SMARTMONTOOLS_TARNAME}.tar.gz" >smartmontools.tar.gz
+curl -L --retry 7 "${SMARTMONTOOLS_BASEURL}/${SMARTMONTOOLS_RELEASE}/${SMARTMONTOOLS_TARNAME}.tar.gz" >smartmontools.tar.gz
 echo "${SMARTMONTOOLS_SHA512}  smartmontools.tar.gz" | sha512sum -c
 tar -zxvf smartmontools.tar.gz
 cd "$SMARTMONTOOLS_TARNAME"

diff --git a/docker/scripts/functions.sh b/docker/scripts/functions.sh
@@ -71,17 +71,22 @@ function fail() {
 	puttink "${tink_host}" phone-home '{"type":"failure", "reason":"'"$1"'"}'
 }
 
+# syntax: rcurl --flags
+rcurl() {
+	# TODO(tstromberg): Add --retry-connrefused once we can upgrade to curl 7.52
+	curl --retry 7 "$@"
+}
+
 # syntax: tink POST 1.2.3.4 phone-home '{"this": "data"}'
 function tink() {
 	local method=$1 tink_host=$2 endpoint=$3 post_data=$4
 
-	curl \
-		-vvvvv \
+	rcurl \
+		-v \
 		--data "${post_data}" \
 		--fail \
 		--header "Content-Type: application/json" \
 		--request "${method}" \
-		--retry 3 \
 		"${tink_host}/${endpoint}"
 }
 
@@ -208,7 +213,7 @@ function bios_inventory() {
 		# Use eclypsium-proxy to reach the auth server from the deprov network
 		echo "Requesting a hollow token from ${hollow_auth_url}"
 		local hollow_token
-		if ! hollow_token=$(curl --request POST \
+		if ! hollow_token=$(rcurl --request POST \
 			--url "${hollow_auth_url}" \
 			--user "${HOLLOW_CLIENT_ID}:${HOLLOW_CLIENT_REQUEST_SECRET}" \
 			--data "grant_type=client_credentials&audience=${hollow_auth_audience}&scope=${hollow_auth_scope}" \
@@ -222,7 +227,7 @@ function bios_inventory() {
 		# Write the bios data to Hollow
 		echo "Writing BIOS feature inventory data to ${hollow_url}"
 		local hollow_response
-		if ! hollow_response=$(curl --request POST \
+		if ! hollow_response=$(rcurl --request POST \
 			--url "${hollow_url}" \
 			--header "Authorization: Bearer $hollow_token" \
 			--header "Content-Type: application/json" \
@@ -427,7 +432,7 @@ function should_stream() {
 	local dest=$2
 
 	available=$(BLOCKSIZE=1 df --output=avail "$dest" | grep -v Avail)
-	img_size=$(curl -s -I "$image" | tr -d '\r' | awk 'tolower($0) ~ /content-length/ { print $2 }')
+	img_size=$(rcurl --silent --head "$image" | tr -d '\r' | awk 'tolower($0) ~ /content-length/ { print $2 }')
 	max_size=$((available - (1024 * 1024 * 1024))) # be safe and allow 1G of leeway
 
 	# img_size == 0 is if server can't stat the file, for example some

diff --git a/docker/scripts/osie.sh b/docker/scripts/osie.sh
@@ -147,7 +147,7 @@ if [[ -z ${cpr_url} ]]; then
 else
 	echo "NOTICE: Custom CPR url found!"
 	echo "Overriding default CPR location with custom cpr_url"
-	if ! curl "$cpr_url" | jq . >$cprconfig; then
+	if ! rcurl "$cpr_url" | jq . >$cprconfig; then
 		phone_home "${tinkerbell}" '{"instance_id":"'"$(jq -r .id "$metadata")"'"}'
 		echo "$0: CPR URL unavailable: $cpr_url" >&2
 		exit 1
@@ -202,10 +202,10 @@ if ! [[ -f /statedir/disks-partioned-image-extracted ]]; then
 		git -C $assetdir checkout FETCH_HEAD
 	elif [[ $image_uri =~ ^https:// ]]; then
 		echo -e "${GREEN}#### Adding custom uri: ${image_uri}${NC}"
-		curl --retry 3 "${image_uri}/image.tar.gz" --output "$assetdir/image.tar.gz"
-		curl --retry 3 "${image_uri}/initrd.tar.gz" --output "$assetdir/initrd.tar.gz"
-		curl --retry 3 "${image_uri}/kernel.tar.gz" --output "$assetdir/kernel.tar.gz"
-		curl --retry 3 "${image_uri}/modules.tar.gz" --output "$assetdir/modules.tar.gz"
+		rcurl "${image_uri}/image.tar.gz" --output "$assetdir/image.tar.gz"
+		rcurl "${image_uri}/initrd.tar.gz" --output "$assetdir/initrd.tar.gz"
+		rcurl "${image_uri}/kernel.tar.gz" --output "$assetdir/kernel.tar.gz"
+		rcurl "${image_uri}/modules.tar.gz" --output "$assetdir/modules.tar.gz"
 	else
 		echo -e "${RED}#### Image URI is not https: ${image_uri}${NC}"
 		exit 1
@@ -529,7 +529,7 @@ logger -s -t phone_home "Making a call to tell the packet API is online."
 # doesn't hurt to log as much as we can in case it fails.
 n=1
 until [ \$n -ge 6 ] || [ "\${PIPESTATUS[0]}" -eq 0 ]; do
-	curl -X PUT -H "Content-Type: application/json" -vs -d '{"instance_id": "$(jq -r .id "$metadata")"}' "$tinkerbell/phone-home" 2>&1 | logger -s -t phone_home
+	curl --retry 7 -X PUT -H "Content-Type: application/json" -vs -d '{"instance_id": "$(jq -r .id "$metadata")"}' "$tinkerbell/phone-home" 2>&1 | logger -s -t phone_home
 
 	if [ "\${PIPESTATUS[0]}" -eq 0 ]; then
 		logger -s -t phone_home "This device has been announced to the packet API."

diff --git a/docker/scripts/suse.sh b/docker/scripts/suse.sh
@@ -107,7 +107,7 @@ logger -s -t phone_home "Making a call to tell the packet API is online."
 # doesn't hurt to log as much as we can in case it fails.
 n=1
 until [ \$n -ge 6 ] || [ "\${PIPESTATUS[0]}" -eq 0 ]; do
-	curl -X PUT -H "Content-Type: application/json" -vs -d '{"instance_id": "$(jq -r .id "$metadata")"}' "$tinkerbell/phone-home" 2>&1 | logger -s -t phone_home
+	curl --retry 7 -X PUT -H "Content-Type: application/json" -vs -d '{"instance_id": "$(jq -r .id "$metadata")"}' "$tinkerbell/phone-home" 2>&1 | logger -s -t phone_home
 	if [ \${PIPESTATUS[0]} -eq 0 ]; then
 		logger -s -t phone_home "This device has been announced to the packet API."
 		break
@@ -146,7 +146,7 @@ if [[ $DOS == "SUSELINUX" ]] || [[ $DOS == "SUSE" ]]; then
 	smtserver=suse-smt.sjc1.packet.net
 	ensure_reachable "$smtserver"
 
-	smt_setup=$(curl http://$smtserver/repo/tools/clientSetup4SMT.sh)
+	smt_setup=$(rcurl http://$smtserver/repo/tools/clientSetup4SMT.sh)
 	echo "$smtserver" >"$TARGET/var/cache/smt-server"
 	if [ -n "$smt_setup" ]; then
 		echo "Retrieived SMTSetup from SMT server..."
@@ -158,7 +158,7 @@ if [[ $DOS == "SUSELINUX" ]] || [[ $DOS == "SUSE" ]]; then
 
 	# Get remote SMT CA from SMT server
 	#TODO: make this use facility once we have more than one!
-	smt_crt=$(curl http://$smtserver/certs/smt.crt.sha)
+	smt_crt=$(rcurl http://$smtserver/certs/smt.crt.sha)
 	if [ -n "$smt_crt" ]; then
 		echo "Retrieived remote SMT certificate SHA1 from SMT server..."
 		echo "$smt_crt" >"$TARGET/var/cache/smt-remote.crt.sha"
@@ -216,6 +216,7 @@ if [[ $DOS == "SUSELINUX" ]] || [[ $DOS == "SUSE" ]]; then
 			local post_data=$3
 
 			curl \
+				--retry 7 \
 				-f \
 				-vvvvv \
 				-X PUT \

diff --git a/docker/scripts/vosie.sh b/docker/scripts/vosie.sh
@@ -105,7 +105,7 @@ if [[ -z ${cpr_url} ]]; then
 else
 	echo "NOTICE: Custom CPR url found!"
 	echo "Overriding default CPR location with custom cpr_url"
-	if ! curl "$cpr_url" | jq . >$cprconfig; then
+	if ! rcurl "$cpr_url" | jq . >$cprconfig; then
 		phone_home "${tinkerbell}" '{"instance_id":"'"$(jq -r .id "$metadata")"'"}'
 		echo "$0: CPR URL unavailable: $cpr_url" >&2
 		exit 1
@@ -492,6 +492,7 @@ function puttink() {
 	local post_data=$3
 
 	curl \
+		--retry 7 \
 		-f \
 		-vvvvv \
 		-X PUT \

diff --git a/docker/scripts/wosie.sh b/docker/scripts/wosie.sh
@@ -148,7 +148,7 @@ set_autofail_stage "writing rootfs to disk"
 tmpfile=/tmp/image.tar.gz
 if should_stream "$image" ${tmpfile%/*}; then
 	echo -e "${GREEN}#### Retrieving and extracting image archive to first disk in one shot${NC}"
-	curl -sL "$image" | pv -bnti 5 | tar -zxOf- | dd bs=512k of="$install_disk"
+	rcurl -sL "$image" | pv -bnti 5 | tar -zxOf- | dd bs=512k of="$install_disk"
 else
 	echo -e "${GREEN}#### Retrieving image archive${NC}"
 	wget --quiet "$image" -O $tmpfile

diff --git a/docker/tests/test_functions.sh b/docker/tests/test_functions.sh
@@ -102,7 +102,7 @@ test_ensure_reachable() {
 test_filter_bad_devs() {
 	local devs
 	devs=$(
-		curl -sf https://raw.githubusercontent.com/torvalds/linux/master/Documentation/admin-guide/devices.txt |
+		curl --retry 7 -sf https://raw.githubusercontent.com/torvalds/linux/master/Documentation/admin-guide/devices.txt |
 			grep -E 'block\s*(Loopback devices|SCSI disk)' |
 			awk '{print $1}'
 		echo -e '251\n253\n259' # virtio disks seen in tests