Retry apt operations to death

Nodes are probably broken if update or install fails. Don't proceed if we can't get past these. Also, instead of ignoring the error off dpkg, use --force depends, which changes the errors to be kinder warnings for anyone looking through the logs.
duda · Mar 24, 2015 · ed56773 · ed56773
1 parent 064b7de
commit ed56773
Showing 1 changed file with 18 additions and 6 deletions.
diff --git a/cluster/gce/configure-vm.sh b/cluster/gce/configure-vm.sh
@@ -115,19 +115,27 @@ download-or-bust() {
 # Install salt from GCS.  See README.md for instructions on how to update these
 # debs.
 install-salt() {
-  apt-get update
+  echo "== Refreshing package database =="
+  until apt-get update; do
+    echo "== apt-get update failed, retrying =="
+    echo sleep 5
+  done
 
   mkdir -p /var/cache/salt-install
   cd /var/cache/salt-install
 
-  TARS=(
+  DEBS=(
     libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
     python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
     salt-common_2014.1.13+ds-1~bpo70+1_all.deb
     salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
   )
   URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
 
+  for deb in "${DEBS[@]}"; do
+    download-or-bust "${URL_BASE}/${deb}"
+  done
+
   # Based on
   # https://major.io/2014/06/26/install-debian-packages-without-starting-daemons/
   # We do this to prevent Salt from starting the salt-minion
@@ -141,13 +149,17 @@ exit 101
 EOF
   chmod 0755 /usr/sbin/policy-rc.d
 
-  for tar in "${TARS[@]}"; do
-    download-or-bust "${URL_BASE}/${tar}"
-    dpkg -i "${tar}" || true
+  for deb in "${DEBS[@]}"; do
+    echo "== Installing ${deb}, ignore dependency complaints (will fix later) =="
+    dpkg --force-depends -i "${deb}"
   done
 
   # This will install any of the unmet dependencies from above.
-  apt-get install -f -y
+  echo "== Installing unmet dependencies =="
+  until apt-get install -f -y; do
+    echo "== apt-get install failed, retrying =="
+    echo sleep 5
+  done
 
   rm /usr/sbin/policy-rc.d
 }