From 9556318c5827dae10b290dfa56d1e1b81622bb57 Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Thu, 26 Mar 2015 10:22:50 -0700
Subject: [PATCH 1/2] Wait for master instance running before adding route

---
 cluster/aws/util.sh | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh
index 62b4885ee719b..52d686a936b89 100644
--- a/cluster/aws/util.sh
+++ b/cluster/aws/util.sh
@@ -330,6 +330,21 @@ function ensure-iam-profiles {
   }
 }
 
+# Wait for instance to be in running state
+function wait-for-instance-running {
+  instance_id=$1
+  while true; do
+    instance_state=$($AWS_CMD describe-instances --instance-ids $instance_id | expect_instance_states running)
+    if [[ "$instance_state" == "" ]]; then
+      break
+    else
+      echo "Waiting for instance ${instance_id} to spawn"
+      echo "Sleeping for 3 seconds..."
+      sleep 3
+    fi
+  done
+}
+
 function kube-up {
   find-release-tars
   upload-server-tars
@@ -467,6 +482,9 @@ function kube-up {
     else
       KUBE_MASTER=${MASTER_NAME}
       KUBE_MASTER_IP=${ip}
+      # We are not able to add a route to the instance until that instance is in "running" state.
+      wait-for-instance-running $master_id
+      sleep 10
       $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MASTER_IP_RANGE} --instance-id $master_id > $LOG
 
       echo -e " ${color_green}[master running @${KUBE_MASTER_IP}]${color_norm}"
@@ -532,19 +550,10 @@ function kube-up {
     # We are not able to add a route to the instance until that instance is in "running" state.
     # This is quite an ugly solution to this problem. In Bash 4 we could use assoc. arrays to do this for
     # all instances at once but we can't be sure we are running Bash 4.
-    while true; do
-      instance_state=$($AWS_CMD describe-instances --instance-ids $minion_id | expect_instance_states running)
-      if [[ "$instance_state" == "" ]]; then
-        echo "Minion ${MINION_NAMES[$i]} running"
-        sleep 10
-        $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MINION_IP_RANGES[$i]} --instance-id $minion_id > $LOG
-        break
-      else
-        echo "Waiting for minion ${MINION_NAMES[$i]} to spawn"
-        echo "Sleeping for 3 seconds..."
-        sleep 3
-      fi
-    done
+    wait-for-instance-running $minion_id
+    echo "Minion ${MINION_NAMES[$i]} running"
+    sleep 10
+    $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MINION_IP_RANGES[$i]} --instance-id $minion_id > $LOG
   done
 
   FAIL=0

From 53bdbec64ef77854c7fcf8f2fce31e56f01d55f2 Mon Sep 17 00:00:00 2001
From: Justin Santa Barbara <justin@fathomdb.com>
Date: Thu, 26 Mar 2015 10:26:23 -0700
Subject: [PATCH 2/2] Launch minions in parallel

---
 cluster/aws/util.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh
index 52d686a936b89..74d13d21b985d 100644
--- a/cluster/aws/util.sh
+++ b/cluster/aws/util.sh
@@ -482,12 +482,13 @@ function kube-up {
     else
       KUBE_MASTER=${MASTER_NAME}
       KUBE_MASTER_IP=${ip}
+      echo -e " ${color_green}[master running @${KUBE_MASTER_IP}]${color_norm}"
+
       # We are not able to add a route to the instance until that instance is in "running" state.
       wait-for-instance-running $master_id
       sleep 10
       $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MASTER_IP_RANGE} --instance-id $master_id > $LOG
 
-      echo -e " ${color_green}[master running @${KUBE_MASTER_IP}]${color_norm}"
       break
     fi
     echo -e " ${color_yellow}[master not working yet]${color_norm}"
@@ -518,7 +519,7 @@ function kube-up {
     sleep 10
   done
 
-
+  MINION_IDS=()
   for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     echo "Starting Minion (${MINION_NAMES[$i]})"
     (
@@ -544,15 +545,19 @@ function kube-up {
     add-tag $minion_id Name ${MINION_NAMES[$i]}
     add-tag $minion_id Role $MINION_TAG
 
-    sleep 3
-    $AWS_CMD modify-instance-attribute --instance-id $minion_id --source-dest-check '{"Value": false}' > $LOG
+    MINION_IDS[$i]=$minion_id
+  done
 
+  # Add routes to minions
+  for (( i=0; i<${#MINION_NAMES[@]}; i++)); do
     # We are not able to add a route to the instance until that instance is in "running" state.
     # This is quite an ugly solution to this problem. In Bash 4 we could use assoc. arrays to do this for
     # all instances at once but we can't be sure we are running Bash 4.
+    minion_id=${MINION_IDS[$i]}
     wait-for-instance-running $minion_id
     echo "Minion ${MINION_NAMES[$i]} running"
     sleep 10
+    $AWS_CMD modify-instance-attribute --instance-id $minion_id --source-dest-check '{"Value": false}' > $LOG
     $AWS_CMD create-route --route-table-id $ROUTE_TABLE_ID --destination-cidr-block ${MINION_IP_RANGES[$i]} --instance-id $minion_id > $LOG
   done