Switch boto to use truncated exponential backoff

heroku · Jun 27, 2014 · 77cb837 · 77cb837
1 parent 6472811
commit 77cb837
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 9 deletions.
diff --git a/boto/connection.py b/boto/connection.py
@@ -901,7 +901,8 @@ def _mexe(self, request, sender=None, override_num_retries=None,
                                               self.is_secure)
         while i <= num_retries:
             # Use binary exponential backoff to desynchronize client requests.
-            next_sleep = random.random() * (2 ** i)
+            next_sleep = min(random.random() * (2 ** i),
+                             boto.config.get('Boto', 'max_retry_delay', 60))
             try:
                 # we now re-sign each request before it is retried
                 boto.log.debug('Token: %s' % self.provider.security_token)

diff --git a/boto/dynamodb/layer1.py b/boto/dynamodb/layer1.py
@@ -173,7 +173,8 @@ def _exponential_time(self, i):
         if i == 0:
             next_sleep = 0
         else:
-            next_sleep = 0.05 * (2 ** i)
+            next_sleep = min(0.05 * (2 ** i),
+                             boto.config.get('Boto', 'max_retry_delay', 60))
         return next_sleep
 
     def list_tables(self, limit=None, start_table=None):

diff --git a/boto/dynamodb2/layer1.py b/boto/dynamodb2/layer1.py
@@ -2123,7 +2123,7 @@ def _retry_handler(self, response, i, next_sleep):
                     'ProvisionedThroughputExceededException',
                     i
                 )
-                next_sleep = self._exponential_time(i)
+                next_sleep = self._truncated_exponential_time(i)
                 i += 1
                 status = (msg, i, next_sleep)
                 if i == self.NumberRetries:
@@ -2150,12 +2150,13 @@ def _retry_handler(self, response, i, next_sleep):
             if actual_crc32 != expected_crc32:
                 msg = ("The calculated checksum %s did not match the expected "
                        "checksum %s" % (actual_crc32, expected_crc32))
-                status = (msg, i + 1, self._exponential_time(i))
+                status = (msg, i + 1, self._truncated_exponential_time(i))
         return status
 
-    def _exponential_time(self, i):
+    def _truncated_exponential_time(self, i):
         if i == 0:
             next_sleep = 0
         else:
-            next_sleep = 0.05 * (2 ** i)
+            next_sleep = min(0.05 * (2 ** i),
+                             boto.config.get('Boto', 'max_retry_delay', 60))
         return next_sleep
diff --git a/boto/route53/connection.py b/boto/route53/connection.py
@@ -530,7 +530,8 @@ def _retry_handler(self, response, i, next_sleep):
                     'PriorRequestNotComplete',
                     i
                 )
-                next_sleep = random.random() * (2 ** i)
+                next_sleep = min(random.random() * (2 ** i),
+                                 boto.config.get('Boto', 'max_retry_delay', 60))
                 i += 1
                 status = (msg, i, next_sleep)
 

diff --git a/boto/utils.py b/boto/utils.py
@@ -237,7 +237,8 @@ def retry_url(url, retry_on_404=True, num_retries=10):
         boto.log.exception('Caught exception reading instance data')
         # If not on the last iteration of the loop then sleep.
         if i + 1 != num_retries:
-            time.sleep(2 ** i)
+            time.sleep(min(2 ** i,
+                           boto.config.get('Boto', 'max_retry_delay', 60)))
     boto.log.error('Unable to read instance data, giving up')
     return ''
 
@@ -318,7 +319,9 @@ def __getitem__(self, key):
                                " for the '%s' try" % (i + 1))
 
                 if i + 1 != self._num_retries:
-                    next_sleep = random.random() * (2 ** i)
+                    next_sleep = min(
+                        random.random() * 2 ** i,
+                        boto.config.get('Boto', 'max_retry_delay', 60))
                     time.sleep(next_sleep)
             else:
                 boto.log.error('Unable to read meta data, giving up')