From 77cb83782ad090bcb9dfa56f1460b87b3049a20b Mon Sep 17 00:00:00 2001 From: Mike Schwartz Date: Fri, 27 Jun 2014 10:09:49 -0700 Subject: [PATCH] Switch boto to use truncated exponential backoff --- boto/connection.py | 3 ++- boto/dynamodb/layer1.py | 3 ++- boto/dynamodb2/layer1.py | 9 +++++---- boto/route53/connection.py | 3 ++- boto/utils.py | 7 +++++-- 5 files changed, 16 insertions(+), 9 deletions(-) diff --git a/boto/connection.py b/boto/connection.py index 051007df64..ebe26cdc31 100644 --- a/boto/connection.py +++ b/boto/connection.py @@ -901,7 +901,8 @@ def _mexe(self, request, sender=None, override_num_retries=None, self.is_secure) while i <= num_retries: # Use binary exponential backoff to desynchronize client requests. - next_sleep = random.random() * (2 ** i) + next_sleep = min(random.random() * (2 ** i), + boto.config.get('Boto', 'max_retry_delay', 60)) try: # we now re-sign each request before it is retried boto.log.debug('Token: %s' % self.provider.security_token) diff --git a/boto/dynamodb/layer1.py b/boto/dynamodb/layer1.py index 317cf43370..01a490200f 100644 --- a/boto/dynamodb/layer1.py +++ b/boto/dynamodb/layer1.py @@ -173,7 +173,8 @@ def _exponential_time(self, i): if i == 0: next_sleep = 0 else: - next_sleep = 0.05 * (2 ** i) + next_sleep = min(0.05 * (2 ** i), + boto.config.get('Boto', 'max_retry_delay', 60)) return next_sleep def list_tables(self, limit=None, start_table=None): diff --git a/boto/dynamodb2/layer1.py b/boto/dynamodb2/layer1.py index 9a1c4adfba..360c7d0851 100644 --- a/boto/dynamodb2/layer1.py +++ b/boto/dynamodb2/layer1.py @@ -2123,7 +2123,7 @@ def _retry_handler(self, response, i, next_sleep): 'ProvisionedThroughputExceededException', i ) - next_sleep = self._exponential_time(i) + next_sleep = self._truncated_exponential_time(i) i += 1 status = (msg, i, next_sleep) if i == self.NumberRetries: @@ -2150,12 +2150,13 @@ def _retry_handler(self, response, i, next_sleep): if actual_crc32 != expected_crc32: msg = ("The calculated checksum %s did not match the expected " "checksum %s" % (actual_crc32, expected_crc32)) - status = (msg, i + 1, self._exponential_time(i)) + status = (msg, i + 1, self._truncated_exponential_time(i)) return status - def _exponential_time(self, i): + def _truncated_exponential_time(self, i): if i == 0: next_sleep = 0 else: - next_sleep = 0.05 * (2 ** i) + next_sleep = min(0.05 * (2 ** i), + boto.config.get('Boto', 'max_retry_delay', 60)) return next_sleep diff --git a/boto/route53/connection.py b/boto/route53/connection.py index 26be071ecf..522dd09cdf 100644 --- a/boto/route53/connection.py +++ b/boto/route53/connection.py @@ -530,7 +530,8 @@ def _retry_handler(self, response, i, next_sleep): 'PriorRequestNotComplete', i ) - next_sleep = random.random() * (2 ** i) + next_sleep = min(random.random() * (2 ** i), + boto.config.get('Boto', 'max_retry_delay', 60)) i += 1 status = (msg, i, next_sleep) diff --git a/boto/utils.py b/boto/utils.py index 7654d228e3..ae900da62f 100644 --- a/boto/utils.py +++ b/boto/utils.py @@ -237,7 +237,8 @@ def retry_url(url, retry_on_404=True, num_retries=10): boto.log.exception('Caught exception reading instance data') # If not on the last iteration of the loop then sleep. if i + 1 != num_retries: - time.sleep(2 ** i) + time.sleep(min(2 ** i, + boto.config.get('Boto', 'max_retry_delay', 60))) boto.log.error('Unable to read instance data, giving up') return '' @@ -318,7 +319,9 @@ def __getitem__(self, key): " for the '%s' try" % (i + 1)) if i + 1 != self._num_retries: - next_sleep = random.random() * (2 ** i) + next_sleep = min( + random.random() * 2 ** i, + boto.config.get('Boto', 'max_retry_delay', 60)) time.sleep(next_sleep) else: boto.log.error('Unable to read meta data, giving up')