Merge github.com:grpc/grpc into backoff2

Reintroduce flakiness flag for tests: lb_policies_test is inherently flaky and I don't have time to deflake this week
yuliblossom · Mar 20, 2016 · de7edf8 · de7edf8
2 parents 3ab6f4d + cb1c389
commit de7edf8
Show file tree

Hide file tree

Showing 23 changed files with 222 additions and 63 deletions.
diff --git a/Makefile b/Makefile
@@ -1540,8 +1540,6 @@ test_c: buildtests_c
 	$(Q) $(BINDIR)/$(CONFIG)/json_test || ( echo test json_test failed ; exit 1 )
 	$(E) "[RUN]     Testing lame_client_test"
 	$(Q) $(BINDIR)/$(CONFIG)/lame_client_test || ( echo test lame_client_test failed ; exit 1 )
-	$(E) "[RUN]     Testing lb_policies_test"
-	$(Q) $(BINDIR)/$(CONFIG)/lb_policies_test || ( echo test lb_policies_test failed ; exit 1 )
 	$(E) "[RUN]     Testing message_compress_test"
 	$(Q) $(BINDIR)/$(CONFIG)/message_compress_test || ( echo test message_compress_test failed ; exit 1 )
 	$(E) "[RUN]     Testing mlog_test"
@@ -1623,6 +1621,8 @@ test_c: buildtests_c
 
 
 flaky_test_c: buildtests_c
+	$(E) "[RUN]     Testing lb_policies_test"
+	$(Q) $(BINDIR)/$(CONFIG)/lb_policies_test || ( echo test lb_policies_test failed ; exit 1 )
 
 
 test_cxx: test_zookeeper buildtests_cxx

diff --git a/build.yaml b/build.yaml
@@ -1673,6 +1673,7 @@ targets:
   - gpr
 - name: lb_policies_test
   cpu_cost: 0.1
+  flaky: true
   build: test
   language: c
   src:
@@ -2462,6 +2463,8 @@ targets:
   - gpr_test_util
   - gpr
   - grpc++_test_config
+  exclude_configs:
+  - tsan
   platforms:
   - mac
   - linux
@@ -2481,6 +2484,8 @@ targets:
   - gpr_test_util
   - gpr
   - grpc++_test_config
+  exclude_configs:
+  - tsan
   platforms:
   - mac
   - linux
@@ -2937,7 +2942,7 @@ node_modules:
   - src/node/ext/server_credentials.cc
   - src/node/ext/timeval.cc
 openssl_fallback:
-  base_uri: http://openssl.org/source/
+  base_uri: https://openssl.org/source/old/1.0.2/
   extraction_dir: openssl-1.0.2f
   tarball: openssl-1.0.2f.tar.gz
 php_config_m4:

diff --git a/include/grpc/compression.h b/include/grpc/compression.h
@@ -55,11 +55,13 @@ GRPCAPI int grpc_compression_algorithm_parse(
 GRPCAPI int grpc_compression_algorithm_name(
     grpc_compression_algorithm algorithm, char **name);
 
-/** Returns the compression algorithm corresponding to \a level.
+/** Returns the compression algorithm corresponding to \a level for the
+ * compression algorithms encoded in the \a accepted_encodings bitset.
  *
  * It abort()s for unknown levels . */
 GRPCAPI grpc_compression_algorithm
-grpc_compression_algorithm_for_level(grpc_compression_level level);
+grpc_compression_algorithm_for_level(grpc_compression_level level,
+                                     uint32_t accepted_encodings);
 
 GRPCAPI void grpc_compression_options_init(grpc_compression_options *opts);
 

diff --git a/src/core/compression/compression_algorithm.c b/src/core/compression/compression_algorithm.c
@@ -128,20 +128,57 @@ grpc_mdelem *grpc_compression_encoding_mdelem(
 /* TODO(dgq): Add the ability to specify parameters to the individual
  * compression algorithms */
 grpc_compression_algorithm grpc_compression_algorithm_for_level(
-    grpc_compression_level level) {
+    grpc_compression_level level, uint32_t accepted_encodings) {
   GRPC_API_TRACE("grpc_compression_algorithm_for_level(level=%d)", 1,
                  ((int)level));
+  if (level > GRPC_COMPRESS_LEVEL_HIGH) {
+    gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level);
+    abort();
+  }
+
+  const size_t num_supported =
+      GPR_BITCOUNT(accepted_encodings) - 1; /* discard NONE */
+  if (level == GRPC_COMPRESS_LEVEL_NONE || num_supported == 0) {
+    return GRPC_COMPRESS_NONE;
+  }
+
+  GPR_ASSERT(level > 0);
+
+  /* Establish a "ranking" or compression algorithms in increasing order of
+   * compression.
+   * This is simplistic and we will probably want to introduce other dimensions
+   * in the future (cpu/memory cost, etc). */
+  const grpc_compression_algorithm algos_ranking[] = {GRPC_COMPRESS_GZIP,
+                                                      GRPC_COMPRESS_DEFLATE};
+
+  /* intersect algos_ranking with the supported ones keeping the ranked order */
+  grpc_compression_algorithm
+      sorted_supported_algos[GRPC_COMPRESS_ALGORITHMS_COUNT];
+  size_t algos_supported_idx = 0;
+  for (size_t i = 0; i < GPR_ARRAY_SIZE(algos_ranking); i++) {
+    const grpc_compression_algorithm alg = algos_ranking[i];
+    for (size_t j = 0; j < num_supported; j++) {
+      if (GPR_BITGET(accepted_encodings, alg) == 1) {
+        /* if \a alg in supported */
+        sorted_supported_algos[algos_supported_idx++] = alg;
+        break;
+      }
+    }
+    if (algos_supported_idx == num_supported) break;
+  }
+
   switch (level) {
     case GRPC_COMPRESS_LEVEL_NONE:
-      return GRPC_COMPRESS_NONE;
+      abort(); /* should have been handled already */
     case GRPC_COMPRESS_LEVEL_LOW:
+      return sorted_supported_algos[0];
     case GRPC_COMPRESS_LEVEL_MED:
+      return sorted_supported_algos[num_supported / 2];
     case GRPC_COMPRESS_LEVEL_HIGH:
-      return GRPC_COMPRESS_DEFLATE;
+      return sorted_supported_algos[num_supported - 1];
     default:
-      gpr_log(GPR_ERROR, "Unknown compression level %d.", (int)level);
       abort();
-  }
+  };
 }
 
 void grpc_compression_options_init(grpc_compression_options *opts) {

diff --git a/src/core/surface/call.c b/src/core/surface/call.c
@@ -1481,3 +1481,11 @@ void *grpc_call_context_get(grpc_call *call, grpc_context_index elem) {
 }
 
 uint8_t grpc_call_is_client(grpc_call *call) { return call->is_client; }
+
+grpc_compression_algorithm grpc_call_compression_for_level(
+    grpc_call *call, grpc_compression_level level) {
+  gpr_mu_lock(&call->mu);
+  const uint32_t accepted_encodings = call->encodings_accepted_by_peer;
+  gpr_mu_unlock(&call->mu);
+  return grpc_compression_algorithm_for_level(level, accepted_encodings);
+}
diff --git a/src/core/surface/call.h b/src/core/surface/call.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -38,7 +38,9 @@
 #include "src/core/channel/context.h"
 #include "src/core/surface/api_trace.h"
 #include "src/core/surface/surface_trace.h"
+
 #include <grpc/grpc.h>
+#include <grpc/impl/codegen/compression_types.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -102,6 +104,11 @@ void *grpc_call_context_get(grpc_call *call, grpc_context_index elem);
 
 uint8_t grpc_call_is_client(grpc_call *call);
 
+/* Return an appropriate compression algorithm for the requested compression \a
+ * level in the context of \a call. */
+grpc_compression_algorithm grpc_call_compression_for_level(
+    grpc_call *call, grpc_compression_level level);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc
@@ -43,6 +43,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/channel/compress_filter.h"
+#include "src/core/surface/call.h"
 #include "src/cpp/common/create_auth_context.h"
 
 namespace grpc {
@@ -197,7 +198,7 @@ bool ServerContext::IsCancelled() const {
 
 void ServerContext::set_compression_level(grpc_compression_level level) {
   const grpc_compression_algorithm algorithm_for_level =
-      grpc_compression_algorithm_for_level(level);
+      grpc_call_compression_for_level(call_, level);
   set_compression_algorithm(algorithm_for_level);
 }
 

diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -166,7 +166,7 @@ extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_im
 typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name);
 extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import;
 #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import
-typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level);
+typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings);
 extern grpc_compression_algorithm_for_level_type grpc_compression_algorithm_for_level_import;
 #define grpc_compression_algorithm_for_level grpc_compression_algorithm_for_level_import
 typedef void(*grpc_compression_options_init_type)(grpc_compression_options *opts);

diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -166,7 +166,7 @@ extern grpc_compression_algorithm_parse_type grpc_compression_algorithm_parse_im
 typedef int(*grpc_compression_algorithm_name_type)(grpc_compression_algorithm algorithm, char **name);
 extern grpc_compression_algorithm_name_type grpc_compression_algorithm_name_import;
 #define grpc_compression_algorithm_name grpc_compression_algorithm_name_import
-typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level);
+typedef grpc_compression_algorithm(*grpc_compression_algorithm_for_level_type)(grpc_compression_level level, uint32_t accepted_encodings);
 extern grpc_compression_algorithm_for_level_type grpc_compression_algorithm_for_level_import;
 #define grpc_compression_algorithm_for_level grpc_compression_algorithm_for_level_import
 typedef void(*grpc_compression_options_init_type)(grpc_compression_options *opts);

diff --git a/templates/tools/dockerfile/apt_get_basic.include b/templates/tools/dockerfile/apt_get_basic.include
@@ -1,4 +1,3 @@
-<%page args="skip_golang=False"/>\
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y ${'\\'}
   autoconf ${'\\'}
@@ -10,9 +9,7 @@ RUN apt-get update && apt-get install -y ${'\\'}
   gcc ${'\\'}
   gcc-multilib ${'\\'}
   git ${'\\'}
-% if not skip_golang:
   golang ${'\\'}
-% endif
   gyp ${'\\'}
   lcov ${'\\'}
   libc6 ${'\\'}

diff --git a/templates/tools/dockerfile/run_tests_addons.include b/templates/tools/dockerfile/run_tests_addons.include
@@ -1,10 +1,7 @@
-<%page args="skip_zookeeper=False"/>\
 <%include file="ccache_setup.include"/>
-% if not skip_zookeeper:
 #======================
 # Zookeeper dependencies
 # TODO(jtattermusch): is zookeeper still needed?
 RUN apt-get install -y libzookeeper-mt-dev
-% endif
 
 RUN mkdir /var/local/jenkins
diff --git a/.../test/cxx_squeeze_x64/Dockerfile.template → ...e/test/cxx_wheezy_x64/Dockerfile.template b/.../test/cxx_squeeze_x64/Dockerfile.template → ...e/test/cxx_wheezy_x64/Dockerfile.template
@@ -29,20 +29,19 @@
   # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
   # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-  FROM debian:squeeze
+  FROM debian:wheezy
 
-  <%include file="../../apt_get_basic.include" args="skip_golang=True"/>
+  <%include file="../../apt_get_basic.include"/>
+  <%include file="../../cxx_deps.include"/>
 
-  # libgflags-dev is not available on squeezy
-  RUN apt-get update && apt-get -y install libgtest-dev libc++-dev clang && apt-get clean
-
-  RUN apt-get update && apt-get -y install python-pip && apt-get clean
-  RUN pip install argparse
+  RUN apt-get update && apt-get install -y ${'\\'}
+    gcc-4.4 ${'\\'}
+    gcc-4.4-multilib
 
   RUN wget ${openssl_fallback.base_uri + openssl_fallback.tarball}
 
-  ENV POST_GIT_STEP tools/dockerfile/test/cxx_squeeze_x64/post-git-setup.sh
+  ENV POST_GIT_STEP tools/dockerfile/test/cxx_wheezy_x64/post-git-setup.sh
 
-  <%include file="../../run_tests_addons.include" args="skip_zookeeper=True"/>
+  <%include file="../../run_tests_addons.include"/>
   # Define the default command.
   CMD ["bash"]
diff --git a/...xx_squeeze_x64/post-git-setup.sh.template → ...cxx_wheezy_x64/post-git-setup.sh.template b/...xx_squeeze_x64/post-git-setup.sh.template → ...cxx_wheezy_x64/post-git-setup.sh.template
diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template
@@ -34,7 +34,13 @@
   <%include file="../../apt_get_basic.include"/>
   #========================
   # Sanity test dependencies
-  RUN apt-get update && apt-get install -y python-pip
+  RUN apt-get update && apt-get install -y ${"\\"}
+        python-pip ${"\\"}
+        autoconf ${"\\"}
+        automake ${"\\"}
+        libtool ${"\\"}
+        curl ${"\\"}
+        python-virtualenv
   RUN pip install simplejson mako
 
   #===================

diff --git a/test/core/compression/compression_test.c b/test/core/compression/compression_test.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -93,18 +93,98 @@ static void test_compression_algorithm_name(void) {
 }
 
 static void test_compression_algorithm_for_level(void) {
-  size_t i;
-  grpc_compression_level levels[] = {
-      GRPC_COMPRESS_LEVEL_NONE, GRPC_COMPRESS_LEVEL_LOW,
-      GRPC_COMPRESS_LEVEL_MED, GRPC_COMPRESS_LEVEL_HIGH};
-  grpc_compression_algorithm algorithms[] = {
-      GRPC_COMPRESS_NONE, GRPC_COMPRESS_DEFLATE, GRPC_COMPRESS_DEFLATE,
-      GRPC_COMPRESS_DEFLATE};
   gpr_log(GPR_DEBUG, "test_compression_algorithm_for_level");
 
-  for (i = 0; i < GPR_ARRAY_SIZE(levels); i++) {
-    GPR_ASSERT(algorithms[i] ==
-               grpc_compression_algorithm_for_level(levels[i]));
+  {
+    /* accept only identity (aka none) */
+    uint32_t accepted_encodings = 0;
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_NONE); /* always */
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_NONE,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_LOW,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_MED,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_HIGH,
+                                                    accepted_encodings));
+  }
+
+  {
+    /* accept only gzip */
+    uint32_t accepted_encodings = 0;
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_NONE); /* always */
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_GZIP);
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_NONE,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_GZIP ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_LOW,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_GZIP ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_MED,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_GZIP ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_HIGH,
+                                                    accepted_encodings));
+  }
+
+  {
+    /* accept only deflate */
+    uint32_t accepted_encodings = 0;
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_NONE); /* always */
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_DEFLATE);
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_NONE,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_DEFLATE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_LOW,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_DEFLATE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_MED,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_DEFLATE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_HIGH,
+                                                    accepted_encodings));
+  }
+
+  {
+    /* accept gzip and deflate */
+    uint32_t accepted_encodings = 0;
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_NONE); /* always */
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_GZIP);
+    GPR_BITSET(&accepted_encodings, GRPC_COMPRESS_DEFLATE);
+
+    GPR_ASSERT(GRPC_COMPRESS_NONE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_NONE,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_GZIP ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_LOW,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_DEFLATE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_MED,
+                                                    accepted_encodings));
+
+    GPR_ASSERT(GRPC_COMPRESS_DEFLATE ==
+               grpc_compression_algorithm_for_level(GRPC_COMPRESS_LEVEL_HIGH,
+                                                    accepted_encodings));
   }
 }