Down integrate to Github

gminor7 · Mar 20, 2019 · 2f864fd · 2f864fd
1 parent 5928445
commit 2f864fd
Show file tree

Hide file tree

Showing 90 changed files with 1,530 additions and 972 deletions.
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -80,11 +80,24 @@ string(REGEX REPLACE     "${protobuf_VERSION_REGEX}" "\\5"
 
 message(STATUS "${protobuf_VERSION_PRERELEASE}")
 
+message(STATUS "${protobuf_VERSION_PRERELEASE}")
+
 # Package version
 set(protobuf_VERSION
   "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
 
 if(protobuf_VERSION_PRERELEASE)
+<<<<<<<
+=======
+  set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")
+else()
+  set(protobuf_VERSION "${protobuf_VERSION}.0")
+endif()
+message(STATUS "${protobuf_VERSION}")
+
+if(protobuf_VERBOSE)
+
+>>>>>>>
   set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")
 else()
   set(protobuf_VERSION "${protobuf_VERSION}.0")

diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in
@@ -115,7 +115,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver.h"
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\util\type_resolver_util.h" include\google\protobuf\util\type_resolver_util.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format.h" include\google\protobuf\wire_format.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite.h" include\google\protobuf\wire_format_lite.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wire_format_lite_inl.h" include\google\protobuf\wire_format_lite_inl.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\wrappers.pb.h" include\google\protobuf\wrappers.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\any.proto" include\google\protobuf\any.proto
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\api.proto" include\google\protobuf\api.proto

diff --git a/conformance/conformance.proto b/conformance/conformance.proto
@@ -119,6 +119,10 @@ message ConformanceRequest {
 
   // Specify details for how to encode jspb.
   JspbEncodingConfig jspb_encoding_options = 6;
+
+  // This can be used in json and text format. If true, testee should print
+  // unknown fields instead of ignore. This feature is optional.
+  bool print_unknown_fields = 9;
 }
 
 // Represents a single test case's output.

diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc
@@ -214,8 +214,10 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
     }
 
     case conformance::TEXT_FORMAT: {
-      GOOGLE_CHECK(TextFormat::PrintToString(*test_message,
-                                             response->mutable_text_payload()));
+      TextFormat::Printer printer;
+      printer.SetHideUnknownFields(!request.print_unknown_fields());
+      GOOGLE_CHECK(printer.PrintToString(*test_message,
+                                         response->mutable_text_payload()));
       break;
     }
 

diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py
@@ -166,7 +166,8 @@ def do_test(request):
         return response
 
     elif request.requested_output_format == conformance_pb2.TEXT_FORMAT:
-      response.text_payload = text_format.MessageToString(test_message)
+      response.text_payload = text_format.MessageToString(
+          test_message, print_unknown_fields=request.print_unknown_fields)
 
   except Exception as e:
     response.runtime_error = str(e)

diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc
@@ -68,6 +68,7 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
       input_format_(input_format),
       output_format_(output_format),
       prototype_message_(prototype_message),
+      prototype_message_for_compare_(prototype_message.New()),
       test_name_(test_name) {
   switch (input_format) {
     case conformance::PROTOBUF: {
@@ -102,7 +103,7 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
 
 Message* ConformanceTestSuite::ConformanceRequestSetting::
     GetTestMessage() const {
-  return prototype_message_.New();
+  return prototype_message_for_compare_->New();
 }
 
 string ConformanceTestSuite::ConformanceRequestSetting::

diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h
@@ -224,6 +224,14 @@ class ConformanceTestSuite {
 
     string ConformanceLevelToString(ConformanceLevel level) const;
 
+    void SetPrintUnknownFields(bool print_unknown_fields) {
+      request_.set_print_unknown_fields(true);
+    }
+
+    void SetPrototypeMessageForCompare(const Message& message) {
+      prototype_message_for_compare_.reset(message.New());
+    }
+
    protected:
     virtual string InputFormatString(conformance::WireFormat format) const;
     virtual string OutputFormatString(conformance::WireFormat format) const;
@@ -234,6 +242,7 @@ class ConformanceTestSuite {
     ::conformance::WireFormat input_format_;
     ::conformance::WireFormat output_format_;
     const Message& prototype_message_;
+    std::unique_ptr<Message> prototype_message_for_compare_;
     string test_name_;
   };
 

diff --git a/conformance/text_format_conformance_suite.cc b/conformance/text_format_conformance_suite.cc
@@ -43,6 +43,7 @@ using conformance::WireFormat;
 using google::protobuf::Message;
 using google::protobuf::TextFormat;
 using protobuf_test_messages::proto2::TestAllTypesProto2;
+using protobuf_test_messages::proto2::UnknownToTestAllTypes;
 using protobuf_test_messages::proto3::TestAllTypesProto3;
 using std::string;
 
@@ -54,8 +55,14 @@ TextFormatConformanceTestSuite::TextFormatConformanceTestSuite() {
 }
 
 bool TextFormatConformanceTestSuite::ParseTextFormatResponse(
-    const ConformanceResponse& response, Message* test_message) {
-  if (!TextFormat::ParseFromString(response.text_payload(), test_message)) {
+    const ConformanceResponse& response,
+    const ConformanceRequestSetting& setting, Message* test_message) {
+  TextFormat::Parser parser;
+  const ConformanceRequest& request = setting.GetRequest();
+  if (request.print_unknown_fields()) {
+    parser.AllowFieldNumber(true);
+  }
+  if (!parser.ParseFromString(response.text_payload(), test_message)) {
     GOOGLE_LOG(ERROR) << "INTERNAL ERROR: internal text->protobuf transcode "
                       << "yielded unparseable proto. Text payload: "
                       << response.text_payload();
@@ -103,7 +110,7 @@ bool TextFormatConformanceTestSuite::ParseResponse(
         return false;
       }
 
-      if (!ParseTextFormatResponse(response, test_message)) {
+      if (!ParseTextFormatResponse(response, setting, test_message)) {
         ReportFailure(
             test_name, level, request, response,
             "TEXT_FORMAT output we received from test was unparseable.");
@@ -171,6 +178,27 @@ void TextFormatConformanceTestSuite::RunValidTextFormatTestWithMessage(
   RunValidInputTest(setting2, input_text);
 }
 
+void TextFormatConformanceTestSuite::RunValidUnknownTextFormatTest(
+    const string& test_name, const Message& message) {
+  string serialized_input;
+  message.SerializeToString(&serialized_input);
+  TestAllTypesProto3 prototype;
+  ConformanceRequestSetting setting1(
+      RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Drop",
+      serialized_input);
+  setting1.SetPrototypeMessageForCompare(message);
+  RunValidBinaryInputTest(setting1, "");
+
+  ConformanceRequestSetting setting2(
+      RECOMMENDED, conformance::PROTOBUF, conformance::TEXT_FORMAT,
+      conformance::TEXT_FORMAT_TEST, prototype, test_name + "_Print",
+      serialized_input);
+  setting2.SetPrototypeMessageForCompare(message);
+  setting2.SetPrintUnknownFields(true);
+  RunValidBinaryInputTest(setting2, serialized_input);
+}
+
 void TextFormatConformanceTestSuite::RunSuiteImpl() {
   RunValidTextFormatTest("HelloWorld", REQUIRED,
                          "optional_string: 'Hello, World!'");
@@ -235,6 +263,29 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() {
                                "Data: { group_int32: 1 }");
   RunValidTextFormatTestProto2("GroupFieldEmpty", REQUIRED,
                                "Data {}");
+
+
+  // Unknown Fields
+  UnknownToTestAllTypes message;
+  // Unable to print unknown Fixed32/Fixed64 fields as if they are known.
+  // Fixed32/Fixed64 fields are not added in the tests.
+  message.set_optional_int32(123);
+  message.set_optional_string("hello");
+  message.set_optional_bool(true);
+  RunValidUnknownTextFormatTest("ScalarUnknownFields", message);
+
+  message.Clear();
+  message.mutable_nested_message()->set_c(111);
+  RunValidUnknownTextFormatTest("MessageUnknownFields", message);
+
+  message.Clear();
+  message.mutable_optionalgroup()->set_a(321);
+  RunValidUnknownTextFormatTest("GroupUnknownFields", message);
+
+  message.add_repeated_int32(1);
+  message.add_repeated_int32(2);
+  message.add_repeated_int32(3);
+  RunValidUnknownTextFormatTest("RepeatedUnknownFields", message);
 }
 
 }  // namespace protobuf

diff --git a/conformance/text_format_conformance_suite.h b/conformance/text_format_conformance_suite.h
@@ -51,9 +51,12 @@ class TextFormatConformanceTestSuite : public ConformanceTestSuite {
                                          ConformanceLevel level,
                                          const string& input_text,
                                          const Message& prototype);
+  void RunValidUnknownTextFormatTest(const string& test_name,
+                                     const Message& message);
   void ExpectParseFailure(const string& test_name, ConformanceLevel level,
                           const string& input);
   bool ParseTextFormatResponse(const conformance::ConformanceResponse& response,
+                               const ConformanceRequestSetting& setting,
                                Message* test_message);
   bool ParseResponse(const conformance::ConformanceResponse& response,
                      const ConformanceRequestSetting& setting,

diff --git a/conformance/text_format_failure_list_java.txt b/conformance/text_format_failure_list_java.txt
@@ -0,0 +1,4 @@
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
+Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
diff --git a/java/core/src/main/java/com/google/protobuf/ByteString.java b/java/core/src/main/java/com/google/protobuf/ByteString.java
@@ -1438,16 +1438,14 @@ final boolean equalsRange(ByteString other, int offset, int length) {
         LiteralByteString lbsOther = (LiteralByteString) other;
         byte[] thisBytes = bytes;
         byte[] otherBytes = lbsOther.bytes;
-        int thisLimit = getOffsetIntoBytes() + length;
-        for (int thisIndex = getOffsetIntoBytes(),
-                otherIndex = lbsOther.getOffsetIntoBytes() + offset;
-            (thisIndex < thisLimit);
-            ++thisIndex, ++otherIndex) {
-          if (thisBytes[thisIndex] != otherBytes[otherIndex]) {
-            return false;
-          }
-        }
-        return true;
+
+        return UnsafeUtil.mismatch(
+                thisBytes,
+                getOffsetIntoBytes(),
+                otherBytes,
+                lbsOther.getOffsetIntoBytes() + offset,
+                length)
+            == -1;
       }
 
       return other.substring(offset, offset + length).equals(substring(0, length));

diff --git a/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java b/java/core/src/main/java/com/google/protobuf/CodedOutputStream.java
@@ -1264,16 +1264,34 @@ public final void writeInt32NoTag(int value) throws IOException {
 
     @Override
     public final void writeUInt32NoTag(int value) throws IOException {
-      if (HAS_UNSAFE_ARRAY_OPERATIONS && spaceLeft() >= MAX_VARINT32_SIZE) {
-        while (true) {
-          if ((value & ~0x7F) == 0) {
-            UnsafeUtil.putByte(buffer, position++, (byte) value);
-            return;
-          } else {
-            UnsafeUtil.putByte(buffer, position++, (byte) ((value & 0x7F) | 0x80));
-            value >>>= 7;
-          }
+      if (HAS_UNSAFE_ARRAY_OPERATIONS
+          && !Android.isOnAndroidDevice()
+          && spaceLeft() >= MAX_VARINT32_SIZE) {
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
+        }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
+        }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
+        }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        if ((value & ~0x7F) == 0) {
+          UnsafeUtil.putByte(buffer, position++, (byte) value);
+          return;
         }
+        UnsafeUtil.putByte(buffer, position++, (byte) (value | 0x80));
+        value >>>= 7;
+        UnsafeUtil.putByte(buffer, position++, (byte) value);
       } else {
         try {
           while (true) {

diff --git a/java/core/src/main/java/com/google/protobuf/Extension.java b/java/core/src/main/java/com/google/protobuf/Extension.java
@@ -30,13 +30,19 @@
 
 package com.google.protobuf;
 
+// TODO(chrisn): Change ContainingType to extend Message
 /**
  * Interface that generated extensions implement.
  *
  * @author liujisi@google.com (Jisi Liu)
  */
 public abstract class Extension<ContainingType extends MessageLite, Type>
     extends ExtensionLite<ContainingType, Type> {
+  // TODO(chrisn): Add package-private constructor.
+
+  /** {@inheritDoc} Overridden to return {@link Message} instead of {@link MessageLite}. */
+  @Override
+  public abstract Message getMessageDefaultInstance();
 
   /** Returns the descriptor of the extension. */
   public abstract Descriptors.FieldDescriptor getDescriptor();

diff --git a/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java b/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java
@@ -448,22 +448,26 @@ public static Builder newBuilder() {
     }
 
     /**
-     * Find a type by its full name. Returns null if it cannot be found in
-     * this {@link TypeRegistry}.
+     * Find a type by its full name. Returns null if it cannot be found in this {@link
+     * TypeRegistry}.
      */
     public Descriptor find(String name) {
       return types.get(name);
     }
 
+    /* @Nullable */
+    Descriptor getDescriptorForTypeUrl(String typeUrl) throws InvalidProtocolBufferException {
+      return find(getTypeName(typeUrl));
+    }
+
     private final Map<String, Descriptor> types;
 
     private TypeRegistry(Map<String, Descriptor> types) {
       this.types = types;
     }
 
-    /**
-     * A Builder is used to build {@link TypeRegistry}.
-     */
+
+    /** A Builder is used to build {@link TypeRegistry}. */
     public static class Builder {
       private Builder() {}
 
@@ -801,15 +805,14 @@ private void printAny(MessageOrBuilder message) throws IOException {
         throw new InvalidProtocolBufferException("Invalid Any type.");
       }
       String typeUrl = (String) message.getField(typeUrlField);
-      String typeName = getTypeName(typeUrl);
-      Descriptor type = registry.find(typeName);
+      Descriptor type = registry.getDescriptorForTypeUrl(typeUrl);
       if (type == null) {
         throw new InvalidProtocolBufferException("Cannot find type for url: " + typeUrl);
       }
       ByteString content = (ByteString) message.getField(valueField);
       Message contentMessage =
           DynamicMessage.getDefaultInstance(type).getParserForType().parseFrom(content);
-      WellKnownTypePrinter printer = wellKnownTypePrinters.get(typeName);
+      WellKnownTypePrinter printer = wellKnownTypePrinters.get(getTypeName(typeUrl));
       if (printer != null) {
         // If the type is one of the well-known types, we use a special
         // formatting.
@@ -1443,7 +1446,7 @@ private void mergeAny(JsonElement json, Message.Builder builder)
         throw new InvalidProtocolBufferException("Missing type url when parsing: " + json);
       }
       String typeUrl = typeUrlElement.getAsString();
-      Descriptor contentType = registry.find(getTypeName(typeUrl));
+      Descriptor contentType = registry.getDescriptorForTypeUrl(typeUrl);
       if (contentType == null) {
         throw new InvalidProtocolBufferException("Cannot resolve type: " + typeUrl);
       }

diff --git a/java/util/src/main/java/com/google/protobuf/util/Timestamps.java b/java/util/src/main/java/com/google/protobuf/util/Timestamps.java
@@ -112,8 +112,9 @@ public int compare(Timestamp t1, Timestamp t2) {
       };
 
   /**
-   * Returns a {@link Comparator} for {@link Timestamp}s which sorts in increasing chronological
-   * order. Nulls and invalid {@link Timestamp}s are not allowed (see {@link #isValid}).
+   * Returns a {@link Comparator} for {@link Timestamp Timestamps} which sorts in increasing
+   * chronological order. Nulls and invalid {@link Timestamp Timestamps} are not allowed (see
+   * {@link #isValid}).
    */
   public static Comparator<Timestamp> comparator() {
     return COMPARATOR;

diff --git a/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java b/java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java
@@ -34,6 +34,7 @@
 import com.google.protobuf.BoolValue;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.BytesValue;
+import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.DoubleValue;
 import com.google.protobuf.FloatValue;
@@ -834,6 +835,7 @@ public void testStruct() throws Exception {
     assertRoundTripEquals(message);
   }
 
+
   public void testAnyFields() throws Exception {
     TestAllTypes content = TestAllTypes.newBuilder().setOptionalInt32(1234).build();
     TestAny message = TestAny.newBuilder().setAnyValue(Any.pack(content)).build();