From dd4c50711096ad26a8af9d87a647c1ef9a916bcc Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 25 Jul 2018 21:46:28 +0000 Subject: [PATCH 1/6] Give a unique category to each test. This change introduce a TestCategory enum to ConformanceRequest. Existing tests are divided into three categories: binary format test, json format test and json format (ignore unknown when parsing) test. For the previous two categories, there is no change to existing testee programs. For tests with the last category, testee programs should either enable ignoring unknown field during json parsing or skip the test. --- conformance/ConformanceJava.java | 7 +++++-- conformance/conformance.proto | 13 ++++++++++++- conformance/conformance_cpp.cc | 8 +++++++- conformance/conformance_php.php | 6 +++++- conformance/conformance_test.cc | 18 ++++++++++++++---- conformance/conformance_test.h | 10 +++------- conformance/failure_list_cpp.txt | 6 ------ conformance/failure_list_java.txt | 6 ------ 8 files changed, 46 insertions(+), 28 deletions(-) diff --git a/conformance/ConformanceJava.java b/conformance/ConformanceJava.java index 596d113a9240b..862a67b70b2bf 100644 --- a/conformance/ConformanceJava.java +++ b/conformance/ConformanceJava.java @@ -235,8 +235,11 @@ private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest re try { TestMessagesProto3.TestAllTypesProto3.Builder builder = TestMessagesProto3.TestAllTypesProto3.newBuilder(); - JsonFormat.parser().usingTypeRegistry(typeRegistry) - .merge(request.getJsonPayload(), builder); + JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry); + if (request.getTestCategory() == JSON_IGNORE_UNKNOWN_PARSING_TEST) { + parser = parser.ignoringUnknownFields(); + } + parser.merge(request.getJsonPayload(), builder); testMessage = builder.build(); } catch (InvalidProtocolBufferException e) { return Conformance.ConformanceResponse.newBuilder().setParseError(e.getMessage()).build(); diff --git a/conformance/conformance.proto b/conformance/conformance.proto index 897e7b387912a..bac8b3249c38b 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -57,6 +57,14 @@ enum WireFormat { JSON = 2; } +enum TestCategory { + BINARY_TEST = 0; // Test binary wire format. + JSON_TEST = 1; // Test json wire format. + JSON_IGNORE_UNKNOWN_PARSING_TEST = 2; // Similar to JSON_TEST. However, + // during parsing json, testee should + // ignore unknown fields. +} + // Represents a single test case's input. The testee should: // // 1. parse this proto (which should always succeed) @@ -83,7 +91,10 @@ message ConformanceRequest { // protobuf_test_messages.proto2.TestAllTypesProto2. string message_type = 4; - bool ignore_unknown_json = 5; + // Each test is given a unique test category. Some category may need spedific + // support in testee programs. Refer to the defintion of TestCategory for + // more information. + TestCategory test_category = 5; } // Represents a single test case's output. diff --git a/conformance/conformance_cpp.cc b/conformance/conformance_cpp.cc index 97ae1a7a209b5..ac2f6dea1bcc5 100644 --- a/conformance/conformance_cpp.cc +++ b/conformance/conformance_cpp.cc @@ -46,6 +46,7 @@ using google::protobuf::DescriptorPool; using google::protobuf::Message; using google::protobuf::MessageFactory; using google::protobuf::util::BinaryToJsonString; +using google::protobuf::util::JsonParseOptions; using google::protobuf::util::JsonToBinaryString; using google::protobuf::util::NewTypeResolverForDescriptorPool; using google::protobuf::util::Status; @@ -112,8 +113,13 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) { case ConformanceRequest::kJsonPayload: { string proto_binary; + JsonParseOptions options; + options.ignore_unknown_fields = + (request.test_category() == + conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST); Status status = JsonToBinaryString(type_resolver, *type_url, - request.json_payload(), &proto_binary); + request.json_payload(), &proto_binary, + options); if (!status.ok()) { response->set_parse_error(string("Parse error: ") + status.error_message().as_string()); diff --git a/conformance/conformance_php.php b/conformance/conformance_php.php index 65483e274fe6c..799cc3e361427 100755 --- a/conformance/conformance_php.php +++ b/conformance/conformance_php.php @@ -3,6 +3,7 @@ require_once("Conformance/WireFormat.php"); require_once("Conformance/ConformanceResponse.php"); require_once("Conformance/ConformanceRequest.php"); +require_once("Conformance/TestCategory.php"); require_once("Protobuf_test_messages/Proto3/ForeignMessage.php"); require_once("Protobuf_test_messages/Proto3/ForeignEnum.php"); require_once("Protobuf_test_messages/Proto3/TestAllTypesProto3.php"); @@ -12,6 +13,7 @@ require_once("GPBMetadata/Conformance.php"); require_once("GPBMetadata/Google/Protobuf/TestMessagesProto3.php"); +use \Conformance\TestCategory; use \Conformance\WireFormat; if (!ini_get("date.timezone")) { @@ -39,7 +41,9 @@ function doTest($request) trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR); } } elseif ($request->getPayload() == "json_payload") { - $ignore_json_unknown = $request->getIgnoreUnknownJson(); + $ignore_json_unknown = + ($request->getTestCategory() == + TestCategory::JSON_IGNORE_UNKNOWN_PARSING_TEST); try { $test_message->mergeFromJsonString($request->getJsonPayload(), $ignore_json_unknown); diff --git a/conformance/conformance_test.cc b/conformance/conformance_test.cc index d9e88ce5eb9d7..74342b6845770 100644 --- a/conformance/conformance_test.cc +++ b/conformance/conformance_test.cc @@ -193,10 +193,11 @@ namespace protobuf { ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( ConformanceLevel level, conformance::WireFormat input_format, - conformance::WireFormat output_format, bool is_proto3, + conformance::WireFormat output_format, + conformance::TestCategory test_category, + bool is_proto3, const string& test_name, const string& input) - : level_(level), input_format_(input_format), - output_format_(output_format), is_proto3_(is_proto3) { + : level_(level), is_proto3_(is_proto3) { auto newTestMessage = [&is_proto3]() { Message* newMessage; if (is_proto3) { @@ -243,6 +244,8 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting( GOOGLE_LOG(FATAL) << "Unspecified output format"; } + request_.set_test_category(test_category); + test_name_ = ConformanceLevelToString(level) + rname + input_format_string + test_name + output_format_string; @@ -465,6 +468,7 @@ void ConformanceTestSuite::ExpectParseFailureForProtoWithProtoVersion ( ConformanceRequest request; ConformanceResponse response; request.set_protobuf_payload(proto); + request.set_test_category(conformance::BINARY_TEST); if (isProto3) { request.set_message_type("protobuf_test_messages.proto3.TestAllTypesProto3"); } else { @@ -511,11 +515,13 @@ void ConformanceTestSuite::RunValidJsonTest( const string& equivalent_text_format) { ConformanceRequestSetting setting1( level, conformance::JSON, conformance::PROTOBUF, + conformance::JSON_TEST, true, test_name, input_json); RunValidInputTest(setting1, equivalent_text_format); ConformanceRequestSetting setting2( level, conformance::JSON, conformance::JSON, + conformance::JSON_TEST, true, test_name, input_json); RunValidInputTest(setting2, equivalent_text_format); } @@ -525,6 +531,7 @@ void ConformanceTestSuite::RunValidJsonTestWithProtobufInput( const string& equivalent_text_format) { ConformanceRequestSetting setting( level, conformance::PROTOBUF, conformance::JSON, + conformance::JSON_TEST, true, test_name, input.SerializeAsString()); RunValidInputTest(setting, equivalent_text_format); } @@ -534,8 +541,8 @@ void ConformanceTestSuite::RunValidJsonIgnoreUnknownTest( const string& equivalent_text_format) { ConformanceRequestSetting setting( level, conformance::JSON, conformance::PROTOBUF, + conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST, true, test_name, input_json); - setting.SetIgnoreUnknownJson(true); RunValidInputTest(setting, equivalent_text_format); } @@ -545,12 +552,14 @@ void ConformanceTestSuite::RunValidProtobufTest( bool isProto3) { ConformanceRequestSetting setting1( level, conformance::PROTOBUF, conformance::PROTOBUF, + conformance::BINARY_TEST, isProto3, test_name, input_protobuf); RunValidInputTest(setting1, equivalent_text_format); if (isProto3) { ConformanceRequestSetting setting2( level, conformance::PROTOBUF, conformance::JSON, + conformance::BINARY_TEST, true, test_name, input_protobuf); RunValidInputTest(setting2, equivalent_text_format); } @@ -561,6 +570,7 @@ void ConformanceTestSuite::RunValidBinaryProtobufTest( const string& input_protobuf, bool isProto3) { ConformanceRequestSetting setting( level, conformance::PROTOBUF, conformance::PROTOBUF, + conformance::BINARY_TEST, isProto3, test_name, input_protobuf); RunValidBinaryInputTest(setting, input_protobuf); } diff --git a/conformance/conformance_test.h b/conformance/conformance_test.h index 685f67fbf2527..2044896f681d1 100644 --- a/conformance/conformance_test.h +++ b/conformance/conformance_test.h @@ -152,7 +152,9 @@ class ConformanceTestSuite { public: ConformanceRequestSetting( ConformanceLevel level, conformance::WireFormat input_format, - conformance::WireFormat output_format, bool is_proto3, + conformance::WireFormat output_format, + conformance::TestCategory test_category, + bool is_proto3, const string& test_name, const string& input); Message* GetTestMessage() const; @@ -169,14 +171,8 @@ class ConformanceTestSuite { return level_; } - void SetIgnoreUnknownJson(bool ignore_unknown_json) { - request_.set_ignore_unknown_json(ignore_unknown_json); - } - private: ConformanceLevel level_; - conformance::WireFormat input_format_; - conformance::WireFormat output_format_; bool is_proto3_; string test_name_; conformance::ConformanceRequest request_; diff --git a/conformance/failure_list_cpp.txt b/conformance/failure_list_cpp.txt index ea8e84738bc09..752fbb5d9b897 100644 --- a/conformance/failure_list_cpp.txt +++ b/conformance/failure_list_cpp.txt @@ -54,9 +54,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64 -Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt index 6f085c66dd19a..dc1f9ba5c9fa3 100644 --- a/conformance/failure_list_java.txt +++ b/conformance/failure_list_java.txt @@ -45,9 +45,3 @@ Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValu Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE -Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput From 666d1076fa902949f4a9f44ed20f400a3239f19f Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 25 Jul 2018 22:30:17 +0000 Subject: [PATCH 2/6] Fix python test --- conformance/conformance_python.py | 6 +- conformance/failure_list_python-post26.txt | 6 -- conformance/failure_list_python.txt | 6 -- conformance/failure_list_python_cpp.txt | 6 -- .../Conformance.cs | 76 ++++++++++++------- 5 files changed, 54 insertions(+), 46 deletions(-) diff --git a/conformance/conformance_python.py b/conformance/conformance_python.py index c5ba2467a6b54..876642bc79953 100755 --- a/conformance/conformance_python.py +++ b/conformance/conformance_python.py @@ -78,7 +78,11 @@ def do_test(request): elif request.WhichOneof('payload') == 'json_payload': try: - json_format.Parse(request.json_payload, test_message) + ignore_unknown_fields = \ + request.test_category == \ + conformance_pb2.JSON_IGNORE_UNKNOWN_PARSING_TEST + json_format.Parse(request.json_payload, test_message, + ignore_unknown_fields) except Exception as e: response.parse_error = str(e) return response diff --git a/conformance/failure_list_python-post26.txt b/conformance/failure_list_python-post26.txt index 60b1146e0cbed..19d99b044a43e 100644 --- a/conformance/failure_list_python-post26.txt +++ b/conformance/failure_list_python-post26.txt @@ -1,8 +1,2 @@ JsonInput.StringFieldSurrogateInWrongOrder JsonInput.StringFieldUnpairedHighSurrogate -Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_python.txt b/conformance/failure_list_python.txt index 5f01b53b65032..e3ce7af75906e 100644 --- a/conformance/failure_list_python.txt +++ b/conformance/failure_list_python.txt @@ -19,9 +19,3 @@ Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_0 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_1 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_2 Required.Proto3.ProtobufInput.IllegalZeroFieldNum_Case_3 -Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/conformance/failure_list_python_cpp.txt b/conformance/failure_list_python_cpp.txt index f80517d993804..a498ad1a3e62d 100644 --- a/conformance/failure_list_python_cpp.txt +++ b/conformance/failure_list_python_cpp.txt @@ -52,9 +52,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64 -Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.cs index 77284824dfc34..1be79d564e6a0 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.cs @@ -24,22 +24,24 @@ public static partial class ConformanceReflection { static ConformanceReflection() { byte[] descriptorData = global::System.Convert.FromBase64String( string.Concat( - "ChFjb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2UiwAEKEkNvbmZvcm1h", + "ChFjb25mb3JtYW5jZS5wcm90bxILY29uZm9ybWFuY2Ui1QEKEkNvbmZvcm1h", "bmNlUmVxdWVzdBIaChBwcm90b2J1Zl9wYXlsb2FkGAEgASgMSAASFgoManNv", "bl9wYXlsb2FkGAIgASgJSAASOAoXcmVxdWVzdGVkX291dHB1dF9mb3JtYXQY", "AyABKA4yFy5jb25mb3JtYW5jZS5XaXJlRm9ybWF0EhQKDG1lc3NhZ2VfdHlw", - "ZRgEIAEoCRIbChNpZ25vcmVfdW5rbm93bl9qc29uGAUgASgIQgkKB3BheWxv", - "YWQisQEKE0NvbmZvcm1hbmNlUmVzcG9uc2USFQoLcGFyc2VfZXJyb3IYASAB", - "KAlIABIZCg9zZXJpYWxpemVfZXJyb3IYBiABKAlIABIXCg1ydW50aW1lX2Vy", - "cm9yGAIgASgJSAASGgoQcHJvdG9idWZfcGF5bG9hZBgDIAEoDEgAEhYKDGpz", - "b25fcGF5bG9hZBgEIAEoCUgAEhEKB3NraXBwZWQYBSABKAlIAEIICgZyZXN1", - "bHQqNQoKV2lyZUZvcm1hdBIPCgtVTlNQRUNJRklFRBAAEgwKCFBST1RPQlVG", - "EAESCAoESlNPThACQiEKH2NvbS5nb29nbGUucHJvdG9idWYuY29uZm9ybWFu", - "Y2ViBnByb3RvMw==")); + "ZRgEIAEoCRIwCg10ZXN0X2NhdGVnb3J5GAUgASgOMhkuY29uZm9ybWFuY2Uu", + "VGVzdENhdGVnb3J5QgkKB3BheWxvYWQisQEKE0NvbmZvcm1hbmNlUmVzcG9u", + "c2USFQoLcGFyc2VfZXJyb3IYASABKAlIABIZCg9zZXJpYWxpemVfZXJyb3IY", + "BiABKAlIABIXCg1ydW50aW1lX2Vycm9yGAIgASgJSAASGgoQcHJvdG9idWZf", + "cGF5bG9hZBgDIAEoDEgAEhYKDGpzb25fcGF5bG9hZBgEIAEoCUgAEhEKB3Nr", + "aXBwZWQYBSABKAlIAEIICgZyZXN1bHQqNQoKV2lyZUZvcm1hdBIPCgtVTlNQ", + "RUNJRklFRBAAEgwKCFBST1RPQlVGEAESCAoESlNPThACKlQKDFRlc3RDYXRl", + "Z29yeRIPCgtCSU5BUllfVEVTVBAAEg0KCUpTT05fVEVTVBABEiQKIEpTT05f", + "SUdOT1JFX1VOS05PV05fUEFSU0lOR19URVNUEAJCIQofY29tLmdvb2dsZS5w", + "cm90b2J1Zi5jb25mb3JtYW5jZWIGcHJvdG8z")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, - new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), }, new pbr::GeneratedClrTypeInfo[] { - new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "RequestedOutputFormat", "MessageType", "IgnoreUnknownJson" }, new[]{ "Payload" }, null, null), + new pbr::GeneratedClrTypeInfo(new[] {typeof(global::Conformance.WireFormat), typeof(global::Conformance.TestCategory), }, new pbr::GeneratedClrTypeInfo[] { + new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceRequest), global::Conformance.ConformanceRequest.Parser, new[]{ "ProtobufPayload", "JsonPayload", "RequestedOutputFormat", "MessageType", "TestCategory" }, new[]{ "Payload" }, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Conformance.ConformanceResponse), global::Conformance.ConformanceResponse.Parser, new[]{ "ParseError", "SerializeError", "RuntimeError", "ProtobufPayload", "JsonPayload", "Skipped" }, new[]{ "Result" }, null, null) })); } @@ -53,6 +55,21 @@ public enum WireFormat { [pbr::OriginalName("JSON")] Json = 2, } + public enum TestCategory { + /// + /// Test binary wire format. + /// + [pbr::OriginalName("BINARY_TEST")] BinaryTest = 0, + /// + /// Test json wire format. + /// + [pbr::OriginalName("JSON_TEST")] JsonTest = 1, + /// + /// Similar to JSON_TEST. However, + /// + [pbr::OriginalName("JSON_IGNORE_UNKNOWN_PARSING_TEST")] JsonIgnoreUnknownParsingTest = 2, + } + #endregion #region Messages @@ -90,7 +107,7 @@ public ConformanceRequest() { public ConformanceRequest(ConformanceRequest other) : this() { requestedOutputFormat_ = other.requestedOutputFormat_; messageType_ = other.messageType_; - ignoreUnknownJson_ = other.ignoreUnknownJson_; + testCategory_ = other.testCategory_; switch (other.PayloadCase) { case PayloadOneofCase.ProtobufPayload: ProtobufPayload = other.ProtobufPayload; @@ -160,14 +177,19 @@ public string MessageType { } } - /// Field number for the "ignore_unknown_json" field. - public const int IgnoreUnknownJsonFieldNumber = 5; - private bool ignoreUnknownJson_; + /// Field number for the "test_category" field. + public const int TestCategoryFieldNumber = 5; + private global::Conformance.TestCategory testCategory_ = 0; + /// + /// Each test is given a unique test category. Some category may need spedific + /// support in testee programs. Refer to the defintion of TestCategory for + /// more information. + /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public bool IgnoreUnknownJson { - get { return ignoreUnknownJson_; } + public global::Conformance.TestCategory TestCategory { + get { return testCategory_; } set { - ignoreUnknownJson_ = value; + testCategory_ = value; } } @@ -207,7 +229,7 @@ public bool Equals(ConformanceRequest other) { if (JsonPayload != other.JsonPayload) return false; if (RequestedOutputFormat != other.RequestedOutputFormat) return false; if (MessageType != other.MessageType) return false; - if (IgnoreUnknownJson != other.IgnoreUnknownJson) return false; + if (TestCategory != other.TestCategory) return false; if (PayloadCase != other.PayloadCase) return false; return Equals(_unknownFields, other._unknownFields); } @@ -219,7 +241,7 @@ public override int GetHashCode() { if (payloadCase_ == PayloadOneofCase.JsonPayload) hash ^= JsonPayload.GetHashCode(); if (RequestedOutputFormat != 0) hash ^= RequestedOutputFormat.GetHashCode(); if (MessageType.Length != 0) hash ^= MessageType.GetHashCode(); - if (IgnoreUnknownJson != false) hash ^= IgnoreUnknownJson.GetHashCode(); + if (TestCategory != 0) hash ^= TestCategory.GetHashCode(); hash ^= (int) payloadCase_; if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); @@ -250,9 +272,9 @@ public void WriteTo(pb::CodedOutputStream output) { output.WriteRawTag(34); output.WriteString(MessageType); } - if (IgnoreUnknownJson != false) { + if (TestCategory != 0) { output.WriteRawTag(40); - output.WriteBool(IgnoreUnknownJson); + output.WriteEnum((int) TestCategory); } if (_unknownFields != null) { _unknownFields.WriteTo(output); @@ -274,8 +296,8 @@ public int CalculateSize() { if (MessageType.Length != 0) { size += 1 + pb::CodedOutputStream.ComputeStringSize(MessageType); } - if (IgnoreUnknownJson != false) { - size += 1 + 1; + if (TestCategory != 0) { + size += 1 + pb::CodedOutputStream.ComputeEnumSize((int) TestCategory); } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); @@ -294,8 +316,8 @@ public void MergeFrom(ConformanceRequest other) { if (other.MessageType.Length != 0) { MessageType = other.MessageType; } - if (other.IgnoreUnknownJson != false) { - IgnoreUnknownJson = other.IgnoreUnknownJson; + if (other.TestCategory != 0) { + TestCategory = other.TestCategory; } switch (other.PayloadCase) { case PayloadOneofCase.ProtobufPayload: @@ -334,7 +356,7 @@ public void MergeFrom(pb::CodedInputStream input) { break; } case 40: { - IgnoreUnknownJson = input.ReadBool(); + testCategory_ = (global::Conformance.TestCategory) input.ReadEnum(); break; } } From 142eb938eda4cd858f927ecf9d5c0a011a1958f8 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Wed, 25 Jul 2018 23:46:26 +0000 Subject: [PATCH 3/6] Fix java --- conformance/ConformanceJava.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conformance/ConformanceJava.java b/conformance/ConformanceJava.java index 862a67b70b2bf..dcdc27e3c18e5 100644 --- a/conformance/ConformanceJava.java +++ b/conformance/ConformanceJava.java @@ -236,7 +236,8 @@ private Conformance.ConformanceResponse doTest(Conformance.ConformanceRequest re TestMessagesProto3.TestAllTypesProto3.Builder builder = TestMessagesProto3.TestAllTypesProto3.newBuilder(); JsonFormat.Parser parser = JsonFormat.parser().usingTypeRegistry(typeRegistry); - if (request.getTestCategory() == JSON_IGNORE_UNKNOWN_PARSING_TEST) { + if (request.getTestCategory() + == Conformance.TestCategory.JSON_IGNORE_UNKNOWN_PARSING_TEST) { parser = parser.ignoringUnknownFields(); } parser.merge(request.getJsonPayload(), builder); From 1928e053d827e5bb82798f32f88759a70fa9716f Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Thu, 26 Jul 2018 01:08:59 +0000 Subject: [PATCH 4/6] Fix csharp --- conformance/failure_list_csharp.txt | 6 ------ csharp/src/Google.Protobuf.Conformance/Program.cs | 3 +++ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/conformance/failure_list_csharp.txt b/conformance/failure_list_csharp.txt index 69c723b5536c0..2a20aa78e7ff4 100644 --- a/conformance/failure_list_csharp.txt +++ b/conformance/failure_list_csharp.txt @@ -1,8 +1,2 @@ Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonFalse.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNull.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonNumber.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonObject.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonString.ProtobufOutput -Required.Proto3.JsonInput.IgnoreUnknownJsonTrue.ProtobufOutput diff --git a/csharp/src/Google.Protobuf.Conformance/Program.cs b/csharp/src/Google.Protobuf.Conformance/Program.cs index 96dc354e99441..1eac00be2fca5 100644 --- a/csharp/src/Google.Protobuf.Conformance/Program.cs +++ b/csharp/src/Google.Protobuf.Conformance/Program.cs @@ -87,6 +87,9 @@ private static ConformanceResponse PerformRequest(ConformanceRequest request, Ty switch (request.PayloadCase) { case ConformanceRequest.PayloadOneofCase.JsonPayload: + if (request.TestCategory == global::Conformance.TestCategory.JsonIgnoreUnknownParsingTest) { + return new ConformanceResponse { Skipped = "CSharp doesn't support skipping unknown fields in json parsing." }; + } var parser = new JsonParser(new JsonParser.Settings(20, typeRegistry)); message = parser.Parse(request.JsonPayload); break; From f6eae58058d81533116c07d3d71730bc7f9ecb99 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Thu, 26 Jul 2018 17:26:56 +0000 Subject: [PATCH 5/6] Update document --- conformance/conformance.proto | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/conformance/conformance.proto b/conformance/conformance.proto index bac8b3249c38b..d838e5780d47f 100644 --- a/conformance/conformance.proto +++ b/conformance/conformance.proto @@ -60,9 +60,12 @@ enum WireFormat { enum TestCategory { BINARY_TEST = 0; // Test binary wire format. JSON_TEST = 1; // Test json wire format. - JSON_IGNORE_UNKNOWN_PARSING_TEST = 2; // Similar to JSON_TEST. However, - // during parsing json, testee should - // ignore unknown fields. + // Similar to JSON_TEST. However, during parsing json, testee should ignore + // unknown fields. This feature is optional. Each implementation can descide + // whether to support it. See + // https://developers.google.com/protocol-buffers/docs/proto3#json_options + // for more detail. + JSON_IGNORE_UNKNOWN_PARSING_TEST = 2; } // Represents a single test case's input. The testee should: @@ -91,7 +94,7 @@ message ConformanceRequest { // protobuf_test_messages.proto2.TestAllTypesProto2. string message_type = 4; - // Each test is given a unique test category. Some category may need spedific + // Each test is given a specific test category. Some category may need spedific // support in testee programs. Refer to the defintion of TestCategory for // more information. TestCategory test_category = 5; From 929ba6af12938afac6d00853c91738f0cd4ee0d2 Mon Sep 17 00:00:00 2001 From: Bo Yang Date: Thu, 26 Jul 2018 21:09:01 +0000 Subject: [PATCH 6/6] Update csharp generated code --- csharp/src/Google.Protobuf.Conformance/Conformance.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/csharp/src/Google.Protobuf.Conformance/Conformance.cs b/csharp/src/Google.Protobuf.Conformance/Conformance.cs index 1be79d564e6a0..6ca52c32af8cd 100644 --- a/csharp/src/Google.Protobuf.Conformance/Conformance.cs +++ b/csharp/src/Google.Protobuf.Conformance/Conformance.cs @@ -65,7 +65,11 @@ public enum TestCategory { /// [pbr::OriginalName("JSON_TEST")] JsonTest = 1, /// - /// Similar to JSON_TEST. However, + /// Similar to JSON_TEST. However, during parsing json, testee should ignore + /// unknown fields. This feature is optional. Each implementation can descide + /// whether to support it. See + /// https://developers.google.com/protocol-buffers/docs/proto3#json_options + /// for more detail. /// [pbr::OriginalName("JSON_IGNORE_UNKNOWN_PARSING_TEST")] JsonIgnoreUnknownParsingTest = 2, } @@ -181,7 +185,7 @@ public string MessageType { public const int TestCategoryFieldNumber = 5; private global::Conformance.TestCategory testCategory_ = 0; /// - /// Each test is given a unique test category. Some category may need spedific + /// Each test is given a specific test category. Some category may need spedific /// support in testee programs. Refer to the defintion of TestCategory for /// more information. ///