[Python-experimental] JSON schema 'null' type should be modeled as 'n…

…one_type' (#6121)

* Handle null type

* Handle null type

* Handle null type. Add 'null' type in the OAS document for testing purpose

* Handle null type. Add 'null' type in the OAS document for testing purpose

* Handle null type. Add 'null' type in the OAS document for testing purpose

* Handle null type. Add 'null' type in the OAS document for testing purpose

* Handle null type. Add 'null' type in the OAS document for testing purpose

* Handle null type. Add 'null' type in the OAS document for testing purpose

* improve documentation

* Handle 'null' type

* Handle 'null' type. Add unit tests

* Add NullType for go

* Add NullType for go

* fix modeling of AnyType for go-experimental

* execute scripts in bin directory

* Add review comments

* Add 'null' type in oneOf

* Improve OAS YAML file for golang openapi3 samples

* 'Any type' includes the null value, so 'isNullable' should be set to TRUE

* 'Any type' includes the null value, so 'isNullable' should be set to TRUE

* Handle AnyType and NullType

* handle anytype for go-experimental

* Log warning instead of error

* anyOf/oneOf

* Change x-golang-is-container extension to x-golang-has-wrapper

* Add code comments

* Handle Object and any type

* Handle Object and any type

* Handle object and any type

* add code comments

* handle additional properties

* handle additional properties

* handle additional properties

* handle anytype and objecttype for go-exerimental

* Move golang changes to a separate branch

* Move golang changes to a separate branch

* Better names for the OAS document test properties

* Move golang changes to a separate branch

* Run samples scripts

* Run samples scripts

* fix unit test issues

* Handle none type

* Fix index out of range exception

* fix formatting issues

* fix formatting issues

* fix formatting issues. Finally figured out how to check formatting in local workspace

* fix formatting issues

* run samples scripts

modules/openapi-generator/src/main/java/org/openapitools/codegen/CodegenProperty.java

@@ -134,7 +134,9 @@ public class CodegenProperty implements Cloneable, IJsonSchemaValidationProperti
     public boolean isUri;
     public boolean isEmail;
     /**
-     * The type is a free-form object, i.e. it is a map of string to values with no declared properties
+     * The type is a free-form object, i.e. it is a map of string to values with no declared properties.
+     * A OAS free-form schema may include the 'additionalProperties' attribute, which puts a constraint
+     * on the type of the undeclared properties.
      */
     public boolean isFreeFormObject;
     /**
@@ -153,6 +155,9 @@ public class CodegenProperty implements Cloneable, IJsonSchemaValidationProperti
     public boolean isDiscriminator;
     public List<String> _enum;
     public Map<String, Object> allowableValues;
+    // If 'additionalProperties' is not set, items is null.
+    // If 'additionalProperties' is set to a type or refers to a type, 'items' provides the type information for
+    // the undeclared properties. 
     public CodegenProperty items;
     public CodegenProperty mostInnerItems;
     public Map<String, Object> vendorExtensions = new HashMap<String, Object>();

modules/openapi-generator/src/main/java/org/openapitools/codegen/DefaultCodegen.java

@@ -2180,7 +2180,14 @@ public CodegenModel fromModel(String name, Schema schema) {
             m.xmlNamespace = schema.getXml().getNamespace();
             m.xmlName = schema.getXml().getName();
         }
-
+        if (ModelUtils.isAnyTypeSchema(schema)) {
+            // The 'null' value is allowed when the OAS schema is 'any type'.
+            // See https://github.com/OAI/OpenAPI-Specification/issues/1389
+            if (Boolean.FALSE.equals(schema.getNullable())) {
+                LOGGER.error("Schema '{}' is any type, which includes the 'null' value. 'nullable' cannot be set to 'false'", name);
+            }
+            m.isNullable = true;
+        }
         if (ModelUtils.isArraySchema(schema)) {
             m.isArrayModel = true;
             m.arrayModelType = fromProperty(name, schema).complexType;
@@ -3030,6 +3037,12 @@ public CodegenProperty fromProperty(String name, Schema p) {
         } else if (ModelUtils.isFreeFormObject(p)) {
             property.isFreeFormObject = true;
         } else if (ModelUtils.isAnyTypeSchema(p)) {
+            // The 'null' value is allowed when the OAS schema is 'any type'.
+            // See https://github.com/OAI/OpenAPI-Specification/issues/1389
+            if (Boolean.FALSE.equals(p.getNullable())) {
+                LOGGER.warn("Schema '{}' is any type, which includes the 'null' value. 'nullable' cannot be set to 'false'", p.getName());
+            }
+            property.isNullable = true;
             property.isAnyType = true;
         } else if (ModelUtils.isArraySchema(p)) {
             // default to string if inner item is undefined

modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientCodegen.java

@@ -143,6 +143,7 @@ public PythonClientCodegen() {
         // map uuid to string for the time being
         typeMapping.put("UUID", "str");
         typeMapping.put("URI", "str");
+        typeMapping.put("null", "none_type");
 
         // from https://docs.python.org/3/reference/lexical_analysis.html#keywords
         setReservedWordsLowerCase(

modules/openapi-generator/src/main/java/org/openapitools/codegen/languages/PythonClientExperimentalCodegen.java

@@ -388,7 +388,9 @@ public Map<String, Object> postProcessAllModels(Map<String, Object> objs) {
                 composedSchemaSets.add(cm.oneOf);
                 for (Set<String> importSet : composedSchemaSets) {
                     for (String otherModelName : importSet) {
-                        cm.imports.add(otherModelName);
+                        if (!languageSpecificPrimitives.contains(otherModelName)) {
+                            cm.imports.add(otherModelName);
+                        }
                     }
                 }
 

modules/openapi-generator/src/main/java/org/openapitools/codegen/utils/OneOfImplementorAdditionalData.java

@@ -75,9 +75,11 @@ public void addFromInterfaceModel(CodegenModel cm, List<Map<String, String>> mod
         // note that we can't just toAdd.removeAll(m.vars) for every interfaceModel,
         // as they might have different value of `hasMore` and thus are not equal
         List<String> omitAdding = new ArrayList<String>();
-        for (CodegenModel m : cm.interfaceModels) {
-            for (CodegenProperty v : m.vars) {
-                omitAdding.add(v.baseName);
+        if (cm.interfaceModels != null) {
+            for (CodegenModel m : cm.interfaceModels) {
+                for (CodegenProperty v : m.vars) {
+                    omitAdding.add(v.baseName);
+                }
             }
         }
         for (CodegenProperty v : toAdd) {

modules/openapi-generator/src/main/resources/python/python-experimental/model_utils.mustache

@@ -1153,7 +1153,7 @@ def get_oneof_instance(self, model_args, constant_args):
             and path to item.
 
     Returns
-        oneof_instance (instance/None)
+        oneof_instance (instance)
     """
     if len(self._composed_schemas['oneOf']) == 0:
         return None
@@ -1162,6 +1162,13 @@ def get_oneof_instance(self, model_args, constant_args):
     # Iterate over each oneOf schema and determine if the input data
     # matches the oneOf schemas.
     for oneof_class in self._composed_schemas['oneOf']:
+        # The composed oneOf schema allows the 'null' type and the input data
+        # is the null value. This is a OAS >= 3.1 feature.
+        if oneof_class is none_type:
+            # skip none_types because we are deserializing dict data.
+            # none_type deserialization is handled in the __new__ method
+            continue
+
         # transform js keys from input data to python keys in fixed_model_args
         fixed_model_args = change_keys_js_to_python(
             model_args, oneof_class)
@@ -1207,9 +1214,11 @@ def get_anyof_instances(self, model_args, constant_args):
     Args:
         self: the class we are handling
         model_args (dict): var_name to var_value
-            used to make instances
+            The input data, e.g. the payload that must match at least one
+            anyOf child schema in the OpenAPI document.
         constant_args (dict): var_name to var_value
-            used to make instances
+            args that every model requires, including configuration, server
+            and path to item.
 
     Returns
         anyof_instances (list)
@@ -1219,6 +1228,13 @@ def get_anyof_instances(self, model_args, constant_args):
         return anyof_instances
 
     for anyof_class in self._composed_schemas['anyOf']:
+        # The composed oneOf schema allows the 'null' type and the input data
+        # is the null value. This is a OAS >= 3.1 feature.
+        if anyof_class is none_type:
+            # skip none_types because we are deserializing dict data.
+            # none_type deserialization is handled in the __new__ method
+            continue
+
         # transform js keys to python keys in fixed_model_args
         fixed_model_args = change_keys_js_to_python(model_args, anyof_class)
 

modules/openapi-generator/src/test/resources/3_0/python-experimental/petstore-with-fake-endpoints-models-for-testing-with-http-signature.yaml

@@ -1250,22 +1250,32 @@ components:
           type: integer
           format: int32
           description: User Status
-        arbitraryObject:
+        objectWithNoDeclaredProps:
           type: object
           description: test code generation for objects
             Value must be a map of strings to values. It cannot be the 'null' value.
-        arbitraryNullableObject:
+        objectWithNoDeclaredPropsNullable:
           type: object
           description: test code generation for nullable objects.
             Value must be a map of strings to values or the 'null' value.
           nullable: true
-        arbitraryTypeValue:
+        anyTypeProp:
           description: test code generation for any type
-            Value can be any type - string, number, boolean, array or object.
-        arbitraryNullableTypeValue:
+            Here the 'type' attribute is not specified, which means the value can be anything,
+            including the null value, string, number, boolean, array or object.
+            See https://github.com/OAI/OpenAPI-Specification/issues/1389
+        # TODO: this should be supported, currently there are some issues in the code generation.
+        #anyTypeExceptNullProp:
+        #  description: any type except 'null'
+        #    Here the 'type' attribute is not specified, which means the value can be anything,
+        #    including the null value, string, number, boolean, array or object.
+        #  not:
+        #    type: 'null'
+        anyTypePropNullable:
           description: test code generation for any type
-            Value can be any type - string, number, boolean, array, object or
-            the 'null' value.
+            Here the 'type' attribute is not specified, which means the value can be anything,
+            including the null value, string, number, boolean, array or object.
+            The 'nullable' attribute does not change the allowed values.
           nullable: true
       xml:
         name: User
@@ -1859,6 +1869,7 @@ components:
           - $ref: '#/components/schemas/banana'
     fruitReq:
        oneOf:
+          - type: 'null'
           - $ref: '#/components/schemas/appleReq'
           - $ref: '#/components/schemas/bananaReq'
     appleReq:

samples/client/petstore/python-experimental/petstore_api/model_utils.py

@@ -1419,7 +1419,7 @@ def get_oneof_instance(self, model_args, constant_args):
             and path to item.
 
     Returns
-        oneof_instance (instance/None)
+        oneof_instance (instance)
     """
     if len(self._composed_schemas['oneOf']) == 0:
         return None
@@ -1428,6 +1428,13 @@ def get_oneof_instance(self, model_args, constant_args):
     # Iterate over each oneOf schema and determine if the input data
     # matches the oneOf schemas.
     for oneof_class in self._composed_schemas['oneOf']:
+        # The composed oneOf schema allows the 'null' type and the input data
+        # is the null value. This is a OAS >= 3.1 feature.
+        if oneof_class is none_type:
+            # skip none_types because we are deserializing dict data.
+            # none_type deserialization is handled in the __new__ method
+            continue
+
         # transform js keys from input data to python keys in fixed_model_args
         fixed_model_args = change_keys_js_to_python(
             model_args, oneof_class)
@@ -1473,9 +1480,11 @@ def get_anyof_instances(self, model_args, constant_args):
     Args:
         self: the class we are handling
         model_args (dict): var_name to var_value
-            used to make instances
+            The input data, e.g. the payload that must match at least one
+            anyOf child schema in the OpenAPI document.
         constant_args (dict): var_name to var_value
-            used to make instances
+            args that every model requires, including configuration, server
+            and path to item.
 
     Returns
         anyof_instances (list)
@@ -1485,6 +1494,13 @@ def get_anyof_instances(self, model_args, constant_args):
         return anyof_instances
 
     for anyof_class in self._composed_schemas['anyOf']:
+        # The composed oneOf schema allows the 'null' type and the input data
+        # is the null value. This is a OAS >= 3.1 feature.
+        if anyof_class is none_type:
+            # skip none_types because we are deserializing dict data.
+            # none_type deserialization is handled in the __new__ method
+            continue
+
         # transform js keys to python keys in fixed_model_args
         fixed_model_args = change_keys_js_to_python(model_args, anyof_class)
 

samples/openapi3/client/petstore/go-experimental/go-petstore/docs/User.md

@@ -321,6 +321,16 @@ SetArbitraryTypeValue sets ArbitraryTypeValue field to given value.
 
 HasArbitraryTypeValue returns a boolean if a field has been set.
 
+### SetArbitraryTypeValueNil
+
+`func (o *User) SetArbitraryTypeValueNil(b bool)`
+
+ SetArbitraryTypeValueNil sets the value for ArbitraryTypeValue to be an explicit nil
+
+### UnsetArbitraryTypeValue
+`func (o *User) UnsetArbitraryTypeValue()`
+
+UnsetArbitraryTypeValue ensures that no value is present for ArbitraryTypeValue, not even an explicit nil
 ### GetArbitraryNullableTypeValue
 
 `func (o *User) GetArbitraryNullableTypeValue() interface{}`

samples/openapi3/client/petstore/go-experimental/go-petstore/model_user.go

@@ -29,7 +29,7 @@ type User struct {
 	// test code generation for nullable objects. Value must be a map of strings to values or the 'null' value.
 	ArbitraryNullableObject map[string]interface{} `json:"arbitraryNullableObject,omitempty"`
 	// test code generation for any type Value can be any type - string, number, boolean, array or object.
-	ArbitraryTypeValue *interface{} `json:"arbitraryTypeValue,omitempty"`
+	ArbitraryTypeValue interface{} `json:"arbitraryTypeValue,omitempty"`
 	// test code generation for any type Value can be any type - string, number, boolean, array, object or the 'null' value.
 	ArbitraryNullableTypeValue interface{} `json:"arbitraryNullableTypeValue,omitempty"`
 }
@@ -372,22 +372,23 @@ func (o *User) SetArbitraryNullableObject(v map[string]interface{}) {
 	o.ArbitraryNullableObject = v
 }
 
-// GetArbitraryTypeValue returns the ArbitraryTypeValue field value if set, zero value otherwise.
+// GetArbitraryTypeValue returns the ArbitraryTypeValue field value if set, zero value otherwise (both if not set or set to explicit null).
 func (o *User) GetArbitraryTypeValue() interface{} {
-	if o == nil || o.ArbitraryTypeValue == nil {
+	if o == nil  {
 		var ret interface{}
 		return ret
 	}
-	return *o.ArbitraryTypeValue
+	return o.ArbitraryTypeValue
 }
 
 // GetArbitraryTypeValueOk returns a tuple with the ArbitraryTypeValue field value if set, nil otherwise
 // and a boolean to check if the value has been set.
+// NOTE: If the value is an explicit nil, `nil, true` will be returned
 func (o *User) GetArbitraryTypeValueOk() (*interface{}, bool) {
 	if o == nil || o.ArbitraryTypeValue == nil {
 		return nil, false
 	}
-	return o.ArbitraryTypeValue, true
+	return &o.ArbitraryTypeValue, true
 }
 
 // HasArbitraryTypeValue returns a boolean if a field has been set.
@@ -401,7 +402,7 @@ func (o *User) HasArbitraryTypeValue() bool {
 
 // SetArbitraryTypeValue gets a reference to the given interface{} and assigns it to the ArbitraryTypeValue field.
 func (o *User) SetArbitraryTypeValue(v interface{}) {
-	o.ArbitraryTypeValue = &v
+	o.ArbitraryTypeValue = v
 }
 
 // GetArbitraryNullableTypeValue returns the ArbitraryNullableTypeValue field value if set, zero value otherwise (both if not set or set to explicit null).

samples/openapi3/client/petstore/python-experimental/docs/User.md

@@ -11,10 +11,10 @@ Name | Type | Description | Notes
 **password** | **str** |  | [optional] 
 **phone** | **str** |  | [optional] 
 **user_status** | **int** | User Status | [optional] 
-**arbitrary_object** | **bool, date, datetime, dict, float, int, list, str** | test code generation for objects Value must be a map of strings to values. It cannot be the 'null' value. | [optional] 
-**arbitrary_nullable_object** | **bool, date, datetime, dict, float, int, list, str, none_type** | test code generation for nullable objects. Value must be a map of strings to values or the 'null' value. | [optional] 
-**arbitrary_type_value** | **bool, date, datetime, dict, float, int, list, str, none_type** | test code generation for any type Value can be any type - string, number, boolean, array or object. | [optional] 
-**arbitrary_nullable_type_value** | **bool, date, datetime, dict, float, int, list, str, none_type** | test code generation for any type Value can be any type - string, number, boolean, array, object or the 'null' value. | [optional] 
+**object_with_no_declared_props** | **bool, date, datetime, dict, float, int, list, str** | test code generation for objects Value must be a map of strings to values. It cannot be the 'null' value. | [optional] 
+**object_with_no_declared_props_nullable** | **bool, date, datetime, dict, float, int, list, str, none_type** | test code generation for nullable objects. Value must be a map of strings to values or the 'null' value. | [optional] 
+**any_type_prop** | **bool, date, datetime, dict, float, int, list, str, none_type** | test code generation for any type Here the 'type' attribute is not specified, which means the value can be anything, including the null value, string, number, boolean, array or object. See https://github.com/OAI/OpenAPI-Specification/issues/1389 | [optional] 
+**any_type_prop_nullable** | **bool, date, datetime, dict, float, int, list, str, none_type** | test code generation for any type Here the 'type' attribute is not specified, which means the value can be anything, including the null value, string, number, boolean, array or object. The 'nullable' attribute does not change the allowed values. | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)
 

samples/openapi3/client/petstore/python-experimental/petstore_api/model_utils.py

@@ -1419,7 +1419,7 @@ def get_oneof_instance(self, model_args, constant_args):
             and path to item.
 
     Returns
-        oneof_instance (instance/None)
+        oneof_instance (instance)
     """
     if len(self._composed_schemas['oneOf']) == 0:
         return None
@@ -1428,6 +1428,13 @@ def get_oneof_instance(self, model_args, constant_args):
     # Iterate over each oneOf schema and determine if the input data
     # matches the oneOf schemas.
     for oneof_class in self._composed_schemas['oneOf']:
+        # The composed oneOf schema allows the 'null' type and the input data
+        # is the null value. This is a OAS >= 3.1 feature.
+        if oneof_class is none_type:
+            # skip none_types because we are deserializing dict data.
+            # none_type deserialization is handled in the __new__ method
+            continue
+
         # transform js keys from input data to python keys in fixed_model_args
         fixed_model_args = change_keys_js_to_python(
             model_args, oneof_class)
@@ -1473,9 +1480,11 @@ def get_anyof_instances(self, model_args, constant_args):
     Args:
         self: the class we are handling
         model_args (dict): var_name to var_value
-            used to make instances
+            The input data, e.g. the payload that must match at least one
+            anyOf child schema in the OpenAPI document.
         constant_args (dict): var_name to var_value
-            used to make instances
+            args that every model requires, including configuration, server
+            and path to item.
 
     Returns
         anyof_instances (list)
@@ -1485,6 +1494,13 @@ def get_anyof_instances(self, model_args, constant_args):
         return anyof_instances
 
     for anyof_class in self._composed_schemas['anyOf']:
+        # The composed oneOf schema allows the 'null' type and the input data
+        # is the null value. This is a OAS >= 3.1 feature.
+        if anyof_class is none_type:
+            # skip none_types because we are deserializing dict data.
+            # none_type deserialization is handled in the __new__ method
+            continue
+
         # transform js keys to python keys in fixed_model_args
         fixed_model_args = change_keys_js_to_python(model_args, anyof_class)
 

samples/openapi3/client/petstore/python-experimental/petstore_api/models/fruit_req.py

@@ -218,5 +218,6 @@ def _composed_schemas():
           'oneOf': [
               apple_req.AppleReq,
               banana_req.BananaReq,
+              none_type,
           ],
         }