Skip to content

Commit

Permalink
fix(type-safe-api): fix model serialisation for nested collections in…
Browse files Browse the repository at this point in the history 
… typescript (#882)

Previously, arrays of arrays or dictionaries of dictionaries (or any combination) were not
serialising/deserialising correctly. There were 2 main problems:

1. Nested arrays/dictionaries of models would be treated as if they were a single array/dictionary
2. Nested dates (even under single level arrays) weren't being serialised/deserialised at all which
could lead to runtime errors
  • Loading branch information
@cogwirrel
cogwirrel authored Nov 10, 2024
1 parent 0e532c0 commit ef749b3
Show file tree
Hide file tree
Showing 7 changed files with 4,952 additions and 1,307 deletions.
20 changes: 12 additions & 8 deletions packages/type-safe-api/scripts/type-safe-api/generators/generate-next.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import _orderBy from "lodash/orderBy";
import _uniq from "lodash/uniq";
import _uniqBy from "lodash/uniqBy";
import _isEqual from "lodash/isEqual";
import _cloneDeepWith from "lodash/cloneDeepWith"
import { OpenAPIV3 } from "openapi-types";
import * as parseOpenapi from "parse-openapi";
import { getOperationResponses } from "parse-openapi/dist/parser/getOperationResponses";
Expand Down Expand Up @@ -302,6 +303,7 @@ const splitAndWriteFiles = (renderedFileContents: string[], outputPath: string)

// Model types which indicate it is composed (ie inherits/mixin's another schema)
const COMPOSED_SCHEMA_TYPES = new Set(["one-of", "any-of", "all-of"]);
const COLLECTION_TYPES = new Set(["array", "dictionary"]);
const PRIMITIVE_TYPES = new Set(["string", "integer", "number", "boolean", "null", "any", "binary", "void"]);

/**
Expand Down Expand Up @@ -496,7 +498,7 @@ const mutateModelWithAdditionalTypes = (model: parseOpenapi.Model) => {
(model as any).javaType = toJavaType(model);
(model as any).pythonName = toPythonName('property', model.name);
(model as any).pythonType = toPythonType(model);
(model as any).isPrimitive = PRIMITIVE_TYPES.has(model.type);
(model as any).isPrimitive = PRIMITIVE_TYPES.has(model.type) && !COMPOSED_SCHEMA_TYPES.has(model.export) && !COLLECTION_TYPES.has(model.export);
};

interface MockDataContext {
Expand Down Expand Up @@ -615,13 +617,17 @@ const _ensureModelLinks = (spec: OpenAPIV3.Document, modelsByName: {[name: strin
if (modelsByName[name] && !model.link) {
model.link = modelsByName[name];
}
} else if (model.link && typeof schema.additionalProperties !== 'boolean') {
_ensureModelLinks(spec, modelsByName, model.link, schema.additionalProperties, visited);
}
} else if (model.export === "array" && 'items' in schema && schema.items) {
if (isRef(schema.items)) {
const name = splitRef(schema.items.$ref)[2];
if (modelsByName[name] && !model.link) {
model.link = modelsByName[name];
}
} else if (model.link) {
_ensureModelLinks(spec, modelsByName, model.link, schema.items, visited);
}
}

Expand Down Expand Up @@ -713,13 +719,12 @@ const buildData = async (inSpec: OpenAPIV3.Document, metadata: any) => {
// In order for the new generator not to be breaking, we apply the same logic here, however this can be removed
// in future since we have control to avoid the duplicate handlers while allowing an operation to be part of
// multiple "services".
let spec = JSON.parse(JSON.stringify(inSpec, (key, value) => {
let spec = _cloneDeepWith(inSpec, (value, key) => {
// Keep only the first tag where we find a tag
if (key === "tags" && value && value.length > 0 && typeof value[0] === "string") {
return [value[0]];
}
return value;
})) as OpenAPIV3.Document;
}) as OpenAPIV3.Document;

// Ensure spec has schemas set
if (!spec?.components?.schemas) {
Expand Down Expand Up @@ -775,16 +780,15 @@ const buildData = async (inSpec: OpenAPIV3.Document, metadata: any) => {

// "Inline" any refs to non objects/enums
const inlinedRefs: Set<string> = new Set();
spec = JSON.parse(JSON.stringify(spec, (k, v) => {
spec = _cloneDeepWith(spec, (v) => {
if (v && typeof v === "object" && v.$ref) {
const resolved = resolveRef(spec, v.$ref);
if (resolved && resolved.type && resolved.type !== "object" && !(resolved.type === "string" && resolved.enum)) {
inlinedRefs.add(v.$ref);
return resolved;
}
}
return v;
}));
});

// Delete the non object schemas that were inlined
[...inlinedRefs].forEach(ref => {
Expand All @@ -808,7 +812,7 @@ const buildData = async (inSpec: OpenAPIV3.Document, metadata: any) => {
faker.setDefaultRefDate(new Date("2021-06-10"));
const mockDataContext: MockDataContext = {
faker,
dereferencedSpec: await SwaggerParser.dereference(structuredClone(spec), { dereference: { circular: 'ignore' }}) as OpenAPIV3.Document,
dereferencedSpec: await SwaggerParser.dereference(structuredClone(spec), { dereference: { circular: 'ignore' } }) as OpenAPIV3.Document,
};

// Augment operations with additional data
Expand Down
112 changes: 90 additions & 22 deletions ...e-safe-api/scripts/type-safe-api/generators/typescript/templates/client/models/models.ejs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ import {
} from './<%= importName %>';
<%_ }); _%>
<%_ const isComposite = model.export === "one-of" || model.export === "any-of" || model.export === "all-of"; _%>
<%_
// Nested arrays of primitives (besides dates) don't need to have .map(...) called to convert them as the base case will be a noop
// eg. an array of arrays of strings doesn't need to be rendered as `value.map(item0 => item0.map(item1 => item1))`
const canShortCircuitConversion = (property) => {
if (["array", "dictionary"].includes(property.export)) {
return canShortCircuitConversion(property.link);
}
return property.isPrimitive && !["date", "date-time"].includes(property.format);
};
_%>
<%_ if (model.export === "enum") { _%>
/**
Expand Down Expand Up @@ -126,19 +136,45 @@ export function <%= model.name %>FromJSONTyped(json: any, ignoreDiscriminator: b
<%_ } else { _%>
return {
<%_
// Renders the appropriate nested function for .map() or mapValues() for arrays and dictionaries for the given type
const renderNestedFromJsonValue = (type, depth = 0) => {
const itemIdentifier = `item${depth}`;
if (type.isPrimitive) {
return `(${itemIdentifier}) => ${["date", "date-time"].includes(type.format) ? `new Date(${itemIdentifier})` : itemIdentifier}`;
} else if (type.export === "array") {
return `(${itemIdentifier}) => ${itemIdentifier}.map(${renderNestedFromJsonValue(type.link, depth + 1)})`;
} else if (type.export === "dictionary") {
return `(${itemIdentifier}) => mapValues(${itemIdentifier}, ${renderNestedFromJsonValue(type.link, depth + 1)})`;
}
return `${type.name || type.type}FromJSON`;
};
// Renders the code to transform a property of the model from its json representation into the model types
const renderFromJsonValue = (property) => {
const value = `json['${property.name}']`;
let rendered = '';
if (canShortCircuitConversion(property)) {
rendered = value;
} else if (property.isPrimitive) {
rendered = ["date", "date-time"].includes(property.format) ? `(new Date(${value}))` : value;
} else if (property.export === "array") {
rendered = `((${value} as Array<any>).map(${renderNestedFromJsonValue(property.link)}))`;
rendered = property.uniqueItems ? `new Set(${rendered})` : rendered;
} else if (property.export === "dictionary") {
rendered = `(mapValues(${value}, ${renderNestedFromJsonValue(property.link)}))`;
} else {
rendered = `${property.type}FromJSON(${value})`;
}
rendered = property.isNullable ? `${value} === null ? null : ${rendered}` : rendered;
rendered = !property.isRequired ? `!exists(json, '${property.name}') ? undefined : ${rendered}` : rendered;
return rendered;
};
_%>
<%_ if (model.export === "dictionary") { _%>
...json,
<%_ } _%>
<%_ model.properties.forEach((property) => { _%>
<%_ if (property.isPrimitive) { _%>
'<%= property.typescriptName %>': <% if (!property.isRequired) { %>!exists(json, '<%- property.name %>') ? undefined : <% } %><% if (["date", "date-time"].includes(property.format) && property.isNullable) { %>json['<%- property.name %>'] === null ? null : <% } %><% if (["date", "date-time"].includes(property.format)) { %>(new Date(json['<%= property.name %>']))<% } else { %>json['<%= property.name %>']<% } %>,
<%_ } else if (property.export === 'array') { _%>
'<%= property.typescriptName %>': <% if (!property.isRequired) { %>!exists(json, '<%- property.name %>') ? undefined : <% } %><% if (property.isNullable) { %>json['<%- property.name %>'] === null ? null : <% } %><%= property.uniqueItems ? 'new Set(' : '' %>((json['<%= property.name %>'] as Array<any>).map(<%= property.type %>FromJSON))<%= property.uniqueItems ? ')' : '' %>,
<%_ } else if (property.export === 'dictionary') { _%>
'<%= property.typescriptName %>': <% if (!property.isRequired) { %>!exists(json, '<%- property.name %>') ? undefined : <% } %><% if (property.isNullable) { %>json['<%- property.name %>'] === null ? null : <% } %>(mapValues(json['<%= property.name %>'], <%= property.type %>FromJSON)),
<%_ } else { _%>
'<%= property.typescriptName %>': <% if (!property.isRequired) { %>!exists(json, '<%- property.name %>') ? undefined : <% } %><% if (property.isNullable) { %>json['<%- property.name %>'] === null ? null : <% } %><%= property.type %>FromJSON(json['<%= property.name %>']),
<%_ } _%>
'<%= property.typescriptName %>': <%- renderFromJsonValue(property) %>,
<%_ }); _%>
};
<%_ } _%>
Expand Down Expand Up @@ -173,24 +209,56 @@ export function <%= model.name %>ToJSON(value?: <%= model.name %> | null): any {
<%_ } else { _%>
return {
<%_
// Render code to convert a date to its string representation
const renderToJsonDateValue = (identifier, format) => {
return `${identifier}.toISOString()${format === 'date' ? '.substr(0,10)' : ''}`;
};
// Renders the appropriate nested function for .map() or mapValues() for arrays and dictionaries for the given type
const renderNestedToJsonValue = (type, depth = 0) => {
const itemIdentifier = `item${depth}`;
if (type.isPrimitive) {
return `(${itemIdentifier}) => ${["date", "date-time"].includes(type.format) ? renderToJsonDateValue(itemIdentifier, type.format) : itemIdentifier}`;
} else if (type.export === "array") {
return `(${itemIdentifier}) => ${itemIdentifier}.map(${renderNestedToJsonValue(type.link, depth + 1)})`;
} else if (type.export === "dictionary") {
return `(${itemIdentifier}) => mapValues(${itemIdentifier}, ${renderNestedToJsonValue(type.link, depth + 1)})`;
}
return `${type.name || type.type}ToJSON`;
};
// Renders the code to transform a property of the model to its json representation from the model types
const renderToJsonValue = (property) => {
const value = `value.${property.typescriptName}`;
let rendered = '';
if (canShortCircuitConversion(property)) {
rendered = value;
} else if (property.isPrimitive) {
rendered = ["date", "date-time"].includes(property.format) ? `(${renderToJsonDateValue(value, property.format)})` : value;
} else if (property.export === "array") {
const prefix = property.uniqueItems ? `Array.from(${value} as Array<any>)` : `(${value} as Array<any>)`;
rendered = `(${prefix}.map(${renderNestedToJsonValue(property.link)}))`;
} else if (property.export === "dictionary") {
rendered = `(mapValues(${value}, ${renderNestedToJsonValue(property.link)}))`;
} else if (property.type !== "any") {
rendered = `${property.type}ToJSON(${value})`;
} else {
rendered = value;
}
if ((property.isPrimitive && ["date", "date-time"].includes(property.format)) || (!property.isPrimitive && ["array", "dictionary"].includes(property.export))) {
rendered = property.isNullable ? `${value} === null ? null : ${rendered}` : rendered;
rendered = !property.isRequired ? `${value} === undefined ? undefined : ${rendered}` : rendered;
}
return rendered;
};
_%>
<%_ if (model.export === "dictionary") { _%>
...value,
<%_ } _%>
<%_ model.properties.forEach((property) => { _%>
<%_ if (!property.isReadOnly) { _%>
<%_ if (property.isPrimitive && ["date", "date-time"].includes(property.format)) { _%>
'<%= property.name %>': <% if (!property.isRequired) { %>value.<%- property.typescriptName %> === undefined ? undefined : <% } %>(<% if (property.isNullable) { %>value.<%- property.typescriptName %> === null ? null : <% } %>value.<%- property.typescriptName %>.toISOString()<% if (property.format === 'date') { %>.substr(0,10)<% } %>),
<%_ } else if (property.isPrimitive) { _%>
'<%= property.name %>': value.<%- property.typescriptName %>,
<%_ } else if (property.export === 'array') { _%>
'<%= property.name %>': <% if (!property.isRequired) { %>value.<%- property.typescriptName %> === undefined ? undefined : <% } %>(<% if (property.isNullable) { %>value.<%- property.typescriptName %> === null ? null : <% } %><% if (property.uniqueItems) { %>Array.from(value.<%- property.typescriptName %> as Set<any>)<% } else { %>(value.<%- property.typescriptName %> as Array<any>)<% } %>.map(<%- property.type %>ToJSON)),
<%_ } else if (property.export === 'dictionary') { _%>
'<%= property.name %>': <% if (!property.isRequired) { %>value.<%- property.typescriptName %> === undefined ? undefined : <% } %>(<% if (property.isNullable) { %>value.<%- property.typescriptName %> === null ? null : <% } %>mapValues(value.<%- property.typescriptName %>, <%- property.type %>ToJSON)),
<%_ } else if (property.type !== 'any') { _%>
'<%= property.name %>': <%- property.type %>ToJSON(value.<%- property.typescriptName %>),
<%_ } else { _%>
'<%= property.name %>': value.<%- property.typescriptName %>,
<%_ } _%>
'<%= property.name %>': <%- renderToJsonValue(property) %>,
<%_ } _%>
<%_ }); _%>
};
Expand Down
100 changes: 100 additions & 0 deletions packages/type-safe-api/test/resources/specs/edge-cases.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,16 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/ArrayOfOneOfs"
/nested-collections:
post:
operationId: nestedCollections
responses:
200:
description: ok
content:
application/json:
schema:
$ref: "#/components/schemas/NestedCollections"
/additional-properties:
post:
operationId: dictionary
Expand Down Expand Up @@ -175,6 +185,96 @@ components:
type: array
items:
$ref: "#/components/schemas/NamedOneOfUnion"
NestedCollections:
type: object
properties:
nestedArrayOfStrings:
type: array
items:
type: array
items:
type: string
nestedArrayOfDates:
type: array
items:
type: array
items:
type: string
format: date
nestedArrayOfObjects:
type: array
items:
type: array
items:
$ref: "#/components/schemas/SomeObject"
fourDimensionalNestedArrayOfObjects:
type: array
items:
type: array
items:
type: array
items:
type: array
items:
$ref: "#/components/schemas/SomeObject"
nestedDictionaryOfStrings:
type: object
additionalProperties:
type: object
additionalProperties:
type: string
nestedDictionaryOfObjects:
type: object
additionalProperties:
type: object
additionalProperties:
$ref: "#/components/schemas/SomeObject"
fourDimensionalNestedDictionaryOfObjects:
type: object
additionalProperties:
type: object
additionalProperties:
type: object
additionalProperties:
type: object
additionalProperties:
$ref: "#/components/schemas/SomeObject"
nestedMixOfDictionariesAndArrays:
type: array
items:
type: object
additionalProperties:
type: array
items:
type: array
items:
type: object
additionalProperties:
type: array
items:
$ref: "#/components/schemas/SomeObject"
cycleArray:
$ref: "#/components/schemas/CycleArray"
cycleDictionary:
$ref: "#/components/schemas/CycleDictionary"
CycleArray:
type: array
items:
$ref: "#/components/schemas/CycleArrayNode"
CycleArrayNode:
type: object
properties:
nodes:
$ref: "#/components/schemas/CycleArray"
CycleDictionary:
type: object
additionalProperties:
$ref: "#/components/schemas/CycleDictionaryNode"
CycleDictionaryNode:
type: object
properties:
nodes:
$ref: "#/components/schemas/CycleDictionary"
AdditionalPropertiesResponse:
type: object
properties:
Expand Down
Loading

0 comments on commit ef749b3

Please sign in to comment.