Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve OpenAPI request/response naming strategy #18321

Closed
wants to merge 32 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
45aee60
Improve OpenAPI request/response naming strategy
averche Dec 12, 2022
691e29b
default mappings
averche Dec 12, 2022
1fad3a1
change the mapped names
averche Dec 14, 2022
ed954bf
add operationImplied flag
averche Dec 14, 2022
a1058a5
Add a test
averche Dec 14, 2022
7a9b75d
changelog
averche Dec 14, 2022
59a692f
requestResponsePrefix -> defaultMountPath
averche Dec 14, 2022
7a39ca9
update -> write
averche Dec 14, 2022
3b90f9a
fix test
averche Dec 14, 2022
91a5134
undo test change
averche Dec 14, 2022
19d5ef0
move the map into openapi_path_mappings.go
averche Dec 19, 2022
2375e5e
Update table with prefix/operation/suffix struct
averche Dec 20, 2022
0d14048
Fix function and test
averche Dec 20, 2022
4d5ccb9
test
averche Dec 20, 2022
ac7fe27
mountPathWithPrefix
averche Dec 20, 2022
0be96b8
mountPathWithPrefix
averche Dec 20, 2022
a0a65bf
comment
averche Dec 20, 2022
8e34fda
Adjust certain path mappings
averche Dec 21, 2022
77684f5
undo script changes
averche Dec 21, 2022
dc6fb71
Merge branch 'main' into ui/openapi-request-response-names
averche Dec 21, 2022
dd5767c
spaces
averche Dec 21, 2022
1e6bfe2
don't need path lib
averche Dec 21, 2022
70ec341
dedup
averche Dec 21, 2022
5165291
change KVv1/KVv2 naming structure
averche Jan 2, 2023
f1b79a1
Merge branch 'main' into ui/openapi-request-response-names
averche Jan 3, 2023
d8d4413
Add back CreateOperationIDs
averche Jan 4, 2023
9ad60e9
comment
averche Jan 4, 2023
66a741b
rename constructOperationID and address PR feedback
averche Jan 5, 2023
9088087
fix tests
averche Jan 5, 2023
f1d7eff
add updateOperationOnly flag
averche Jan 5, 2023
27da1e3
fix test
averche Jan 5, 2023
98dae14
remove kv -> secret hack
averche Jan 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions changelog/18321.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:improvement
openapi: Improve request/response naming strategy
```
13 changes: 4 additions & 9 deletions sdk/framework/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -530,14 +530,9 @@ func (b *Backend) handleRootHelp(req *logical.Request) (*logical.Response, error
return nil, err
}

// Plugins currently don't have a direct knowledge of their own "type"
// (e.g. "kv", "cubbyhole"). It defaults to the name of the executable but
// can be overridden when the plugin is mounted. Since we need this type to
// form the request & response full names, we are passing it as an optional
// request parameter to the plugin's root help endpoint. If specified in
// the request, the type will be used as part of the request/response body
// names in the OAS document.
requestResponsePrefix := req.GetString("requestResponsePrefix")
// If specified in the request, the mount path will be used as part of the
// request/response body names and operation id's in the OpenAPI document.
mountPathWithPrefix := req.GetString("mount_path_with_prefix")

// Build OpenAPI response for the entire backend
vaultVersion := "unknown"
Expand All @@ -550,7 +545,7 @@ func (b *Backend) handleRootHelp(req *logical.Request) (*logical.Response, error
}

doc := NewOASDocument(vaultVersion)
if err := documentPaths(b, requestResponsePrefix, doc); err != nil {
if err := documentPaths(b, mountPathWithPrefix, doc); err != nil {
b.Logger().Warn("error generating OpenAPI", "error", err)
}

Expand Down
108 changes: 80 additions & 28 deletions sdk/framework/openapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,9 +215,9 @@ var (
)

// documentPaths parses all paths in a framework.Backend into OpenAPI paths.
func documentPaths(backend *Backend, requestResponsePrefix string, doc *OASDocument) error {
func documentPaths(backend *Backend, mountPathWithPrefix string, doc *OASDocument) error {
for _, p := range backend.Paths {
if err := documentPath(p, backend.SpecialPaths(), requestResponsePrefix, backend.BackendType, doc); err != nil {
if err := documentPath(p, backend.SpecialPaths(), mountPathWithPrefix, backend.BackendType, doc); err != nil {
return err
}
}
Expand All @@ -226,7 +226,7 @@ func documentPaths(backend *Backend, requestResponsePrefix string, doc *OASDocum
}

// documentPath parses a framework.Path into one or more OpenAPI paths.
func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix string, backendType logical.BackendType, doc *OASDocument) error {
func documentPath(p *Path, specialPaths *logical.Paths, mountPathWithPrefix string, backendType logical.BackendType, doc *OASDocument) error {
var sudoPaths []string
var unauthPaths []string

Expand Down Expand Up @@ -265,12 +265,8 @@ func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix st
// Body fields will be added to individual operations.
pathFields, bodyFields := splitFields(p.Fields, path)

defaultMountPath := requestResponsePrefix
if requestResponsePrefix == "kv" {
defaultMountPath = "secret"
}

if defaultMountPath != "system" && defaultMountPath != "identity" {
if mountPathWithPrefix != "sys" && mountPathWithPrefix != "identity" {
defaultMountPath := mountPathWithPrefix[strings.LastIndex(mountPathWithPrefix, "/")+1:]
p := OASParameter{
Name: fmt.Sprintf("%s_mount_path", defaultMountPath),
Description: "Path where the backend was mounted; the endpoint path will be offset by the mount path",
Expand Down Expand Up @@ -324,6 +320,12 @@ func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix st
// Process each supported operation by building up an Operation object
// with descriptions, properties and examples from the framework.Path data.
for opType, opHandler := range operations {
operationID, updateOperationOnly := constructOperationID(opType, mountPathWithPrefix, path)

if updateOperationOnly && opType != logical.UpdateOperation {
continue
Copy link
Contributor

@maxb maxb Jan 6, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I lack the permissions to unresolve the previous conversation, so I need to copy/paste my comment into a new one instead:

Sorry, but the new solution is still very problematic for my usecase, as I rely on the OpenAPI to determine the complete set of operations that a given path supports, and this would erase that information.

Additionally, it is likely to prove confusing to people referring to the OpenAPI or Swagger UI to guide their understanding of already-written code, and find it is calling an operation which is now erased, and so seems mysterious and undocumented.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fair enough, I suppose I can disambiguate the names in some other way. It's just a shame that the libraries will have multiple methods generated which will result in exactly the same behaviour.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The more I think about it, the more I think the library code generation is going to need a way to apply tweaks to the canonical OpenAPI document, to vary the generation process. Let me explain further:

Today, as you have identified, there are various APIs in Vault which are deprecated and replaced by newer APIs. It would be nice if we could exclude the historical cruft from the generated client libraries, whilst it remains important to still acknowledge the APIs exist, to help people understand existing code, and understand the impact of wildcards in policies.

We could (and IMO should) add a simple "deprecated" flag to deprecated APIs, and use that to render them as crossed out in the API explorer UI (something Swagger UI already supports). Then, at the birth of a new autogenerated client library, it would make sense to exclude all these.

But wait! This only works out nice and simply, for the first version of the autogenerated client library. Users expect API stability from their libraries. If we simply exclude all deprecated APIs, then the moment any future API deprecation in Vault would result in the API being removed from the next version of the autogenerated client library, breaking compatibility.

The problem is actually deeper still. There may be a bugfix to how an API is represented in Vault, but that may trigger an API removal in an autogenerated client library if we're not careful. Here is an example:

There is currently this path regexp with the PKI secrets engine: "keys/generate/(internal|exported|kms)" and this currently generates three separate APIs. However, the same three values are represented in multiple other PKI secrets engine APIs in this other style: "issuers/generate/intermediate/"+framework.GenericNameRegex("exported") where the parameter captured by the GenericNameRegex must have one of the three values, internal, exported or kms. This other pattern only generates one OpenAPI operation. I intend to submit a PR to make the PKI secrets engine internally consistent in how it handles this parameter. When I do, what is currently three operations in the OpenAPI will change to one. If this was after version 1 of an autogenerated client library, there would be a need for the library to maintain deprecated compatibility implementations of the old operations!

There are lots more examples to be found in my #18554 PR - various incorrect parameters being fixed or removed, for example.

In summary, then: the OpenAPI straight from Vault is a great starting point for client library generation, but if the library's API is to meet the stability guarantees a developer would hope for, it will be an important requirement to be able to support library specific overrides.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I definitely agree that we will want to support some sort of library-specific overrides. As I mentioned before we have been entertaining an idea of a dedicated repository for generating and hosting openapi.json files. The repository will likely contain two flavours: one for library generation and one for documentation purposes. It is likely that the repository will use sdk/openapi.go directly (and without loading the actual plugins) rather than going though the API endpoint.

You bring up very good points regarding deprecation & backwards compatibility issues. Since our conversation last week, I had some time to think about it. In the case of an API endpoint being deprecated and removed from the library-specific API, it should definitely result in a new major version of OpenAPI-for-libraries spec (and consequently a new major version of the libraries). This seems to be a fairly established pattern for OpenAPI-generated libraries. For examples Stripe's go library is currently on major version 74 using OpenAPI version v217. The client code does not need to upgrade right away but if an when they do, they could expect some breaking code changes due to the volatile nature of code generation.

Regarding bug fixes resulting in removal of certain paths, hopefully we can fix all such issues before the 1.0 release. If not, it is still not catastrophic as we can simply bump up the major version for both OpenAPI spec and the generated libraries.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fast-paced new major versions of libraries does work particularly well in the Go world, since with Go modules, a final compiled application can depend upon multiple major versions if it needs to.

In other programming language ecosystems, such as .NET, each compiled application can only depend on one version of each library, so if application A depends on libraries B and C, and B and C need incompatible versions of D, this is a problem. (The "diamond dependency" problem, as if you draw out the dependency graph between A, B, C and D it forms a diamond shape.)

Because of this, what's considered as perfectly reasonable for Go may be seen as less so for .NET. Interestingly, stripe-dotnet has had 30 major versions in the same time stripe-go has gone through 46!

In Vault and Stripe's cases, this might be OK, as both tend to be mostly used for custom business logic for a specific use-case, and unlikely to be required from multiple places deep in a dependency tree, as a common utility library would be.

One thing that might be worth considering, is making sure the library generation system has the capability to apply tweaks on top of the canonical OpenAPI document fetched from the Vault project. As an example, let's imagine that Vault 1.14 hypothetically cleans up the specification of some APIs, in a way which is perfectly compatible at the HTTP layer, but results in incompatible changes to generated code - whilst at the same time adding new APIs that you want to release to users ASAP. You might approach this by keeping a manually curated "overlay" or "patch" in the library generation repo, which temporarily preserves the older forms alongside the newer forms, until such time you need to declare a library major version increment in the future.

}
averche marked this conversation as resolved.
Show resolved Hide resolved

props := opHandler.Properties()
if props.Unpublished {
continue
Expand All @@ -348,6 +350,7 @@ func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix st
op.Summary = props.Summary
op.Description = props.Description
op.Deprecated = props.Deprecated
op.OperationID = operationID

// Add any fields not present in the path as body parameters for POST.
if opType == logical.CreateOperation || opType == logical.UpdateOperation {
Expand Down Expand Up @@ -395,7 +398,7 @@ func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix st

// Set the final request body. Only JSON request data is supported.
if len(s.Properties) > 0 || s.Example != nil {
requestName := constructRequestResponseName(path, requestResponsePrefix, "Request")
requestName := operationID + "Request"
doc.Components.Schemas[requestName] = s
op.RequestBody = &OASRequestBody{
Required: true,
Expand Down Expand Up @@ -502,7 +505,7 @@ func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix st
}

if len(resp.Fields) != 0 {
responseName := constructRequestResponseName(path, requestResponsePrefix, "Response")
responseName := operationID + "Response"
doc.Components.Schemas[responseName] = responseSchema
content = OASContent{
"application/json": &OASMediaTypeObject{
Expand Down Expand Up @@ -534,31 +537,73 @@ func documentPath(p *Path, specialPaths *logical.Paths, requestResponsePrefix st
return nil
}

// constructRequestResponseName joins the given path with prefix & suffix into
// a CamelCase request or response name.
// constructOperationID joins the given inputs into a title case operationID,
// which is also used as a root for request and response names.
//
// For example, path=/config/lease/{name}, prefix="secret", suffix="request"
// will result in "SecretConfigLeaseRequest"
func constructRequestResponseName(path, prefix, suffix string) string {
var b strings.Builder
// For many mount + path combinations, which would otherwise result in
// difficult-to-read names, the function uses a custom lookup table to
// construct the name instead.
//
// Examples of generated operation ID's:
// - KVv2Write
// - KVv2Read
// - GoogleCloudLogin
// - GoogleCloudWriteRole
//
// The function also returns "update operation only" indicator, it is set to
// true if the operation ID should result in a single update (POST) operation
// for this path. For example, the following 3 operations, will be collapsed
// into a single operation "Deregister" and will be mapped into a single
// update (POST) operation:
//
// PUT /v1/gcpkms/keys/deregister/:key
// POST /v1/gcpkms/keys/deregister/:key
// DELETE /v1/gcpkms/keys/deregister/:key
func constructOperationID(operation logical.Operation, mount, path string) (string, bool) {
// For most request names "write" seems to be more applicable in place of "update"
var operationStr string
if operation == logical.UpdateOperation {
averche marked this conversation as resolved.
Show resolved Hide resolved
operationStr = "write"
} else {
operationStr = string(operation)
}

title := cases.Title(language.English)
updateOperationOnly := false

b.WriteString(title.String(prefix))
var parts []string

// split the path by / _ - separators
for _, token := range strings.FieldsFunc(path, func(r rune) bool {
return r == '/' || r == '_' || r == '-'
}) {
// exclude request fields
if !strings.ContainsAny(token, "{}") {
b.WriteString(title.String(token))
if mapping, ok := knownPathMappings[knownPathKey{mount: mount, path: path}]; ok {
// If the mapping exists, construct a [prefix][operation][suffix] name
if mapping.prefix != "" {
parts = append(parts, mapping.prefix)
}

if mapping.operation != "" {
parts = append(parts, mapping.operation)
} else {
parts = append(parts, nonWordRe.Split(strings.ToLower(operationStr), -1)...)
}

if mapping.suffix != "" {
parts = append(parts, mapping.suffix)
}

updateOperationOnly = mapping.updateOperationOnly
} else {
// Else, fall back to [mount][operation][path] split by non-word characters
parts = append(parts, nonWordRe.Split(strings.ToLower(mount), -1)...)
parts = append(parts, nonWordRe.Split(strings.ToLower(operationStr), -1)...)
parts = append(parts, nonWordRe.Split(strings.ToLower(path), -1)...)
}

b.WriteString(suffix)
// Title case everything & join the result into a string
title := cases.Title(language.English, cases.NoLower)

return b.String()
for i, s := range parts {
parts[i] = title.String(s)
}

return strings.Join(parts, ""), updateOperationOnly
}

func specialPathMatch(path string, specialPaths []string) bool {
Expand Down Expand Up @@ -784,6 +829,9 @@ func cleanResponse(resp *logical.Response) *cleanedResponse {
// postSysToolsRandomUrlbytes_2
//
// An optional user-provided suffix ("context") may also be appended.
//
// Deprecated: operationID's are now populated using constructOperationID();
// This function is here for backwards compatibility with older plugins
func (d *OASDocument) CreateOperationIDs(context string) {
averche marked this conversation as resolved.
Show resolved Hide resolved
// title caser
title := cases.Title(language.English)
Expand Down Expand Up @@ -814,6 +862,10 @@ func (d *OASDocument) CreateOperationIDs(context string) {
continue
}

if oasOperation.OperationID != "" {
continue
}

// Discard "_mount_path" from any {thing_mount_path} parameters
path = strings.Replace(path, "_mount_path", "", 1)

Expand Down
Loading