Improve support for x-form-urlencoded content usage with http:Client

ballerina-tests/tests/http_client_data_binding.bal

@@ -396,7 +396,8 @@ service /backend on clientDBBackendListener {
     }
 
     resource function get xmltype() returns http:NotFound {
-        return {body: xml `<test>Bad Request</test>`};
+        xml payload = xml`<test>Bad Request</test>`;
+        return {body: payload};
     }
 
     resource function get jsontype() returns http:InternalServerError {

ballerina-tests/tests/http_client_url_encoded_content_test.bal

@@ -0,0 +1,107 @@
+// Copyright (c) 2021 WSO2 Inc. (http://www.wso2.org) All Rights Reserved.
+//
+// WSO2 Inc. licenses this file to you under the Apache License,
+// Version 2.0 (the "License"); you may not use this file except
+// in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// NOTE: All the tokens/credentials used in this test are dummy tokens/credentials and used only for testing purposes.
+
+import ballerina/http;
+import ballerina/url;
+import ballerina/mime;
+import ballerina/test;
+
+final http:Client clientUrlEncodedTestClient = check new(string`http://localhost:${clientFormUrlEncodedTestPort.toString()}/databinding`);
+final string expectedResponse = "URL_ENCODED_key1=value1&key2=value2";
+final readonly & map<string> payload = {
+    "key1": "value1",
+    "key2": "value2"
+};
+
+service /databinding on new http:Listener(clientFormUrlEncodedTestPort) {
+    resource function 'default .(http:Request req) returns string|error {
+        string contentType = req.getContentType();
+        contentType = contentType == mime:APPLICATION_FORM_URLENCODED ? "URL_ENCODED": "INVALID";
+        string payload = check req.getTextPayload();
+        string decodedContent = check url:decode(payload, "UTF-8");
+        return string`${contentType}_${decodedContent}`;
+    }
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithPost() returns error? {
+    string response = check clientUrlEncodedTestClient->post("", payload, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertEquals(response, expectedResponse, msg = "Found unexpected output");
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithPut() returns error? {
+    string response = check clientUrlEncodedTestClient->put("", payload, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertEquals(response, expectedResponse, msg = "Found unexpected output");
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithDelete() returns error? {
+    string response = check clientUrlEncodedTestClient->delete("", payload, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertEquals(response, expectedResponse, msg = "Found unexpected output");
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithPatch() returns error? {
+    string response = check clientUrlEncodedTestClient->patch("", payload, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertEquals(response, expectedResponse, msg = "Found unexpected output");
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithExecute() returns error? {
+    string response = check clientUrlEncodedTestClient->execute("POST", "", payload, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertEquals(response, expectedResponse, msg = "Found unexpected output");
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithIntPayload() returns error? {
+    string|error response = clientUrlEncodedTestClient->post("", 10, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertTrue(response is error, "Found unexpected output");
+    if response is error {
+        test:assertEquals(response.message(), "unsupported content for application/x-www-form-urlencoded media type", msg = "Found unexpected output");
+    }
+}
+
+@test:Config {
+    groups: ["urlEncodedContent"]
+}
+isolated function testUrlContentWithJsonPayload() returns error? {
+    json jsonPayload = {
+        "key1": "val1",
+        "key2": [
+            "val2.1", "val2.2"
+        ]
+    };
+    string|error response = clientUrlEncodedTestClient->post("", jsonPayload, mediaType = mime:APPLICATION_FORM_URLENCODED);
+    test:assertTrue(response is error, "Found unexpected output");
+    if response is error {
+        test:assertEquals(response.message(), "unsupported content for application/x-www-form-urlencoded media type", msg = "Found unexpected output");
+    }
+}

ballerina-tests/tests/test_service_ports.bal

@@ -156,6 +156,7 @@ const int requestInterceptorByteArrayPayloadBindingTestPort = 9606;
 const int requestInterceptorWithQueryParamTestPort = 9607;
 const int requestInterceptorServiceConfigTestPort1 = 9608;
 const int requestInterceptorServiceConfigTestPort2 = 9609;
+const int clientFormUrlEncodedTestPort = 9610;
 
 //HTTP2
 const int serverPushTestPort1 = 9701;

ballerina/Ballerina.toml

@@ -12,7 +12,7 @@ distribution = "2201.0.0"
 path = "../native/build/libs/http-native-2.2.0-SNAPSHOT.jar"
 
 [[platform.java11.dependency]]
-path = "./lib/mime-native-2.2.0-20211220-222200-7bd9f71.jar"
+path = "./lib/mime-native-2.2.0-20211222-181300-dbf5417.jar"
 
 [[platform.java11.dependency]]
 path = "./lib/netty-common-4.1.71.Final.jar"

ballerina/http_client_endpoint.bal

@@ -73,7 +73,7 @@ public client isolated class Client {
 
     private isolated function processPost(string path, RequestMessage message, TargetType targetType,
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         Response|ClientError response = self.httpClient->post(path, req);
         if (observabilityEnabled && response is Response) {
@@ -100,7 +100,7 @@ public client isolated class Client {
 
     private isolated function processPut(string path, RequestMessage message, TargetType targetType,
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         Response|ClientError response = self.httpClient->put(path, req);
         if (observabilityEnabled && response is Response) {
@@ -127,7 +127,7 @@ public client isolated class Client {
 
     private isolated function processPatch(string path, RequestMessage message, TargetType targetType,
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         Response|ClientError response = self.httpClient->patch(path, req);
         if (observabilityEnabled && response is Response) {
@@ -154,7 +154,7 @@ public client isolated class Client {
 
     private isolated function processDelete(string path, RequestMessage message, TargetType targetType,
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         Response|ClientError response = self.httpClient->delete(path, req);
         if (observabilityEnabled && response is Response) {
@@ -243,7 +243,7 @@ public client isolated class Client {
     private isolated function processExecute(string httpVerb, string path, RequestMessage message,
             TargetType targetType, string? mediaType, map<string|string[]>? headers)
             returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         Response|ClientError response = self.httpClient->execute(httpVerb, path, req);
         if (observabilityEnabled && response is Response) {
@@ -283,7 +283,7 @@ public client isolated class Client {
     # + message - An HTTP outbound request or any allowed payload
     # + return - An `http:HttpFuture` that represents an asynchronous service invocation or else an `http:ClientError` if the submission fails
     remote isolated function submit(string httpVerb, string path, RequestMessage message) returns HttpFuture|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, ());
         return self.httpClient->submit(httpVerb, path, req);
 
     }

ballerina/http_commons.bal

@@ -21,6 +21,8 @@ import ballerina/io;
 import ballerina/observe;
 import ballerina/time;
 import ballerina/log;
+import ballerina/lang.'string as strings;
+import ballerina/url;
 
 final boolean observabilityEnabled = observe:isObservabilityEnabled();
 
@@ -37,7 +39,7 @@ public isolated function parseHeader(string headerValue) returns HeaderValue[]|C
     name: "parseHeader"
 } external;
 
-isolated function buildRequest(RequestMessage message) returns Request|ClientError {
+isolated function buildRequest(RequestMessage message, string? mediaType) returns Request|ClientError {
     Request request = new;
     if (message is ()) {
         request.noEntityBody = true;
@@ -56,16 +58,46 @@ isolated function buildRequest(RequestMessage message) returns Request|ClientErr
     } else if (message is mime:Entity[]) {
         request.setBodyParts(message);
     } else {
-        var result = trap val:toJson(message);
-        if (result is error) {
-            return error InitializingOutboundRequestError("json conversion error: " + result.message(), result);
-        } else {
-            request.setJsonPayload(result);
+        match mediaType {
+            mime:APPLICATION_FORM_URLENCODED => {
+                string payload = check processUrlEncodedContent(message);
+                request.setTextPayload(payload, mime:APPLICATION_FORM_URLENCODED);
+            }
+            _ => {
+                json payload = check processJsonContent(message);
+                request.setJsonPayload(payload);
+            }
         }
     }
     return request;
 }
 
+isolated function processUrlEncodedContent(anydata message) returns string|ClientError {
+    if message is map<string> {
+        do {
+            string[] messageParams = [];
+            foreach var ['key, value] in message.entries() {
+                string encodedValue = check url:encode(value, "UTF-8");
+                string entry = string`${'key}=${encodedValue}`;
+                messageParams.push(entry);
+            }
+            return strings:'join("&", ...messageParams);
+        } on fail var e {
+            return error InitializingOutboundRequestError("content encoding error: " + e.message(), e);
+        }
+    } else {
+        return error InitializingOutboundRequestError("unsupported content for application/x-www-form-urlencoded media type");
+    }
+}
+
+isolated function processJsonContent(anydata message) returns json|ClientError {
+    var result = trap val:toJson(message);
+    if (result is error) {
+        return error InitializingOutboundRequestError("json conversion error: " + result.message(), result);
+    }
+    return result;
+}
+
 isolated function buildResponse(ResponseMessage message) returns Response|ListenerError {
     Response response = new;
     if (message is ()) {

ballerina/resiliency_failover_client.bal

@@ -82,7 +82,7 @@ public client isolated class FailoverClient {
 
     private isolated function processPost(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performFailoverAction(path, req, HTTP_POST);
         if (result is HttpFuture) {
@@ -110,7 +110,7 @@ public client isolated class FailoverClient {
 
     private isolated function processPut(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performFailoverAction(path, req, HTTP_PUT);
         if (result is HttpFuture) {
@@ -138,7 +138,7 @@ public client isolated class FailoverClient {
 
     private isolated function processPatch(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performFailoverAction(path, req, HTTP_PATCH);
         if (result is HttpFuture) {
@@ -166,7 +166,7 @@ public client isolated class FailoverClient {
 
     private isolated function processDelete(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performFailoverAction(path, req, HTTP_DELETE);
         if (result is HttpFuture) {
@@ -259,7 +259,7 @@ public client isolated class FailoverClient {
     private isolated function processExecute(string httpVerb, string path, RequestMessage message,
             TargetType targetType, string? mediaType, map<string|string[]>? headers) 
             returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performExecuteAction(path, req, httpVerb);
         if (result is HttpFuture) {
@@ -302,7 +302,7 @@ public client isolated class FailoverClient {
     # + return - An `http:HttpFuture` that represents an asynchronous service invocation or else an `http:ClientError` if the submission
     #            fails
     remote isolated function submit(string httpVerb, string path, RequestMessage message) returns HttpFuture|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, ());
         var result = self.performExecuteAction(path, req, "SUBMIT", verb = httpVerb);
         if (result is Response) {
             return getInvalidTypeError();

ballerina/resiliency_load_balance_client.bal

@@ -74,7 +74,7 @@ public client isolated class LoadBalanceClient {
 
     private isolated function processPost(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performLoadBalanceAction(path, req, HTTP_POST);
         return processResponse(result, targetType);
@@ -98,7 +98,7 @@ public client isolated class LoadBalanceClient {
 
     private isolated function processPut(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performLoadBalanceAction(path, req, HTTP_PUT);
         return processResponse(result, targetType);
@@ -122,7 +122,7 @@ public client isolated class LoadBalanceClient {
 
     private isolated function processPatch(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performLoadBalanceAction(path, req, HTTP_PATCH);
         return processResponse(result, targetType);
@@ -146,7 +146,7 @@ public client isolated class LoadBalanceClient {
 
     private isolated function processDelete(string path, RequestMessage message, TargetType targetType, 
             string? mediaType, map<string|string[]>? headers) returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performLoadBalanceAction(path, req, HTTP_DELETE);
         return processResponse(result, targetType);
@@ -222,7 +222,7 @@ public client isolated class LoadBalanceClient {
     private isolated function processExecute(string httpVerb, string path, RequestMessage message,
             TargetType targetType, string? mediaType, map<string|string[]>? headers) 
             returns Response|PayloadType|ClientError {
-        Request req = check buildRequest(message);
+        Request req = check buildRequest(message, mediaType);
         populateOptions(req, mediaType, headers);
         var result = self.performLoadBalanceExecuteAction(path, req, httpVerb);
         return processResponse(result, targetType);

changelog.md

@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Changed
 - [Rename RequestContext add function to set](https://github.com/ballerina-platform/ballerina-standard-library/issues/2414)
 - [Only allow default path in interceptors engaged at listener level](https://github.com/ballerina-platform/ballerina-standard-library/issues/2452)
+- [Provide a better way to send with `application/x-www-form-urlencoded`](https://github.com/ballerina-platform/ballerina-standard-library/issues/1705)
 
 ## [2.0.1] - 2021-11-20