From 018ef5fec12735acb419ba5e258bdd9b9e412130 Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Fri, 9 Aug 2024 11:26:43 +0530 Subject: [PATCH 01/14] Add tests --- ballerina/Dependencies.toml | 34 ++++++++++++++++++++++- ballerina/tests/tests.bal | 55 +++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 ballerina/tests/tests.bal diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index cfce4f4..7e859ba 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -96,7 +96,7 @@ modules = [ [[package]] org = "ballerina" name = "io" -version = "1.6.0" +version = "1.6.1" dependencies = [ {org = "ballerina", name = "jballerina.java"}, {org = "ballerina", name = "lang.value"} @@ -147,6 +147,15 @@ dependencies = [ {org = "ballerina", name = "jballerina.java"} ] +[[package]] +org = "ballerina" +name = "lang.error" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"} +] + [[package]] org = "ballerina" name = "lang.int" @@ -205,6 +214,9 @@ dependencies = [ {org = "ballerina", name = "lang.value"}, {org = "ballerina", name = "observe"} ] +modules = [ + {org = "ballerina", packageName = "log", moduleName = "log"} +] [[package]] org = "ballerina" @@ -245,6 +257,9 @@ dependencies = [ {org = "ballerina", name = "io"}, {org = "ballerina", name = "jballerina.java"} ] +modules = [ + {org = "ballerina", packageName = "os", moduleName = "os"} +] [[package]] org = "ballerina" @@ -255,6 +270,20 @@ dependencies = [ {org = "ballerina", name = "time"} ] +[[package]] +org = "ballerina" +name = "test" +version = "0.0.0" +scope = "testOnly" +dependencies = [ + {org = "ballerina", name = "jballerina.java"}, + {org = "ballerina", name = "lang.array"}, + {org = "ballerina", name = "lang.error"} +] +modules = [ + {org = "ballerina", packageName = "test", moduleName = "test"} +] + [[package]] org = "ballerina" name = "time" @@ -293,6 +322,9 @@ version = "2.0.1" dependencies = [ {org = "ballerina", name = "constraint"}, {org = "ballerina", name = "http"}, + {org = "ballerina", name = "log"}, + {org = "ballerina", name = "os"}, + {org = "ballerina", name = "test"}, {org = "ballerina", name = "url"}, {org = "ballerinai", name = "observe"} ] diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal new file mode 100644 index 0000000..6e8d9c9 --- /dev/null +++ b/ballerina/tests/tests.bal @@ -0,0 +1,55 @@ +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/test; +import ballerina/os; + +configurable boolean isLiveServer = ?; +configurable string token = isLiveServer ? os:getEnv("token") : "test"; +final string mockServiceUrl = "http://localhost:9090"; + +@test:Config{ + groups: ["live_tests", "mock_tests"] +} +isolated function testChatCompletion() returns error? { + + // Create an OpenAI chat client with the appropriate service URL. + Client openAIChat; + if isLiveServer { + openAIChat = check new ({auth: {token}}); + } else { + openAIChat = check new ({auth: {token}}, mockServiceUrl); + } + + // Create a chat completion request. + CreateChatCompletionRequest request = { + model: "gpt-4o-mini", + messages: [{"role": "user", "content": "This is a test message"}] + }; + + do{ + // Call the API. + CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); + + // Check the response. + string? content = response.choices[0].message.content; + test:assertTrue(content !is (), msg = "An error occurred with response content"); + + } on fail error e { + test:assertFail(msg = "An error occurred: " + e.message()); + } + +} From 20b74c83cd73bd14bc8330c036a662c878fb247c Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Fri, 9 Aug 2024 11:26:52 +0530 Subject: [PATCH 02/14] Add Mock Server --- ballerina/tests/mock_service.bal | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 ballerina/tests/mock_service.bal diff --git a/ballerina/tests/mock_service.bal b/ballerina/tests/mock_service.bal new file mode 100644 index 0000000..9baec6b --- /dev/null +++ b/ballerina/tests/mock_service.bal @@ -0,0 +1,64 @@ +// Copyright (c) 2024, WSO2 LLC. (http://www.wso2.com). +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import ballerina/http; +import ballerina/log; + +listener http:Listener httpListener = new (9090); + +http:Service mockService = service object { + resource function post chat/completions(@http:Payload CreateChatCompletionRequest payload) returns CreateChatCompletionResponse|http:BadRequest { + + // Validate the request payload + if (payload.messages[0]["content"].toString() is "" || payload.model.toString() is "") { + return http:BAD_REQUEST; + } + + // Mock response + CreateChatCompletionResponse response = { + id: "chatcmpl-00000", + choices: [ + { + finish_reason: "stop", + index: 0, + message: {"content": "Test message received! How can I assist you today?", "role": "assistant", "refusal": null}, + logprobs: null + } + ], + created: 1723091495, + model: "gpt-4o-mini-2024-07-18", + system_fingerprint: "fp_48196bc67a", + "object": "chat.completion", + "usage": {"completion_tokens": 11, "prompt_tokens": 13, "total_tokens": 24} + }; + + return response; + } +}; + +function init() returns error? { + + // This is to avoid starting the mock server when running the tests on live server + if isLiveServer { + log:printInfo("Skiping mock server initialization as the tests are running on live server"); + return; + } + + // Start the mock server + log:printInfo("Initiating mock server..."); + check httpListener.attach(mockService, "/"); + check httpListener.'start(); +} \ No newline at end of file From a37dc61be9c5e9a79d267de6b3dbfc2f8ea98095 Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Fri, 9 Aug 2024 11:45:15 +0530 Subject: [PATCH 03/14] Add instructions on running tests --- ballerina/tests/README.md | 89 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 ballerina/tests/README.md diff --git a/ballerina/tests/README.md b/ballerina/tests/README.md new file mode 100644 index 0000000..2962bfd --- /dev/null +++ b/ballerina/tests/README.md @@ -0,0 +1,89 @@ +# Running Tests + +## Prerequisites + +You need an API token from OpenAI. + +To obtain this, refer to the [Ballerina OpenAI Chat Connector](https://github.com/ballerina-platform/module-ballerinax-openai.chat/blob/main/ballerina/Module.md). + +## Test Environments + +There are two test environments for running the `openai.chat` connector tests. The default environment is a mock server for the OpenAI API. The other environment is the actual OpenAI API. + +You can run the tests in either of these environments, and each has its own compatible set of tests. + +| Test Groups | Environment | +|-------------|---------------------------------------------------| +| mock_tests | Mock server for OpenAI API (Default Environment) | +| live_tests | OpenAI API | + +## Running Tests in the Mock Server + +To execute the tests on the mock server, ensure that the `isLiveServer` environment variable is either set to `false` or left unset before initiating the tests. + +This environment variable can be configured within the `Config.toml` file located in the `tests` directory or specified as an environment variable. + +### Using a `Config.toml` File + +Create a `Config.toml` file in the `tests` directory with the following content: + +```toml +isLiveServer = false +``` + +### Using Environment Variables + +Alternatively, you can set the environment variable directly. + +For Linux or macOS: + +```bash +export isLiveServer=false +``` + +For Windows: + +```bash +setx isLiveServer false +``` + +Then, run the following command to execute the tests: + +```bash +./gradlew clean test +``` + +## Running Tests Against the OpenAI Live API + +### Using a `Config.toml` File + +Create a `Config.toml` file in the `tests` directory and add your authentication credentials: + +```toml +isLiveServer = true +token = "" +``` + +### Using Environment Variables + +Alternatively, you can set your authentication credentials as environment variables. + +For Linux or macOS: + +```bash +export isLiveServer=true +export token="" +``` + +For Windows: + +```bash +setx isLiveServer true +setx token +``` + +Then, run the following command to execute the tests: + +```bash +./gradlew clean test +``` \ No newline at end of file From 24686d2053e5a6fc9feebcf2b5d670a2fdff10eb Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Fri, 9 Aug 2024 14:18:55 +0530 Subject: [PATCH 04/14] Fix missing newline at end of file in mock_service.bal --- ballerina/tests/mock_service.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/tests/mock_service.bal b/ballerina/tests/mock_service.bal index 9baec6b..44c7af4 100644 --- a/ballerina/tests/mock_service.bal +++ b/ballerina/tests/mock_service.bal @@ -61,4 +61,4 @@ function init() returns error? { log:printInfo("Initiating mock server..."); check httpListener.attach(mockService, "/"); check httpListener.'start(); -} \ No newline at end of file +} From de3a288e508a904f77c1c0d0e032841c27fd7bd3 Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Mon, 12 Aug 2024 10:30:25 +0530 Subject: [PATCH 05/14] Remove unnecessary comments and log statements in mock_service.bal and tests.bal --- ballerina/tests/mock_service.bal | 2 -- ballerina/tests/tests.bal | 4 ---- 2 files changed, 6 deletions(-) diff --git a/ballerina/tests/mock_service.bal b/ballerina/tests/mock_service.bal index 44c7af4..f871f49 100644 --- a/ballerina/tests/mock_service.bal +++ b/ballerina/tests/mock_service.bal @@ -51,13 +51,11 @@ http:Service mockService = service object { function init() returns error? { - // This is to avoid starting the mock server when running the tests on live server if isLiveServer { log:printInfo("Skiping mock server initialization as the tests are running on live server"); return; } - // Start the mock server log:printInfo("Initiating mock server..."); check httpListener.attach(mockService, "/"); check httpListener.'start(); diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index 6e8d9c9..c11afdc 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -26,7 +26,6 @@ final string mockServiceUrl = "http://localhost:9090"; } isolated function testChatCompletion() returns error? { - // Create an OpenAI chat client with the appropriate service URL. Client openAIChat; if isLiveServer { openAIChat = check new ({auth: {token}}); @@ -34,17 +33,14 @@ isolated function testChatCompletion() returns error? { openAIChat = check new ({auth: {token}}, mockServiceUrl); } - // Create a chat completion request. CreateChatCompletionRequest request = { model: "gpt-4o-mini", messages: [{"role": "user", "content": "This is a test message"}] }; do{ - // Call the API. CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); - // Check the response. string? content = response.choices[0].message.content; test:assertTrue(content !is (), msg = "An error occurred with response content"); From 69e4e46ff937db81c41cdbee4d10a61954ad3bb6 Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Mon, 12 Aug 2024 10:32:23 +0530 Subject: [PATCH 06/14] chore: Remove unnecessary whitespaces in mock_service.bal --- ballerina/tests/mock_service.bal | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ballerina/tests/mock_service.bal b/ballerina/tests/mock_service.bal index f871f49..6c29918 100644 --- a/ballerina/tests/mock_service.bal +++ b/ballerina/tests/mock_service.bal @@ -21,13 +21,13 @@ listener http:Listener httpListener = new (9090); http:Service mockService = service object { resource function post chat/completions(@http:Payload CreateChatCompletionRequest payload) returns CreateChatCompletionResponse|http:BadRequest { - + // Validate the request payload if (payload.messages[0]["content"].toString() is "" || payload.model.toString() is "") { return http:BAD_REQUEST; } - // Mock response + // Mock response CreateChatCompletionResponse response = { id: "chatcmpl-00000", choices: [ @@ -45,12 +45,11 @@ http:Service mockService = service object { "usage": {"completion_tokens": 11, "prompt_tokens": 13, "total_tokens": 24} }; - return response; + return response; } }; function init() returns error? { - if isLiveServer { log:printInfo("Skiping mock server initialization as the tests are running on live server"); return; From c2bdaf28d353e378681d8e103c80a97256ca536e Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Mon, 12 Aug 2024 15:25:08 +0530 Subject: [PATCH 07/14] change client initialization in test.bal --- ballerina/tests/tests.bal | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index c11afdc..3765368 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -20,18 +20,19 @@ import ballerina/os; configurable boolean isLiveServer = ?; configurable string token = isLiveServer ? os:getEnv("token") : "test"; final string mockServiceUrl = "http://localhost:9090"; +final Client openAIChat = check initClient(); + +function initClient() returns Client|error { + if isLiveServer { + return new ({auth: {token}}); + } + return new ({auth: {token}}, mockServiceUrl); +} @test:Config{ groups: ["live_tests", "mock_tests"] } isolated function testChatCompletion() returns error? { - - Client openAIChat; - if isLiveServer { - openAIChat = check new ({auth: {token}}); - } else { - openAIChat = check new ({auth: {token}}, mockServiceUrl); - } CreateChatCompletionRequest request = { model: "gpt-4o-mini", From 8d7856149683aa536f5d9dac5ac6a88e32e776e0 Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Mon, 12 Aug 2024 15:27:55 +0530 Subject: [PATCH 08/14] Update variable name for OpenAI chat client initialization in tests.bal --- ballerina/tests/tests.bal | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index 3765368..ebc6c00 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -20,7 +20,7 @@ import ballerina/os; configurable boolean isLiveServer = ?; configurable string token = isLiveServer ? os:getEnv("token") : "test"; final string mockServiceUrl = "http://localhost:9090"; -final Client openAIChat = check initClient(); +final Client openaiChat = check initClient(); function initClient() returns Client|error { if isLiveServer { @@ -40,7 +40,7 @@ isolated function testChatCompletion() returns error? { }; do{ - CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); + CreateChatCompletionResponse response = check openaiChat->/chat/completions.post(request); string? content = response.choices[0].message.content; test:assertTrue(content !is (), msg = "An error occurred with response content"); From 66b01b86c8f4ac02725c9696c21b8df6af43470b Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Mon, 12 Aug 2024 21:48:08 +0530 Subject: [PATCH 09/14] fix formatting issues --- ballerina/tests/mock_service.bal | 1 - ballerina/tests/tests.bal | 11 +++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ballerina/tests/mock_service.bal b/ballerina/tests/mock_service.bal index 6c29918..1b6b125 100644 --- a/ballerina/tests/mock_service.bal +++ b/ballerina/tests/mock_service.bal @@ -44,7 +44,6 @@ http:Service mockService = service object { "object": "chat.completion", "usage": {"completion_tokens": 11, "prompt_tokens": 13, "total_tokens": 24} }; - return response; } }; diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index ebc6c00..69ef9fe 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -14,8 +14,8 @@ // specific language governing permissions and limitations // under the License. -import ballerina/test; import ballerina/os; +import ballerina/test; configurable boolean isLiveServer = ?; configurable string token = isLiveServer ? os:getEnv("token") : "test"; @@ -29,7 +29,7 @@ function initClient() returns Client|error { return new ({auth: {token}}, mockServiceUrl); } -@test:Config{ +@test:Config { groups: ["live_tests", "mock_tests"] } isolated function testChatCompletion() returns error? { @@ -39,14 +39,13 @@ isolated function testChatCompletion() returns error? { messages: [{"role": "user", "content": "This is a test message"}] }; - do{ + do { CreateChatCompletionResponse response = check openaiChat->/chat/completions.post(request); - + string? content = response.choices[0].message.content; test:assertTrue(content !is (), msg = "An error occurred with response content"); - + } on fail error e { test:assertFail(msg = "An error occurred: " + e.message()); } - } From 382ed1cf1ffd493d881d5b58f5dfe486db7e290f Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Tue, 13 Aug 2024 11:34:30 +0530 Subject: [PATCH 10/14] Revert "Update variable name for OpenAI chat client initialization in tests.bal" --- ballerina/tests/tests.bal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index 69ef9fe..9267ee2 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -20,7 +20,7 @@ import ballerina/test; configurable boolean isLiveServer = ?; configurable string token = isLiveServer ? os:getEnv("token") : "test"; final string mockServiceUrl = "http://localhost:9090"; -final Client openaiChat = check initClient(); +final Client openAIChat = check initClient(); function initClient() returns Client|error { if isLiveServer { @@ -39,9 +39,9 @@ isolated function testChatCompletion() returns error? { messages: [{"role": "user", "content": "This is a test message"}] }; - do { - CreateChatCompletionResponse response = check openaiChat->/chat/completions.post(request); - + do{ + CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); + string? content = response.choices[0].message.content; test:assertTrue(content !is (), msg = "An error occurred with response content"); From 1697ccffd0ee3e32a857c186b618ac8a1d9edb4e Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Tue, 13 Aug 2024 11:40:28 +0530 Subject: [PATCH 11/14] Update isLiveServer configuration --- ballerina/tests/tests.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index 9267ee2..094a4d6 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -17,7 +17,7 @@ import ballerina/os; import ballerina/test; -configurable boolean isLiveServer = ?; +configurable boolean isLiveServer = os:getEnv("isLiveServer") == "true"; configurable string token = isLiveServer ? os:getEnv("token") : "test"; final string mockServiceUrl = "http://localhost:9090"; final Client openAIChat = check initClient(); From 5a826ff12bb5d1fd3983d9752c66c561798af24b Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Tue, 13 Aug 2024 11:41:47 +0530 Subject: [PATCH 12/14] fix: Fix formatting issues in tests.bal --- ballerina/tests/tests.bal | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index 094a4d6..8088243 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -39,9 +39,9 @@ isolated function testChatCompletion() returns error? { messages: [{"role": "user", "content": "This is a test message"}] }; - do{ + do { CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); - + string? content = response.choices[0].message.content; test:assertTrue(content !is (), msg = "An error occurred with response content"); From 6ccc0378a3a8b40f36e83c9d2d984889138fa2c7 Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Tue, 13 Aug 2024 20:03:48 +0530 Subject: [PATCH 13/14] chore: Remove unnecessary parentheses in mock_service.bal --- ballerina/tests/mock_service.bal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballerina/tests/mock_service.bal b/ballerina/tests/mock_service.bal index 1b6b125..f70be36 100644 --- a/ballerina/tests/mock_service.bal +++ b/ballerina/tests/mock_service.bal @@ -23,7 +23,7 @@ http:Service mockService = service object { resource function post chat/completions(@http:Payload CreateChatCompletionRequest payload) returns CreateChatCompletionResponse|http:BadRequest { // Validate the request payload - if (payload.messages[0]["content"].toString() is "" || payload.model.toString() is "") { + if payload.messages[0]["content"].toString() is "" || payload.model.toString() is "" { return http:BAD_REQUEST; } From 3ffa8c8db40c7b5e7669c35d43d4bd6099440feb Mon Sep 17 00:00:00 2001 From: manodyaSenevirathne Date: Tue, 13 Aug 2024 21:54:01 +0530 Subject: [PATCH 14/14] Refactor testChatCompletion function in tests.bal --- ballerina/tests/tests.bal | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/ballerina/tests/tests.bal b/ballerina/tests/tests.bal index 8088243..0a5276a 100644 --- a/ballerina/tests/tests.bal +++ b/ballerina/tests/tests.bal @@ -33,19 +33,12 @@ function initClient() returns Client|error { groups: ["live_tests", "mock_tests"] } isolated function testChatCompletion() returns error? { - CreateChatCompletionRequest request = { model: "gpt-4o-mini", messages: [{"role": "user", "content": "This is a test message"}] }; - - do { - CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); - - string? content = response.choices[0].message.content; - test:assertTrue(content !is (), msg = "An error occurred with response content"); - - } on fail error e { - test:assertFail(msg = "An error occurred: " + e.message()); - } + CreateChatCompletionResponse response = check openAIChat->/chat/completions.post(request); + test:assertTrue(response.choices.length() > 0, msg = "Expected at least one completion choice"); + string? content = response.choices[0].message.content; + test:assertTrue(content !is (), msg = "Expected content in the completion response"); }