How to use the triton server to generate SSE STREAM effect similar to ChatGPT ？

[WackyGem]

I'm learning to use tritonclient for bloom model inference, the server is working, here is my client code, I want to achieve chatgpt like effect but don't know how to change it, plz help me,thanks 🙏.

import unittest

import argparse
import numpy as np
import tritonclient.grpc as grpcclient
import tritonclient.http as httpclient

from tritonclient.utils import np_to_triton_dtype


def create_inference_server_client(protocol, url, concurrency, verbose):
    client_util = httpclient if protocol == "http" else grpcclient
    if protocol == "http":
        return client_util.InferenceServerClient(url,
                                                 concurrency=concurrency,
                                                 verbose=verbose)
    elif protocol == "grpc":
        return client_util.InferenceServerClient(url, verbose=verbose)


def prepare_tensor(name, input, protocol):
    client_util = httpclient if protocol == "http" else grpcclient
    t = client_util.InferInput(
        name, input.shape, np_to_triton_dtype(input.dtype))
    t.set_data_from_numpy(input)
    return t


class TestAny(unittest.TestCase):

    def test_any(self):
        model_name = "ensemble"

        FLAGS = argparse.Namespace(verbose=False,
                                   url="localhost:8001",
                                   beam_width=1,
                                   topk=40,
                                   topp=0.95,
                                   protocol="grpc",
                                   return_log_probs=False,
                                   output_len=500,
                                   model_variant='bloom')
        with create_inference_server_client(FLAGS.protocol,
                                            FLAGS.url,
                                            concurrency=1,
                                            verbose=FLAGS.verbose,
                                            ) as client:
            input0 = [
                ["Exercise can enhance the body‘s immunity and"],
                ["Reading can enhance knowledge, improve skills, enrich"]
            ]
            bad_words_list = np.array([
                [""],
                [""],
            ], dtype=object)
            stop_words_list = np.array([
                [""],
                [""],
            ], dtype=object)
            input0_data = np.array(input0).astype(object)
            output0_len = np.ones_like(input0).astype(np.uint32) * 300
            runtime_top_k = (20 * np.ones([input0_data.shape[0], 1])).astype(np.uint32)
            runtime_top_p = 0.95 * np.ones([input0_data.shape[0], 1]).astype(np.float32)
            beam_search_diversity_rate = 0 * np.ones([input0_data.shape[0], 1]).astype(np.float32)
            temperature = 1 * np.ones([input0_data.shape[0], 1]).astype(np.float32)
            len_penalty = 2 * np.ones([input0_data.shape[0], 1]).astype(np.float32)
            repetition_penalty = 1 * np.ones([input0_data.shape[0], 1]).astype(np.float32)
            random_seed = 0 * np.ones([input0_data.shape[0], 1]).astype(np.uint64)
            inputs = [
                prepare_tensor("INPUT_0", input0_data, FLAGS.protocol),
                prepare_tensor("INPUT_1", output0_len, FLAGS.protocol),
                prepare_tensor("INPUT_2", bad_words_list, FLAGS.protocol),
                prepare_tensor("INPUT_3", stop_words_list, FLAGS.protocol),
                prepare_tensor("runtime_top_k", runtime_top_k, FLAGS.protocol),
                prepare_tensor("runtime_top_p", runtime_top_p, FLAGS.protocol),
                prepare_tensor("beam_search_diversity_rate", beam_search_diversity_rate, FLAGS.protocol),
                prepare_tensor("temperature", temperature, FLAGS.protocol),
                prepare_tensor("len_penalty", len_penalty, FLAGS.protocol),
                prepare_tensor("repetition_penalty", repetition_penalty, FLAGS.protocol),
                prepare_tensor("random_seed", random_seed, FLAGS.protocol),
            ]
            try:
                result = client.infer(model_name, inputs,
                                      outputs=[grpcclient.InferRequestedOutput(name="OUTPUT_0")],)
                output0 = [x.decode('utf-8') for x in result.as_numpy("OUTPUT_0")]
                print("============After ensemble============")
                print(f"output: \n {output0}", flush=True)
                print(f"sequence_length: \n{result.as_numpy('sequence_length')}")
                if FLAGS.return_log_probs:
                    print(f"cum_log_probs: \n{result.as_numpy('cum_log_probs')}")
                    print(f"output_log_probs: \n{result.as_numpy('output_log_probs')}")
            except Exception as e:
                print(e)