Updated agent and text files to wire in text (typing)

Fixed the load on product page 3 using a suspense.

.github/workflows/main.old

@@ -0,0 +1,48 @@
+# name: CI
+
+# on:
+#   push:
+#     branches:
+#     - main
+#   pull_request:
+
+# jobs:
+#   build:
+#     runs-on: ubuntu-latest
+#     env:
+#       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#     steps:
+#       - uses: actions/checkout@v4
+
+#       - name: Bazel Setup
+#         uses: bazelbuild/setup-bazelisk@v3
+
+#       - name: Mount bazel cache
+#         uses: actions/cache@v4
+#         with:
+#           path: "/home/runner/.cache/bazel"
+#           key: bazel
+
+#       - name: "Build"
+#         run: bazel build //...
+
+#       - name: Checkout Docs Branch
+#         if: github.ref == 'refs/heads/main'
+#         uses: actions/checkout@v4
+#         with:
+#           ref: 'docs'
+#           clean: 'false'
+
+#       - name: Expand Docs
+#         if: github.ref == 'refs/heads/main'
+#         run: |
+#           echo "Unzipping Docs Site"
+#           tar -xf "${GITHUB_WORKSPACE}/bazel-out/k8-fastbuild/bin/docs/docs.tar"
+
+#       - name: Check-in Latest Docs
+#         if: github.ref == 'refs/heads/main'
+#         uses: EndBug/add-and-commit@v9
+#         with:
+#           author_name: "Build Robot"
+
+

.trunk/.gitignore

@@ -0,0 +1,9 @@
+*out
+*logs
+*actions
+*notifications
+*tools
+plugins
+user_trunk.yaml
+user.yaml
+tmp

.trunk/configs/.hadolint.yaml

@@ -0,0 +1,4 @@
+# Following source doesn't work in most setups
+ignored:
+  - SC1090
+  - SC1091

.trunk/configs/.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+profile=black

.trunk/configs/.markdownlint.yaml

@@ -0,0 +1,2 @@
+# Prettier friendly markdownlint config (all formatting rules disabled)
+extends: markdownlint/style/prettier

.trunk/configs/.yamllint.yaml

@@ -0,0 +1,7 @@
+rules:
+  quoted-strings:
+    required: only-when-needed
+    extra-allowed: ["{|}"]
+  key-duplicates: {}
+  octal-values:
+    forbid-implicit-octal: true

.trunk/configs/ruff.toml

@@ -0,0 +1,5 @@
+# Generic, formatter-friendly config.
+select = ["B", "D3", "E", "F"]
+
+# Never enforce `E501` (line length violations). This should be handled by formatters.
+ignore = ["E501"]

.trunk/trunk.yaml

@@ -0,0 +1,43 @@
+# This file controls the behavior of Trunk: https://docs.trunk.io/cli
+# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
+version: 0.1
+cli:
+  version: 1.22.8
+# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
+plugins:
+  sources:
+    - id: trunk
+      ref: v1.6.4
+      uri: https://github.com/trunk-io/plugins
+# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
+runtimes:
+  enabled:
+    - [email protected]
+    - [email protected]
+# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
+lint:
+  enabled:
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - git-diff-check
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+    - [email protected]
+actions:
+  disabled:
+    - trunk-announce
+    - trunk-check-pre-push
+    - trunk-fmt-pre-commit
+  enabled:
+    - trunk-upgrade-available

demos/digital-commerce/apps/api/src/events/text-agent.ts

@@ -0,0 +1,35 @@
+// Copyright 2024 Google, LLC
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     https://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { ChatPromptRequest } from "libs/model/src/lib/api";
+import { Socket } from "socket.io";
+import sessionManager from "../state";
+import { extractTextCandidates } from "../utils";
+
+export default (socket: Socket) => async ({sessionID, prompt, value}: ChatPromptRequest) => {
+    const session = sessionManager.getSession(sessionID);
+
+    const model = session.groundedModel;
+
+    const chatPrompt = prompt + `\nProduct Data JSON: ${prompt}\nExample JSON output: {prompt: '${value}', response: 'Some generated response'} where the response value is in markdown format.`;
+
+    model
+        .generateContent({
+            contents: [{role: 'user', parts: [{text: chatPrompt}]}],
+        })
+        .then((result) => {
+            const value = extractTextCandidates(result);
+            socket.emit('agent:response', value);
+        });
+}

demos/digital-commerce/apps/api/src/events/voice-agent.ts

@@ -0,0 +1,78 @@
+// Copyright 2024 Google, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {Socket} from 'socket.io';
+import {deleteFile, saveFile} from '../storage';
+import sessionManager from '../state';
+
+import speech from '@google-cloud/speech';
+import {extractTextCandidates} from '../utils';
+import {api} from 'model';
+
+const speechClient = new speech.SpeechClient();
+export const AUDIO_EXTENSION = 'webm';
+
+export default (socket: Socket) =>
+    async ({sessionID, type, prompt, value}: api.AudioPromptRequest) => {
+        const session = sessionManager.getSession(sessionID);
+
+        const model = session.groundedModel;
+
+        const handleFile = async (filUri: string) => {
+            console.log(`file handle ${filUri}`);
+            const [resp] = await speechClient.recognize({
+                config: {
+                    encoding: 'WEBM_OPUS',
+                    sampleRateHertz: 48000,
+                    languageCode: 'en-US',
+                    enableAutomaticPunctuation: true
+                },
+                audio: {uri: filUri},
+            });
+
+            if (resp && resp.results) {
+                const transcription = resp.results
+                    .map((result) =>
+                        result.alternatives && result.alternatives.length > 0 ? result.alternatives[0].transcript : ''
+                    )
+                    .join('\n');
+
+                socket.emit('voice:transcript', {
+                    message: transcription,
+                });
+
+                const audioPrompt = transcription + `\nProduct Data JSON: ${prompt}\nExample JSON output: {prompt: '${transcription}', response: 'Some generated response'} where the response value is in markdown format.`;
+
+                model
+                    .generateContent({
+                        contents: [{role: 'user', parts: [{text: audioPrompt}]}],
+                    })
+                    .then((result) => {
+                        const value = extractTextCandidates(result);
+                        socket.emit('agent:response', value);
+                    })
+                    .finally(() => {
+                        deleteFile(filUri);
+                    });
+            }
+        };
+
+        if (model) {
+            await saveFile(sessionID, 'voice-transcript', type, value, AUDIO_EXTENSION, handleFile);
+        } else {
+            socket.emit('voice:error', {
+                message: 'invalid session',
+            });
+        }
+    };

demos/digital-commerce/apps/demo/src/pages/Settings.tsx

@@ -60,7 +60,7 @@ const Settings = () => {
     initialValues: config,
     validationSchema: SettingSchema,
     onSubmit: async (values: Config) => {
-      AxiosInstance.post(`/registration`, { sessionID: sessionID, config: values } as api.ConfigurationRequest)
+      AxiosInstance.post('/registration', { sessionID: sessionID, config: values } as api.ConfigurationRequest)
         .then((resp) => {
           if (resp.status === 201 || resp.status === 202) {
             const reg = resp.data as api.ConfigurationResponse;

demos/digital-commerce/apps/demo/src/pages/products/Step3.tsx

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import React, { ReactNode, useContext, useEffect, useMemo, useState } from 'react';
+import React, { ReactNode, Suspense, useContext, useEffect, useMemo, useState } from 'react';
 
 import {
   Box,
@@ -202,29 +202,29 @@ const Step3 = () => {
         </Grid>
       </Grid>
 
-      <Tabs value={selectedTab} onChange={handleTabChange}>
-        <Tab key="selected_tab_0" label={product.base.language} {...a11yProps(0)} />
-        {product.alternatives?.map((entry, idx) => (
-          <Tab key={`selected_tab_${idx + 1}`} label={entry.base.language} {...a11yProps(idx + 1)} />
-        ))}
-      </Tabs>
-
-      {product && product.base && product.base.name ? (
-        <React.Fragment>
-          <ProductTabPanel key="product_panel_0" value={selectedTab} index={0}>
-            <ProductDetail product={product} />
-          </ProductTabPanel>
-
+      <Suspense>
+        <Tabs value={selectedTab} onChange={handleTabChange}>
+          <Tab key="selected_tab_0" label={product.base.language} {...a11yProps(0)} />
           {product.alternatives?.map((entry, idx) => (
-            <ProductTabPanel key={`product_panel_${idx + 1}`} value={selectedTab} index={idx + 1}>
-              <ProductDetail product={entry} />
-            </ProductTabPanel>
+            <Tab key={`selected_tab_${idx + 1}`} label={entry.base.language} {...a11yProps(idx + 1)} />
           ))}
-        </React.Fragment>
-      ) : (
-        <></>
-      )}
+        </Tabs>
+        {product && product.base && product.base.name ? (
+          <React.Fragment>
+            <ProductTabPanel key="product_panel_0" value={selectedTab} index={0}>
+              <ProductDetail product={product} />
+            </ProductTabPanel>
 
+            {product.alternatives?.map((entry, idx) => (
+              <ProductTabPanel key={`product_panel_${idx + 1}`} value={selectedTab} index={idx + 1}>
+                <ProductDetail product={entry} />
+              </ProductTabPanel>
+            ))}
+          </React.Fragment>
+        ) : (
+          <></>
+        )}
+      </Suspense>
       <GoogleBackdrop backdrop={backdrop} setBackdrop={setBackdrop} />
     </React.Fragment>
   );

demos/digital-commerce/apps/demo/src/utils/WebClient.ts

@@ -0,0 +1,26 @@
+// Copyright 2024 Google, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import axios from 'axios';
+
+const API_BASE = import.meta.env.VITE_API_URL_BASE;
+
+const AxiosInstance = axios.create({
+  baseURL: API_BASE,
+  timeout: 10000,
+});
+AxiosInstance.defaults.headers.common['Content-Type'] = 'application/json';
+
+
+export default AxiosInstance;