Added Batch Processing Preview

Added Batch Processing
Added Imagen 3 to Batch
Added Error checking and stream events to batch

demos/digital-commerce/.vscode/settings.json

@@ -1,6 +1,8 @@
 {
     "cSpell.words": [
         "appspot",
-        "nohighlight"
+        "gtin",
+        "nohighlight",
+        "recordrtc"
     ]
 }

demos/digital-commerce/apps/api/src/events/batch-stream.ts

@@ -0,0 +1,182 @@
+// Copyright 2024 Google, LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { GoogleAuth } from 'google-auth-library';
+import { Socket } from 'socket.io';
+import sessionManager from '../state';
+import { extractTextCandidates } from '../utils';
+import { BatchPromptRequest } from 'libs/model/src/lib/api';
+import { BaseProduct, BatchProduct, Category, Image, Product } from 'model';
+import axios from 'axios';
+import { GenerativeModel } from '@google-cloud/vertexai';
+
+const GCP_IMAGE_MODEL=process.env.GCP_IMAGE_MODEL
+const GCP_PROJECT_ID=process.env.GCP_PROJECT_ID
+const GCP_LOCATION=process.env.GCP_LOCATION
+
+interface SimpleProduct {
+  language: string;
+  name: string;
+  category: string;
+  description: string;
+  seoHtmlHeader: string;
+  attributeValues: { name: string; value: string | number | boolean }[];
+}
+
+interface SimpleImage {
+  bytesBase64Encoded: string;
+  mimeType: string;
+}
+
+interface ImageResponse {
+  predictions: SimpleImage[];
+}
+
+const generate_image = (product: Product, socket: Socket) => {
+  // TODO - reactor once the API is supported in main stream, until then use axios.
+  // If you HAVE NOT been approved for imagen 3, you may need to change to an older model: imagegeneration@006
+
+  const auth = new GoogleAuth();
+  auth
+    .getAccessToken()
+    .then((token) => {
+      const url = `https://${GCP_LOCATION}-aiplatform.googleapis.com/v1/projects/${GCP_PROJECT_ID}/locations/${GCP_LOCATION}/publishers/google/models/${GCP_IMAGE_MODEL}:predict`;
+      const req_body = {
+        instances: [
+          {
+            prompt: `Given the following JSON product details create an image of the product in an environment suitable for seeling the product on an e-commerce web site. Natural Lighting, High Contrast, 35mm, Vivid.\nProduct Details:\n${JSON.stringify(
+              product
+            )}`,
+          },
+        ],
+        parameters: {
+          sampleCount: 1,
+        },
+      };
+
+      axios
+        .post(url, req_body, {
+          headers: {
+            Authorization: `Bearer ${token}`,
+            'X-Goog-User-Project': GCP_PROJECT_ID,
+          },
+        })
+        .then((resp) => {
+          const data = resp.data as ImageResponse;
+          if (!product.images) {
+            product.images = [];
+          }
+          product.images.push(
+            ...data.predictions.map((s) => ({ base64: s.bytesBase64Encoded, type: s.mimeType } as Image))
+          );
+        })
+        .catch((e) => {
+          console.error(e);
+          socket.emit('batch:error', { message: `Failed to generate image for: ${product.base.name}` });
+        })
+        .finally(() => socket.emit('batch:response', { message: product }));
+    })
+    .catch((e) => {
+      console.error(e);
+      socket.emit('batch:response', { message: product });
+    });
+};
+
+const generate_product = ({
+  model,
+  socket,
+  value,
+  prompt,
+  incrementor,
+  count,
+}: {
+  model: GenerativeModel;
+  socket: Socket;
+  value: BatchProduct;
+  prompt: string;
+  incrementor: () => void;
+  count: number;
+}) => {
+  model.generateContent({ contents: [{ role: 'user', parts: [{ text: prompt }] }] }).then((result) => {
+    try {
+      const resultText = extractTextCandidates(result);
+      const obj = JSON.parse(resultText) as SimpleProduct;
+      const product = {
+        base: {
+          language: obj.language,
+          name: obj.name,
+          description: obj.description,
+          seoHtmlHeader: obj.seoHtmlHeader,
+          attributeValues: obj.attributeValues,
+        } as BaseProduct,
+        category: { name: obj.category } as Category,
+      } as Product;
+      generate_image(product, socket);
+      incrementor()
+    } catch (e) {
+      if (count < 3) {
+        socket.emit('batch:warn', { message: `Retrying process for item: ${value.name}` });
+        generate_product({ model: model, socket: socket, value: value, prompt: prompt, incrementor: incrementor, count: count + 1 });
+      } else {
+        socket.emit('batch:error', { message: `Failed to process: ${value.name}` });
+      }
+    }
+  });
+};
+
+export default (socket: Socket) =>
+  async ({ sessionID, values }: BatchPromptRequest) => {
+    const session = sessionManager.getSession(sessionID);
+    const model = session.groundedModel;
+
+    const exampleOutput = {
+      language: 'EN-US',
+      name: '',
+      category: '',
+      description: '',
+      seoHtmlHeader: '',
+      attributeValues: [{ name: '', value: '' }],
+    } as SimpleProduct;
+
+    const example = JSON.stringify(exampleOutput);
+
+    const processRequestCount = values.length;
+    let processed = 0;
+
+    const incrementProcessed = (): void => {
+        processed = processed +1;
+        if (processed === processRequestCount) {
+            socket.emit('batch:complete', { message: `Processed: ${processed} of ${processRequestCount}` });
+        }
+    }
+
+    values.forEach((value) => {
+      const prompt =
+        `Given the following product information follow the steps outlined below and produce a valid JSON response using the example output:
+GTIN: ${value.gtin}
+Product Name: ${value.name}
+Short description: ${value.short_description}
+
+- Find the top category and it's top 25 attributes and all matching values for this product, if there is not a matching value, do not include the attribute.
+- The category hierarchy must be 4 levels deep, separated by ' > ' character as the category name.
+- Write an enriched product description in markdown format for a retailers online catalog as the description.
+- Write the HTML SEO description and keywords the product in plain text/html no Markdown as seoHtmlHeader.
+- If the product is edible, include nutritional as additional attributeValues.
+
+Example Output:
+${example}
+`.trim();
+      generate_product({ model: model, socket: socket, value: value, prompt: prompt, incrementor: incrementProcessed, count: 0 });
+    });
+  };

demos/digital-commerce/apps/api/src/example-app.yaml

@@ -26,6 +26,9 @@ env_variables:
   STORAGE_BUCKET: ""
   FIRESTORE_DB_ID: ""
   FIRESTORE_COLLECTION: ""
+  GCP_PROJECT_ID: ""
+  GCP_LOCATION: "us-central1"
+  GCP_IMAGE_MODEL: "imagen-3.0-fast-generate-001"
 
 liveness_check:
   path: "/liveness_check"

demos/digital-commerce/apps/api/src/main.ts

@@ -24,7 +24,11 @@ import registrationHandler from './routes/register';
 import imageHandlers from './routes/images';
 import textHandlers from './routes/text';
 import videoHandlers from './routes/video';
+import batchHandlers from './routes/batch';
+
 import voiceStream from './events/voice-prompt';
+import batchStream from './events/batch-stream';
+
 
 config()
 
@@ -54,11 +58,13 @@ app.use('/api/registration', registrationHandler);
 app.use('/api/text', textHandlers);
 app.use('/api/images', imageHandlers);
 app.use('/api/video', videoHandlers);
+app.use('/api/batch', batchHandlers);
 
 
 io.on('connection', (socket: Socket) => {
     console.log(`Connected: ${socket.id}`);
     socket.on('voice:request', voiceStream(socket));
+    socket.on('batch:request', batchStream(socket));
 });
 
 const port = process.env.PORT || 3000;

demos/digital-commerce/apps/api/src/routes/batch.ts

@@ -0,0 +1,41 @@
+// Copyright 2024 Google, LLC
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     https://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Request, Response, Router } from "express";
+import sessionManager from '../state';
+import { BatchProduct } from "model";
+
+const router = Router();
+
+
+// Initial products, may be added onto in the UI
+const MIXED_PRODUCTS: BatchProduct[] = [
+    {id: 1, gtin: "00012345678905", name: "Apple iPhone 15 Pro Max", short_description: "Smartphone with A17 Pro chip and Super Retina XDR display"},
+    {id: 2, gtin: "00023456789014", name: "Samsung Galaxy Tab S9 Ultra", short_description: "Tablet with Dynamic AMOLED 2X display and Snapdragon 8 Gen 2 for Galaxy processor"},
+    {id: 3, gtin: "0003456789013", name: "Sony WH-1000XM5", short_description: "Wireless noise-cancelling headphones with industry-leading noise cancellation"},
+    {id: 4, gtin: "0004567890122", name: "Bose QuietComfort Earbuds II", short_description: "True wireless noise-cancelling earbuds with CustomTune technology"},
+    {id: 5, gtin: "0005678901231", name: "LG C3 OLED TV", short_description: "Smart TV with α9 AI Processor Gen6 and 4K self-lit OLED evo panel"},
+    {id: 6, gtin: "0006789012340", name: "Dyson V15 Detect Absolute", short_description: "Cordless vacuum cleaner with laser dust detection and HEPA filtration"},
+    {id: 7, gtin: "0007890123459", name: "KitchenAid Artisan Stand Mixer", short_description: "5-quart stand mixer with 10 speeds and tilt-head design"},
+    {id: 8, gtin: "0008901234568", name: "Nespresso Vertuo Next", short_description: "Single-serve coffee machine with centrifusion technology"},
+    {id: 9, gtin: "0009012345677", name: "Nike Air Zoom Pegasus 40", short_description: "Running shoes with React foam and Zoom Air unit"},
+    {id: 10, gtin: "0010123456786", name: "LEGO Star Wars Millennium Falcon", short_description: "7,541-piece LEGO set of the iconic Star Wars spaceship"},
+]
+
+router.get('/', (req: Request, resp: Response) => {
+    resp.status(200)
+    resp.json(MIXED_PRODUCTS)
+})
+
+export default router;

demos/digital-commerce/apps/api/src/utils.ts

@@ -18,7 +18,9 @@ import {api} from 'model';
 
 export const extractTextCandidates = (result: GenerateContentResult): string => {
     if (result.response.candidates) {
-        return result.response.candidates[0].content.parts[0].text;
+        const text = result.response.candidates[0].content.parts[0].text;
+        const cleanedString = text.replace(/\\(?!["\\\/bfnrt])/g, "\\\\");
+        return cleanedString;
     } else {
         return 'no content';
     }

demos/digital-commerce/apps/demo/src/components/Header.tsx

@@ -70,6 +70,17 @@ const Header = () => {
                             disabled={!config || !config.customerName || config.customerName === ''}>
                             Products
                         </Button>
+                        <Button 
+                            sx={{
+                                my: 2,
+                                display: 'block',
+                                color: '#333333',
+                                fontFamily: 'Google Sans',
+                            }}
+                            onClick={() => nav('/batch', {replace: true})}
+                            disabled={!config || !config.customerName || config.customerName === ''}>
+                            Batch
+                        </Button>
                         <Button
                             sx={{
                                 my: 2,

demos/digital-commerce/apps/demo/src/components/ProductAccordion.tsx

@@ -0,0 +1,86 @@
+// Copyright 2024 Google, LLC
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     https://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Product } from 'model';
+import { Accordion,
+    AccordionDetails,
+    AccordionSummary, Typography, Grid,
+    Paper} from "@mui/material";
+import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
+import React from "react";
+
+
+const MarkdownPreview = React.lazy(() => import('@uiw/react-markdown-preview'));
+
+const ProductAccordionPanel = ({ index, product }: { index: number; product: Product }) => {
+    return (
+      <Accordion>
+        <AccordionSummary
+          expandIcon={<ExpandMoreIcon />}
+          aria-controls={`panel-content-${index}`}
+          id={`panel-header-${index}`}>
+          {product.base.name}
+        </AccordionSummary>
+        <AccordionDetails sx={{ p: 1, borderRadius: '10px' }}>
+          <Typography variant="overline">{product.category.name}</Typography>
+          <Grid container spacing={2}>
+            <Grid item xs={8}>
+              <MarkdownPreview
+                source={product.base.description}
+                style={{ backgroundColor: '#fff', color: '#666', marginBottom: '2em' }}
+              />
+            </Grid>
+            <Grid item xs={4}>
+              {product.images ? (
+                product.images.map((i) => (
+                  <Paper elevation={5} sx={{borderRadius: '10px', display: 'flex', flexGrow: 1}}>
+                      <img src={`data:${i.type};base64,${i.base64}`} width="100%" style={{objectFit: 'cover', borderRadius: '10px'}}/>
+                  </Paper>
+                ))
+              ) : (
+                <></>
+              )}
+            </Grid>
+          </Grid>
+
+          {product.base.attributeValues ? (
+            <React.Fragment>
+              <Typography variant="h6">Attributes</Typography>
+              <Grid container spacing={2}>
+                {product.base.attributeValues.map((a) => (
+                  <Grid item xs={3}>
+                    <Typography variant="overline">{a.name}</Typography>
+                    <br />
+                    <Typography variant="caption">{a.value}</Typography>
+                  </Grid>
+                ))}
+              </Grid>
+            </React.Fragment>
+          ) : (
+            <></>
+          )}
+
+          <Typography variant="h6" sx={{ mt: 2 }}>
+            SEO
+          </Typography>
+          <MarkdownPreview
+            source={`\`\`\`html\n${product.base.seoHtmlHeader}\`\`\``}
+            style={{ margin: '2em', backgroundColor: '#fff', color: '#666', marginBottom: '2em' }}
+          />
+        </AccordionDetails>
+      </Accordion>
+    );
+  };
+
+  export default ProductAccordionPanel;