diff --git a/apps/nextjs/src/app/image-spike/[slug]/images.tsx b/apps/nextjs/src/app/image-spike/[slug]/images.tsx index 4b898e4ca..0d9ff58cf 100644 --- a/apps/nextjs/src/app/image-spike/[slug]/images.tsx +++ b/apps/nextjs/src/app/image-spike/[slug]/images.tsx @@ -2,6 +2,7 @@ import { useCallback, useState } from "react"; +import { type ValidationResult } from "@oakai/api/src/imageGen/validateImageWithOpenAI"; import type { Resource } from "ai-apps/image-alt-generation/types"; import Image from "next/image"; import Link from "next/link"; @@ -9,6 +10,11 @@ import Link from "next/link"; import LoadingWheel from "@/components/LoadingWheel"; import { trpc } from "@/utils/trpc"; +type ValidatedImage = { + imageData: ImageResponse; + validationResult: ValidationResult; +}; + export interface PageData { id?: string; path?: string; @@ -27,6 +33,9 @@ interface ImageResponse { alt?: string; license: string; photographer?: string; + appropriatenessScore?: number; + appropriatenessReasoning?: string; + imageSource?: string; } type ImagesFromCloudinary = { @@ -49,6 +58,10 @@ interface ComparisonColumn { const ImagesPage = ({ pageData }: { pageData: PageData }) => { const [selectedImagePrompt, setSelectedImagePrompt] = useState(""); + const [theBestImage, setTheBestImage] = useState<{ + data: ValidatedImage | null | undefined; + status: string; + } | null>(null); const [comparisonColumns, setComparisonColumns] = useState< ComparisonColumn[] >([ @@ -61,34 +74,269 @@ const ImagesPage = ({ pageData }: { pageData: PageData }) => { }, ]); - if ( - !pageData?.lessonPlan?.cycle1?.explanation?.imagePrompt || - !pageData?.lessonPlan?.cycle2?.explanation?.imagePrompt || - !pageData?.lessonPlan?.cycle3?.explanation?.imagePrompt - ) { - return "Please choose a lesson with all three learning cycles"; - } - - const slideTexts = { - cycle1: pageData?.lessonPlan?.cycle1?.explanation?.imagePrompt, - cycle2: pageData?.lessonPlan?.cycle2?.explanation?.imagePrompt, - cycle3: pageData?.lessonPlan?.cycle3?.explanation?.imagePrompt, - }; - + // Move all mutation declarations to component level + const cloudinaryMutation = + trpc.cloudinaryRouter.getCloudinaryImagesBySearchExpression.useMutation(); + const flickrMutation = trpc.imageSearch.getImagesFromFlickr.useMutation(); + const unsplashMutation = trpc.imageSearch.getImagesFromUnsplash.useMutation(); + const stableDiffusionCoreMutation = trpc.imageGen.stableDifCore.useMutation(); + const daleMutation = trpc.imageGen.openAi.useMutation(); + const validateImage = trpc.imageGen.validateImage.useMutation(); + const validateImagesInParallel = + trpc.imageGen.validateImagesInParallel.useMutation(); + // Other mutations for the comparison feature const trpcMutations = { - "Custom Pipeline": trpc.imageGen.customPipeline.useMutation(), + "Custom Pipeline Show all with Reasoning": + trpc.imageGen.customPipelineWithReasoning.useMutation(), "Stable Diffusion Ultra": trpc.imageGen.stableDifUltra.useMutation(), "Stable Diffusion 3.0 & 3.5": trpc.imageGen.stableDif3.useMutation(), - "Stable Diffusion Core": trpc.imageGen.stableDifCore.useMutation(), - "DAL-E": trpc.imageGen.openAi.useMutation(), - Flickr: trpc.imageSearch.getImagesFromFlickr.useMutation(), - Unsplash: trpc.imageSearch.getImagesFromUnsplash.useMutation(), - Cloudinary: - trpc.cloudinaryRouter.getCloudinaryImagesBySearchExpression.useMutation(), - Google: trpc.imageSearch.getImagesFromGoogle.useMutation(), - "Simple Google": trpc.imageSearch.simpleGoogleSearch.useMutation(), + "Stable Diffusion Core": stableDiffusionCoreMutation, + "DAL-E": daleMutation, + Flickr: flickrMutation, + Unsplash: unsplashMutation, + Cloudinary: cloudinaryMutation, + // Google: trpc.imageSearch.getImagesFromGoogle.useMutation(), + // "Simple Google": trpc.imageSearch.simpleGoogleSearch.useMutation(), }; + const findTheBestImage = useCallback(async () => { + let validImages: ValidatedImage[] = []; + + function updateStatus(status: string) { + console.log("[Status Update]:", status); + setTheBestImage((prev) => ({ + data: prev?.data, + status, + })); + } + + updateStatus("Searching Cloudinary..."); + try { + // Try Cloudinary first + console.log( + "[Cloudinary] Starting search with prompt:", + selectedImagePrompt, + ); + const cloudinaryImages = await cloudinaryMutation.mutateAsync({ + searchExpression: selectedImagePrompt, + }); + console.log("[Cloudinary] Raw response:", cloudinaryImages); + + const cloudinaryData = ( + cloudinaryImages as ImagesFromCloudinary + ).resources.map((image) => ({ + id: image.public_id, + url: image.url, + license: "Cloudinary", + })); + console.log("[Cloudinary] Processed data:", cloudinaryData); + + console.log("[Validation] Starting Cloudinary validation"); + + const validateCloudinaryResponse = + await validateImagesInParallel.mutateAsync({ + images: cloudinaryData, + searchExpression: selectedImagePrompt, + }); + + console.log( + "[Validation] Cloudinary validation results:", + validateCloudinaryResponse, + ); + + validImages = [ + ...validateCloudinaryResponse.filter( + (image) => image.validationResult.isValid, + ), + ]; + console.log("[Cloudinary] Valid images:", validImages); + + // If we don't have enough valid images, try Flickr + if (validImages.length < 1) { + updateStatus("Nothing appropriate on cloudinary, searching Flickr..."); + console.log("[Flickr] Starting search"); + const flickrImages = await flickrMutation.mutateAsync({ + searchExpression: selectedImagePrompt, + }); + console.log("[Flickr] Raw response:", flickrImages); + + console.log("[Validation] Starting Flickr validation"); + + const validateFlickrResponse = + await validateImagesInParallel.mutateAsync({ + images: flickrImages, + searchExpression: selectedImagePrompt, + }); + + console.log( + "[Validation] Flickr validation results:", + validateFlickrResponse, + ); + + validImages = [ + ...validImages, + ...validateFlickrResponse.filter( + (image) => image.validationResult.isValid, + ), + ]; + console.log("[Flickr] Updated valid images:", validImages); + } + + // If still not enough, try Unsplash + if (validImages.length < 1) { + updateStatus("Nothing appropriate on Flickr, searching Unsplash..."); + console.log("[Unsplash] Starting search"); + const unsplashImages = await unsplashMutation.mutateAsync({ + searchExpression: selectedImagePrompt, + }); + console.log("[Unsplash] Raw response:", unsplashImages); + + console.log("[Validation] Starting Unsplash validation"); + const validateUnsplashResponse = + await validateImagesInParallel.mutateAsync({ + images: unsplashImages, + searchExpression: selectedImagePrompt, + }); + console.log( + "[Validation] Unsplash validation results:", + validateUnsplashResponse, + ); + + validImages = [ + ...validImages, + ...validateUnsplashResponse.filter( + (image) => image.validationResult.isValid, + ), + ]; + console.log("[Unsplash] Updated valid images:", validImages); + } + + // If still not enough, try Stable Diffusion Core + if (validImages.length < 1) { + updateStatus( + "Nothing appropriate on Unsplash, generating an image using Stable Diffusion Core...", + ); + console.log("[StableDiffusion] Starting generation"); + const stableDiffusionCoreImages = + await stableDiffusionCoreMutation.mutateAsync({ + searchExpression: selectedImagePrompt, + }); + console.log( + "[StableDiffusion] Raw response:", + stableDiffusionCoreImages, + ); + + const stableDiffusionCoreData = [ + { + id: "1", + url: stableDiffusionCoreImages, + license: "Stable Diffusion Core", + }, + ]; + console.log( + "[StableDiffusion] Processed data:", + stableDiffusionCoreData, + ); + + console.log("[Validation] Starting Stable Diffusion validation"); + const validateStableDiffusionResponse = + await validateImagesInParallel.mutateAsync({ + images: stableDiffusionCoreData, + searchExpression: selectedImagePrompt, + }); + console.log( + "[Validation] Stable Diffusion validation results:", + validateStableDiffusionResponse, + ); + + validImages = [ + ...validImages, + ...validateStableDiffusionResponse.filter( + (image) => image.validationResult.isValid, + ), + ]; + console.log("[StableDiffusion] Updated valid images:", validImages); + } + + // If still not enough, try DALL-E as last resort + if (validImages.length < 1) { + updateStatus( + "Stable Diffusion Core did not generate a usable image, generating an image using DAL-E...", + ); + console.log("[DALL-E] Starting generation"); + const daleImages = await daleMutation.mutateAsync({ + searchExpression: selectedImagePrompt, + }); + console.log("[DALL-E] Raw response:", daleImages); + + const daleData = [ + { + id: "1", + url: daleImages, + license: "OpenAI", + }, + ]; + console.log("[DALL-E] Processed data:", daleData); + + console.log("[Validation] Starting DALL-E validation"); + const validateDaleResponse = await validateImagesInParallel.mutateAsync( + { + images: daleData, + searchExpression: selectedImagePrompt, + }, + ); + console.log( + "[Validation] DALL-E validation results:", + validateDaleResponse, + ); + + validImages = [ + ...validImages, + ...validateDaleResponse.filter( + (image) => image.validationResult.isValid, + ), + ]; + console.log("[DALL-E] Updated valid images:", validImages); + } + + // Sort all valid images by appropriateness score if we have more than one + if (validImages.length > 1) { + validImages.sort( + (a, b) => + b.validationResult.metadata.appropriatenessScore - + a.validationResult.metadata.appropriatenessScore, + ); + console.log("[Final] Sorted valid images:", validImages); + } + + // Return the best image or null if no valid images were found + const finalAnswer = validImages.length > 0 ? validImages[0] : null; + console.log("[Final] Selected best image:", finalAnswer); + + setTheBestImage({ + data: finalAnswer, + status: finalAnswer + ? "The best image has been found!" + : "No appropriate images found.", + }); + return finalAnswer; + } catch (error) { + console.error("[Error] Error in findTheBestImage:", error); + setTheBestImage({ + data: null, + status: "An error occurred while searching for images.", + }); + return null; + } + }, [ + selectedImagePrompt, + cloudinaryMutation, + flickrMutation, + unsplashMutation, + stableDiffusionCoreMutation, + daleMutation, + ]); + const fetchImages = useCallback( async (columnId: string, source: string, prompt: string) => { if (!trpcMutations[source]) { @@ -108,7 +356,11 @@ const ImagesPage = ({ pageData }: { pageData: PageData }) => { searchExpression: prompt, }); - if ( + if (source === "Custom Pipeline With Reasoning") { + updateColumn(columnId, { + imageSearchBatch: response as ImageResponse[], + }); + } else if ( source === "Stable Diffusion Ultra" || source === "Stable Diffusion 3.0 & 3.5" || source === "Stable Diffusion Core" || @@ -162,6 +414,20 @@ const ImagesPage = ({ pageData }: { pageData: PageData }) => { [trpcMutations], ); + if ( + !pageData?.lessonPlan?.cycle1?.explanation?.imagePrompt || + !pageData?.lessonPlan?.cycle2?.explanation?.imagePrompt || + !pageData?.lessonPlan?.cycle3?.explanation?.imagePrompt + ) { + return "Please choose a lesson with all three learning cycles"; + } + + const slideTexts = { + cycle1: pageData?.lessonPlan?.cycle1?.explanation?.imagePrompt, + cycle2: pageData?.lessonPlan?.cycle2?.explanation?.imagePrompt, + cycle3: pageData?.lessonPlan?.cycle3?.explanation?.imagePrompt, + }; + const updateColumn = ( columnId: string, updates: Partial, @@ -205,6 +471,7 @@ const ImagesPage = ({ pageData }: { pageData: PageData }) => { selectedImagePrompt === prompt ? "bg-black text-white" : "" }`} onClick={() => { + setTheBestImage(null); setSelectedImagePrompt(prompt); setComparisonColumns((prev) => prev.map((col) => ({ @@ -231,18 +498,66 @@ const ImagesPage = ({ pageData }: { pageData: PageData }) => { : "Select an image prompt to view it here"}

-
- {selectedImagePrompt && comparisonColumns.length < 3 && ( - +
+ {selectedImagePrompt && ( + <> + {comparisonColumns.length < 3 && ( + + )} + + )}
- - {selectedImagePrompt && ( + {theBestImage && ( +
+

+ {theBestImage.status} +

+ {theBestImage.data && ( +
+ {theBestImage.data.imageData.alt +

License: {theBestImage.data.imageData.license}

+

+ Relavance Score:{" "} + { + theBestImage.data.validationResult.metadata + .appropriatenessScore + } +

+

+ Reasoning:{" "} + { + theBestImage.data.validationResult.metadata + .validationReasoning + } +

+ {theBestImage.data.imageData.photographer && ( +

By: {theBestImage.data.imageData.photographer}

+ )} + {theBestImage.data.imageData.title && ( +

Title: {theBestImage.data.imageData.title}

+ )} +
+ )} +
+ )} + {!theBestImage && selectedImagePrompt && (
{comparisonColumns.map((column, index) => (
@@ -290,7 +605,6 @@ const ImagesPage = ({ pageData }: { pageData: PageData }) => { {column.imageSearchBatch && (
{column.imageSearchBatch?.map((image, i) => { - console.log("image", image.url); return (
{ height={400} />

License: {image.license}

+

+ Relavance Score: {image.appropriatenessScore ?? ""} +

+

Reasoning: {image.appropriatenessReasoning ?? ""}

{image.photographer &&

By: {image.photographer}

} {image.title &&

Title: {image.title}

}
diff --git a/packages/api/src/imageGen/validateImageWithOpenAI.ts b/packages/api/src/imageGen/validateImageWithOpenAI.ts new file mode 100644 index 000000000..0774ec874 --- /dev/null +++ b/packages/api/src/imageGen/validateImageWithOpenAI.ts @@ -0,0 +1,109 @@ +import OpenAI from "openai"; +import { z } from "zod"; + +export interface ValidationResult { + isValid: boolean; + metadata: { + imageContent: string; + promptUsed: string; + appropriatenessScore: number; + validationReasoning: string; + }; +} + +function isTheImageBase64(url: string) { + const base64Regex = /^data:image\/(png|jpe?g|gif);base64,/; + if (base64Regex.test(url)) { + return `f"data:image/jpeg;base64,${url}`; + } + return url; +} + +export async function validateImageWithOpenAI( + imageUrl: string, + prompt: string, +): Promise { + console.log(`[OpenAI Validation] Starting validation for image: ${imageUrl}`); + console.log(`[OpenAI Validation] Prompt: ${prompt}`); + + try { + const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + baseURL: process.env.HELICONE_EU_HOST, + }); + + console.log("[OpenAI Validation] Sending request to OpenAI"); + + const response = await openai.chat.completions.create({ + model: "gpt-4o", + messages: [ + { + role: "system", + + content: + "You are an image validator assessing if images are suitable for a classroom setting. " + + "Always return your output as strict JSON. Do not include any additional text outside the JSON format. " + + "The JSON should have the following structure:\n" + + `{ + "imageContent": "Description of the image content", + "prompt": "The provided prompt", + "validationReasoning": "Start by listing various image examples that could be used to match this prompt. Then describe exactly what is in this image. Finally provide reasoning for or against why this image is suitable for the prompt according to your own reasoning. And finally give it a relevance score. Feel free to be strict.", + "appropriatenessScore": 1-10, + "valid": true or false + }`, + }, + { + role: "user", + content: [ + { + type: "text", + text: `Prompt: "${prompt}"\nAnalyze this image and determine if it's suitable.`, + }, + { + type: "image_url", + image_url: { + url: isTheImageBase64(imageUrl), + }, + }, + ], + }, + ], + temperature: 0.2, + }); + + const fullResponse = response?.choices?.[0]?.message?.content?.trim() || ""; + console.log( + `[OpenAI Validation] Full validation response:\n${fullResponse}`, + ); + + const jsonStart = fullResponse.indexOf("{"); + const jsonEnd = fullResponse.lastIndexOf("}"); + if (jsonStart === -1 || jsonEnd === -1) { + throw new Error( + "[OpenAI Validation] Response does not contain valid JSON.", + ); + } + const jsonResponse = fullResponse.slice(jsonStart, jsonEnd + 1); + const responseSchema = z.object({ + imageContent: z.string().optional(), + prompt: z.string().optional(), + appropriatenessScore: z.number().optional(), + validationReasoning: z.string().optional(), + valid: z.boolean().optional(), + }); + const validMatch = responseSchema.parse(JSON.parse(jsonResponse)); + + return { + isValid: validMatch.valid || false, + metadata: { + imageContent: validMatch.imageContent || "", + promptUsed: prompt, + appropriatenessScore: validMatch.appropriatenessScore || 0, + validationReasoning: validMatch.validationReasoning || "", + }, + }; + } catch (error) { + console.error("[OpenAI Validation] Error during validation:", error); + throw new Error(`[OpenAI Validation] Validation failed: ${error}`); + } +} diff --git a/packages/api/src/router/imageGen.ts b/packages/api/src/router/imageGen.ts index e3239ae77..ef6bb998f 100644 --- a/packages/api/src/router/imageGen.ts +++ b/packages/api/src/router/imageGen.ts @@ -2,6 +2,10 @@ import OpenAI from "openai"; import { v4 as uuidv4 } from "uuid"; import { z } from "zod"; +import { + validateImageWithOpenAI, + type ValidationResult, +} from "../imageGen/validateImageWithOpenAI"; import { protectedProcedure } from "../middleware/auth"; import { router } from "../trpc"; import { @@ -10,341 +14,203 @@ import { type ImageResponse, } from "./imageSearch"; -interface ValidationResult { - imageData: ImageResponse; - isValid: boolean; -} - +// Constants const STABLE_DIF_API_KEY = process.env.STABLE_DIF_API_KEY; const MAX_PARALLEL_CHECKS = 5; -async function validateImageWithOpenAI( - imageUrl: string, - prompt: string, -): Promise { - console.log(`[OpenAI Validation] Starting validation for image: ${imageUrl}`); - console.log(`[OpenAI Validation] Prompt: ${prompt}`); - - try { - const openai = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - baseURL: process.env.HELICONE_EU_HOST, - }); - - console.log("[OpenAI Validation] Sending request to OpenAI"); - const response = await openai.chat.completions.create({ - model: "gpt-4", - messages: [ - { - role: "system", - content: - "You are an image validator assessing if images are suitable for a classroom setting. " + - "Consider an image valid if it generally relates to the prompt and is classroom-appropriate, " + - "even if it's not a perfect match. Respond in this format:\n" + - "VALID: true/false\n" + - "REASONING: brief explanation of your decision", - }, - { - role: "user", - content: `Prompt: "${prompt}"\nImage URL: ${imageUrl}\nIs this image suitable and related to the prompt?`, - }, - ], - temperature: 0.2, - }); - - const fullResponse = response?.choices?.[0]?.message?.content?.trim() || ""; - console.log( - `[OpenAI Validation] Full validation response:\n${fullResponse}`, - ); - - // Parse the response with safe defaults - let isValid = false; - let reasoning = "No reasoning provided"; - - // Check for valid response pattern - if (fullResponse) { - const validMatch = fullResponse.match(/VALID:\s*(true|false)/i); - const reasoningMatch = fullResponse.match(/REASONING:\s*(.*)/i); - - if (validMatch && validMatch[1]) { - isValid = validMatch[1].toLowerCase() === "true"; - } else { - console.warn( - "[OpenAI Validation] Could not parse VALID status from response", - ); - } - - if (reasoningMatch && reasoningMatch[1]) { - reasoning = reasoningMatch[1]; - } else { - console.warn( - "[OpenAI Validation] Could not parse REASONING from response", - ); - } - } else { - console.warn("[OpenAI Validation] Received empty response from OpenAI"); - } - - console.log(`[OpenAI Validation] Parsed result:`, { - isValid, - reasoning, - prompt, - imageUrl, - }); - - return isValid; - } catch (error) { - console.error("[OpenAI Validation] Error during validation:", error); - if (error instanceof Error) { - console.error("[OpenAI Validation] Error details:", { - message: error.message, - stack: error.stack, - }); - } - return false; - } +// Types +interface ValidatedImage { + id: string; + url: string; + appropriatenessScore: number; + appropriatenessReasoning: string; + imageSource: string; + license: string; + photographer: string; + alt?: string; + title?: string; } -async function validateImagesInParallel( +// Utility Functions +export async function validateImagesInParallel( images: ImageResponse[], searchExpression: string, -): Promise { +): Promise<{ imageData: ImageResponse; validationResult: ValidationResult }[]> { console.log( - `[Parallel Validation] Starting parallel validation for ${images.length} images`, - ); - console.log( - `[Parallel Validation] Will check up to ${MAX_PARALLEL_CHECKS} images`, + `[Validation] Starting parallel validation for ${images.length} images`, ); const imagesToCheck = images.slice(0, MAX_PARALLEL_CHECKS); - console.log( - `[Parallel Validation] Actually checking ${imagesToCheck.length} images`, - ); const validationPromises = imagesToCheck.map(async (image) => { - console.log(`[Parallel Validation] Validating image: ${image.url}`); try { const isValid = await validateImageWithOpenAI( image.url, searchExpression, ); - console.log( - `[Parallel Validation] Validation result for ${image.url}: ${isValid}`, - ); - return { imageData: image, isValid }; + return { imageData: image, validationResult: isValid }; } catch (error) { - console.error( - `[Parallel Validation] Error validating ${image.url}:`, - error, - ); - return { imageData: image, isValid: false }; + console.error(`[Validation] Error validating ${image.url}:`, error); + return { + imageData: image, + validationResult: { + isValid: false, + metadata: { + imageContent: "Error validating image", + promptUsed: searchExpression, + appropriatenessScore: 0, + validationReasoning: "Error validating", + }, + }, + }; } }); - const results = await Promise.all(validationPromises); - console.log( - `[Parallel Validation] Completed validation for ${results.length} images`, - ); - console.log( - "[Parallel Validation] Results:", - results.map((r) => ({ url: r.imageData.url, isValid: r.isValid })), + return Promise.all(validationPromises); +} + +async function generateStabilityImage( + endpoint: "core" | "ultra" | "sd3", + searchExpression: string, + outputFormat: "webp" | "jpeg" = "webp", +): Promise { + const formData = new FormData(); + formData.append("prompt", searchExpression); + formData.append("output_format", outputFormat); + + const response = await fetch( + `https://api.stability.ai/v2beta/stable-image/generate/${endpoint}`, + { + method: "POST", + body: formData, + headers: { + Authorization: `Bearer ${STABLE_DIF_API_KEY}`, + Accept: "image/*", + }, + }, ); - return results; + + if (!response.ok) { + throw new Error(`Image generation failed with status ${response.status}`); + } + + const imageBuffer = await response.arrayBuffer(); + return `data:image/${outputFormat};base64,${Buffer.from(imageBuffer).toString("base64")}`; } + +// Image Source Functions async function tryFlickrImages( searchExpression: string, -): Promise { - console.log( - `[Flickr] Attempting to fetch Flickr images for: ${searchExpression}`, - ); +): Promise { try { const flickrResponse = await flickrImages({ searchExpression }); - console.log( - `[Flickr] Received ${flickrResponse.length} images from Flickr`, - ); + if (flickrResponse.length === 0) return null; - if (flickrResponse.length === 0) { - console.log("[Flickr] No images returned from Flickr"); - return null; - } - - console.log("[Flickr] Starting validation of Flickr images"); const validationResults = await validateImagesInParallel( flickrResponse, searchExpression, ); + const firstValidImage = validationResults.find( + (result) => result.validationResult.isValid, + ); - const firstValidImage = validationResults.find((result) => result.isValid); - if (firstValidImage) { - console.log( - `[Flickr] Found valid image: ${firstValidImage.imageData.url}`, - ); - return { - ...firstValidImage.imageData, - license: `Flickr - ${firstValidImage.imageData.license}`, - photographer: firstValidImage.imageData.photographer || "Flickr User", - }; - } - - console.log("[Flickr] No valid images found from Flickr"); - return null; + if (!firstValidImage) return null; + + return { + ...firstValidImage.imageData, + id: uuidv4(), + appropriatenessScore: + firstValidImage.validationResult.metadata.appropriatenessScore, + appropriatenessReasoning: + firstValidImage.validationResult.metadata.validationReasoning, + license: `Flickr - ${firstValidImage.imageData.license}`, + photographer: firstValidImage.imageData.photographer || "Flickr User", + imageSource: "Flickr", + }; } catch (error) { - console.error("[Flickr] Error fetching/validating Flickr images:", error); - if (error instanceof Error) { - console.error("[Flickr] Error details:", { - message: error.message, - stack: error.stack, - }); - } + console.error("[Flickr] Error:", error); return null; } } async function tryUnsplashImages( searchExpression: string, -): Promise { - console.log( - `[Unsplash] Attempting to fetch Unsplash images for: ${searchExpression}`, - ); +): Promise { try { const unsplashResponse = await unsplashImages({ searchExpression }); - console.log( - `[Unsplash] Received ${unsplashResponse.length} images from Unsplash`, - ); - - if (unsplashResponse.length === 0) { - console.log("[Unsplash] No images returned from Unsplash"); - return null; - } + if (unsplashResponse.length === 0) return null; - console.log("[Unsplash] Starting validation of Unsplash images"); const validationResults = await validateImagesInParallel( unsplashResponse, searchExpression, ); - - const firstValidImage = validationResults.find((result) => result.isValid); - if (firstValidImage) { - console.log( - `[Unsplash] Found valid image: ${firstValidImage.imageData.url}`, - ); - return { - ...firstValidImage.imageData, - license: "Unsplash License", - photographer: - firstValidImage.imageData.photographer || "Unsplash Photographer", - }; - } - - console.log("[Unsplash] No valid images found from Unsplash"); - return null; - } catch (error) { - console.error( - "[Unsplash] Error fetching/validating Unsplash images:", - error, - ); - if (error instanceof Error) { - console.error("[Unsplash] Error details:", { - message: error.message, - stack: error.stack, - }); - } - return null; - } -} - -async function generateStabilityImage( - endpoint: string, - searchExpression: string, - outputFormat: "webp" | "jpeg" = "webp", -): Promise { - console.log(`[Stability ${endpoint}] Starting image generation`); - console.log(`[Stability ${endpoint}] Prompt: ${searchExpression}`); - - const formData = new FormData(); - formData.append("prompt", searchExpression); - formData.append("output_format", outputFormat); - - try { - console.log(`[Stability ${endpoint}] Sending request to Stability AI`); - const response = await fetch( - `https://api.stability.ai/v2beta/stable-image/generate/${endpoint}`, - { - method: "POST", - body: formData, - headers: { - Authorization: `Bearer ${STABLE_DIF_API_KEY}`, - Accept: "image/*", - }, - }, + const firstValidImage = validationResults.find( + (result) => result.validationResult.isValid, ); - if (!response.ok) { - console.error(`[Stability ${endpoint}] API error:`, { - status: response.status, - statusText: response.statusText, - }); - throw new Error(`Image generation failed with status ${response.status}`); - } - - console.log(`[Stability ${endpoint}] Successfully received image response`); - const imageBuffer = await response.arrayBuffer(); - const base64Image = `data:image/${outputFormat};base64,${Buffer.from(imageBuffer).toString("base64")}`; - console.log(`[Stability ${endpoint}] Successfully encoded image to base64`); - return base64Image; + if (!firstValidImage) return null; + + return { + ...firstValidImage.imageData, + id: uuidv4(), + appropriatenessScore: + firstValidImage.validationResult.metadata.appropriatenessScore, + appropriatenessReasoning: + firstValidImage.validationResult.metadata.validationReasoning, + license: "Unsplash License", + imageSource: "Unsplash", + photographer: + firstValidImage.imageData.photographer || "Unsplash Photographer", + }; } catch (error) { - console.error(`[Stability ${endpoint}] Error:`, error); - throw error; + console.error("[Unsplash] Error:", error); + return null; } } async function tryStabilityCore( searchExpression: string, -): Promise { - console.log("[StabilityCore] Starting image generation attempt"); +): Promise { try { const imageUrl = await generateStabilityImage("core", searchExpression); - console.log("[StabilityCore] Image generated, starting validation"); - const isValid = await validateImageWithOpenAI(imageUrl, searchExpression); - console.log(`[StabilityCore] Validation result: ${isValid}`); + console.log("**************************************************"); + console.log("imageUrl", imageUrl); + console.log("**************************************************"); - if (isValid) { - console.log("[StabilityCore] Image validated successfully"); - return { - id: uuidv4(), - url: imageUrl, - title: `AI Generated: ${searchExpression}`, - alt: searchExpression, - license: "Stability AI - Core", - photographer: "Generated by Stability AI", - }; - } - console.log("[StabilityCore] Image validation failed"); - return null; + const validationResponse = await validateImageWithOpenAI( + imageUrl, + searchExpression, + ); + + if (!validationResponse.isValid) return null; + + return { + id: uuidv4(), + url: imageUrl, + title: `AI Generated: ${searchExpression}`, + alt: searchExpression, + license: "Stability AI - Core", + photographer: "Generated by Stability AI", + appropriatenessScore: validationResponse.metadata.appropriatenessScore, + appropriatenessReasoning: validationResponse.metadata.validationReasoning, + imageSource: "Stability AI", + }; } catch (error) { - console.error("[StabilityCore] Error during generation/validation:", error); - if (error instanceof Error) { - console.error("[StabilityCore] Error details:", { - message: error.message, - stack: error.stack, - }); - } + console.error("[StabilityCore] Error:", error); return null; } } async function tryDallE( searchExpression: string, -): Promise { - console.log("[DALL-E] Starting image generation attempt"); +): Promise { try { const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.HELICONE_EU_HOST, }); - console.log("[DALL-E] Sending request to OpenAI"); const response = await openai.images.generate({ model: "dall-e-3", prompt: searchExpression, @@ -352,254 +218,242 @@ async function tryDallE( size: "1024x1024", }); - console.log("[DALL-E] Response received:", response); + if (!response?.data?.[0]?.url) return null; - if (response?.data?.[0]?.url) { - const imageUrl = response.data[0].url; - console.log("[DALL-E] Image generated, starting validation"); - - const isValid = await validateImageWithOpenAI(imageUrl, searchExpression); - console.log(`[DALL-E] Validation result: ${isValid}`); + const imageUrl = response.data[0].url; + const validationResponse = await validateImageWithOpenAI( + imageUrl, + searchExpression, + ); - if (isValid) { - console.log("[DALL-E] Image validated successfully"); - return { - id: uuidv4(), - url: imageUrl, - title: `AI Generated: ${searchExpression}`, - alt: searchExpression, - license: "OpenAI DALL-E 3", - photographer: "Generated by DALL-E 3", - }; - } - } - console.log("[DALL-E] No valid image generated"); - return null; + if (!validationResponse.isValid) return null; + + return { + id: uuidv4(), + url: imageUrl, + title: `AI Generated: ${searchExpression}`, + alt: searchExpression, + license: "OpenAI DALL-E 3", + imageSource: "OpenAI DALL-E 3", + photographer: "Generated by DALL-E 3", + appropriatenessScore: validationResponse.metadata.appropriatenessScore, + appropriatenessReasoning: validationResponse.metadata.validationReasoning, + }; } catch (error) { - console.error("[DALL-E] Error during generation/validation:", error); - if (error instanceof Error) { - console.error("[DALL-E] Error details:", { - message: error.message, - stack: error.stack, - }); - } + console.error("[DALL-E] Error:", error); return null; } } -export const imageGen = router({ - customPipeline: protectedProcedure - .input( - z.object({ - searchExpression: z.string(), - }), - ) - .mutation(async ({ input }): Promise => { - const { searchExpression } = input; - console.log( - `[Pipeline] Starting custom pipeline for prompt: ${searchExpression}`, - ); - try { - console.log("[Pipeline] Trying Flickr..."); - const flickrResult = await tryFlickrImages(searchExpression); - if (flickrResult) { - console.log("[Pipeline] Flickr success, returning result"); - return flickrResult.url; - } - console.log("[Pipeline] Flickr attempt failed, trying Unsplash..."); - - const unsplashResult = await tryUnsplashImages(searchExpression); - if (unsplashResult) { - console.log("[Pipeline] Unsplash success, returning result"); - return unsplashResult.url; - } - console.log( - "[Pipeline] Unsplash attempt failed, trying Stability Core...", +// Router Definition +export const imageGen = router({ + customPipelineWithReasoning: protectedProcedure + .input(z.object({ searchExpression: z.string() })) + .mutation(async ({ input }): Promise => { + const results: ValidatedImage[] = []; + + // Get and validate Flickr images + const flickrResponse = await flickrImages({ + searchExpression: input.searchExpression, + }); + if (flickrResponse.length > 0) { + const validationResults = await validateImagesInParallel( + flickrResponse, + input.searchExpression, ); - - const stabilityResult = await tryStabilityCore(searchExpression); - if (stabilityResult) { - console.log("[Pipeline] Stability Core success, returning result"); - return stabilityResult.url; - } - console.log( - "[Pipeline] Stability Core attempt failed, trying DALL-E...", + results.push( + ...validationResults.map((result) => ({ + id: uuidv4(), + url: result.imageData.url, + appropriatenessScore: + result.validationResult.metadata.appropriatenessScore, + appropriatenessReasoning: + result.validationResult.metadata.validationReasoning, + imageSource: "Flickr", + license: `Flickr - ${result.imageData.license}`, + photographer: result.imageData.photographer || "Flickr User", + title: result.imageData.title, + alt: result.imageData.alt, + })), ); + } - const dalleResult = await tryDallE(searchExpression); - if (dalleResult) { - console.log("[Pipeline] DALL-E success, returning result"); - return dalleResult.url; - } + // Get and validate Unsplash images + const unsplashResponse = await unsplashImages({ + searchExpression: input.searchExpression, + }); + if (unsplashResponse.length > 0) { + const validationResults = await validateImagesInParallel( + unsplashResponse, + input.searchExpression, + ); + results.push( + ...validationResults.map((result) => ({ + id: uuidv4(), + url: result.imageData.url, + appropriatenessScore: + result.validationResult.metadata.appropriatenessScore, + appropriatenessReasoning: + result.validationResult.metadata.validationReasoning, + imageSource: "Unsplash", + license: "Unsplash License", + photographer: + result.imageData.photographer || "Unsplash Photographer", + title: result.imageData.title, + alt: result.imageData.alt, + })), + ); + } - console.log("[Pipeline] All attempts failed"); - throw new Error("No suitable images found from any provider."); + // Try Stability AI + try { + const imageUrl = await generateStabilityImage( + "core", + input.searchExpression, + ); + const validationResponse = await validateImageWithOpenAI( + imageUrl, + input.searchExpression, + ); + results.push({ + id: uuidv4(), + url: imageUrl, + title: `AI Generated: ${input.searchExpression}`, + alt: input.searchExpression, + license: "Stability AI - Core", + photographer: "Generated by Stability AI", + appropriatenessScore: + validationResponse.metadata.appropriatenessScore, + appropriatenessReasoning: + validationResponse.metadata.validationReasoning, + imageSource: "Stability AI", + }); } catch (error) { - console.error("[Pipeline] Pipeline error:", error); - if (error instanceof Error) { - console.error("[Pipeline] Error details:", { - message: error.message, - stack: error.stack, - }); - throw new Error(`Image pipeline failed: ${error.message}`); - } - throw new Error("Image generation failed. Please try again later."); + console.error("[StabilityCore] Error:", error); } - }), - stableDifUltra: protectedProcedure - .input( - z.object({ - searchExpression: z.string(), - }), - ) - .mutation(async ({ input }) => { - const { searchExpression } = input; + // Try DALL-E try { - // Construct the form data - const formData = new FormData(); - formData.append("prompt", searchExpression); - formData.append("output_format", "webp"); - - // Make the request using fetch - const response = await fetch( - `https://api.stability.ai/v2beta/stable-image/generate/ultra`, - { - method: "POST", - body: formData, - headers: { - Authorization: `Bearer ${STABLE_DIF_API_KEY}`, - Accept: "image/*", - }, - }, - ); + const openai = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + baseURL: process.env.HELICONE_EU_HOST, + }); - if (!response.ok) { - throw new Error( - `Image generation failed with status ${response.status}`, + const response = await openai.images.generate({ + model: "dall-e-3", + prompt: input.searchExpression, + n: 1, + size: "1024x1024", + }); + + if (response?.data?.[0]?.url) { + const imageUrl = response.data[0].url; + const validationResponse = await validateImageWithOpenAI( + imageUrl, + input.searchExpression, ); + results.push({ + id: uuidv4(), + url: imageUrl, + title: `AI Generated: ${input.searchExpression}`, + alt: input.searchExpression, + license: "OpenAI DALL-E 3", + imageSource: "OpenAI DALL-E 3", + photographer: "Generated by DALL-E 3", + appropriatenessScore: + validationResponse.metadata.appropriatenessScore, + appropriatenessReasoning: + validationResponse.metadata.validationReasoning, + }); } - // return "https://unsplash.com/photos/1"; - // Response is expected to be an image buffer - const imageBuffer = await response.arrayBuffer(); - const imageUrl = `data:image/webp;base64,${Buffer.from(imageBuffer).toString("base64")}`; - - return imageUrl; } catch (error) { - throw new Error("Image generation failed. Please try again later."); + console.error("[DALL-E] Error:", error); } - }), - stableDif3: protectedProcedure - .input( - z.object({ - searchExpression: z.string(), - }), - ) - .mutation(async ({ input }) => { - const { searchExpression } = input; - try { - // Construct the form data - const formData = new FormData(); - formData.append("prompt", searchExpression); - formData.append("output_format", "jpeg"); - - // Make the request using fetch - const response = await fetch( - `https://api.stability.ai/v2beta/stable-image/generate/sd3`, - { - method: "POST", - body: formData, - headers: { - Authorization: `Bearer ${STABLE_DIF_API_KEY}`, - Accept: "image/*", - }, - }, - ); + // Return all results, even if some have low appropriateness scores + if (results.length === 0) { + throw new Error("No images found from any provider."); + } - const imageBuffer = await response.arrayBuffer(); + return results; + }), - const imageUrl = `data:image/jpeg;base64,${Buffer.from(imageBuffer).toString("base64")}`; - return imageUrl; - } catch (error) { - throw new Error("Image generation failed. Please try again later."); + customPipeline: protectedProcedure + .input(z.object({ searchExpression: z.string() })) + .mutation(async ({ input }) => { + const providers = [ + tryFlickrImages, + tryUnsplashImages, + tryStabilityCore, + tryDallE, + ]; + + for (const provider of providers) { + try { + const result = await provider(input.searchExpression); + if (result) { + return { + url: result.url, + appropriatenessScore: result.appropriatenessScore, + appropriatenessReasoning: result.appropriatenessReasoning, + imageSource: result.imageSource, + }; + } + } catch (error) { + console.error(`Provider error:`, error); + } } + + throw new Error("No suitable images found from any provider."); }), - stableDifCore: protectedProcedure - .input( - z.object({ - searchExpression: z.string(), - }), - ) + + stableDifUltra: protectedProcedure + .input(z.object({ searchExpression: z.string() })) .mutation(async ({ input }) => { - const { searchExpression } = input; + return generateStabilityImage("ultra", input.searchExpression); + }), - try { - // Construct the form data - const formData = new FormData(); - formData.append("prompt", searchExpression); - formData.append("output_format", "webp"); - - // Make the request using fetch - const response = await fetch( - "https://api.stability.ai/v2beta/stable-image/generate/core", - { - method: "POST", - body: formData, - headers: { - Authorization: `Bearer ${STABLE_DIF_API_KEY}`, - Accept: "image/*", - }, - }, - ); + stableDif3: protectedProcedure + .input(z.object({ searchExpression: z.string() })) + .mutation(async ({ input }) => { + return generateStabilityImage("sd3", input.searchExpression, "jpeg"); + }), - if (!response.ok) { - throw new Error( - `Image generation failed with status ${response.status}`, - ); - } - // return "https://unsplash.com/photos/1"; - // Response is expected to be an image buffer - const imageBuffer = await response.arrayBuffer(); - const imageUrl = `data:image/webp;base64,${Buffer.from(imageBuffer).toString("base64")}`; + stableDifCore: protectedProcedure + .input(z.object({ searchExpression: z.string() })) + .mutation(async ({ input }) => { + return generateStabilityImage("core", input.searchExpression); + }), - return imageUrl; - } catch (error) { + openAi: protectedProcedure + .input(z.object({ searchExpression: z.string() })) + .mutation(async ({ input }) => { + const result = await tryDallE(input.searchExpression); + if (!result) throw new Error("Image generation failed. Please try again later."); - } + return result.url; }), - openAi: protectedProcedure + validateImage: protectedProcedure + .input(z.object({ imageUrl: z.string(), prompt: z.string() })) + .mutation(async ({ input }) => { + return validateImageWithOpenAI(input.imageUrl, input.prompt); + }), + validateImagesInParallel: protectedProcedure .input( z.object({ + images: z.array( + z.object({ + id: z.string(), + url: z.string(), + license: z.string(), + photographer: z.string().optional(), + alt: z.string().optional(), + }), + ), searchExpression: z.string(), }), ) .mutation(async ({ input }) => { - const { searchExpression } = input; - - try { - const openai = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - baseURL: process.env.HELICONE_EU_HOST, - }); - const response = await openai.images.generate({ - model: "dall-e-3", - prompt: searchExpression, - n: 1, - size: "1024x1024", - }); - - if (!response || response === undefined) { - throw new Error("No response from OpenAI"); - } - console.log("response", response); - - const image_url = response?.data[0]?.url; - - return image_url; - } catch (error) { - // console.error("Error generating image:", error); - throw new Error("Image generation failed. Please try again later."); - } + return validateImagesInParallel(input.images, input.searchExpression); }), });