From 9f588b95884ddc36c6e544d55c28fade2fe01fa8 Mon Sep 17 00:00:00 2001
From: Paul Arterburn <paularterburn@gmail.com>
Date: Mon, 13 May 2024 12:16:17 -0600
Subject: [PATCH] Analyze attached images

---
 app/models/concerns/entry/ai_assistant.rb | 30 +++++++++++++++++++----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/app/models/concerns/entry/ai_assistant.rb b/app/models/concerns/entry/ai_assistant.rb
index db1b08b9..b5129b6f 100644
--- a/app/models/concerns/entry/ai_assistant.rb
+++ b/app/models/concerns/entry/ai_assistant.rb
@@ -66,6 +66,7 @@ def as_life_coach
 - Its responses should include any resources that may be relevant to the DabbleMeGPT's analysis and provide the user with links to those resources that would be helpful.
 - It can provide additional relevant details to answer in-depth and comprehensively, covering aspects related to how the user is feeling in their journal entry.
 - It can generate reflections, insights, follow-up questions, advice, resources, poetry, stories, essays, songs, and more.
+#{image_url_cdn.present? ? "- It can analyze the attached image as part of the entry and include it in its response." : ""}
 
 On its output format:
 - Do not output code or code block syntax because the interface does not support code output.
@@ -104,16 +105,35 @@ def as_life_coach
     def entry_body
       entry_token_count = as_life_coach.to_s.length.to_f / 4
       conversation = []
-      text_bodies_for_ai.each do |body|
+      text_bodies_for_ai.each_with_index do |body, index|
         entry_token_count += body.length.to_f / 4
         tokens_left = max_tokens - entry_token_count - MAX_RESPONSE_TOKENS
         break if tokens_left <= 0
 
         role = body.starts_with?("||DabbleMeGPT||") ? "assistant" : "user"
-        conversation << {
-          role: role,
-          content: body.gsub("||DabbleMeGPT||", "").first(tokens_left)
-        }
+
+        if index == 0 && image_url_cdn.present?
+          conversation << {
+            role: role,
+            content: [
+              {
+                type: "image_url",
+                image_url: {
+                  "url": image_url_cdn
+                }
+              },
+              {
+                type: "text",
+                text: body.gsub("||DabbleMeGPT||", "").first(tokens_left)
+              }
+            ]
+          }
+        else
+          conversation << {
+            role: role,
+            content: body.gsub("||DabbleMeGPT||", "").first(tokens_left)
+          }
+        end
       end
       conversation.compact
     end