From ad0f0bee2da0e956f72332b96aab553e7da369d9 Mon Sep 17 00:00:00 2001 From: Michael Genson <71845777+michael-genson@users.noreply.github.com> Date: Mon, 23 Sep 2024 04:04:36 -0500 Subject: [PATCH] feat: OpenAI Custom Headers/Params and Debug Page (#4227) Co-authored-by: Kuchenpirat <24235032+Kuchenpirat@users.noreply.github.com> --- .../installation/backend-config.md | 5 +- docs/docs/overrides/api.html | 2 +- .../Layout/LayoutParts/AppSidebar.vue | 3 +- frontend/lang/messages/en-US.json | 6 +- frontend/layouts/admin.vue | 19 ++- frontend/lib/api/admin/admin-debug.ts | 21 +++ frontend/lib/api/client-admin.ts | 3 + frontend/lib/api/types/admin.ts | 4 + frontend/pages/admin/debug/openai.vue | 127 ++++++++++++++++++ frontend/pages/admin/{ => debug}/parser.vue | 0 mealie/core/settings/settings.py | 6 +- mealie/routes/admin/__init__.py | 2 + mealie/routes/admin/admin_debug.py | 52 +++++++ mealie/schema/admin/__init__.py | 2 + mealie/schema/admin/debug.py | 6 + mealie/services/openai/openai.py | 7 +- mealie/services/openai/prompts/debug.txt | 1 + .../services/parser_services/openai/parser.py | 18 ++- mealie/services/recipe/recipe_service.py | 8 +- tests/utils/api_routes/__init__.py | 2 + 20 files changed, 277 insertions(+), 17 deletions(-) create mode 100644 frontend/lib/api/admin/admin-debug.ts create mode 100644 frontend/pages/admin/debug/openai.vue rename frontend/pages/admin/{ => debug}/parser.vue (100%) create mode 100644 mealie/routes/admin/admin_debug.py create mode 100644 mealie/schema/admin/debug.py create mode 100644 mealie/services/openai/prompts/debug.txt diff --git a/docs/docs/documentation/getting-started/installation/backend-config.md b/docs/docs/documentation/getting-started/installation/backend-config.md index 5ad9fc3ba44..ddfa236a561 100644 --- a/docs/docs/documentation/getting-started/installation/backend-config.md +++ b/docs/docs/documentation/getting-started/installation/backend-config.md @@ -105,18 +105,21 @@ For usage, see [Usage - OpenID Connect](../authentication/oidc.md) :octicons-tag-24: v1.7.0 Mealie supports various integrations using OpenAI. For more information, check out our [OpenAI documentation](./open-ai.md). +For custom mapping variables (e.g. OPENAI_CUSTOM_HEADERS) you should pass values as JSON encoded strings (e.g. `OPENAI_CUSTOM_PARAMS='{"k1": "v1", "k2": "v2"}'`) | Variables | Default | Description | | ---------------------------- | :-----: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | OPENAI_BASE_URL | None | The base URL for the OpenAI API. If you're not sure, leave this empty to use the standard OpenAI platform | | OPENAI_API_KEY | None | Your OpenAI API Key. Enables OpenAI-related features | | OPENAI_MODEL | gpt-4o | Which OpenAI model to use. If you're not sure, leave this empty | +| OPENAI_CUSTOM_HEADERS | None | Custom HTTP headers to add to all OpenAI requests. This should generally be left empty unless your custom service requires them | +| OPENAI_CUSTOM_PARAMS | None | Custom HTTP query params to add to all OpenAI requests. This should generally be left empty unless your custom service requires them | | OPENAI_ENABLE_IMAGE_SERVICES | True | Whether to enable OpenAI image services, such as creating recipes via image. Leave this enabled unless your custom model doesn't support it, or you want to reduce costs | | OPENAI_WORKERS | 2 | Number of OpenAI workers per request. Higher values may increase processing speed, but will incur additional API costs | | OPENAI_SEND_DATABASE_DATA | True | Whether to send Mealie data to OpenAI to improve request accuracy. This will incur additional API costs | | OPENAI_REQUEST_TIMEOUT | 60 | The number of seconds to wait for an OpenAI request to complete before cancelling the request. Leave this empty unless you're running into timeout issues on slower hardware | -### Themeing +### Theming Setting the following environmental variables will change the theme of the frontend. Note that the themes are the same for all users. This is a break-change when migration from v0.x.x -> 1.x.x. diff --git a/docs/docs/overrides/api.html b/docs/docs/overrides/api.html index c1b7fc8ada2..ed6136a1666 100644 --- a/docs/docs/overrides/api.html +++ b/docs/docs/overrides/api.html @@ -14,7 +14,7 @@
diff --git a/frontend/components/Layout/LayoutParts/AppSidebar.vue b/frontend/components/Layout/LayoutParts/AppSidebar.vue index 44d86f6d586..c55d5612400 100644 --- a/frontend/components/Layout/LayoutParts/AppSidebar.vue +++ b/frontend/components/Layout/LayoutParts/AppSidebar.vue @@ -84,13 +84,12 @@ {{ nav.title }} - + {{ child.icon }} {{ child.title }} - diff --git a/frontend/lang/messages/en-US.json b/frontend/lang/messages/en-US.json index 2dd9dcee3c2..72ec75defa9 100644 --- a/frontend/lang/messages/en-US.json +++ b/frontend/lang/messages/en-US.json @@ -1246,7 +1246,11 @@ "here-are-a-few-things-to-help-you-get-started": "Here are a few things to help you get started with Mealie", "restore-from-v1-backup": "Have a backup from a previous instance of Mealie v1? You can restore it here.", "manage-profile-or-get-invite-link": "Manage your own profile, or grab an invite link to share with others." - } + }, + "debug-openai-services": "Debug OpenAI Services", + "debug-openai-services-description": "Use this page to debug OpenAI services. You can test your OpenAI connection and see the results here. If you have image services enabled, you can also provide an image.", + "run-test": "Run Test", + "test-results": "Test Results" }, "profile": { "welcome-user": "👋 Welcome, {0}!", diff --git a/frontend/layouts/admin.vue b/frontend/layouts/admin.vue index 16547f9cd21..41633791984 100644 --- a/frontend/layouts/admin.vue +++ b/frontend/layouts/admin.vue @@ -92,10 +92,23 @@ export default defineComponent({ restricted: true, }, { - icon: $globals.icons.slotMachine, - to: "/admin/parser", - title: i18n.tc("sidebar.parser"), + icon: $globals.icons.robot, + title: i18n.tc("recipe.debug"), restricted: true, + children: [ + { + icon: $globals.icons.robot, + to: "/admin/debug/openai", + title: i18n.tc("admin.openai"), + restricted: true, + }, + { + icon: $globals.icons.slotMachine, + to: "/admin/debug/parser", + title: i18n.tc("sidebar.parser"), + restricted: true, + }, + ] }, ]; diff --git a/frontend/lib/api/admin/admin-debug.ts b/frontend/lib/api/admin/admin-debug.ts new file mode 100644 index 00000000000..bfbc4cbc4a2 --- /dev/null +++ b/frontend/lib/api/admin/admin-debug.ts @@ -0,0 +1,21 @@ +import { BaseAPI } from "../base/base-clients"; +import { DebugResponse } from "~/lib/api/types/admin"; + +const prefix = "/api"; + +const routes = { + openai: `${prefix}/admin/debug/openai`, +}; + +export class AdminDebugAPI extends BaseAPI { + async debugOpenAI(fileObject: Blob | File | undefined = undefined, fileName = "") { + let formData: FormData | null = null; + if (fileObject) { + formData = new FormData(); + formData.append("image", fileObject); + formData.append("extension", fileName.split(".").pop() ?? ""); + } + + return await this.requests.post(routes.openai, formData); + } +} diff --git a/frontend/lib/api/client-admin.ts b/frontend/lib/api/client-admin.ts index bf151d390ed..27463043084 100644 --- a/frontend/lib/api/client-admin.ts +++ b/frontend/lib/api/client-admin.ts @@ -5,6 +5,7 @@ import { AdminGroupsApi } from "./admin/admin-groups"; import { AdminBackupsApi } from "./admin/admin-backups"; import { AdminMaintenanceApi } from "./admin/admin-maintenance"; import { AdminAnalyticsApi } from "./admin/admin-analytics"; +import { AdminDebugAPI } from "./admin/admin-debug"; import { ApiRequestInstance } from "~/lib/api/types/non-generated"; export class AdminAPI { @@ -15,6 +16,7 @@ export class AdminAPI { public backups: AdminBackupsApi; public maintenance: AdminMaintenanceApi; public analytics: AdminAnalyticsApi; + public debug: AdminDebugAPI; constructor(requests: ApiRequestInstance) { this.about = new AdminAboutAPI(requests); @@ -24,6 +26,7 @@ export class AdminAPI { this.backups = new AdminBackupsApi(requests); this.maintenance = new AdminMaintenanceApi(requests); this.analytics = new AdminAnalyticsApi(requests); + this.debug = new AdminDebugAPI(requests); Object.freeze(this); } diff --git a/frontend/lib/api/types/admin.ts b/frontend/lib/api/types/admin.ts index 6f42b40d443..4f02f2d527c 100644 --- a/frontend/lib/api/types/admin.ts +++ b/frontend/lib/api/types/admin.ts @@ -173,6 +173,10 @@ export interface CustomPageOut { categories?: RecipeCategoryResponse[]; id: number; } +export interface DebugResponse { + success: boolean; + response?: string | null; +} export interface EmailReady { ready: boolean; } diff --git a/frontend/pages/admin/debug/openai.vue b/frontend/pages/admin/debug/openai.vue new file mode 100644 index 00000000000..522cea20fc6 --- /dev/null +++ b/frontend/pages/admin/debug/openai.vue @@ -0,0 +1,127 @@ + + + diff --git a/frontend/pages/admin/parser.vue b/frontend/pages/admin/debug/parser.vue similarity index 100% rename from frontend/pages/admin/parser.vue rename to frontend/pages/admin/debug/parser.vue diff --git a/mealie/core/settings/settings.py b/mealie/core/settings/settings.py index 0b33f5a88cb..0e73ee51d76 100644 --- a/mealie/core/settings/settings.py +++ b/mealie/core/settings/settings.py @@ -3,7 +3,7 @@ import secrets from datetime import datetime, timezone from pathlib import Path -from typing import NamedTuple +from typing import Any, NamedTuple from dateutil.tz import tzlocal from pydantic import field_validator @@ -305,6 +305,10 @@ def OIDC_READY(self) -> bool: """Your OpenAI API key. Required to enable OpenAI features""" OPENAI_MODEL: str = "gpt-4o" """Which OpenAI model to send requests to. Leave this unset for most usecases""" + OPENAI_CUSTOM_HEADERS: dict[str, str] = {} + """Custom HTTP headers to send with each OpenAI request""" + OPENAI_CUSTOM_PARAMS: dict[str, Any] = {} + """Custom HTTP parameters to send with each OpenAI request""" OPENAI_ENABLE_IMAGE_SERVICES: bool = True """Whether to enable image-related features in OpenAI""" OPENAI_WORKERS: int = 2 diff --git a/mealie/routes/admin/__init__.py b/mealie/routes/admin/__init__.py index c0a9839e47f..c0ef59a9dcc 100644 --- a/mealie/routes/admin/__init__.py +++ b/mealie/routes/admin/__init__.py @@ -3,6 +3,7 @@ from . import ( admin_about, admin_backups, + admin_debug, admin_email, admin_maintenance, admin_management_groups, @@ -19,3 +20,4 @@ router.include_router(admin_email.router, tags=["Admin: Email"]) router.include_router(admin_backups.router, tags=["Admin: Backups"]) router.include_router(admin_maintenance.router, tags=["Admin: Maintenance"]) +router.include_router(admin_debug.router, tags=["Admin: Debug"]) diff --git a/mealie/routes/admin/admin_debug.py b/mealie/routes/admin/admin_debug.py new file mode 100644 index 00000000000..65bb654904f --- /dev/null +++ b/mealie/routes/admin/admin_debug.py @@ -0,0 +1,52 @@ +import os +import shutil + +from fastapi import APIRouter, File, UploadFile + +from mealie.core.dependencies.dependencies import get_temporary_path +from mealie.routes._base import BaseAdminController, controller +from mealie.schema.admin.debug import DebugResponse +from mealie.services.openai import OpenAILocalImage, OpenAIService + +router = APIRouter(prefix="/debug") + + +@controller(router) +class AdminDebugController(BaseAdminController): + @router.post("/openai", response_model=DebugResponse) + async def debug_openai(self, image: UploadFile | None = File(None)): + if not self.settings.OPENAI_ENABLED: + return DebugResponse(success=False, response="OpenAI is not enabled") + if image and not self.settings.OPENAI_ENABLE_IMAGE_SERVICES: + return DebugResponse( + success=False, response="Image was provided, but OpenAI image services are not enabled" + ) + + with get_temporary_path() as temp_path: + if image: + with temp_path.joinpath(image.filename).open("wb") as buffer: + shutil.copyfileobj(image.file, buffer) + local_image_path = temp_path.joinpath(image.filename) + local_images = [OpenAILocalImage(filename=os.path.basename(local_image_path), path=local_image_path)] + else: + local_images = None + + try: + openai_service = OpenAIService() + prompt = openai_service.get_prompt("debug") + + message = "Hello, checking to see if I can reach you." + if local_images: + message = f"{message} Here is an image to test with:" + + response = await openai_service.get_response( + prompt, message, images=local_images, force_json_response=False + ) + return DebugResponse(success=True, response=f'OpenAI is working. Response: "{response}"') + + except Exception as e: + self.logger.exception(e) + return DebugResponse( + success=False, + response=f'OpenAI request failed. Full error has been logged. {e.__class__.__name__}: "{e}"', + ) diff --git a/mealie/schema/admin/__init__.py b/mealie/schema/admin/__init__.py index 85d33599605..b399d46f8e9 100644 --- a/mealie/schema/admin/__init__.py +++ b/mealie/schema/admin/__init__.py @@ -1,6 +1,7 @@ # This file is auto-generated by gen_schema_exports.py from .about import AdminAboutInfo, AppInfo, AppStartupInfo, AppStatistics, AppTheme, CheckAppConfig, OIDCInfo from .backup import AllBackups, BackupFile, BackupOptions, CreateBackup, ImportJob +from .debug import DebugResponse from .email import EmailReady, EmailSuccess, EmailTest from .maintenance import MaintenanceLogs, MaintenanceStorageDetails, MaintenanceSummary from .migration import ChowdownURL, MigrationFile, MigrationImport, Migrations @@ -49,4 +50,5 @@ "EmailReady", "EmailSuccess", "EmailTest", + "DebugResponse", ] diff --git a/mealie/schema/admin/debug.py b/mealie/schema/admin/debug.py new file mode 100644 index 00000000000..e653a2bb628 --- /dev/null +++ b/mealie/schema/admin/debug.py @@ -0,0 +1,6 @@ +from mealie.schema._mealie import MealieModel + + +class DebugResponse(MealieModel): + success: bool + response: str | None = None diff --git a/mealie/services/openai/openai.py b/mealie/services/openai/openai.py index 64cd4180eb6..09c391d56b8 100644 --- a/mealie/services/openai/openai.py +++ b/mealie/services/openai/openai.py @@ -90,6 +90,8 @@ def __init__(self) -> None: base_url=settings.OPENAI_BASE_URL, api_key=settings.OPENAI_API_KEY, timeout=settings.OPENAI_REQUEST_TIMEOUT, + default_headers=settings.OPENAI_CUSTOM_HEADERS, + default_query=settings.OPENAI_CUSTOM_PARAMS, ) super().__init__() @@ -176,6 +178,5 @@ async def get_response( if not response.choices: return None return response.choices[0].message.content - except Exception: - self.logger.exception("OpenAI Request Failed") - return None + except Exception as e: + raise Exception(f"OpenAI Request Failed. {e.__class__.__name__}: {e}") from e diff --git a/mealie/services/openai/prompts/debug.txt b/mealie/services/openai/prompts/debug.txt new file mode 100644 index 00000000000..c6fb0bdb173 --- /dev/null +++ b/mealie/services/openai/prompts/debug.txt @@ -0,0 +1 @@ +You are a simple chatbot being used for debugging purposes. diff --git a/mealie/services/parser_services/openai/parser.py b/mealie/services/parser_services/openai/parser.py index 77e1c022935..0e01931bef5 100644 --- a/mealie/services/parser_services/openai/parser.py +++ b/mealie/services/parser_services/openai/parser.py @@ -80,10 +80,20 @@ async def _parse(self, ingredients: list[str]) -> OpenAIIngredients: tasks.append(service.get_response(prompt, message, force_json_response=True)) # re-combine chunks into one response - responses_json = await asyncio.gather(*tasks) - responses = [ - OpenAIIngredients.parse_openai_response(response_json) for response_json in responses_json if responses_json - ] + try: + responses_json = await asyncio.gather(*tasks) + except Exception as e: + raise Exception("Failed to call OpenAI services") from e + + try: + responses = [ + OpenAIIngredients.parse_openai_response(response_json) + for response_json in responses_json + if responses_json + ] + except Exception as e: + raise Exception("Failed to parse OpenAI response") from e + if not responses: raise Exception("No response from OpenAI") diff --git a/mealie/services/recipe/recipe_service.py b/mealie/services/recipe/recipe_service.py index d9efbaca87e..b420216c73f 100644 --- a/mealie/services/recipe/recipe_service.py +++ b/mealie/services/recipe/recipe_service.py @@ -487,7 +487,13 @@ async def build_recipe_from_images(self, images: list[Path], translate_language: if translate_language: message += f" Please translate the recipe to {translate_language}." - response = await openai_service.get_response(prompt, message, images=openai_images, force_json_response=True) + try: + response = await openai_service.get_response( + prompt, message, images=openai_images, force_json_response=True + ) + except Exception as e: + raise Exception("Failed to call OpenAI services") from e + try: openai_recipe = OpenAIRecipe.parse_openai_response(response) recipe = self._convert_recipe(openai_recipe) diff --git a/tests/utils/api_routes/__init__.py b/tests/utils/api_routes/__init__.py index 587ca4abef3..dc096b387e7 100644 --- a/tests/utils/api_routes/__init__.py +++ b/tests/utils/api_routes/__init__.py @@ -11,6 +11,8 @@ """`/api/admin/backups`""" admin_backups_upload = "/api/admin/backups/upload" """`/api/admin/backups/upload`""" +admin_debug_openai = "/api/admin/debug/openai" +"""`/api/admin/debug/openai`""" admin_email = "/api/admin/email" """`/api/admin/email`""" admin_groups = "/api/admin/groups"