feat: support searxng (#216)

Co-Authored-By: Minghan Zhang <[email protected]>
coaidev · Jun 27, 2024 · b0a684c · b0a684c
1 parent 04ad6af
commit b0a684c
Show file tree

Hide file tree

Showing 13 changed files with 353 additions and 52 deletions.
diff --git a/addition/web/call.go b/addition/web/call.go
@@ -11,7 +11,7 @@ import (
 type Hook func(message []globals.Message, token int) (string, error)
 
 func toWebSearchingMessage(message []globals.Message) []globals.Message {
-	data := GenerateSearchResult(message[len(message)-1].Content)
+	data, _ := GenerateSearchResult(message[len(message)-1].Content)
 
 	return utils.Insert(message, 0, globals.Message{
 		Role: globals.System,
@@ -35,7 +35,7 @@ func ToChatSearched(instance *conversation.Conversation, restart bool) []globals
 func ToSearched(enable bool, message []globals.Message) []globals.Message {
 	if enable {
 		return toWebSearchingMessage(message)
-	} else {
-		return message
 	}
+
+	return message
 }
diff --git a/addition/web/search.go b/addition/web/search.go
@@ -3,10 +3,14 @@ package web
 import (
 	"chat/globals"
 	"chat/utils"
+	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"strconv"
 	"strings"
+
+	"github.com/gin-gonic/gin"
 )
 
 type SearXNGResponse struct {
@@ -50,7 +54,7 @@ func formatResponse(data *SearXNGResponse) string {
 func createURLParams(query string) string {
 	params := url.Values{}
 
-	params.Add("q", url.QueryEscape(query))
+	params.Add("q", query)
 	params.Add("format", "json")
 	params.Add("safesearch", strconv.Itoa(globals.SearchSafeSearch))
 	if len(globals.SearchEngines) > 0 {
@@ -73,19 +77,41 @@ func createSearXNGRequest(query string) (*SearXNGResponse, error) {
 	return utils.MapToRawStruct[SearXNGResponse](data)
 }
 
-func GenerateSearchResult(q string) string {
+func GenerateSearchResult(q string) (string, error) {
 	res, err := createSearXNGRequest(q)
 	if err != nil {
-		globals.Warn(fmt.Sprintf("[web] failed to get search result: %s (query: %s)", err.Error(), q))
-		return ""
+		globals.Warn(fmt.Sprintf("[web] failed to get search result: %s (query: %s)", err.Error(), utils.Extract(q, 20, "...")))
+
+		content := fmt.Sprintf("search failed: %s", err.Error())
+		return content, errors.New(content)
 	}
 
 	content := formatResponse(res)
 	globals.Debug(fmt.Sprintf("[web] search result: %s (query: %s)", utils.Extract(content, 50, "..."), q))
 
 	if globals.SearchCrop {
 		globals.Debug(fmt.Sprintf("[web] crop search result length %d to %d max", len(content), globals.SearchCropLength))
-		return utils.Extract(content, globals.SearchCropLength, "...")
+		return utils.Extract(content, globals.SearchCropLength, "..."), nil
+	}
+	return content, nil
+}
+
+func TestSearch(c *gin.Context) {
+	// get `query` param from query
+	query := c.Query("query")
+
+	fmt.Println(query)
+
+	res, err := GenerateSearchResult(query)
+	if err != nil {
+		c.JSON(http.StatusOK, gin.H{
+			"status": false,
+			"error":  err.Error(),
+		})
+	} else {
+		c.JSON(http.StatusOK, gin.H{
+			"status": true,
+			"result": res,
+		})
 	}
-	return content
 }
diff --git a/admin/router.go b/admin/router.go
@@ -1,13 +1,17 @@
 package admin
 
 import (
+	"chat/addition/web"
 	"chat/channel"
+
 	"github.com/gin-gonic/gin"
 )
 
 func Register(app *gin.RouterGroup) {
 	channel.Register(app)
 
+	app.GET("/admin/config/test/search", web.TestSearch)
+
 	app.GET("/admin/analytics/info", InfoAPI)
 	app.GET("/admin/analytics/model", ModelAnalysisAPI)
 	app.GET("/admin/analytics/request", RequestAnalysisAPI)

diff --git a/app/src/admin/api/system.ts b/app/src/admin/api/system.ts
@@ -2,6 +2,10 @@ import { CommonResponse } from "@/api/common.ts";
 import { getErrorMessage } from "@/utils/base.ts";
 import axios from "axios";
 
+export type TestWebSearchResponse = CommonResponse & {
+  result: string;
+};
+
 export type whiteList = {
   enabled: boolean;
   custom: string;
@@ -30,7 +34,11 @@ export type MailState = {
 
 export type SearchState = {
   endpoint: string;
-  query: number;
+  crop: boolean;
+  crop_len: number;
+  engines: string[];
+  image_proxy: boolean;
+  safe_search: number;
 };
 
 export type SiteState = {
@@ -72,10 +80,16 @@ export async function getConfig(): Promise<SystemResponse> {
   try {
     const response = await axios.get("/admin/config/view");
     const data = response.data as SystemResponse;
-    if (data.status) {
-      data.data &&
-        (data.data.mail.white_list.white_list =
-          data.data.mail.white_list.white_list || commonWhiteList);
+    if (data.status && data.data) {
+      // init system data pre-format
+
+      data.data.mail.white_list.white_list =
+        data.data.mail.white_list.white_list || commonWhiteList;
+      data.data.search.engines = data.data.search.engines || [];
+      data.data.search.crop_len =
+        data.data.search.crop_len && data.data.search.crop_len > 0
+          ? data.data.search.crop_len
+          : 1000;
     }
 
     return data;
@@ -104,6 +118,19 @@ export async function updateRootPassword(
   }
 }
 
+export async function testWebSearching(
+  query: string,
+): Promise<TestWebSearchResponse> {
+  try {
+    const response = await axios.get(
+      `/admin/config/test/search?query=${encodeURIComponent(query)}`,
+    );
+    return response.data as TestWebSearchResponse;
+  } catch (e) {
+    return { status: false, error: getErrorMessage(e), result: "" };
+  }
+}
+
 export const commonWhiteList: string[] = [
   "gmail.com",
   "outlook.com",
@@ -151,8 +178,12 @@ export const initialSystemState: SystemProps = {
     },
   },
   search: {
-    endpoint: "https://duckduckgo-api.vercel.app",
-    query: 5,
+    endpoint: "",
+    crop: false,
+    crop_len: 1000,
+    engines: [],
+    image_proxy: false,
+    safe_search: 0,
   },
   common: {
     article: [],

diff --git a/app/src/components/ui/combo-box.tsx b/app/src/components/ui/combo-box.tsx
@@ -21,20 +21,26 @@ type ComboBoxProps = {
   value: string;
   onChange: (value: string) => void;
   list: string[];
+  listTranslated?: string;
   placeholder?: string;
   defaultOpen?: boolean;
   className?: string;
+  classNameContent?: string;
   align?: "start" | "end" | "center" | undefined;
+  hideSearchBar?: boolean;
 };
 
 export function Combobox({
   value,
   onChange,
   list,
+  listTranslated,
   placeholder,
   defaultOpen,
   className,
+  classNameContent,
   align,
+  hideSearchBar,
 }: ComboBoxProps) {
   const { t } = useTranslation();
   const [open, setOpen] = React.useState(defaultOpen ?? false);
@@ -43,7 +49,7 @@ export function Combobox({
     const seq = [...list, value ?? ""].filter((v) => v);
     const set = new Set(seq);
     return [...set];
-  }, [list]);
+  }, [list, value]);
 
   return (
     <Popover open={open} onOpenChange={setOpen}>
@@ -54,20 +60,29 @@ export function Combobox({
           aria-expanded={open}
           className={cn("w-[320px] max-w-[60vw] justify-between", className)}
         >
-          {value || (placeholder ?? "")}
+          {value
+            ? listTranslated
+              ? t(`${listTranslated}.${value}`)
+              : value
+            : placeholder ?? ""}
           <ChevronsUpDown className="ml-2 h-4 w-4 shrink-0 opacity-50" />
         </Button>
       </PopoverTrigger>
-      <PopoverContent className="w-[320px] max-w-[60vw] p-0" align={align}>
+      <PopoverContent
+        className={cn("w-[320px] max-w-[60vw] p-0", classNameContent)}
+        align={align}
+      >
         <Command>
-          <CommandInput placeholder={placeholder} />
+          {!hideSearchBar && <CommandInput placeholder={placeholder} />}
           <CommandEmpty>{t("admin.empty")}</CommandEmpty>
           <CommandList>
             {valueList.map((key) => (
               <CommandItem
                 key={key}
                 value={key}
                 onSelect={() => {
+                  if (key === value) return setOpen(false);
+
                   onChange(key);
                   setOpen(false);
                 }}
@@ -78,12 +93,12 @@ export function Combobox({
                     key === value ? "opacity-100" : "opacity-0",
                   )}
                 />
-                {key}
+                {listTranslated ? t(`${listTranslated}.${key}`) : key}
               </CommandItem>
             ))}
           </CommandList>
         </Command>
       </PopoverContent>
     </Popover>
   );
-}
+}
diff --git a/app/src/components/ui/multi-combobox.tsx b/app/src/components/ui/multi-combobox.tsx
@@ -76,7 +76,7 @@ export function MultiCombobox({
       </PopoverTrigger>
       <PopoverContent className="w-[320px] max-w-[60vw] p-0" align={align}>
         <Command>
-          {disabledSearch && <CommandInput placeholder={searchPlaceholder} />}
+          {!disabledSearch && <CommandInput placeholder={searchPlaceholder} />}
           <CommandEmpty>{t("admin.empty")}</CommandEmpty>
           <CommandList className={`thin-scrollbar`}>
             {valueList.map((key) => (

diff --git a/app/src/components/ui/number-input.tsx b/app/src/components/ui/number-input.tsx
@@ -67,6 +67,7 @@ const NumberInput = React.forwardRef<HTMLInputElement, NumberInputProps>(
 
     return (
       <Input
+        {...props}
         ref={ref}
         className={cn(
           "number-input transition",
@@ -90,4 +91,4 @@ const NumberInput = React.forwardRef<HTMLInputElement, NumberInputProps>(
 );
 NumberInput.displayName = "NumberInput";
 
-export { NumberInput };
+export { NumberInput };
diff --git a/app/src/resources/i18n/cn.json b/app/src/resources/i18n/cn.json
@@ -718,8 +718,25 @@
       "searchEndpoint": "搜索接入点",
       "searchQuery": "最大搜索结果数",
       "searchQueryTip": "最大搜索结果数，默认为 5",
-      "searchTip": "DuckDuckGo 联网搜索接入点，不填则无法正常使用联网功能\nDuckDuckGo API 项目搭建：[duckduckgo-api](https://github.com/binjie09/duckduckgo-api)",
-      "searchPlaceholder": "DuckDuckGo 接入点 (格式仅需填写 https://example.com)",
+      "searchCrop": "开启结果截断",
+      "searchCropTip": "开启结果截断，开启后搜索结果内容的字符数如果超过最大结果字符数，则内容后面会被截断",
+      "searchCropLen": "最大结果字符数",
+      "searchEngines": "搜索引擎设置",
+      "searchEnginesPlaceholder": "已选 {{length}} 个搜索引擎",
+      "searchEnginesSearchPlaceholder": "请输入搜索引擎名称，如：Google",
+      "searchEnginesEmptyTip": "设置搜索引擎为空时，默认使用 SearXNG 内默认配置的搜索引擎",
+      "searchTest": "搜索测试",
+      "searchTestTip": "搜索测试，输入查询内容进行搜索测试",
+      "searchSafeSearch": "安全搜索模式",
+      "searchSafeSearchModes": {
+        "none": "关闭",
+        "moderation": "中等",
+        "strict": "严格"
+      },
+      "searchImageProxy": "开启图片代理",
+      "searchImageProxyTip": "图片代理，开启后搜索引擎返回的图片将会通过 SearXNG 服务节点代理加载",
+      "searchTip": "[SearXNG](https://github.com/searxng/searxng) 开源搜索引擎提供联网搜索能力。SearXNG Docker 私有化部署示例：[SearXNG Docker](https://github.com/zmh-program/searxng)",
+      "searchPlaceholder": "SearXNG 服务接入点 (例如 http://ip:7980)",
       "closeRegistration": "暂停注册",
       "closeRegistrationTip": "暂停注册，关闭后新用户将无法注册",
       "closeRelay": "关闭中转 API",

diff --git a/app/src/resources/i18n/en.json b/app/src/resources/i18n/en.json
@@ -519,7 +519,7 @@
       "mailPass": "Password",
       "searchEndpoint": "Search Endpoint",
       "searchQuery": "Max Search Results",
-      "searchTip": "DuckDuckGo network search access point, if not filled in, you will not be able to use the network function normally.\nDuckDuckGo API Project Build: [duckduckgo-api] (https://github.com/binjie09/duckduckgo-api)",
+      "searchTip": "[SearXNG](https://github.com/searxng/searxng) open source search engine that provides networked search capabilities. SearXNG Docker Privatization Deployment Example: [SearXNG Docker](https://github.com/zmh-program/searxng)",
       "mailFrom": "Sender",
       "test": "Test outgoing",
       "updateRoot": "Change Root Password",
@@ -575,14 +575,31 @@
       "relayPlan": "Subscription Quota Support Staging API",
       "relayPlanTip": "Subscription quota supports the transit API, after opening the transit API billing will give priority to the use of user subscription quota\n(Tip: Subscription is a quota of times, the model of billing for tokens may affect the cost)",
       "searchQueryTip": "Maximum number of search results, default is 5",
-      "searchPlaceholder": "DuckDuckGo Access Point (Format only https://example.com)",
+      "searchPlaceholder": "SearXNG Service Access Point (e.g. http://ip: 7980)",
       "image_store": "Picture storage",
       "image_storeTip": "Images generated by the OpenAI channel DALL-E will be stored on the server to prevent invalidation of the images",
       "image_storeNoBackend": "No backend domain configured, cannot enable image storage",
       "closeRelay": "Turn off Staging API",
       "closeRelayTip": "Turn off the staging API, the staging API will not be available after turning off",
       "debugMode": "debugging mode",
-      "debugModeTip": "Debug mode, after turning on, the log will output detailed request parameters and other logs for troubleshooting"
+      "debugModeTip": "Debug mode, after turning on, the log will output detailed request parameters and other logs for troubleshooting",
+      "searchCrop": "Turn on results truncation",
+      "searchCropTip": "Turn on result truncation, if the number of characters in the search result content exceeds the maximum number of characters, the content will be truncated",
+      "searchCropLen": "Maximum Result Characters",
+      "searchEngines": "Search Engine Settings",
+      "searchEnginesPlaceholder": "{{length}} search engines selected",
+      "searchEnginesSearchPlaceholder": "Please enter the search engine name, ex: Google",
+      "searchEnginesEmptyTip": "When the search engine is empty, the default search engine configured in SearXNG is used by default",
+      "searchSafeSearch": "SafeSearch Mode",
+      "searchSafeSearchModes": {
+        "none": "Turn off",
+        "moderation": "Medium",
+        "strict": "Demanding"
+      },
+      "searchImageProxy": "Turn on image proxy",
+      "searchImageProxyTip": "Image proxy, the image returned by the search engine after opening will be loaded through the SearXNG service node proxy",
+      "searchTest": "Search Quizzes",
+      "searchTestTip": "Search test, enter the query for search test"
     },
     "user": "Users",
     "invitation-code": "Invitation Code",