feat: add difylocal service

.env.template

@@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub.
 
 ### Services ##############################################
 # the enable service list
-ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "django")
+ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django")
 
 ###########################################################
 ################ Containers Customization #################

.work/config/.env.amd64

@@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub.
 
 ### Services ##############################################
 # the enable service list
-ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "django")
+ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django")
 
 ###########################################################
 ################ Containers Customization #################

.work/config/.env.arm64

@@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub.
 
 ### Services ##############################################
 # the enable service list
-ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "django")
+ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django")
 
 ###########################################################
 ################ Containers Customization #################

difylocal/.env

@@ -0,0 +1,17 @@
+### difylocal ############################################
+
+# -------------- image config ---------------
+# official image
+IMAGE_OFFICIAL_DIFYLOCAL_NAME=python
+IMAGE_OFFICIAL_DIFYLOCAL_VERSION=3.10.0
+# basic image
+IMAGE_BASIC_DIFYLOCAL_VERSION=0.0.1
+# app image
+IMAGE_APP_DIFYLOCAL_VERSION=latest
+# -------------- service config --------------
+DIFYLOCAL_HOST_WEB_PORT=4300
+DIFYLOCAL_CONTAINER_WEB_PORT=3000
+DIFYLOCAL_HOST_API_PORT=4301
+DIFYLOCAL_CONTAINER_API_PORT=5001
+DIFYLOCAL_HOST_CODE_EXECUTION_ENDPOINT_PORT=4302
+DIFYLOCAL_CONTAINER_CODE_EXECUTION_ENDPOINT_PORT=8194

difylocal/README.md

@@ -0,0 +1,47 @@
+> Since dify relies on too many services, the difylocal service will have the following characteristics, which are different from other services.
+> 1. ```difylocal``` only provides the nodejs and python environment for running dify, and does not have any other dependencies.
+> 2. you should clone dify to ```difylocal/data```, and run it by source code: https://docs.dify.ai/v/zh-hans/getting-started/install-self-hosted/local-source-code
+>
+> ![image](https://github.com/WGrape/sparrow/assets/35942268/5f12c6c5-3e2e-497f-a257-4888439a9b9f)
+
+
+## What is this directory
+This is the service directory, it includes these directories and files as bellow.
+
+## 1. Directories
+
+### (1) Commit to Git.
+
+#### /hooks
+
+```/hooks```: This is the hook actions while the service/container start/stop.
+
+#### /init
+
+```/init```: This is init scripts after the container created, is executed by ```./hooks/after_start.sh```
+
+#### /make_app_image
+```/make_app_image```: This is the ```sparrow-app-*``` image making directory.
+
+#### /make_basic_image
+```/make_basic_image```: This is the ```sparrow-basic-*``` image making directory.
+
+#### /templates
+```/templates```: this is the template directory for the service, the ```template``` means all static and unchanging files.
+
+### (2)Not commit to Git.
+
+#### /data
+```data```: This is the data storage directory for the service/container, you mustn't create any files in the ```data``` directory, because it may cause service/container start failed.
+
+#### /logs
+- ```logs```: This is the log storage directory for the service/container.
+
+## 2. Files
+
+#### /.env
+```/.env```: this is the environment variables file.
+
+#### /docker-compose.yml
+```/docker-compose.yml```: this is the ```docker-compose.yml``` file.
+

difylocal/docker-compose.yml

@@ -0,0 +1,25 @@
+services:
+  ### difylocal #########################################################
+  difylocal:
+    hostname: difylocal
+    image: sparrow-app-difylocal:${IMAGE_APP_DIFYLOCAL_VERSION} # pull_or_build_app_image(local image=>remote image=>build image)
+    container_name: sparrow_container_${CONTAINER_NAMESPACE}_difylocal
+    platform: ${FROM_PLATFORM}
+    restart: always
+    environment:
+      TZ: ${TZ}
+      LANG: C.UTF-8
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - ./difylocal/:/home/sparrow/difylocal/
+      - ./difylocal/data/:/var/data/difylocal/ # data persistence, use a common data persistence directory /var/data/{SERVICE}
+    ports:
+      # You don't need to expose the port in the Dockerfile.
+      - "${DIFYLOCAL_HOST_WEB_PORT}:${DIFYLOCAL_CONTAINER_WEB_PORT}"
+      - "${DIFYLOCAL_HOST_API_PORT}:${DIFYLOCAL_CONTAINER_API_PORT}"
+      - "${DIFYLOCAL_HOST_CODE_EXECUTION_ENDPOINT_PORT}:${DIFYLOCAL_CONTAINER_CODE_EXECUTION_ENDPOINT_PORT}"
+    networks:
+      - backend
+    # only for debug
+    # a command is needed here, otherwise the container cannot be started successfully.
+    command: "tail -f /dev/null"

difylocal/hooks/README.md

@@ -0,0 +1,2 @@
+## What is this directory
+This is the hook actions while the service/container start/stop.

difylocal/hooks/after_start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# include sdk of sparrow.
+. .work/include/sdk.sh

difylocal/hooks/after_stop.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# include sdk of sparrow.
+. .work/include/sdk.sh

difylocal/hooks/before_start.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# include sdk of sparrow.
+. .work/include/sdk.sh

difylocal/hooks/before_stop.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# include sdk of sparrow.
+. .work/include/sdk.sh

difylocal/init/README.md

@@ -0,0 +1,2 @@
+## What is this directory
+This is init scripts after the container created, is executed by ```./hooks/after_start.sh```.

difylocal/make_app_image/Dockerfile

@@ -0,0 +1,14 @@
+ARG FROM_PLATFORM
+ARG IMAGE_BASIC_DIFYLOCAL_VERSION
+FROM --platform=${FROM_PLATFORM} sparrow-basic-difylocal:${IMAGE_BASIC_DIFYLOCAL_VERSION} AS builder
+
+# RUN sed -i 's#https://deb.nodesource.com/node_12.x/#https://mirrors.tuna.tsinghua.edu.cn/nodesource/deb_12.x/#' /etc/apt/sources.list.d/nodesource.list
+
+# RUN apt-get update && apt-get install nodejs=18.19.0 && node -v
+RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
+RUN apt-get update && apt-get install -y nodejs && node -v
+
+RUN npm config set registry https://registry.npmmirror.com
+RUN npm config get registry
+
+WORKDIR /var/data/difylocal

difylocal/make_app_image/README.md

@@ -0,0 +1,2 @@
+## What is this directory
+This is the ```sparrow-app-*``` image making directory.

difylocal/make_app_image/run.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# cd base dir of project.
+# the process is in the same shell as the starting shell.
+cd $SPARROW_BASE_PATH
+
+# define this service.
+service_name="difylocal"
+
+print_stage "try to pull/build sparrow-basic-${service_name} $IMAGE_BASIC_DIFYLOCAL_VERSION"
+if pull basic $service_name $IMAGE_BASIC_DIFYLOCAL_VERSION; then
+    print_info "pull success"
+elif . "./${service_name}/make_basic_image/run.sh"; then
+    print_info "build success"
+else
+    print_error "pull/build failed"
+    exit 1
+fi
+
+cd $service_name/
+if ! docker build \
+  --build-arg FROM_PLATFORM=${FROM_PLATFORM} \
+  --build-arg IMAGE_BASIC_DIFYLOCAL_VERSION=${IMAGE_BASIC_DIFYLOCAL_VERSION} \
+  \
+  -f ./make_app_image/Dockerfile \
+  -t sparrow-app-$service_name:${IMAGE_APP_DIFYLOCAL_VERSION} . ; then
+    print_error "build image failed"
+    exit 1
+fi
+
+cd ../

difylocal/make_basic_image/Dockerfile

@@ -0,0 +1,4 @@
+ARG FROM_PLATFORM
+ARG IMAGE_OFFICIAL_DIFYLOCAL_NAME
+ARG IMAGE_OFFICIAL_DIFYLOCAL_VERSION
+FROM --platform=${FROM_PLATFORM} ${IMAGE_OFFICIAL_DIFYLOCAL_NAME}:${IMAGE_OFFICIAL_DIFYLOCAL_VERSION} AS builder

difylocal/make_basic_image/README.md

@@ -0,0 +1,2 @@
+## What is this directory
+This is the ```sparrow-basic-*``` image making directory.

difylocal/make_basic_image/run.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+# cd base dir of project.
+# the process is in the same shell as the starting shell.
+cd $SPARROW_BASE_PATH
+
+# define this service.
+service_name="difylocal"
+
+cd $service_name/
+if ! docker build \
+  --build-arg FROM_PLATFORM=${FROM_PLATFORM} \
+  --build-arg IMAGE_OFFICIAL_DIFYLOCAL_NAME=${IMAGE_OFFICIAL_DIFYLOCAL_NAME} \
+  --build-arg IMAGE_OFFICIAL_DIFYLOCAL_VERSION=${IMAGE_OFFICIAL_DIFYLOCAL_VERSION} \
+  \
+  -f ./make_basic_image/Dockerfile \
+  -t sparrow-basic-$service_name:${IMAGE_BASIC_DIFYLOCAL_VERSION} . ; then
+    print_error "build image failed"
+    exit 1
+fi
+
+cd ../

difylocal/templates/.env_of_api

@@ -0,0 +1,151 @@
+# Server Edition
+EDITION=SELF_HOSTED
+
+# Your App secret key will be used for securely signing the session cookie
+# Make sure you are changing this key for your deployment with a strong key.
+# You can generate a strong key using `openssl rand -base64 42`.
+# Alternatively you can set it with `SECRET_KEY` environment variable.
+SECRET_KEY=DkE9ZoNcRMLcowTVeo/+SpyWsOGGRwH9EDUJEqlUiEwQNzPpXOuvUu6j
+
+# Console API base URL
+CONSOLE_API_URL=http://127.0.0.1:5001
+CONSOLE_WEB_URL=http://127.0.0.1:3000
+
+# Service API base URL
+SERVICE_API_URL=http://127.0.0.1:5001
+
+# Web APP base URL
+APP_WEB_URL=http://127.0.0.1:3000
+
+# Files URL
+FILES_URL=http://127.0.0.1:5001
+
+# celery configuration
+CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
+
+# redis configuration
+REDIS_HOST=172.20.0.1
+REDIS_PORT=3100
+REDIS_USERNAME=
+REDIS_PASSWORD=secret_redis
+REDIS_DB=0
+
+# PostgreSQL database configuration
+DB_USERNAME=default
+DB_PASSWORD=secret
+DB_HOST=172.20.0.1
+DB_PORT=2900
+DB_DATABASE=
+
+# Storage configuration
+# use for store upload files, private keys...
+# storage type: local, s3, azure-blob
+STORAGE_TYPE=local
+STORAGE_LOCAL_PATH=storage
+S3_ENDPOINT=https://your-bucket-name.storage.s3.clooudflare.com
+S3_BUCKET_NAME=your-bucket-name
+S3_ACCESS_KEY=your-access-key
+S3_SECRET_KEY=your-secret-key
+S3_REGION=your-region
+# Azure Blob Storage configuration
+AZURE_BLOB_ACCOUNT_NAME=your-account-name
+AZURE_BLOB_ACCOUNT_KEY=your-account-key
+AZURE_BLOB_CONTAINER_NAME=yout-container-name
+AZURE_BLOB_ACCOUNT_URL=https://<your_account_name>.blob.core.windows.net
+
+# CORS configuration
+WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
+
+# Vector database configuration, support: weaviate, qdrant, milvus
+VECTOR_STORE=weaviate
+
+# Weaviate configuration
+WEAVIATE_ENDPOINT=http://localhost:8080
+WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
+WEAVIATE_GRPC_ENABLED=false
+WEAVIATE_BATCH_SIZE=100
+
+# Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=difyai123456
+QDRANT_CLIENT_TIMEOUT=20
+
+# Milvus configuration
+MILVUS_HOST=127.0.0.1
+MILVUS_PORT=19530
+MILVUS_USER=root
+MILVUS_PASSWORD=Milvus
+MILVUS_SECURE=false
+
+# Upload configuration
+UPLOAD_FILE_SIZE_LIMIT=15
+UPLOAD_FILE_BATCH_LIMIT=5
+UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
+
+# Model Configuration
+MULTIMODAL_SEND_IMAGE_FORMAT=base64
+
+# Mail configuration, support: resend, smtp
+MAIL_TYPE=
+MAIL_DEFAULT_SEND_FROM=no-reply <[email protected]>
+RESEND_API_KEY=
+RESEND_API_URL=https://api.resend.com
+# smtp configuration
+SMTP_SERVER=smtp.gmail.com
+SMTP_PORT=587
+SMTP_USERNAME=123
+SMTP_PASSWORD=abc
+SMTP_USE_TLS=false
+
+# Sentry configuration
+SENTRY_DSN=
+
+# DEBUG
+DEBUG=false
+SQLALCHEMY_ECHO=false
+
+# Notion import configuration, support public and internal
+NOTION_INTEGRATION_TYPE=public
+NOTION_CLIENT_SECRET=you-client-secret
+NOTION_CLIENT_ID=you-client-id
+NOTION_INTERNAL_SECRET=you-internal-secret
+
+# Hosted Model Credentials
+HOSTED_OPENAI_API_KEY=
+HOSTED_OPENAI_API_BASE=
+HOSTED_OPENAI_API_ORGANIZATION=
+HOSTED_OPENAI_TRIAL_ENABLED=false
+HOSTED_OPENAI_QUOTA_LIMIT=200
+HOSTED_OPENAI_PAID_ENABLED=false
+
+HOSTED_AZURE_OPENAI_ENABLED=false
+HOSTED_AZURE_OPENAI_API_KEY=
+HOSTED_AZURE_OPENAI_API_BASE=
+HOSTED_AZURE_OPENAI_QUOTA_LIMIT=200
+
+HOSTED_ANTHROPIC_API_BASE=
+HOSTED_ANTHROPIC_API_KEY=
+HOSTED_ANTHROPIC_TRIAL_ENABLED=false
+HOSTED_ANTHROPIC_QUOTA_LIMIT=600000
+HOSTED_ANTHROPIC_PAID_ENABLED=false
+
+ETL_TYPE=dify
+UNSTRUCTURED_API_URL=
+
+SSRF_PROXY_HTTP_URL=
+SSRF_PROXY_HTTPS_URL=
+
+BATCH_UPLOAD_LIMIT=10
+KEYWORD_DATA_SOURCE_TYPE=database
+
+# CODE EXECUTION CONFIGURATION
+CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
+CODE_EXECUTION_API_KEY=dify-sandbox
+CODE_MAX_NUMBER=9223372036854775807
+CODE_MIN_NUMBER=-9223372036854775808
+CODE_MAX_STRING_LENGTH=80000
+TEMPLATE_TRANSFORM_MAX_LENGTH=80000
+CODE_MAX_STRING_ARRAY_LENGTH=30
+CODE_MAX_OBJECT_ARRAY_LENGTH=30
+CODE_MAX_NUMBER_ARRAY_LENGTH=1000

difylocal/templates/.env_of_web

@@ -0,0 +1,15 @@
+# For production release, change this to PRODUCTION
+NEXT_PUBLIC_DEPLOY_ENV=DEVELOPMENT
+# The deployment edition, SELF_HOSTED or CLOUD
+NEXT_PUBLIC_EDITION=SELF_HOSTED
+# The base URL of console application, refers to the Console base URL of WEB service if console domain is
+# different from api or web app domain.
+# example: http://cloud.dify.ai/console/api
+NEXT_PUBLIC_API_PREFIX=http://localhost:5001/console/api
+# The URL for Web APP, refers to the Web App base URL of WEB service if web app domain is different from
+# console or api domain.
+# example: http://udify.app/api
+NEXT_PUBLIC_PUBLIC_API_PREFIX=http://localhost:5001/api
+
+# SENTRY
+NEXT_PUBLIC_SENTRY_DSN=