Release: Ingestion, Libraries

.azure-template.env

@@ -1,18 +1,20 @@
-OPENAI_API_TYPE=azure
-OPENAI_API_BASE=https://alkemio-gpt.openai.azure.com/
-OPENAI_API_KEY=azure-openai-key
 OPENAI_API_VERSION=2023-05-15
+AZURE_OPENAI_ENDPOINT=https://alkemio-gpt.openai.azure.com
+AZURE_OPENAI_API_KEY=azure-openai-key
+LLM_DEPLOYMENT_NAME=deploy-gpt-35-turbo
+EMBEDDINGS_DEPLOYMENT_NAME=embedding
 RABBITMQ_HOST=localhost
 RABBITMQ_USER=admin
 RABBITMQ_PASSWORD=super-secure-pass
 AI_MODEL_TEMPERATURE=0.3
-AI_MODEL_NAME=gpt-35-turbo
-AI_DEPLOYMENT_NAME=deploy-gpt-35-turbo
-AI_EMBEDDINGS_DEPLOYMENT_NAME=embedding
 AI_SOURCE_WEBSITE=https://www.alkemio.org
+AI_SOURCE_WEBSITE2=https://welcome.alkem.io
 AI_LOCAL_PATH=~/alkemio/data
-AI_WEBSITE_REPO=https://github.com/alkem-io/website.git
+AI_WEBSITE_REPO=github.com/alkem-io/website.git
+AI_WEBSITE_REPO2=github.com/alkem-io/welcome-site.git
+AI_GITHUB_USER=github-user-for-website-cloning
+AI_GITHUB_PAT=github-user-for-website-cloning
 LANGCHAIN_TRACING_V2=true
 LANGCHAIN_ENDPOINT="https://api.smith.langchain.com"
 LANGCHAIN_API_KEY="langsmith-api-key"
-LANGCHAIN_PROJECT="guidance-engine"
+LANGCHAIN_PROJECT="guidance-engine"

.github/workflows/build-deploy-k8s-dev-azure.yml

@@ -9,10 +9,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout GitHub Action"
-        uses: actions/checkout@master
+        uses: actions/checkout@v3.0.2
 
       - name: "Login into ACR"
-        uses: azure/docker-login@v1
+        uses: azure/docker-login@v1.0.1
         with:
           login-server: ${{ secrets.REGISTRY_LOGIN_SERVER }}
           username: ${{ secrets.REGISTRY_USERNAME }}
@@ -27,32 +27,31 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout GitHub Action"
-        uses: actions/checkout@master
+        uses: actions/checkout@v3.0.2
 
       - name: "Login via Azure CLI"
-        uses: azure/login@v1
+        uses: azure/login@v1.4.7
         with:
           creds: ${{ secrets.AZURE_CRED_K8S_NEW }}
 
-      - uses: Azure/aks-set-context@v1
+      - uses: Azure/aks-set-context@v3.2
         with:
-          creds: ${{ secrets.AZURE_CRED_K8S_NEW }}
           cluster-name: ${{ secrets.CLUSTER_NAME }}
           resource-group: ${{ secrets.RESOURCE_GROUP_K8S }}
 
-      - uses: Azure/k8s-create-secret@v1
+      - uses: Azure/k8s-create-secret@v4.0
         with:
           container-registry-url: ${{ secrets.REGISTRY_LOGIN_SERVER }}
           container-registry-username: ${{ secrets.REGISTRY_USERNAME }}
           container-registry-password: ${{ secrets.REGISTRY_PASSWORD }}
           secret-name: alkemio-guidance-engine-secret
 
-      - uses: azure/setup-kubectl@v3
+      - uses: azure/setup-kubectl@v3.2
         with:
           version: "v1.22.0" # default is latest stable, fixing it to a compatible version
         id: install
 
-      - uses: Azure/k8s-deploy@v1
+      - uses: Azure/k8s-deploy@v4.10
         with:
           manifests: |
             manifests/25-genai-deployment-dev.yaml

.github/workflows/build-deploy-k8s-sandbox-azure.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout GitHub Action"
-        uses: actions/checkout@master
+        uses: actions/checkout@v3.0.2
 
       - name: "Login into ACR"
-        uses: azure/docker-login@v1
+        uses: azure/docker-login@v1.0.1
         with:
           login-server: ${{ secrets.REGISTRY_LOGIN_SERVER }}
           username: ${{ secrets.REGISTRY_USERNAME }}
@@ -25,32 +25,31 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout GitHub Action"
-        uses: actions/checkout@master
+        uses: actions/checkout@v3.0.2
 
       - name: "Login via Azure CLI"
-        uses: azure/login@v1
+        uses: azure/login@v1.4.7
         with:
           creds: ${{ secrets.AZURE_CRED_K8S_NEW }}
 
-      - uses: Azure/aks-set-context@v1
+      - uses: Azure/aks-set-context@v3.2
         with:
-          creds: ${{ secrets.AZURE_CRED_K8S_NEW }}
           cluster-name: k8s-sandbox
           resource-group: res-grp-k8s-sandbox
 
-      - uses: Azure/k8s-create-secret@v1
+      - uses: Azure/k8s-create-secret@v4.0
         with:
           container-registry-url: ${{ secrets.REGISTRY_LOGIN_SERVER }}
           container-registry-username: ${{ secrets.REGISTRY_USERNAME }}
           container-registry-password: ${{ secrets.REGISTRY_PASSWORD }}
           secret-name: alkemio-guidance-engine-secret
 
-      - uses: azure/setup-kubectl@v3
+      - uses: azure/setup-kubectl@v3.2
         with:
           version: "v1.22.0" # default is latest stable, fixing it to a compatible version
         id: install
 
-      - uses: Azure/k8s-deploy@v1
+      - uses: Azure/k8s-deploy@v4.10
         with:
           manifests: |
             manifests/25-genai-deployment-dev.yaml

.github/workflows/build-deploy-k8s-test-azure.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout GitHub Action"
-        uses: actions/checkout@master
+        uses: actions/checkout@v3.0.2
 
       - name: "Login into ACR"
-        uses: azure/docker-login@v1
+        uses: azure/docker-login@v1.0.1
         with:
           login-server: ${{ secrets.REGISTRY_LOGIN_SERVER }}
           username: ${{ secrets.REGISTRY_USERNAME }}
@@ -26,32 +26,31 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout GitHub Action"
-        uses: actions/checkout@master
+        uses: actions/checkout@v3.0.2
 
       - name: "Login via Azure CLI"
-        uses: azure/login@v1
+        uses: azure/login@v1.4.7
         with:
           creds: ${{ secrets.AZURE_CRED_K8S_NEW }}
 
-      - uses: Azure/aks-set-context@v1
+      - uses: Azure/aks-set-context@v3.2
         with:
-          creds: ${{ secrets.AZURE_CRED_K8S_NEW }}
           cluster-name: k8s-test
           resource-group: res-grp-k8s-test
 
-      - uses: Azure/k8s-create-secret@v1
+      - uses: Azure/k8s-create-secret@v4.0
         with:
           container-registry-url: ${{ secrets.REGISTRY_LOGIN_SERVER }}
           container-registry-username: ${{ secrets.REGISTRY_USERNAME }}
           container-registry-password: ${{ secrets.REGISTRY_PASSWORD }}
           secret-name: alkemio-guidance-engine-secret
 
-      - uses: azure/setup-kubectl@v3
+      - uses: azure/setup-kubectl@v3.2
         with:
           version: "v1.22.0" # default is latest stable, fixing it to a compatible version
         id: install
 
-      - uses: Azure/k8s-deploy@v1
+      - uses: Azure/k8s-deploy@v4.10
         with:
           manifests: |
             manifests/25-genai-deployment-dev.yaml

.github/workflows/build-release-docker-hub.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3.0.2
       - name: Prepare
         id: prep
         run: |
@@ -35,13 +35,13 @@ jobs:
           elif [ "${{ github.event_name }}" = "push" ]; then
             TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}"
           fi
-          echo ::set-output name=version::${VERSION}
-          echo ::set-output name=tags::${TAGS}
-          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "created={$(date -u +'%Y-%m-%dT%H:%M:%SZ')}" >> $GITHUB_OUTPUT
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3.0.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3.0.0
       - name: Login to DockerHub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v1
@@ -50,7 +50,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5.0.0
         with:
           context: .
           file: ./Dockerfile

.gitignore

@@ -5,3 +5,4 @@ azure.env
 /__pycache__/*
 /vectordb/*
 local.env
+docker-compose-local.yaml

Dockerfile

@@ -4,8 +4,8 @@ FROM python:3.11-slim-bookworm
 # Set the working directory in the container to /app
 WORKDIR /app
 
-ARG GO_VERSION=1.21.1
-ARG HUGO_VERSION=0.118.2
+ARG GO_VERSION=1.21.5
+ARG HUGO_VERSION=0.121.1
 ARG ARCHITECTURE=amd64
 
 # install git, go and hugo

README.md

@@ -60,8 +60,8 @@ There is a draft implementation for the interaction language of the model (this
 
 ### Docker 
 The following command can be used to build the container from the Docker CLI (default architecture is amd64, so `--build-arg ARCHITECTURE=arm64` for amd64 builds):
-`docker build --build-arg ARCHITECTURE=arm64 --no-cache -t alkemio/guidance-engine:v0.2.0 .`
-`docker build--no-cache -t alkemio/guidance-engine:v0.2.0 .`
+`docker build --build-arg ARCHITECTURE=arm64 --no-cache -t alkemio/guidance-engine:v0.4.0 .`
+`docker build --no-cache -t alkemio/guidance-engine:v0.2.0 .`
 The Dockerfile has some self-explanatory configuration arguments.
 
 The following command can be used to start the container from the Docker CLI:
@@ -70,22 +70,28 @@ where `.env` based on `.azure-template.env`
 Alternatively use `docker-compose up -d`.
 
 with:
-- `OPENAI_API_KEY`: a valid OpenAI API key
-- `OPENAI_API_TYPE`: a valid OpenAI API type. For Azure, the value is `azure`
+- `AZURE_OPENAI_API_KEY`: a valid OpenAI API key
 - `OPENAI_API_VERSION`: a valid Azure OpenAI version. At the moment of writing, latest is `2023-05-15`
-- `OPENAI_API_BASE`: a valid Azure OpenAI base URL, e.g. `https://{your-azure-resource-name}.openai.azure.com/`
+- `AZURE_OPENAI_ENDPOINT`: a valid Azure OpenAI base URL, e.g. `https://{your-azure-resource-name}.openai.azure.com/`
 - `RABBITMQ_HOST`: the RabbitMQ host name
 - `RABBITMQ_USER`: the RabbitMQ user
 - `RABBITMQ_PASSWORD`: the RabbitMQ password
 - `AI_MODEL_TEMPERATURE`: the `temperature` of the model, use value between 0 and 1. 1 means more randomized answer, closer to 0 - a stricter one
-- `AI_MODEL_NAME`: the model name in Azure
-- `AI_DEPLOYMENT_NAME`: the AI gpt model deployment name in Azure
-- `AI_EMBEDDINGS_DEPLOYMENT_NAME`: the AI embeddings model deployment name in Azure
-- `AI_SOURCE_WEBSITE`: the URL of the website that contains the source data (for references only)
+- `LLM_DEPLOYMENT_NAME`: the AI gpt model deployment name in Azure
+- `EMBEDDINGS_DEPLOYMENT_NAME`: the AI embeddings model deployment name in Azure
+- `AI_SOURCE_WEBSITE`: the URL of the foundation website that contains the source data (for references only)
+- `AI_SOURCE_WEBSITE2`: the URL of the welcome website that contains the source data (for references only)
 - `AI_LOCAL_PATH`: local file path for storing data
-- `AI_WEBSITE_REPO`: url of the Git repository containing the website source data, based on Hugo
+- `AI_WEBSITE_REPO`: url of the Git repository containing the foundation website source data, based on Hugo - without https
+- `AI_WEBSITE_REPO2`: url of the Git repository containing the welcome website source data, based on Hugo - without https
+- `AI_GITHUB_USER` : Github user used for cloning website repos
+- `AI_GITHUB_PAT` : Personal access token for cloning website repos
+- `LANGCHAIN_TRACING_V2` : enable Langchain tracing
+- `LANGCHAIN_ENDPOINT` : Langchain tracing endpoint (e.g. "https://api.smith.langchain.com")
+- `LANGCHAIN_API_KEY` : Langchain tracing API key
+- `LANGCHAIN_PROJECT` : Langchain tracing project name (e.g. "guidance-engine")
 
-You can find sample values in `.azure-template.env` and `.openai-template.env`. Configure them and create `.env` file with the updated settings.
+You can find sample values in `.azure-template.env`. Configure them and create `.env` file with the updated settings.
 
 ### Python & Poetry
 The project requires Python & Poetry installed. The minimum version dependencies can be found at `pyproject.toml`.
@@ -102,9 +108,7 @@ The following tasks are still outstanding:
 - assess overall quality and performance of the model and make improvements as and when required.
 - assess the need to summarize the chat history to avoid exceeding the prompt token limit.
 - update the yaml manifest.
-- add error handling.
 - perform extensive testing, in particular in multi-user scenarios.
-- look at improvements of the ingestion. As a minimum the service engine should not consume queries whilst the ingestion is ongoing, as thatwill lead to errors.
-- look at the use of `temperature` for the `QARetrievalChain`. It is not so obvious how this is handled.
+- look at improvements of the ingestion. As a minimum the service engine should not consume queries whilst the ingestion is ongoing, as that will lead to errors.
 - look at the possibility to implement reinforcement learning.
-- return the actual LLM costs and token usage for queries.
+

ai_utils.py

@@ -1,4 +1,4 @@
-from langchain.embeddings import OpenAIEmbeddings
+from langchain.embeddings import AzureOpenAIEmbeddings
 from langchain.vectorstores import FAISS
 from langchain.llms import AzureOpenAI
 from langchain.prompts import PromptTemplate
@@ -8,7 +8,7 @@
 from langchain.chains.conversational_retrieval.prompts import QA_PROMPT
 import logging
 import def_ingest
-from config import config, website_source_path, website_generated_path, vectordb_path, local_path, generate_website, LOG_LEVEL
+from config import config, website_source_path, website_generated_path, website_source_path2, website_generated_path2, vectordb_path, local_path, generate_website, LOG_LEVEL
 
 import os
 
@@ -17,7 +17,7 @@
 
 # Create handlers
 c_handler = logging.StreamHandler()
-f_handler = logging.FileHandler(local_path+'/app.log')
+f_handler = logging.FileHandler(os.path.join(os.path.expanduser(local_path),'app.log'))
 
 c_handler.setLevel(level=getattr(logging, LOG_LEVEL))
 f_handler.setLevel(logging.ERROR)
@@ -118,12 +118,18 @@ def get_language_by_code(language_code):
     template=chat_template, input_variables=["question", "context", "chat_history"]
 )
 
-generic_llm = AzureOpenAI(deployment_name=os.environ["AI_DEPLOYMENT_NAME"], model_name=os.environ["AI_MODEL_NAME"],
+
+generic_llm = AzureOpenAI(azure_deployment=os.environ["LLM_DEPLOYMENT_NAME"],
                             temperature=0, verbose=verbose_models)
 
 question_generator = LLMChain(llm=generic_llm, prompt=custom_question_prompt, verbose=verbose_models)
 
-embeddings = OpenAIEmbeddings(deployment=os.environ["AI_EMBEDDINGS_DEPLOYMENT_NAME"], chunk_size=1)
+
+embeddings = AzureOpenAIEmbeddings(
+    azure_deployment=config['embeddings_deployment_name'],
+    openai_api_version=config['openai_api_version'],
+    chunk_size=1
+)
 
 # Check if the vector database exists
 if os.path.exists(vectordb_path+"/index.pkl"):
@@ -132,19 +138,20 @@ def get_language_by_code(language_code):
     # ingest data
     if generate_website:
         def_ingest.clone_and_generate(config['website_repo'], website_generated_path, website_source_path)
-    def_ingest.mainapp(config['source_website'])
+        def_ingest.clone_and_generate(config['website_repo2'], website_generated_path2, website_source_path2)
+    def_ingest.mainapp(config['source_website'], config['source_website2'])
 
 vectorstore = FAISS.load_local(vectordb_path, embeddings)
 retriever = vectorstore.as_retriever()
 
-chat_llm = AzureChatOpenAI(deployment_name=os.environ["AI_DEPLOYMENT_NAME"],
-                            model_name=os.environ["AI_MODEL_NAME"], temperature=os.environ["AI_MODEL_TEMPERATURE"],
+chat_llm = AzureChatOpenAI(azure_deployment=os.environ["LLM_DEPLOYMENT_NAME"],
+                            temperature=os.environ["AI_MODEL_TEMPERATURE"],
                             max_tokens=max_token_limit)
 
 doc_chain = load_qa_chain(generic_llm, chain_type="stuff", prompt=QA_PROMPT, verbose=verbose_models)
 
 def translate_answer(answer, language):
-    translate_llm = AzureChatOpenAI(deployment_name=os.environ["AI_DEPLOYMENT_NAME"], model_name=os.environ["AI_MODEL_NAME"],
+    translate_llm = AzureChatOpenAI(azure_deployment=os.environ["LLM_DEPLOYMENT_NAME"],
                                 temperature=0, verbose=verbose_models)
     prompt = translation_prompt.format(answer=answer, language=language)
     return translate_llm(prompt)