GenAI-Workshop-Demo1c.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyOwoVZqW2vLZLJnEwqQT0YH"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","source":["!pip install gradio pandas numpy langchain sentence-transformers scikit-learn datasets"],"metadata":{"id":"dnv7PACR-XQo"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Using Embeddings to test Cosine Similarities\n","\n","\n"],"metadata":{"id":"FHYqhdpcMNzp"}},{"cell_type":"code","source":["import datasets\n","from datasets import load_dataset"],"metadata":{"id":"2miaYaR_FMwf","executionInfo":{"status":"ok","timestamp":1712764007717,"user_tz":240,"elapsed":315,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["hf_data = load_dataset('botchagalupe/ProfoundDeming')\n","\n","dataset = \"\"\n","for item in hf_data ['train']:\n","    dataset += item['text'] + \" \"  # Adding a space after each text part\n","\n","print(\"Data Ingested\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EyznJb5bFN6m","executionInfo":{"status":"ok","timestamp":1712764013683,"user_tz":240,"elapsed":2784,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"5e91e3a7-912d-4a66-b096-2d532e68eae6"},"execution_count":7,"outputs":[{"output_type":"stream","name":"stdout","text":["Data Ingested\n"]}]},{"cell_type":"code","source":["import gradio as gr\n","import pandas as pd\n","import numpy as np\n","from langchain.text_splitter import RecursiveCharacterTextSplitter\n","from sentence_transformers import SentenceTransformer\n","from sklearn.metrics.pairwise import cosine_similarity"],"metadata":{"id":"JG4XDq74BYHs","executionInfo":{"status":"ok","timestamp":1712764018724,"user_tz":240,"elapsed":288,"user":{"displayName":"John Willis","userId":"15271974867993570949"}}},"execution_count":8,"outputs":[]},{"cell_type":"code","execution_count":9,"metadata":{"id":"YC90ofwN95hi","colab":{"base_uri":"https://localhost:8080/","height":648},"executionInfo":{"status":"ok","timestamp":1712764038082,"user_tz":240,"elapsed":16735,"user":{"displayName":"John Willis","userId":"15271974867993570949"}},"outputId":"bc4f5974-d09d-45cf-ea9c-b4fdf492fcb2"},"outputs":[{"output_type":"stream","name":"stdout","text":["Setting queue=True in a Colab notebook requires sharing enabled. Setting `share=True` (you can turn this off by setting `share=False` in `launch()` explicitly).\n","\n","Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n","Running on public URL: https://d630cf4a59623a18d4.gradio.live\n","\n","This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["<div><iframe src=\"https://d630cf4a59623a18d4.gradio.live\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"]},"metadata":{}}],"source":["# Constants for default values\n","DEFAULT_CHUNK_SIZE = 400\n","DEFAULT_CHUNK_OVERLAP = 50\n","DEFAULT_NUM_CHUNKS = 10\n","\n","# nomic-ai/nomic-embed-text-v1',trust_remote_code=True\n","\n","# Initialize the sentence transformer model for embeddings\n","#model = SentenceTransformer('all-MiniLM-L6-v2')\n","model = SentenceTransformer('BAAI/bge-small-en-v1.5')\n","#model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')\n","#model = SentenceTransformer('nomic-ai/nomic-embed-text-v1',trust_remote_code=True)\n","\n","\n","def tokenize_text(method, text, chunk_size, chunk_overlap, num_chunks):\n","    \"\"\"\n","    Tokenizes the input text based on the selected method and provided parameters.\n","    \"\"\"\n","    num_chunks = int(num_chunks)\n","    output = []\n","\n","    # Ensure text is provided\n","    if not text.strip():\n","        return pd.DataFrame(columns=['Chunk #', 'Text Chunk', 'Character Count', 'Token Count'])\n","\n","    if method == \"RecursiveCharacterTextSplitter\":\n","        text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap, length_function=len, is_separator_regex=False)\n","        tokenized_texts = text_splitter.split_text(text)[:num_chunks]\n","        for i, chunk in enumerate(tokenized_texts):\n","            output.append({\n","                'Chunk #': i,\n","                'Text Chunk': chunk,\n","                'Character Count': len(chunk),\n","                'Token Count': len(chunk.split())\n","            })\n","\n","    df = pd.DataFrame(output)\n","    return df\n","\n","def calculate_embeddings(df):\n","    \"\"\"\n","    Calculates embeddings for each text chunk in the dataframe.\n","    \"\"\"\n","    if df.empty:\n","        return df\n","\n","    chunks = df['Text Chunk'].tolist()\n","    embeddings = model.encode(chunks)\n","    df['Embeddings'] = embeddings.tolist()\n","    return df\n","\n","def search_similar_chunks(query, df_with_embeddings):\n","    \"\"\"\n","    Search for chunks similar to the query embedding.\n","    \"\"\"\n","    # Compute the query embedding\n","    query_embedding = model.encode([query])[0]\n","\n","    # Calculate similarity scores\n","    chunk_embeddings = np.vstack(df_with_embeddings['Embeddings'])\n","    similarity_scores = cosine_similarity([query_embedding], chunk_embeddings)[0]\n","\n","    # Insert similarity scores into the dataframe after 'Chunk #'\n","    df_with_embeddings.insert(1, 'Similarity', similarity_scores)\n","\n","    # Return the dataframe sorted by similarity scores in descending order\n","    return df_with_embeddings.sort_values(by='Similarity', ascending=False)\n","\n","def process_and_embed(method, text, chunk_size, chunk_overlap, num_chunks):\n","    \"\"\"\n","    Tokenizes the text and calculates embeddings.\n","    \"\"\"\n","    df = tokenize_text(method, text, chunk_size, chunk_overlap, num_chunks)\n","    df_with_embeddings = calculate_embeddings(df)\n","    return df_with_embeddings\n","\n","def update_output(method, text, chunk_size, chunk_overlap, num_chunks, query):\n","    df_with_embeddings = process_and_embed(method, text, chunk_size, chunk_overlap, num_chunks)\n","    if query:\n","        df_with_embeddings = search_similar_chunks(query, df_with_embeddings)\n","        # Update the headers to reflect the new column order after similarity search\n","        return df_with_embeddings[['Chunk #', 'Similarity', 'Text Chunk', 'Character Count', 'Token Count', 'Embeddings']]\n","    return df_with_embeddings[['Chunk #', 'Text Chunk', 'Character Count', 'Token Count', 'Embeddings']]\n","\n","iface = gr.Interface(\n","    fn=update_output,\n","    inputs=[\n","        gr.Dropdown(label=\"Select Tokenization Method\", choices=[\"RecursiveCharacterTextSplitter\"]),\n","        #gr.Textbox(label=\"Enter Text\", lines=10, placeholder=\"Type or paste text here.\"),\n","        gr.Textbox(label=\"Enter Text\", value=dataset),\n","        gr.Number(label=\"Chunk Size\", value=DEFAULT_CHUNK_SIZE),\n","        gr.Number(label=\"Chunk Overlap\", value=DEFAULT_CHUNK_OVERLAP),\n","        gr.Number(label=\"Number of Chunks to Display\", value=DEFAULT_NUM_CHUNKS),\n","        gr.Textbox(label=\"Enter Query for Similarity Search\", lines=2, placeholder=\"Type your query here.\")\n","    ],\n","    outputs=gr.Dataframe(height=900),\n","    title=\"Text Tokenization and Embedding Tool\",\n","    description=\"A tool for tokenizing text and calculating embeddings. Now with similarity search feature.\"\n",")\n","\n","if __name__ == \"__main__\":\n","    iface.launch()\n"]}]}