Add evaluate_ragas component (#715)

Putting it as a draft because will probably change the base Dockerfile
for one that already has torch dependencies, instead of installing them
in the requirements.txt

---------

Co-authored-by: Robbe Sneyders <[email protected]>

components/evaluate_ragas/Dockerfile

@@ -0,0 +1,30 @@
+FROM --platform=linux/amd64 python:3.8-slim as base
+
+# System dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install git -y
+
+# Install requirements
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Install Fondant
+# This is split from other requirements to leverage caching
+ARG FONDANT_VERSION=main
+RUN pip3 install fondant[component,aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+
+# Set the working directory to the component folder
+WORKDIR /component
+COPY src/ src/
+
+FROM base as test
+COPY tests/ tests/
+RUN pip3 install --no-cache-dir -r tests/requirements.txt
+ARG OPENAI_KEY
+ENV OPENAI_KEY=${OPENAI_KEY}
+RUN python -m pytest tests
+
+FROM base
+WORKDIR /component/src
+ENTRYPOINT ["fondant", "execute", "main"]

components/evaluate_ragas/README.md

@@ -0,0 +1,53 @@
+# retriever_eval_ragas
+
+### Description
+Component that evaluates the retriever using RAGAS
+
+### Inputs / outputs
+
+**This component consumes:**
+
+- text: string
+- retrieved_chunks: list<item: string>
+
+**This component produces no data.**
+
+### Arguments
+
+The component takes the following arguments to alter its behavior:
+
+| argument | type | description | default |
+| -------- | ---- | ----------- | ------- |
+| module | str | Module from which the LLM is imported. Defaults to langchain.llms | langchain.llms |
+| llm_name | str | Name of the selected llm | / |
+| llm_kwargs | dict | Arguments of the selected llm | / |
+
+### Usage
+
+You can add this component to your pipeline using the following code:
+
+```python
+from fondant.pipeline import Pipeline
+
+
+pipeline = Pipeline(...)
+
+dataset = pipeline.read(...)
+
+dataset = dataset.apply(
+    "evaluate_ragas",
+    arguments={
+        # Add arguments
+        # "module": "langchain.llms",
+        # "llm_name": ,
+        # "llm_kwargs": {},
+    }
+)
+```
+
+### Testing
+
+You can run the tests using docker with BuildKit. From this directory, run:
+```
+docker build . --target test
+```

components/evaluate_ragas/fondant_component.yaml

@@ -0,0 +1,31 @@
+name: retriever_eval_ragas
+description: Component that evaluates the retriever using RAGAS
+image: fndnt/retriever_eval:dev
+tags:
+  - Text processing
+
+consumes:
+  text:
+    type: string
+  retrieved_chunks:
+    type: array
+    items:
+      type: string
+
+produces:
+  additionalProperties: true
+  # Overwrite with metrics to be computed by ragas
+  # (https://docs.ragas.io/en/latest/concepts/metrics/index.html)
+
+
+args:
+  module:
+    description: Module from which the LLM is imported. Defaults to langchain.llms
+    type: str
+    default: "langchain.llms"
+  llm_name:
+    description: Name of the selected llm
+    type: str
+  llm_kwargs:
+    description: Arguments of the selected llm
+    type: dict

components/evaluate_ragas/requirements.txt

@@ -0,0 +1 @@
+ragas==0.0.21

components/evaluate_ragas/src/main.py

@@ -0,0 +1,81 @@
+import typing as t
+
+import pandas as pd
+from datasets import Dataset
+from fondant.component import PandasTransformComponent
+from ragas import evaluate
+from ragas.llms import LangchainLLM
+
+
+class RetrieverEval(PandasTransformComponent):
+    def __init__(
+        self,
+        *,
+        module: str,
+        llm_name: str,
+        llm_kwargs: dict,
+        produces: t.Dict[str, t.Any],
+        **kwargs,
+    ) -> None:
+        """
+        Args:
+            module: Module from which the LLM is imported. Defaults to langchain.llms
+            llm_name: Name of the selected llm
+            llm_kwargs: Arguments of the selected llm
+            produces: RAGAS metrics to compute.
+            kwargs: Unhandled keyword arguments passed in by Fondant.
+        """
+        self.llm = self.extract_llm(
+            module=module,
+            model_name=llm_name,
+            model_kwargs=llm_kwargs,
+        )
+        self.gpt_wrapper = LangchainLLM(llm=self.llm)
+        self.metric_functions = self.extract_metric_functions(
+            metrics=list(produces.keys()),
+        )
+        self.set_llm(self.metric_functions)
+
+    # import the metric functions selected
+    @staticmethod
+    def import_from(module, name):
+        module = __import__(module, fromlist=[name])
+        return getattr(module, name)
+
+    def extract_llm(self, module, model_name, model_kwargs):
+        module = self.import_from(module, model_name)
+        return module(**model_kwargs)
+
+    def extract_metric_functions(self, metrics: list):
+        functions = []
+        for metric in metrics:
+            functions.append(self.import_from("ragas.metrics", metric))
+        return functions
+
+    def set_llm(self, metric_functions: list):
+        for metric_function in metric_functions:
+            metric_function.llm = self.gpt_wrapper
+
+    # evaluate the retriever
+    @staticmethod
+    def create_hf_ds(dataframe: pd.DataFrame):
+        dataframe = dataframe.rename(
+            columns={"text": "question", "retrieved_chunks": "contexts"},
+        )
+        return Dataset.from_pandas(dataframe)
+
+    def ragas_eval(self, dataset):
+        return evaluate(dataset=dataset, metrics=self.metric_functions)
+
+    def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        hf_dataset = self.create_hf_ds(
+            dataframe=dataframe[["text", "retrieved_chunks"]],
+        )
+        if "id" in hf_dataset.column_names:
+            hf_dataset = hf_dataset.remove_columns("id")
+
+        result = self.ragas_eval(dataset=hf_dataset)
+        results_df = result.to_pandas()
+        results_df = results_df.set_index(dataframe.index)
+
+        return results_df

components/evaluate_ragas/tests/component_test.py

@@ -0,0 +1,117 @@
+import os
+
+import pandas as pd
+import pyarrow as pa
+from main import RetrieverEval
+
+
+def test_transform():
+    input_dataframe = pd.DataFrame(
+        {
+            "text": [
+                "Lorem ipsum dolor sit amet, consectetur adipiscing elit?",
+                "Sed massa massa, interdum a porttitor sit amet, semper eget nunc?",
+            ],
+            "retrieved_chunks": [
+                [
+                    "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \
+                        Quisque ut efficitur neque. Aenean mollis eleifend est, \
+                        eu laoreet magna egestas quis. Cras id sagittis erat. \
+                        Aliquam vel blandit arcu. Morbi ac nulla ullamcorper, \
+                        rutrum neque nec, pellentesque diam. Nulla nec tempor \
+                        enim. Suspendisse a volutpat leo, quis varius dolor.",
+                    "Curabitur placerat ultrices mauris et lobortis. Maecenas \
+                        laoreet tristique sagittis. Integer facilisis eleifend \
+                        dolor, quis fringilla orci eleifend ac. Vestibulum nunc \
+                        odio, tincidunt ut augue et, ornare vehicula sapien. Orci \
+                        varius natoque penatibus et magnis dis parturient montes, \
+                        nascetur ridiculus mus. Sed auctor felis lacus, rutrum \
+                        tempus ligula viverra ac. Curabitur pharetra mauris et \
+                        ornare pulvinar. Suspendisse a ultricies nisl. Mauris \
+                        sit amet odio condimentum, venenatis orci vitae, \
+                        tincidunt purus. Ut ullamcorper convallis ligula ac \
+                        posuere. In efficitur enim ac lacus dignissim congue. \
+                        Nam turpis augue, aliquam et velit sit amet, varius \
+                        euismod ante. Duis volutpat nisl sit amet auctor tempus.\
+                            Vivamus in eros ex.",
+                ],
+                [
+                    "am leo massa, ultricies eu viverra ac, commodo non sapien. \
+                        Mauris et mauris sollicitudin, ultricies ex ac, luctus \
+                        nulla.",
+                    "Cras tincidunt facilisis mi, ac eleifend justo lobortis ut. \
+                        In lobortis cursus ante et faucibus. Vestibulum auctor \
+                        felis at odio varius, ac vulputate leo dictum. \
+                        Phasellus in augue ante. Aliquam aliquam mauris \
+                        sed tellus egestas fermentum.",
+                ],
+            ],
+        },
+    )
+
+    component = RetrieverEval(
+        module="langchain.llms",
+        llm_name="OpenAI",
+        llm_kwargs={"openai_api_key": os.environ["OPENAI_KEY"]},
+        produces={
+            "context_precision": pa.float32(),
+            "context_relevancy": pa.float32(),
+        },
+    )
+
+    output_dataframe = component.transform(input_dataframe)
+
+    expected_output_dataframe = pd.DataFrame(
+        {
+            "question": [
+                "Lorem ipsum dolor sit amet, consectetur adipiscing elit?",
+                "Sed massa massa, interdum a porttitor sit amet, semper eget nunc?",
+            ],
+            "contexts": [
+                [
+                    "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \
+                        Quisque ut efficitur neque. Aenean mollis eleifend est, \
+                        eu laoreet magna egestas quis. Cras id sagittis erat. \
+                        Aliquam vel blandit arcu. Morbi ac nulla ullamcorper, \
+                        rutrum neque nec, pellentesque diam. Nulla nec tempor \
+                        enim. Suspendisse a volutpat leo, quis varius dolor.",
+                    "Curabitur placerat ultrices mauris et lobortis. Maecenas \
+                        laoreet tristique sagittis. Integer facilisis eleifend \
+                        dolor, quis fringilla orci eleifend ac. Vestibulum nunc \
+                        odio, tincidunt ut augue et, ornare vehicula sapien. Orci \
+                        varius natoque penatibus et magnis dis parturient montes, \
+                        nascetur ridiculus mus. Sed auctor felis lacus, rutrum \
+                        tempus ligula viverra ac. Curabitur pharetra mauris et \
+                        ornare pulvinar. Suspendisse a ultricies nisl. Mauris \
+                        sit amet odio condimentum, venenatis orci vitae, \
+                        tincidunt purus. Ut ullamcorper convallis ligula ac \
+                        posuere. In efficitur enim ac lacus dignissim congue. \
+                        Nam turpis augue, aliquam et velit sit amet, varius \
+                        euismod ante. Duis volutpat nisl sit amet auctor tempus.\
+                            Vivamus in eros ex.",
+                ],
+                [
+                    "am leo massa, ultricies eu viverra ac, commodo non sapien. \
+                        Mauris et mauris sollicitudin, ultricies ex ac, luctus \
+                        nulla.",
+                    "Cras tincidunt facilisis mi, ac eleifend justo lobortis ut. \
+                        In lobortis cursus ante et faucibus. Vestibulum auctor \
+                        felis at odio varius, ac vulputate leo dictum. \
+                        Phasellus in augue ante. Aliquam aliquam mauris \
+                        sed tellus egestas fermentum.",
+                ],
+            ],
+            "context_precision": 0.15,
+            "context_relevancy": 0.35,
+        },
+    )
+
+    # Check if columns are the same
+    columns_equal = expected_output_dataframe.columns.equals(output_dataframe.columns)
+
+    # Check if data types within each column match
+    dtypes_match = expected_output_dataframe.dtypes.equals(output_dataframe.dtypes)
+
+    # Check if both conditions are met
+    assert columns_equal
+    assert dtypes_match

components/evaluate_ragas/tests/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+pythonpath = ../src

components/evaluate_ragas/tests/requirements.txt

@@ -0,0 +1 @@
+pytest==7.4.2