From 555c47fa07fffeaef86d7844d269bc9e1caa4903 Mon Sep 17 00:00:00 2001
From: Roman Solomatin <36135455+Samoed@users.noreply.github.com>
Date: Sun, 24 Nov 2024 14:15:38 +0300
Subject: [PATCH 1/3] align readme with current mteb

---
 README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ef87ec4370..df7cc1f388 100644
--- a/README.md
+++ b/README.md
@@ -50,6 +50,8 @@ model_name = "average_word_embeddings_komninos"
 # model_name = "sentence-transformers/all-MiniLM-L6-v2"
 
 model = SentenceTransformer(model_name)
+# or directly from mteb:
+model = mteb.get_model(model_name)
 tasks = mteb.get_tasks(tasks=["Banking77Classification"])
 evaluation = mteb.MTEB(tasks=tasks)
 results = evaluation.run(model, output_folder=f"results/{model_name}")
@@ -220,9 +222,13 @@ Note that the public leaderboard uses the test splits for all datasets except MS
 Models should implement the following interface, implementing an `encode` function taking as inputs a list of sentences, and returning a list of embeddings (embeddings can be `np.array`, `torch.tensor`, etc.). For inspiration, you can look at the [mteb/mtebscripts repo](https://github.com/embeddings-benchmark/mtebscripts) used for running diverse models via SLURM scripts for the paper.
 
 ```python
+import mteb
 from mteb.encoder_interface import PromptType
+from mteb.models.wrapper import Wrapper
+import numpy as np
+
 
-class CustomModel:
+class CustomModel(Wrapper):
     def encode(
         self,
         sentences: list[str],
@@ -244,7 +250,7 @@ class CustomModel:
         pass
 
 model = CustomModel()
-tasks = mteb.get_task("Banking77Classification")
+tasks = mteb.get_tasks(tasks=["Banking77Classification"])
 evaluation = MTEB(tasks=tasks)
 evaluation.run(model)
 ```

From 7ab1306d6c50c14c539a9536b48df1697988d518 Mon Sep 17 00:00:00 2001
From: Roman Solomatin <36135455+Samoed@users.noreply.github.com>
Date: Tue, 26 Nov 2024 12:00:49 +0300
Subject: [PATCH 2/3] align with mieb branch

---
 README.md               | 9 ++-------
 mteb/evaluation/MTEB.py | 5 ++---
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index df7cc1f388..3c659bbde5 100644
--- a/README.md
+++ b/README.md
@@ -46,12 +46,8 @@ from sentence_transformers import SentenceTransformer
 
 # Define the sentence-transformers model name
 model_name = "average_word_embeddings_komninos"
-# or directly from huggingface:
-# model_name = "sentence-transformers/all-MiniLM-L6-v2"
 
-model = SentenceTransformer(model_name)
-# or directly from mteb:
-model = mteb.get_model(model_name)
+model = mteb.get_model(model_name) # if the model is not implemented in MTEB it will be eq. to SentenceTransformer(model_name)
 tasks = mteb.get_tasks(tasks=["Banking77Classification"])
 evaluation = mteb.MTEB(tasks=tasks)
 results = evaluation.run(model, output_folder=f"results/{model_name}")
@@ -224,11 +220,10 @@ Models should implement the following interface, implementing an `encode` functi
 ```python
 import mteb
 from mteb.encoder_interface import PromptType
-from mteb.models.wrapper import Wrapper
 import numpy as np
 
 
-class CustomModel(Wrapper):
+class CustomModel:
     def encode(
         self,
         sentences: list[str],
diff --git a/mteb/evaluation/MTEB.py b/mteb/evaluation/MTEB.py
index 05a3c02ba4..1374c1ce11 100644
--- a/mteb/evaluation/MTEB.py
+++ b/mteb/evaluation/MTEB.py
@@ -13,7 +13,7 @@
 from typing import Any
 
 import datasets
-from sentence_transformers import SentenceTransformer
+from sentence_transformers import CrossEncoder, SentenceTransformer
 
 from mteb.encoder_interface import Encoder
 from mteb.model_meta import ModelMeta
@@ -23,7 +23,6 @@
 from ..abstasks import AbsTask
 from ..load_results.task_results import TaskResult
 from ..models.sentence_transformer_wrapper import SentenceTransformerWrapper
-from ..models.wrapper import Wrapper
 from ..tasks import *
 from . import LangMapping
 
@@ -363,7 +362,7 @@ def run(
 
         meta = self.create_model_meta(model)
         output_path = self.create_output_folder(meta, output_folder)
-        if not isinstance(model, Wrapper):
+        if isinstance(model, (SentenceTransformer, CrossEncoder)):
             model = SentenceTransformerWrapper(model)
 
         if output_path:

From 7b9397f0e59bd83fe3299d37b48654e2172ccd64 Mon Sep 17 00:00:00 2001
From: Roman Solomatin <36135455+Samoed@users.noreply.github.com>
Date: Tue, 26 Nov 2024 19:40:09 +0300
Subject: [PATCH 3/3] fix test

---
 tests/test_benchmark/mock_models.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_benchmark/mock_models.py b/tests/test_benchmark/mock_models.py
index e0b9cf69df..6b26cf67d4 100644
--- a/tests/test_benchmark/mock_models.py
+++ b/tests/test_benchmark/mock_models.py
@@ -33,7 +33,7 @@ def encode(self, sentences, prompt_name: str | None = None, **kwargs):
         return torch.randn(len(sentences), 10).numpy()
 
 
-class MockTorchbf16Encoder(mteb.Encoder):
+class MockTorchbf16Encoder(SentenceTransformer):
     def __init__(self):
         pass