feat: enable codecarbon by default

mteb/cli.py

@@ -265,8 +265,8 @@ def add_run_parser(subparsers) -> None:
     parser.add_argument(
         "--co2_tracker",
         type=bool,
-        default=False,
-        help="Enable CO₂ tracker, disabled by default",
+        default=True,
+        help="Enable CO₂ tracker, enabled by default",
     )
     parser.add_argument(
         "--eval_splits",

mteb/evaluation/MTEB.py

@@ -13,6 +13,7 @@
 from typing import Any
 
 import datasets
+from codecarbon import EmissionsTracker
 from sentence_transformers import SentenceTransformer
 
 from mteb.encoder_interface import Encoder
@@ -316,7 +317,7 @@ def run(
         eval_splits=None,
         overwrite_results: bool = False,
         raise_error: bool = True,
-        co2_tracker: bool = False,
+        co2_tracker: bool = True,
         encode_kwargs: dict[str, Any] = {},
         **kwargs,
     ) -> list[TaskResult]:
@@ -412,13 +413,6 @@ def run(
                 kg_co2_emissions: int | None = 0 if co2_tracker else None
                 for split in task_eval_splits:
                     if co2_tracker:
-                        try:
-                            from codecarbon import EmissionsTracker
-                        except ImportError:
-                            raise ImportError(
-                                "To use the CO2 emissions tracker, please install codecarbon using 'pip install codecarbon'"
-                            )
-
                         with EmissionsTracker(
                             save_to_file=False, save_to_api=False, logging_logger=logger
                         ) as tracker:

pyproject.toml

@@ -40,6 +40,7 @@ dependencies = [
     "typing_extensions>=0.0.0",
     "eval_type_backport>=0.0.0",
     "polars>=0.20.22",
+    "codecarbon>=2.0.0",
 ]
 
 
@@ -54,7 +55,6 @@ mteb = "mteb.cli:main"
 [project.optional-dependencies]
 dev = ["ruff==0.6.4", # locked so we don't get PRs which fail only due to a lint update
 "pytest", "pytest-xdist", "pytest-coverage"]
-codecarbon = ["codecarbon"]
 speedtask = ["GPUtil>=1.4.0", "psutil>=5.9.8"]
 peft = ["peft>=0.11.0"]
 leaderboard = ["gradio>=4.44.0", "gradio_rangeslider>=0.0.6"]