[Test] Add xformer and flash attn tests

tests/basic_correctness/test_basic_correctness.py

@@ -14,6 +14,7 @@
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("max_tokens", [5])
 @pytest.mark.parametrize("enforce_eager", [False, True])
+@pytest.mark.parametrize("attn_backend", ["XFORMER", "FLASH"])
 def test_models(
     hf_runner,
     vllm_runner,
@@ -22,7 +23,10 @@ def test_models(
     dtype: str,
     max_tokens: int,
     enforce_eager: bool,
+    attn_backend: str,
+    monkeypatch,
 ) -> None:
+    monkeypatch.setenv("VLLM_ATTENTION_BACKEND", attn_backend)
     hf_model = hf_runner(model, dtype=dtype)
     hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
     del hf_model

vllm/attention/selector.py

@@ -1,4 +1,5 @@
 import enum
+import os
 from functools import lru_cache
 from typing import Type
 
@@ -10,6 +11,8 @@
 
 logger = init_logger(__name__)
 
+VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
+
 
 class _Backend(enum.Enum):
     FLASH_ATTN = enum.auto()
@@ -75,4 +78,15 @@ def _which_attn_to_use(dtype: torch.dtype) -> _Backend:
             "Cannot use FlashAttention backend because the flash_attn package "
             "is not found. Please install it for better performance.")
         return _Backend.XFORMERS
-    return _Backend.FLASH_ATTN
+
+    backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
+    if backend_by_env_var == "XFORMER":
+        return _Backend.XFORMERS
+    elif backend_by_env_var == "FLASH":
+        return _Backend.FLASH_ATTN
+    elif backend_by_env_var is None:
+        # Default case.
+        return _Backend.FLASH_ATTN
+    else:
+        raise AssertionError(
+            f"{VLLM_ATTENTION_BACKEND}={backend_by_env_var} is not supported.")