From 72b74f364b4ccfe3d91e684ef6c6b787f085f208 Mon Sep 17 00:00:00 2001 From: slaren Date: Sun, 7 Jan 2024 00:33:51 +0100 Subject: [PATCH] cuda : do not create buffer types for devices that don't exist (fixes usage without CUDA devices available) --- ggml-cuda.cu | 5 ++++- ggml-cuda.h | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ggml-cuda.cu b/ggml-cuda.cu index 0969c26e603fc..032d7e98689e1 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -9568,6 +9568,10 @@ static ggml_backend_buffer_type_i ggml_backend_cuda_buffer_type_interface = { ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device) { // FIXME: this is not thread safe + if (device >= ggml_backend_cuda_get_device_count()) { + return nullptr; + } + static struct ggml_backend_buffer_type ggml_backend_cuda_buffer_types[GGML_CUDA_MAX_DEVICES]; static bool ggml_backend_cuda_buffer_type_initialized = false; @@ -9793,7 +9797,6 @@ ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * ten // FIXME: this is not thread safe static std::map, struct ggml_backend_buffer_type> buft_map; - std::array tensor_split_arr = {}; bool all_zero = tensor_split == nullptr || std::all_of(tensor_split, tensor_split + GGML_CUDA_MAX_DEVICES, [](float x) { return x == 0.0f; }); diff --git a/ggml-cuda.h b/ggml-cuda.h index 81d52040e0db1..d19cbf3fdd04b 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -47,8 +47,6 @@ GGML_API int ggml_backend_cuda_get_device_count(void); GGML_API void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size); GGML_API void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total); - - #ifdef __cplusplus } #endif