InternLM 20B Support

[hatrexltd]

Traceback (most recent call last):
File "/root/miniconda3/envs/vllm/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/root/miniconda3/envs/vllm/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/home/ubuntu/vllm/vllm/entrypoints/openai/api_server.py", line 613, in
engine = AsyncLLMEngine.from_engine_args(engine_args)
File "/home/ubuntu/vllm/vllm/engine/async_llm_engine.py", line 487, in from_engine_args
engine = cls(engine_args.worker_use_ray,
File "/home/ubuntu/vllm/vllm/engine/async_llm_engine.py", line 270, in init
self.engine = self._init_engine(*args, **kwargs)
File "/home/ubuntu/vllm/vllm/engine/async_llm_engine.py", line 306, in _init_engine
return engine_class(*args, **kwargs)
File "/home/ubuntu/vllm/vllm/engine/llm_engine.py", line 113, in init
self._init_cache()
File "/home/ubuntu/vllm/vllm/engine/llm_engine.py", line 193, in _init_cache
num_blocks = self._run_workers(
File "/home/ubuntu/vllm/vllm/engine/llm_engine.py", line 704, in _run_workers
all_outputs = ray.get(all_outputs)
File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/ray/_private/auto_init_hook.py", line 24, in auto_init_wrapper
return fn(*args, **kwargs)
File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(*args, **kwargs)
File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/ray/_private/worker.py", line 2547, in get
raise value.as_instanceof_cause()
ray.exceptions.RayTaskError(RuntimeError): ray::RayWorker.execute_method() (pid=337309, ip=10.19.136.100, actor_id=436e9ee99d472a7e607b250401000000, repr=<vllm.engine.ray_utils.RayWorker object at 0x7fb71822abe0>)
File "/home/ubuntu/vllm/vllm/engine/ray_utils.py", line 32, in execute_method
return executor(*args, **kwargs)
File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/home/ubuntu/vllm/vllm/worker/worker.py", line 111, in profile_num_available_blocks
self.model(
File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/ubuntu/vllm/vllm/model_executor/models/internlm.py", line 240, in forward
next_tokens = self.sampler(self.lm_head.weight, hidden_states,
File "/root/miniconda3/envs/vllm/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/ubuntu/vllm/vllm/model_executor/layers/sampler.py", line 86, in forward
sample_results = _sample(probs, logprobs, input_metadata)
File "/home/ubuntu/vllm/vllm/model_executor/layers/sampler.py", line 441, in _sample
sample_results = _random_sample(seq_groups, is_prompts,
File "/home/ubuntu/vllm/vllm/model_executor/layers/sampler.py", line 326, in _random_sample
random_samples = torch.multinomial(probs,
RuntimeError: probability tensor contains either inf, nan or element < 0

When I run InternLM 20B Chat, here is the error.

[ak646260187]

I got the same error! T_T

[ak646260187]

may be vllm-0.1.7 can solve this error

[gesanqiu]

#1496 shall fix this issue. I stated detailedly in the PR, maybe not the right way, so still need help.