From 4835f57faef0e77e2a9a56f846873f53a1111502 Mon Sep 17 00:00:00 2001
From: "S.P. Mohanty" <spmohanty91@gmail.com>
Date: Wed, 12 Jun 2024 22:16:04 +0000
Subject: [PATCH] ensure workers use ray
---
models/vanilla_llama3_baseline.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/models/vanilla_llama3_baseline.py b/models/vanilla_llama3_baseline.py
index 88724f5..eafb310 100644
--- a/models/vanilla_llama3_baseline.py
+++ b/models/vanilla_llama3_baseline.py
@@ -50,6 +50,7 @@ class Llama3_8B_ZeroShotModel(ShopBenchBaseModel):
# initialize the model with vllm
self.llm = vllm.LLM(
self.model_name,
+ worker_use_ray=True,
tensor_parallel_size=VLLM_TENSOR_PARALLEL_SIZE,
gpu_memory_utilization=VLLM_GPU_MEMORY_UTILIZATION,
trust_remote_code=True,
--
GitLab