12288

models/vanilla_llama3_baseline.py

@@ -64,7 +64,7 @@ class Llama3_70B_ZeroShotModel(ShopBenchBaseModel):
             # dtype="half", # note: bfloat16 is not supported on nvidia-T4 GPUs
             enforce_eager=True,
             quantization="AWQ",
-            max_model_len=16384,
+            max_model_len=12288,
             enable_prefix_caching=True
         )
         self.tokenizer = self.llm.get_tokenizer()