55-12

models/vanilla_llama3_baseline.py

@@ -64,7 +64,7 @@ class Llama3_70B_ZeroShotModel(ShopBenchBaseModel):
             # dtype="half", # note: bfloat16 is not supported on nvidia-T4 GPUs
             enforce_eager=True,
             quantization="AWQ",
-            max_model_len=16384,
+            max_model_len=8192,
             enable_prefix_caching=True
         )
         self.tokenizer = self.llm.get_tokenizer()
@@ -335,7 +335,7 @@ class Llama3_70B_ZeroShotModel(ShopBenchBaseModel):
             formatted_prompts = []
             for prompt in prompts:
                 if self.named_keys[0] in prompt:                        # NER
-                    self.max_new_tokens = 15
+                    self.max_new_tokens = 12
                     formatted_prompts.append(ner_prompt + prompt)
 
                 elif self.ranking_keys[0] in prompt:                    # ranking 
@@ -347,7 +347,7 @@ class Llama3_70B_ZeroShotModel(ShopBenchBaseModel):
                     formatted_prompts.append(retrieval_prompt + prompt)
 
                 else:                                                   # general prompt(generation)
-                    self.max_new_tokens = 50
+                    self.max_new_tokens = 55
                     formatted_prompts.append(other_prompt + prompt)
 
         return formatted_prompts