From b63e95c002845d6e90adc9f5dd7bb5b91e911ae6 Mon Sep 17 00:00:00 2001
From: Fanyou Wu <fanyou.wu@outlook.com>
Date: Mon, 17 Jun 2024 04:59:19 -0400
Subject: [PATCH] Update
---
models/llama_loader.py | 7 +-
models/prompt_template/resource/simple_qa.txt | 81 +------------------
models/v4.py | 2 +-
3 files changed, 6 insertions(+), 84 deletions(-)
diff --git a/models/llama_loader.py b/models/llama_loader.py
index 48d4188..65f22fc 100644
--- a/models/llama_loader.py
+++ b/models/llama_loader.py
@@ -39,7 +39,7 @@ class VLLMConfig:
gpu_memory_utilization: float = 0.85
trust_remote_code = True
dtype: str = "half"
- enforce_eager: bool = True
+ enforce_eager: bool = False
n: int = 1
top_p: float = 0.9
temperature: float = 0 # less than 1e-5 is greedy, 0.1
@@ -51,7 +51,7 @@ class VLLMConfig:
include_stop_str_in_output: bool = True
stop: List[str] = field(default_factory=lambda: ["</answer>"])
disable_custom_all_reduce: bool = True
- max_seq_len_to_capture: int = 1
+ max_seq_len_to_capture: int = 8192
@property
def model_kwargs(self) -> dict:
return {
@@ -62,7 +62,8 @@ class VLLMConfig:
"enforce_eager": self.enforce_eager,
"worker_use_ray": self.worker_use_ray,
"disable_custom_all_reduce": self.disable_custom_all_reduce,
- "max_seq_len_to_capture": self.max_seq_len_to_capture
+ "max_seq_len_to_capture": self.max_seq_len_to_capture,
+ "max_num_seqs": 8,
}
def sampling_args(self, override_kwargs: dict = {}) -> vllm.SamplingParams:
diff --git a/models/prompt_template/resource/simple_qa.txt b/models/prompt_template/resource/simple_qa.txt
index 760ba51..83c947f 100644
--- a/models/prompt_template/resource/simple_qa.txt
+++ b/models/prompt_template/resource/simple_qa.txt
@@ -1,85 +1,6 @@
<|begin_of_text|><|start_header_id|>user<|end_header_id|>
-how many ex-dividend date are announced each year by agnc
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-agnc pays dividend every month. that means they have to announce ex-dividend date 12 times a year
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-what are the top 10 holdings in spy?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-the top 10 holdings in spy are microsoft, apple, nvidia, amazon, meta, alphabet, tesla, broadcom, bershire hathaway
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-what was the worldwide box office haul for i am legend?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-585410052
-
-<|start_header_id|>user<|end_header_id|>
-
-who is older, leonardo dicaprio or brad pitt?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-brad pitt
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-what is the first song on drake's second album?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-over my dead body is the first song on drake's second album, take care.
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-who are the original members of the band eagles?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-randy meisner, bernie leadon, don henley, and glenn frey
-
-<|start_header_id|>user<|end_header_id|>
-
-how many times has justin bieber been named as one of the most influential people in the world by time magazine?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-justin bieber was named as one of the most influential people in the world by time magazine once, in 2011.
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-can you tell me the year yellowhammer was designated as the state bird of florida?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-invalid question
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-which new york knicks players went to college at villanova?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-the new york knicks players who went to college at villanova are jalen brunson, josh hart, and donte divincenzo.
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-how many tour de france titles has jonas vingegaard won?
-
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-
-jonas vingegaard has won two tour de france titles.
-
-<|eot_id|><|start_header_id|>user<|end_header_id|>
-
-{query}
+Question: {query}
<|eot_id|><|start_header_id|>assistant<|end_header_id|>
diff --git a/models/v4.py b/models/v4.py
index f7fd2ab..eda7d28 100644
--- a/models/v4.py
+++ b/models/v4.py
@@ -337,7 +337,7 @@ class V4(Base):
responses = self.llm_inference_with_skip_flag(
inputs,
[False] * len(queries),
- max_tokens = 128,
+ max_tokens = 96,
stop=["<|eot_id|>"],
include_stop_str_in_output=False,
)
--
GitLab