diff --git a/models/README.md b/models/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c14c68a3435f7f0f180b952c0dec09f349b92867
--- /dev/null
+++ b/models/README.md
@@ -0,0 +1,32 @@
+# Guide to Writing Your Own Models
+
+## Model Code Organization
+For a streamlined experience, we suggest placing the code for all your models within the `models` directory. This is a recommendation for organizational purposes, but it's not a strict requirement.
+
+## Model Base Class
+Your models should inherit from the `ShopBenchBaseModel` class found in [base_model.py](base_model.py). We provide an example model, `dummy_model.py`, to illustrate how you might structure your own model. Crucially, your model class must implement the `predict` method.
+
+## Configuring Your Model
+To ensure your model is recognized and utilized correctly, please specify your model class name in the [`user_config.py`](user_config.py) file, using the `UserAgent` configuration.
+
+## Model Inputs and Outputs
+
+### Inputs
+Your model will receive two pieces of information for every task:
+- `prompt` (`str`): This is the specific task's input prompt.
+- `is_multiple_choice` (`bool`): This indicates whether the task is a multiple choice question.
+
+### Outputs
+The output from your model's `predict` function should always be a string. Depending on the task, this could be:
+- A single integer (in the range [0, 3]) for multiple choice tasks.
+- A comma-separated list of integers for ranking tasks.
+- A comma-separated list of named entities for Named Entity Recognition (NER) tasks.
+
+For more information on how these responses are processed, please see [parsers.py](../parsers.py).
+
+### Task Type
+Note that the type of task will not be explicitly provided to your model. However, you can infer the task type from the prompt provided.
+
+## Internet Access
+Your model will not have access to the internet during evaluation. As such, you'll need to include any necessary model weights directly in your repository before submission. Ensure that your Model class is self-contained and fully operational without internet access.
+
diff --git a/models/base_model.py b/models/base_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..68cf66b8b1931b1b03965725185490f20ca7f26c
--- /dev/null
+++ b/models/base_model.py
@@ -0,0 +1,23 @@
+class ShopBenchBaseModel:
+ def __init__(self):
+ pass
+
+ def predict(self, prompt: str, is_multiple_choice: bool) -> str:
+ """
+ Generates a prediction based on the input prompt and task type.
+
+ For multiple choice tasks, it randomly selects a choice.
+ For other tasks, it returns a list of integers as a string,
+ representing the model's prediction in a format compatible with task-specific parsers.
+
+ Args:
+ prompt (str): The input prompt for the model.
+ is_multiple_choice (bool): Indicates whether the task is a multiple choice question.
+
+ Returns:
+ str: The prediction as a string representing a single integer[0, 3] for multiple choice tasks,
+ or a string representing a comma separated list of integers for Ranking, Retrieval tasks,
+ or a string representing a comma separated list of named entities for Named Entity Recognition tasks.
+ Please refer to parsers.py for more details on how these responses will be parsed by the evaluator.
+ """
+ raise NotImplementedError("predict method not implemented")
diff --git a/models/dummy_model.py b/models/dummy_model.py
index 0e0bf1c807befbd3c518fe74bec2b12da3f39b5d..dda6f2b7da607befad8776c1151b8e182a54b511 100644
--- a/models/dummy_model.py
+++ b/models/dummy_model.py
@@ -1,43 +1,48 @@
-from typing import List
+from typing import List, Union
import random
import os
-# please use this seed consistently across your code
+from .base_model import ShopBenchBaseModel
+
+# Set a consistent seed for reproducibility
AICROWD_RUN_SEED = int(os.getenv("AICROWD_RUN_SEED", 3142))
-class DummyModel:
+class DummyModel(ShopBenchBaseModel):
"""
- TODO
+ A dummy model implementation for ShopBench, illustrating how to handle both
+ multiple choice and other types of tasks like Ranking, Retrieval, and Named Entity Recognition.
+ This model uses a consistent random seed for reproducible results.
"""
def __init__(self):
- """Initialize your models here"""
+ """Initializes the model and sets the random seed for consistency."""
random.seed(AICROWD_RUN_SEED)
def predict(self, prompt: str, is_multiple_choice: bool) -> str:
"""
- Standard inferface for all tasks and tracks.
+ Generates a prediction based on the input prompt and task type.
- The goal is for your model to be able to infer the task type,
- and respond with a string that is compatible with the task specific parser.
+ For multiple choice tasks, it randomly selects a choice.
+ For other tasks, it returns a list of integers as a string,
+ representing the model's prediction in a format compatible with task-specific parsers.
+ Args:
+ prompt (str): The input prompt for the model.
+ is_multiple_choice (bool): Indicates whether the task is a multiple choice question.
- Note: Even if the development dataset has the task_type information,
- During the actual evaluations, your code will only have access to the prompt,
- and the boolean variable indicating if its a multiple choice question.
+ Returns:
+ str: The prediction as a string representing a single integer[0, 3] for multiple choice tasks,
+ or a string representing a comma separated list of integers for Ranking, Retrieval tasks,
+ or a string representing a comma separated list of named entities for Named Entity Recognition tasks.
+ Please refer to parsers.py for more details on how these responses will be parsed by the evaluator.
"""
+ possible_responses = [1, 2, 3, 4]
- potential_response = [1, 2, 3, 4]
if is_multiple_choice:
- return str(random.choice(potential_response))
+ # Randomly select one of the possible responses for multiple choice tasks
+ return str(random.choice(possible_responses))
else:
- # For Ranking, Retrieval, and Named Entity Recognition tasks
- # the expected response is a string that can be parsed with
- # `ast.literal_eval` (see parsers.py for more details)
- random.shuffle(potential_response)
- return str(potential_response)
-
- # Note: For the generation task, the expected response is a string
- # And, as this is a dummy response, we are just returning the
- # shuffled version of list, but in your case, it can be any string
+ # For other tasks, shuffle the possible responses and return as a string
+ random.shuffle(possible_responses)
+ return str(possible_responses)
diff --git a/models/dummy_model_old.py b/models/dummy_model_old.py
deleted file mode 100644
index 2eb84d3e52d0deb7d8f633090b5a703e79636446..0000000000000000000000000000000000000000
--- a/models/dummy_model_old.py
+++ /dev/null
@@ -1,52 +0,0 @@
-from typing import List
-
-
-class DummyModel:
- """
- Note to participants:
- Example class to show the different functions to be implemented for each type of task
- Make sure to follow the data types as mentioned in the function definitions
- """
- def __init__(self):
- """ Initialize your models here """
- pass
-
- def task_multichoice(self, task_prompt: str) -> int:
- """
- Task method for Multiple choice questions
- Input - Task Prompt (includes choices)
- Output - Single integer index among ones given in the input
- """
- return 0
-
- def task_ranking(self, task_prompt: str) -> List[int]:
- """
- Task method for Ranking
- Input - Task Prompt (includes items to rank)
- Output - Ordered List of ranks for each item
- """
- return [1, 0, 2, 3]
-
- def task_generation(self, task_prompt: str) -> str:
- """
- Task method for Generation
- Input - Task Prompt describing the required generation
- Output - Generated text as per task prompt
- """
- return "This is a test"
-
- def task_retrieval(self, task_prompt: str) -> List[int]:
- """
- Task method for Generation
- Input - Task Prompt describing the items which need to be selected from (includes indexes of items)
- Output - Unordered list of indexes selected (must be a python list even if single item)
- """
- return [0, 1, 2]
-
- def task_named_entity_recognition(self, task_prompt: str) -> List[str]:
- """
- Task method for Named Entity Recognition
- Input - Task Prompt describing the named entity recognition task
- Output - Unordered list of one or more entity names (must be a python list even if single item)
- """
- return ["food", "gpu"]
\ No newline at end of file
diff --git a/parsers.py b/parsers.py
index 2dd7e5e011485a93160f8cb0c97d19e28de8c4d2..9f79af831ec8145fba31512cdca18c45ee683167 100644
--- a/parsers.py
+++ b/parsers.py
@@ -45,6 +45,10 @@ class ShoppingBenchTaskParsers:
"named_entity_recognition": self._parse_named_entity_recognition,
}
+ assert isinstance(
+ response, str
+ ), f"Response must be a string, but got {type(response)}"
+
# Attempt to retrieve the appropriate parser method for the task type.
parser_method = task_parser_methods.get(self.task_type)
@@ -253,5 +257,3 @@ if __name__ == "__main__":
) # failure case - not tolerant to [ if quotes not used
# - extra '[' characters added to boundary elems]): ['[New York', 'ShopBench', 'Amazon]']
# Expected output: ['[New York', 'ShopBench', 'Amazon]']
-
- print(ner_parser.parse("[4, 1, 2, 3]"))