diff --git a/models/README.md b/models/README.md index 86f3f16f8b85d5a728f3b732de4f6e25c24a9c61..486e493d5dfa18dc55289f99f438f53ce64c57f9 100644 --- a/models/README.md +++ b/models/README.md @@ -21,6 +21,7 @@ The output from your model's `predict` function should always be a string. Depen - A single integer (in the range [0, 3]) for multiple choice tasks. - A comma-separated list of integers for ranking tasks. - A comma-separated list of named entities for Named Entity Recognition (NER) tasks. +- (unconstrained) generated response for the generation tasks For more information on how these responses are processed, please see [parsers.py](../parsers.py). diff --git a/models/base_model.py b/models/base_model.py index 68cf66b8b1931b1b03965725185490f20ca7f26c..dc41a235da95d5124def29bd923cccfc43d27730 100644 --- a/models/base_model.py +++ b/models/base_model.py @@ -18,6 +18,7 @@ class ShopBenchBaseModel: str: The prediction as a string representing a single integer[0, 3] for multiple choice tasks, or a string representing a comma separated list of integers for Ranking, Retrieval tasks, or a string representing a comma separated list of named entities for Named Entity Recognition tasks. + or a string representing the (unconstrained) generated response for the generation tasks Please refer to parsers.py for more details on how these responses will be parsed by the evaluator. """ raise NotImplementedError("predict method not implemented") diff --git a/models/dummy_model.py b/models/dummy_model.py index dda6f2b7da607befad8776c1151b8e182a54b511..e126fe5f42e70be99adc823afd4fe99c79581fa1 100644 --- a/models/dummy_model.py +++ b/models/dummy_model.py @@ -35,6 +35,7 @@ class DummyModel(ShopBenchBaseModel): str: The prediction as a string representing a single integer[0, 3] for multiple choice tasks, or a string representing a comma separated list of integers for Ranking, Retrieval tasks, or a string representing a comma separated list of named entities for Named Entity Recognition tasks. + or a string representing the (unconstrained) generated response for the generation tasks Please refer to parsers.py for more details on how these responses will be parsed by the evaluator. """ possible_responses = [1, 2, 3, 4] @@ -46,3 +47,5 @@ class DummyModel(ShopBenchBaseModel): # For other tasks, shuffle the possible responses and return as a string random.shuffle(possible_responses) return str(possible_responses) + # Note: As this is dummy model, we are returning random responses for non-multiple choice tasks. + # For generation tasks, this should ideally return an unconstrained string.