From 4e2c31162ab7069c99671ec9fdf9f6b0fd23f07f Mon Sep 17 00:00:00 2001
From: "S.P. Mohanty" <spmohanty91@gmail.com>
Date: Mon, 18 Mar 2024 06:25:39 +0000
Subject: [PATCH] add generation task response to docs
---
models/README.md | 1 +
models/base_model.py | 1 +
models/dummy_model.py | 3 +++
3 files changed, 5 insertions(+)
diff --git a/models/README.md b/models/README.md
index 86f3f16..486e493 100644
--- a/models/README.md
+++ b/models/README.md
@@ -21,6 +21,7 @@ The output from your model's `predict` function should always be a string. Depen
- A single integer (in the range [0, 3]) for multiple choice tasks.
- A comma-separated list of integers for ranking tasks.
- A comma-separated list of named entities for Named Entity Recognition (NER) tasks.
+- (unconstrained) generated response for the generation tasks
For more information on how these responses are processed, please see [parsers.py](../parsers.py).
diff --git a/models/base_model.py b/models/base_model.py
index 68cf66b..dc41a23 100644
--- a/models/base_model.py
+++ b/models/base_model.py
@@ -18,6 +18,7 @@ class ShopBenchBaseModel:
str: The prediction as a string representing a single integer[0, 3] for multiple choice tasks,
or a string representing a comma separated list of integers for Ranking, Retrieval tasks,
or a string representing a comma separated list of named entities for Named Entity Recognition tasks.
+ or a string representing the (unconstrained) generated response for the generation tasks
Please refer to parsers.py for more details on how these responses will be parsed by the evaluator.
"""
raise NotImplementedError("predict method not implemented")
diff --git a/models/dummy_model.py b/models/dummy_model.py
index dda6f2b..e126fe5 100644
--- a/models/dummy_model.py
+++ b/models/dummy_model.py
@@ -35,6 +35,7 @@ class DummyModel(ShopBenchBaseModel):
str: The prediction as a string representing a single integer[0, 3] for multiple choice tasks,
or a string representing a comma separated list of integers for Ranking, Retrieval tasks,
or a string representing a comma separated list of named entities for Named Entity Recognition tasks.
+ or a string representing the (unconstrained) generated response for the generation tasks
Please refer to parsers.py for more details on how these responses will be parsed by the evaluator.
"""
possible_responses = [1, 2, 3, 4]
@@ -46,3 +47,5 @@ class DummyModel(ShopBenchBaseModel):
# For other tasks, shuffle the possible responses and return as a string
random.shuffle(possible_responses)
return str(possible_responses)
+ # Note: As this is dummy model, we are returning random responses for non-multiple choice tasks.
+ # For generation tasks, this should ideally return an unconstrained string.
--
GitLab