Skip to content
Snippets Groups Projects
Commit 19f6afde authored by jyotish's avatar jyotish
Browse files

Merge branch 'chore/render-method-renaming' into 'master'

chore: rename `render_status_update` -> `render_current_status_as_markdown`

See merge request !2
parents 987ace72 bcf0cec1
No related branches found
No related tags found
1 merge request!2chore: rename `render_status_update` -> `render_current_status_as_markdown`
......@@ -29,8 +29,14 @@ class AIcrowdEvaluator:
ground_truth_data_path=Constants.GROUND_TRUTH_PATH,
predictions_file_path=Constants.PREDICTIONS_FILE_PATH,
)
def render_status_update(self) -> str:
def live_evaluate(self, ground_truth_path: str, predictions_file_path: str) -> dict:
# Not compulsory but encouraged to do so.
# For long running evaluations, this will help the participants understand the performance
# of their models before the completion of the evaluation.
return self.evaluator.scoring(ground_truth_path, prediction_file_path, partial_score=True)
def render_current_status_as_markdown(self) -> str:
"""This method updates the GitLab issue page comment with the current evaluation
progress. The content returned should be in markdown.
......@@ -55,10 +61,9 @@ class AIcrowdEvaluator:
progress_percent = len(predictions) * 100 / total_image_count
# Get live scores for predictions so far
scores = self.evaluator.scoring(
scores = self.live_evaluate(
Constants.GROUND_TRUTH_PATH,
Constants.PREDICTIONS_FILE_PATH,
partial_score=True,
)
except:
progress_percent = 0
......
......@@ -78,8 +78,15 @@ class AIcrowdEvaluator:
def __init__(self, **kwargs):
pass
def render_status_update(self) -> str:
def render_current_status_as_markdown(self) -> str:
return ""
def live_evaluate(self) -> str:
# Not compulsory but encouraged to do so.
# For long running evaluations, this will help the participants understand the performance
# of their models before the completion of the evaluation.
# Refer example evaluator.py at https://gitlab.aicrowd.com/-/tree/master/evaluator.py
return {}
def evaluate(self):
scores = {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment