Skip to content
Snippets Groups Projects
Commit c8714275 authored by MasterScrat's avatar MasterScrat
Browse files

Merge branch 'fix-evaluation-reset'

parents c4c041b3 d6c85eda
No related branches found
No related tags found
No related merge requests found
......@@ -43,14 +43,14 @@ m.patch()
# CONSTANTS
########################################################
INTIAL_PLANNING_TIMEOUT = int(os.getenv(
"FLATLAND_INITIAL_PLANNING_TIMEOUT",
5 * 60)) # 5 mins
"FLATLAND_INITIAL_PLANNING_TIMEOUT",
5 * 60)) # 5 mins
PER_STEP_TIMEOUT = int(os.getenv(
"FLATLAND_PER_STEP_TIMEOUT",
5)) # 5 seconds
"FLATLAND_PER_STEP_TIMEOUT",
5)) # 5 seconds
DEFAULT_COMMAND_TIMEOUT = int(os.getenv(
"FLATLAND_DEFAULT_COMMAND_TIMEOUT",
1 * 60)) # 1 min
"FLATLAND_DEFAULT_COMMAND_TIMEOUT",
1 * 60)) # 1 min
# This applies to the rest of the commands
......@@ -150,6 +150,7 @@ class FlatlandRemoteEvaluationService:
self.env = False
self.env_renderer = False
self.reward = 0
self.simulation_done = True
self.simulation_count = -1
self.simulation_env_file_paths = []
self.simulation_rewards = []
......@@ -249,7 +250,7 @@ class FlatlandRemoteEvaluationService:
) for x in env_paths])
return env_paths
def instantiate_evaluation_metadata(self):
"""
This instantiates a pandas dataframe to record
......@@ -261,9 +262,9 @@ class FlatlandRemoteEvaluationService:
"""
self.evaluation_metadata_df = None
metadata_file_path = os.path.join(
self.test_env_folder,
"metadata.csv"
)
self.test_env_folder,
"metadata.csv"
)
if os.path.exists(metadata_file_path):
self.evaluation_metadata_df = pd.read_csv(metadata_file_path)
self.evaluation_metadata_df["filename"] = \
......@@ -293,7 +294,7 @@ class FlatlandRemoteEvaluationService:
for the **previous** episode in the metadata_df if it exists.
"""
if self.evaluation_metadata_df is not None and len(self.simulation_env_file_paths) > 0:
last_simulation_env_file_path = self.simulation_env_file_paths[-1]
_row = self.evaluation_metadata_df.loc[
......@@ -373,7 +374,7 @@ class FlatlandRemoteEvaluationService:
packed message, and consider the timeouts, etc when trying to
fetch a new command.
"""
COMMAND_TIMEOUT = DEFAULT_COMMAND_TIMEOUT
"""
Handle case specific timeouts :
......@@ -402,7 +403,7 @@ class FlatlandRemoteEvaluationService:
If the user has already done an env_submit call, then the timeout
can be an arbitrarily large number.
"""
COMMAND_TIMEOUT = 10**6
COMMAND_TIMEOUT = 10 ** 6
@timeout_decorator.timeout(
COMMAND_TIMEOUT,
......@@ -486,7 +487,14 @@ class FlatlandRemoteEvaluationService:
Handles a ENV_CREATE command from the client
TODO: Add a high level summary of everything thats happening here.
"""
if not self.simulation_done:
# trying to reset a simulation before finishing the previous one
_command_response = self._error_template("CAN'T CREATE NEW ENV BEFORE PREVIOUS IS DONE")
self.send_response(_command_response, command)
raise Exception(_command_response['payload'])
self.simulation_count += 1
self.simulation_done = False
if self.simulation_count < len(self.env_file_paths):
"""
There are still test envs left that are yet to be evaluated
......@@ -622,6 +630,8 @@ class FlatlandRemoteEvaluationService:
)
if done["__all__"]:
self.simulation_done = True
# Compute percentage complete
complete = 0
for i_agent in range(self.env.get_num_agents()):
......@@ -631,6 +641,12 @@ class FlatlandRemoteEvaluationService:
percentage_complete = complete * 1.0 / self.env.get_num_agents()
self.simulation_percentage_complete[-1] = percentage_complete
print("Evaluation finished in {} timesteps. Percentage agents done: {:.3f}. Normalized reward: {:.3f}.".format(
self.simulation_steps[-1],
self.simulation_percentage_complete[-1],
self.simulation_rewards_normalized[-1]
))
# Record Frame
if self.visualize:
"""
......@@ -673,10 +689,10 @@ class FlatlandRemoteEvaluationService:
min_key = "{}_min".format(metric_name)
max_key = "{}_max".format(metric_name)
print("\t - {}\t => min: {} || mean: {} || max: {}".format(
metric_name,
self.stats[min_key],
self.stats[mean_key],
self.stats[max_key]))
metric_name,
self.stats[min_key],
self.stats[mean_key],
self.stats[max_key]))
print("=" * 100)
# Register simulation time of the last episode
......@@ -732,7 +748,7 @@ class FlatlandRemoteEvaluationService:
if self.result_output_path:
self.evaluation_metadata_df.to_csv(self.result_output_path)
print("Wrote output results to : {}".format(self.result_output_path))
# Upload the metadata file to S3
if aicrowd_helpers.is_grading() and aicrowd_helpers.is_aws_configured():
metadata_s3_key = aicrowd_helpers.upload_to_s3(
......@@ -848,9 +864,9 @@ class FlatlandRemoteEvaluationService:
print("Self.Reward : ", self.reward)
print("Current Simulation : ", self.simulation_count)
if self.env_file_paths and \
self.simulation_count < len(self.env_file_paths):
self.simulation_count < len(self.env_file_paths):
print("Current Env Path : ",
self.env_file_paths[self.simulation_count])
self.env_file_paths[self.simulation_count])
try:
if command['type'] == messages.FLATLAND_RL.PING:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment