From d6c85edaa32e238edc5f9b4cb31a718856137bc1 Mon Sep 17 00:00:00 2001
From: flaurent <florian.laurent@gmail.com>
Date: Wed, 8 Jul 2020 00:09:21 +0200
Subject: [PATCH] Prevent creation of new environment before the previous one
is done, added extra logging
---
flatland/evaluators/service.py | 56 ++++++++++++++++++++++------------
1 file changed, 36 insertions(+), 20 deletions(-)
diff --git a/flatland/evaluators/service.py b/flatland/evaluators/service.py
index 5a98aac6..60d8d461 100644
--- a/flatland/evaluators/service.py
+++ b/flatland/evaluators/service.py
@@ -43,14 +43,14 @@ m.patch()
# CONSTANTS
########################################################
INTIAL_PLANNING_TIMEOUT = int(os.getenv(
- "FLATLAND_INITIAL_PLANNING_TIMEOUT",
- 5 * 60)) # 5 mins
+ "FLATLAND_INITIAL_PLANNING_TIMEOUT",
+ 5 * 60)) # 5 mins
PER_STEP_TIMEOUT = int(os.getenv(
- "FLATLAND_PER_STEP_TIMEOUT",
- 5)) # 5 seconds
+ "FLATLAND_PER_STEP_TIMEOUT",
+ 5)) # 5 seconds
DEFAULT_COMMAND_TIMEOUT = int(os.getenv(
- "FLATLAND_DEFAULT_COMMAND_TIMEOUT",
- 1 * 60)) # 1 min
+ "FLATLAND_DEFAULT_COMMAND_TIMEOUT",
+ 1 * 60)) # 1 min
# This applies to the rest of the commands
@@ -150,6 +150,7 @@ class FlatlandRemoteEvaluationService:
self.env = False
self.env_renderer = False
self.reward = 0
+ self.simulation_done = True
self.simulation_count = -1
self.simulation_env_file_paths = []
self.simulation_rewards = []
@@ -249,7 +250,7 @@ class FlatlandRemoteEvaluationService:
) for x in env_paths])
return env_paths
-
+
def instantiate_evaluation_metadata(self):
"""
This instantiates a pandas dataframe to record
@@ -261,9 +262,9 @@ class FlatlandRemoteEvaluationService:
"""
self.evaluation_metadata_df = None
metadata_file_path = os.path.join(
- self.test_env_folder,
- "metadata.csv"
- )
+ self.test_env_folder,
+ "metadata.csv"
+ )
if os.path.exists(metadata_file_path):
self.evaluation_metadata_df = pd.read_csv(metadata_file_path)
self.evaluation_metadata_df["filename"] = \
@@ -293,7 +294,7 @@ class FlatlandRemoteEvaluationService:
for the **previous** episode in the metadata_df if it exists.
"""
if self.evaluation_metadata_df is not None and len(self.simulation_env_file_paths) > 0:
-
+
last_simulation_env_file_path = self.simulation_env_file_paths[-1]
_row = self.evaluation_metadata_df.loc[
@@ -373,7 +374,7 @@ class FlatlandRemoteEvaluationService:
packed message, and consider the timeouts, etc when trying to
fetch a new command.
"""
-
+
COMMAND_TIMEOUT = DEFAULT_COMMAND_TIMEOUT
"""
Handle case specific timeouts :
@@ -402,7 +403,7 @@ class FlatlandRemoteEvaluationService:
If the user has already done an env_submit call, then the timeout
can be an arbitrarily large number.
"""
- COMMAND_TIMEOUT = 10**6
+ COMMAND_TIMEOUT = 10 ** 6
@timeout_decorator.timeout(
COMMAND_TIMEOUT,
@@ -486,7 +487,14 @@ class FlatlandRemoteEvaluationService:
Handles a ENV_CREATE command from the client
TODO: Add a high level summary of everything thats happening here.
"""
+ if not self.simulation_done:
+ # trying to reset a simulation before finishing the previous one
+ _command_response = self._error_template("CAN'T CREATE NEW ENV BEFORE PREVIOUS IS DONE")
+ self.send_response(_command_response, command)
+ raise Exception(_command_response['payload'])
+
self.simulation_count += 1
+ self.simulation_done = False
if self.simulation_count < len(self.env_file_paths):
"""
There are still test envs left that are yet to be evaluated
@@ -622,6 +630,8 @@ class FlatlandRemoteEvaluationService:
)
if done["__all__"]:
+ self.simulation_done = True
+
# Compute percentage complete
complete = 0
for i_agent in range(self.env.get_num_agents()):
@@ -631,6 +641,12 @@ class FlatlandRemoteEvaluationService:
percentage_complete = complete * 1.0 / self.env.get_num_agents()
self.simulation_percentage_complete[-1] = percentage_complete
+ print("Evaluation finished in {} timesteps. Percentage agents done: {:.3f}. Normalized reward: {:.3f}.".format(
+ self.simulation_steps[-1],
+ self.simulation_percentage_complete[-1],
+ self.simulation_rewards_normalized[-1]
+ ))
+
# Record Frame
if self.visualize:
"""
@@ -673,10 +689,10 @@ class FlatlandRemoteEvaluationService:
min_key = "{}_min".format(metric_name)
max_key = "{}_max".format(metric_name)
print("\t - {}\t => min: {} || mean: {} || max: {}".format(
- metric_name,
- self.stats[min_key],
- self.stats[mean_key],
- self.stats[max_key]))
+ metric_name,
+ self.stats[min_key],
+ self.stats[mean_key],
+ self.stats[max_key]))
print("=" * 100)
# Register simulation time of the last episode
@@ -732,7 +748,7 @@ class FlatlandRemoteEvaluationService:
if self.result_output_path:
self.evaluation_metadata_df.to_csv(self.result_output_path)
print("Wrote output results to : {}".format(self.result_output_path))
-
+
# Upload the metadata file to S3
if aicrowd_helpers.is_grading() and aicrowd_helpers.is_aws_configured():
metadata_s3_key = aicrowd_helpers.upload_to_s3(
@@ -848,9 +864,9 @@ class FlatlandRemoteEvaluationService:
print("Self.Reward : ", self.reward)
print("Current Simulation : ", self.simulation_count)
if self.env_file_paths and \
- self.simulation_count < len(self.env_file_paths):
+ self.simulation_count < len(self.env_file_paths):
print("Current Env Path : ",
- self.env_file_paths[self.simulation_count])
+ self.env_file_paths[self.simulation_count])
try:
if command['type'] == messages.FLATLAND_RL.PING:
--
GitLab