Merge branch 'fix-evaluation-reset'

flatland/evaluators/service.py

@@ -43,14 +43,14 @@ m.patch()
 # CONSTANTS
 ########################################################
 INTIAL_PLANNING_TIMEOUT = int(os.getenv(
-                                "FLATLAND_INITIAL_PLANNING_TIMEOUT",
-                                5 * 60))  # 5 mins
+    "FLATLAND_INITIAL_PLANNING_TIMEOUT",
+    5 * 60))  # 5 mins
 PER_STEP_TIMEOUT = int(os.getenv(
-                                "FLATLAND_PER_STEP_TIMEOUT",
-                                5))  # 5 seconds
+    "FLATLAND_PER_STEP_TIMEOUT",
+    5))  # 5 seconds
 DEFAULT_COMMAND_TIMEOUT = int(os.getenv(
-                                "FLATLAND_DEFAULT_COMMAND_TIMEOUT",
-                                1 * 60))  # 1 min
+    "FLATLAND_DEFAULT_COMMAND_TIMEOUT",
+    1 * 60))  # 1 min
 # This applies to the rest of the commands
 
 
@@ -150,6 +150,7 @@ class FlatlandRemoteEvaluationService:
         self.env = False
         self.env_renderer = False
         self.reward = 0
+        self.simulation_done = True
         self.simulation_count = -1
         self.simulation_env_file_paths = []
         self.simulation_rewards = []
@@ -249,7 +250,7 @@ class FlatlandRemoteEvaluationService:
         ) for x in env_paths])
 
         return env_paths
-    
+
     def instantiate_evaluation_metadata(self):
         """
             This instantiates a pandas dataframe to record
@@ -261,9 +262,9 @@ class FlatlandRemoteEvaluationService:
         """
         self.evaluation_metadata_df = None
         metadata_file_path = os.path.join(
-                self.test_env_folder,
-                "metadata.csv"
-            )
+            self.test_env_folder,
+            "metadata.csv"
+        )
         if os.path.exists(metadata_file_path):
             self.evaluation_metadata_df = pd.read_csv(metadata_file_path)
             self.evaluation_metadata_df["filename"] = \
@@ -293,7 +294,7 @@ class FlatlandRemoteEvaluationService:
         for the **previous** episode in the metadata_df if it exists.
         """
         if self.evaluation_metadata_df is not None and len(self.simulation_env_file_paths) > 0:
-            
+
             last_simulation_env_file_path = self.simulation_env_file_paths[-1]
 
             _row = self.evaluation_metadata_df.loc[
@@ -373,7 +374,7 @@ class FlatlandRemoteEvaluationService:
         packed message, and consider the timeouts, etc when trying to
         fetch a new command.
         """
-        
+
         COMMAND_TIMEOUT = DEFAULT_COMMAND_TIMEOUT
         """
         Handle case specific timeouts :
@@ -402,7 +403,7 @@ class FlatlandRemoteEvaluationService:
             If the user has already done an env_submit call, then the timeout 
             can be an arbitrarily large number.
             """
-            COMMAND_TIMEOUT = 10**6
+            COMMAND_TIMEOUT = 10 ** 6
 
         @timeout_decorator.timeout(
             COMMAND_TIMEOUT,
@@ -486,7 +487,14 @@ class FlatlandRemoteEvaluationService:
         Handles a ENV_CREATE command from the client
         TODO: Add a high level summary of everything thats happening here.
         """
+        if not self.simulation_done:
+            # trying to reset a simulation before finishing the previous one
+            _command_response = self._error_template("CAN'T CREATE NEW ENV BEFORE PREVIOUS IS DONE")
+            self.send_response(_command_response, command)
+            raise Exception(_command_response['payload'])
+
         self.simulation_count += 1
+        self.simulation_done = False
         if self.simulation_count < len(self.env_file_paths):
             """
             There are still test envs left that are yet to be evaluated 
@@ -622,6 +630,8 @@ class FlatlandRemoteEvaluationService:
             )
 
         if done["__all__"]:
+            self.simulation_done = True
+
             # Compute percentage complete
             complete = 0
             for i_agent in range(self.env.get_num_agents()):
@@ -631,6 +641,12 @@ class FlatlandRemoteEvaluationService:
             percentage_complete = complete * 1.0 / self.env.get_num_agents()
             self.simulation_percentage_complete[-1] = percentage_complete
 
+            print("Evaluation finished in {} timesteps. Percentage agents done: {:.3f}. Normalized reward: {:.3f}.".format(
+                self.simulation_steps[-1],
+                self.simulation_percentage_complete[-1],
+                self.simulation_rewards_normalized[-1]
+            ))
+
         # Record Frame
         if self.visualize:
             """
@@ -673,10 +689,10 @@ class FlatlandRemoteEvaluationService:
                 min_key = "{}_min".format(metric_name)
                 max_key = "{}_max".format(metric_name)
                 print("\t - {}\t => min: {} || mean: {} || max: {}".format(
-                            metric_name,
-                            self.stats[min_key],
-                            self.stats[mean_key],
-                            self.stats[max_key]))
+                    metric_name,
+                    self.stats[min_key],
+                    self.stats[mean_key],
+                    self.stats[max_key]))
         print("=" * 100)
 
         # Register simulation time of the last episode
@@ -732,7 +748,7 @@ class FlatlandRemoteEvaluationService:
         if self.result_output_path:
             self.evaluation_metadata_df.to_csv(self.result_output_path)
             print("Wrote output results to : {}".format(self.result_output_path))
-            
+
             # Upload the metadata file to S3 
             if aicrowd_helpers.is_grading() and aicrowd_helpers.is_aws_configured():
                 metadata_s3_key = aicrowd_helpers.upload_to_s3(
@@ -848,9 +864,9 @@ class FlatlandRemoteEvaluationService:
                 print("Self.Reward : ", self.reward)
                 print("Current Simulation : ", self.simulation_count)
                 if self.env_file_paths and \
-                        self.simulation_count < len(self.env_file_paths):
+                    self.simulation_count < len(self.env_file_paths):
                     print("Current Env Path : ",
-                        self.env_file_paths[self.simulation_count])
+                          self.env_file_paths[self.simulation_count])
 
             try:
                 if command['type'] == messages.FLATLAND_RL.PING: