Pass controller inference time from client to server

flatland/evaluators/client.py

@@ -253,15 +253,16 @@ class FlatlandRemoteClient(object):
     def env_step(self, action, render=False):
         """
             Respond with [observation, reward, done, info]
-        """
+        """        
+        # We use the last_env_step_time as an approximate measure of the inference time
+        approximate_inference_time = time.time() - self.last_env_step_time
+        self.update_running_stats("inference_time(approx)", approximate_inference_time)
+
         _request = {}
         _request['type'] = messages.FLATLAND_RL.ENV_STEP
         _request['payload'] = {}
         _request['payload']['action'] = action
-        
-        # We use the last_env_step_time as an approximate measure of the inference time
-        approximate_inference_time = time.time() - self.last_env_step_time
-        self.update_running_stats("inference_time(approx)", approximate_inference_time)
+        _request['payload']['inference_time'] = approximate_inference_time
 
         # Relay the action in a non-blocking way to the server
         # so that it can start doing an env.step on it in ~ parallel

flatland/evaluators/service.py

@@ -196,7 +196,6 @@ class FlatlandRemoteEvaluationService:
             self.stats[max_key] = scalar
             self.stats[counter_key] = 1
 
-
     def get_env_filepaths(self):
         """
         Gathers a list of all available rail env files to be used
@@ -246,8 +245,9 @@ class FlatlandRemoteEvaluationService:
             )
         if os.path.exists(metadata_file_path):
             self.evaluation_metadata_df = pd.read_csv(metadata_file_path)
-            self.evaluation_metadata_df["filename"] = self.evaluation_metadata_df["test_id"] + \
-                                                    "/" + self.evaluation_metadata_df["env_id"] + ".pkl"
+            self.evaluation_metadata_df["filename"] = \
+                self.evaluation_metadata_df["test_id"] + \
+                "/" + self.evaluation_metadata_df["env_id"] + ".pkl"
             self.evaluation_metadata_df = self.evaluation_metadata_df.set_index("filename")
 
             # Add custom columns
@@ -458,7 +458,6 @@ class FlatlandRemoteEvaluationService:
                                malfunction_generator_and_process_data=malfunction_from_file(test_env_file_path),
                                obs_builder_object=DummyObservationBuilder())
 
-
             if self.begin_simulation:
                 # If begin simulation has already been initialized
                 # atleast once
@@ -548,6 +547,9 @@ class FlatlandRemoteEvaluationService:
                 has done['__all__']==True")
 
         action = _payload['action']
+        inference_time = _payload['inference_time']
+        self.update_running_stats("controller_inference_time", inference_time)
+
         time_start = time.time()
         _observation, all_rewards, done, info = self.env.step(action)
         time_diff = time.time() - time_start
@@ -625,7 +627,6 @@ class FlatlandRemoteEvaluationService:
                             self.stats[max_key]))
         print("=" * 100)
 
-
         # Register simulation time of the last episode
         self.simulation_times.append(time.time() - self.begin_simulation)
         # Compute the evaluation metadata for the last episode